Files
vigilcare-clinical/scripts/run-notification-pipeline-tests.sh
T
voltsrage 2a3ef62a7d
CI / frontend (push) Failing after 57s
CI / backend (push) Failing after 6m27s
Add deployment
2026-08-05 00:26:20 +08:00

475 lines
16 KiB
Bash

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}"
BASE_URL="${BASE_URL:-http://localhost:5270}"
RABBITMQ_MGMT_URL="${RABBITMQ_MGMT_URL:-http://localhost:15674}"
RABBITMQ_USER="${RABBITMQ_USER:-guest}"
RABBITMQ_PASS="${RABBITMQ_PASS:-guest}"
MINIO_ENDPOINT="${MINIO_ENDPOINT:-localhost:9005}"
MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-minioadmin}"
MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-minioadmin}"
MINIO_BUCKET="${MINIO_BUCKET:-vigilcare}"
PGHOST="${PGHOST:-localhost}"
PGPORT="${PGPORT:-5436}"
PGDATABASE="${PGDATABASE:-vigilcare}"
PGUSER="${PGUSER:-postgres}"
PGPASSWORD="${PGPASSWORD:-password}"
RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}"
KAFKA_READY_WAIT_SECS="${KAFKA_READY_WAIT_SECS:-30}"
PAGING_PIPELINE_WAIT_SECS="${PAGING_PIPELINE_WAIT_SECS:-30}"
MINIO_WAIT_SECS="${MINIO_WAIT_SECS:-15}"
ACK_PIPELINE_WAIT_SECS="${ACK_PIPELINE_WAIT_SECS:-10}"
SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")"
RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
EXPECTED_QUEUES=(
"alerts.paging.queue"
"alerts.paging.dlq"
"alerts.escalation.queue"
"notifications.discharge.queue"
"notifications.appointment.queue"
)
TMP_FILES=()
cleanup() {
local f
for f in "${TMP_FILES[@]}"; do
rm -f "${f}" "${f}.status" 2>/dev/null || true
done
}
trap cleanup EXIT
if ! command -v curl >/dev/null 2>&1; then
echo "Missing dependency: curl"
exit 1
fi
if ! command -v jq >/dev/null 2>&1; then
echo "Missing dependency: jq"
exit 1
fi
if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then
echo "Missing dependency: docker compose (${COMPOSE_FILE})"
exit 1
fi
compose() {
docker compose -f "${COMPOSE_FILE}" "$@"
}
kafka_exec() {
compose exec -T kafka "$@"
}
psql_cmd() {
local sql="$1"
if command -v psql >/dev/null 2>&1; then
PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
else
compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
fi
}
request() {
local method="$1"
local url="$2"
local body="${3:-}"
local tmp_body
tmp_body="$(mktemp)"
TMP_FILES+=("${tmp_body}")
local status
if [[ -n "${body}" ]]; then
status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}" \
-H "Content-Type: application/json" -d "${body}")"
else
status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}")"
fi
echo "${status}" > "${tmp_body}.status"
echo "${tmp_body}"
}
assert_status() {
local expected="$1"
local body_file="$2"
local status
status="$(cat "${body_file}.status")"
if [[ "${status}" != "${expected}" ]]; then
echo "Expected HTTP ${expected}, got ${status}"
echo "Response body:"
cat "${body_file}"
echo
return 1
fi
}
rabbit_api() {
curl -sS -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" "${RABBITMQ_MGMT_URL}/api/${1}"
}
queue_field() {
local queue="$1"
local field="$2"
rabbit_api "queues/%2F/${queue}" | jq -r ".${field} // 0"
}
wait_for_kafka() {
local elapsed=0
while (( elapsed < KAFKA_READY_WAIT_SECS )); do
if kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1; then
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
echo "Kafka did not become ready within ${KAFKA_READY_WAIT_SECS}s"
return 1
}
wait_for_outbox_processed() {
local outbox_id="$1"
local elapsed=0
while (( elapsed < RELAY_WAIT_SECS )); do
local processed
processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")"
if [[ "${processed}" == "t" ]]; then
return 0
fi
sleep 1
elapsed=$((elapsed + 1))
done
echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s"
return 1
}
wait_for_alert_outbox_relayed() {
local encounter_id="$1"
local elapsed=0
while (( elapsed < RELAY_WAIT_SECS )); do
local outbox_id
outbox_id="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = 'alert.generated' AND partition_key = '${encounter_id}' ORDER BY created_at DESC LIMIT 1")"
if [[ -n "${outbox_id}" ]]; then
local processed
processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")"
if [[ "${processed}" == "t" ]]; then
return 0
fi
fi
sleep 1
elapsed=$((elapsed + 1))
done
echo "alert.generated outbox for encounter ${encounter_id} was not relayed within ${RELAY_WAIT_SECS}s"
return 1
}
wait_for_paging_activity() {
local elapsed=0
while (( elapsed < PAGING_PIPELINE_WAIT_SECS )); do
local unacked ready total
unacked="$(queue_field "alerts.paging.queue" "messages_unacknowledged")"
ready="$(queue_field "alerts.paging.queue" "messages_ready")"
total="$(queue_field "alerts.paging.queue" "messages")"
if [[ "${unacked}" -ge 1 || "${ready}" -ge 1 || "${total}" -ge 1 ]]; then
return 0
fi
sleep 1
elapsed=$((elapsed + 1))
done
echo "No paging activity on alerts.paging.queue within ${PAGING_PIPELINE_WAIT_SECS}s"
return 1
}
wait_for_alert_status() {
local alert_id="$1"
local expected="$2"
local timeout_secs="$3"
local elapsed=0
while (( elapsed < timeout_secs )); do
local status
status="$(psql_cmd "SELECT status FROM clinical_alerts WHERE id = '${alert_id}'")"
if [[ "${status}" == "${expected}" ]]; then
return 0
fi
sleep 1
elapsed=$((elapsed + 1))
done
local final
final="$(psql_cmd "SELECT status FROM clinical_alerts WHERE id = '${alert_id}'")"
echo "Alert ${alert_id} did not reach status ${expected} within ${timeout_secs}s (last status=${final:-unknown})"
return 1
}
minio_object_exists() {
local object_key="$1"
if command -v mc >/dev/null 2>&1; then
mc alias set local "http://${MINIO_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" --api S3v4 >/dev/null 2>&1 || true
mc stat "local/${MINIO_BUCKET}/${object_key}" >/dev/null 2>&1
return $?
fi
docker run --rm --network host --entrypoint /bin/sh minio/mc:latest \
-c "mc alias set local http://${MINIO_ENDPOINT} ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} >/dev/null 2>&1 && mc stat local/${MINIO_BUCKET}/${object_key}" \
>/dev/null 2>&1
}
create_patient() {
local payload='{"firstName":"Notify","lastName":"Verifier","dateOfBirth":"1972-03-18","gender":"Female"}'
local resp
resp="$(request POST "${BASE_URL}/api/v1/patients" "${payload}")"
assert_status "201" "${resp}"
jq -r '.data.id' "${resp}"
}
create_encounter() {
local patient_id="$1"
local suffix="$2"
local payload
payload="$(jq -nc \
--arg physician "Dr. Notify ${suffix}" \
'{encounterType:"Inpatient",department:"ICU",attendingPhysician:$physician}')"
local resp
resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${payload}")"
assert_status "201" "${resp}"
jq -r '.data.id' "${resp}"
}
ingest_critical_potassium() {
local encounter_id="$1"
local idempotency_key="$2"
local payload
payload="$(jq -nc \
--arg recordedAt "${RECORDED_AT}" \
--arg key "${idempotency_key}" \
'{observations:[{observationCode:"POTASSIUM_MEQ_L",value:2.1,unit:"mEq/L",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')"
local resp
resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")"
assert_status "201" "${resp}"
if [[ "$(jq -r '.data.alertGenerated' "${resp}")" != "true" ]]; then
echo "Expected critical potassium ingest to generate an alert"
exit 1
fi
jq -r '.data.alertId' "${resp}"
}
TOTAL_STEPS=8
echo "Running RabbitMQ + MinIO notification pipeline verification against ${BASE_URL}"
echo "Script run id: ${SCRIPT_RUN_ID}"
echo ""
echo "[0/${TOTAL_STEPS}] Preflight — API, Postgres, Kafka, RabbitMQ, and MinIO reachable"
preflight_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
if [[ "${preflight_status}" != "200" ]]; then
echo "API not reachable at ${BASE_URL} (HTTP ${preflight_status})."
echo "Start the API with: dotnet run --project VigilCareClinicalAPI"
exit 1
fi
if ! psql_cmd "SELECT 1" >/dev/null 2>&1; then
echo "Postgres not reachable on ${PGHOST}:${PGPORT}."
echo "Start the stack with: docker compose up -d"
exit 1
fi
if ! wait_for_kafka; then
exit 1
fi
rabbit_health="$(rabbit_api "health/checks/alarms" 2>/dev/null | jq -r '.status // empty' || true)"
if [[ -z "${rabbit_health}" ]]; then
echo "RabbitMQ management API not reachable at ${RABBITMQ_MGMT_URL}."
echo "Start RabbitMQ with: docker compose up -d rabbitmq"
exit 1
fi
if ! curl -sS -o /dev/null -w "%{http_code}" "http://${MINIO_ENDPOINT}/minio/health/live" | grep -q '^200$'; then
echo "MinIO not reachable at http://${MINIO_ENDPOINT}."
echo "Start MinIO with: docker compose up -d minio"
exit 1
fi
echo "OK: API, Postgres, Kafka, RabbitMQ, and MinIO are up"
echo ""
echo "[1/${TOTAL_STEPS}] Verifying RabbitMQ topology (exchange, five queues, DLQ arguments)"
exchange_name="$(rabbit_api "exchanges/%2F/clinical.notifications.exchange" | jq -r '.name // empty')"
if [[ "${exchange_name}" != "clinical.notifications.exchange" ]]; then
echo "Exchange clinical.notifications.exchange not found. Start the API so RabbitMqTopologyProvisioner runs."
exit 1
fi
for queue in "${EXPECTED_QUEUES[@]}"; do
queue_name="$(rabbit_api "queues/%2F/${queue}" | jq -r '.name // empty')"
if [[ "${queue_name}" != "${queue}" ]]; then
echo "Queue ${queue} not found (got '${queue_name}')."
exit 1
fi
done
dlq_ttl_ms="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-message-ttl"] // empty')"
dlq_dlx="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-dead-letter-exchange"] // empty')"
dlq_dlk="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-dead-letter-routing-key"] // empty')"
paging_dlk="$(rabbit_api "queues/%2F/alerts.paging.queue" | jq -r '.arguments["x-dead-letter-routing-key"] // empty')"
if [[ -z "${dlq_ttl_ms}" ]]; then
echo "alerts.paging.dlq is missing x-message-ttl"
exit 1
fi
if [[ "${dlq_dlx}" != "clinical.notifications.exchange" ]]; then
echo "Expected DLQ x-dead-letter-exchange=clinical.notifications.exchange, got '${dlq_dlx}'"
exit 1
fi
if [[ "${dlq_dlk}" != "alerts.escalation" ]]; then
echo "Expected DLQ x-dead-letter-routing-key=alerts.escalation, got '${dlq_dlk}'"
exit 1
fi
if [[ "${paging_dlk}" != "alerts.paging.dlq" ]]; then
echo "Expected paging queue DLK=alerts.paging.dlq, got '${paging_dlk}'"
exit 1
fi
ttl_sec=$(( dlq_ttl_ms / 1000 ))
ESCALATION_WAIT_SECS="${ESCALATION_WAIT_SECS:-$(( ttl_sec * 2 + 25 ))}"
echo "OK: exchange + 5 queues provisioned; DLQ TTL=${dlq_ttl_ms}ms (escalation wait budget=${ESCALATION_WAIT_SECS}s)"
echo ""
echo "[2/${TOTAL_STEPS}] Verifying non-alert observation does not enqueue paging jobs"
patient_no_page="$(create_patient)"
encounter_no_page="$(create_encounter "${patient_no_page}" "NoPage")"
paging_before="$(queue_field "alerts.paging.queue" "messages")"
normal_payload="$(jq -nc \
--arg recordedAt "${RECORDED_AT}" \
--arg key "notify-normal-${SCRIPT_RUN_ID}" \
'{observations:[{observationCode:"POTASSIUM_MEQ_L",value:3.8,unit:"mEq/L",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')"
resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_no_page}/observations" "${normal_payload}")"
assert_status "201" "${resp}"
if [[ "$(jq -r '.data.alertGenerated' "${resp}")" != "false" ]]; then
echo "Expected normal potassium ingest to skip alert generation"
exit 1
fi
sleep 3
paging_after="$(queue_field "alerts.paging.queue" "messages")"
if [[ "${paging_after}" -gt "${paging_before}" ]]; then
echo "Paging queue grew after non-alert ingest (${paging_before} -> ${paging_after})"
exit 1
fi
echo "OK: normal observation produced no paging queue traffic"
echo ""
echo "[3/${TOTAL_STEPS}] Verifying critical alert reaches alerts.paging.queue via Kafka bridge"
patient_page="$(create_patient)"
encounter_page="$(create_encounter "${patient_page}" "Page")"
alert_page_id="$(ingest_critical_potassium "${encounter_page}" "notify-critical-page-${SCRIPT_RUN_ID}")"
if ! wait_for_alert_outbox_relayed "${encounter_page}"; then
exit 1
fi
if ! wait_for_paging_activity; then
echo "Hint: confirm NotificationPublisherService is running and alert.generated severity is Critical."
exit 1
fi
echo "OK: critical alert ${alert_page_id} reached alerts.paging.queue (worker holding or ready message present)"
echo ""
echo "[4/${TOTAL_STEPS}] Verifying acknowledge-before-timeout path (no escalation)"
ack_payload='{"clinicianId":"DR-SCRIPT","note":"Acknowledged from notification verification script."}'
resp="$(request POST "${BASE_URL}/api/v1/alerts/${alert_page_id}/acknowledge" "${ack_payload}")"
assert_status "200" "${resp}"
sleep "${ACK_PIPELINE_WAIT_SECS}"
if ! wait_for_alert_status "${alert_page_id}" "ACKNOWLEDGED" 5; then
exit 1
fi
echo "OK: alert ${alert_page_id} is ACKNOWLEDGED (not escalated)"
echo ""
echo "[5/${TOTAL_STEPS}] Verifying full DLQ escalation path (unacknowledged critical alert)"
patient_escalate="$(create_patient)"
encounter_escalate="$(create_encounter "${patient_escalate}" "Escalate")"
alert_escalate_id="$(ingest_critical_potassium "${encounter_escalate}" "notify-critical-escalate-${SCRIPT_RUN_ID}")"
if ! wait_for_alert_outbox_relayed "${encounter_escalate}"; then
exit 1
fi
if ! wait_for_paging_activity; then
exit 1
fi
echo "Waiting up to ${ESCALATION_WAIT_SECS}s for paging timeout + DLQ TTL + escalation worker..."
if ! wait_for_alert_status "${alert_escalate_id}" "ESCALATED" "${ESCALATION_WAIT_SECS}"; then
echo "Hint: default DLQ TTL is 300000ms (~10 min total). Set PagingAckTimeoutMs=5000 in appsettings.json and restart the API for a faster run, or raise ESCALATION_WAIT_SECS."
exit 1
fi
echo "OK: alert ${alert_escalate_id} escalated to ESCALATED in PostgreSQL"
echo ""
echo "[6/${TOTAL_STEPS}] Verifying discharge summary upload to MinIO"
patient_discharge="$(create_patient)"
encounter_discharge="$(create_encounter "${patient_discharge}" "Discharge")"
discharge_payload='{"status":"Discharged"}'
resp="$(request PATCH "${BASE_URL}/api/v1/encounters/${encounter_discharge}/status" "${discharge_payload}")"
assert_status "200" "${resp}"
discharge_outbox_id="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = 'encounter.status.changed' AND partition_key = '${encounter_discharge}' ORDER BY created_at DESC LIMIT 1")"
if [[ -z "${discharge_outbox_id}" ]]; then
echo "No encounter.status.changed outbox row for encounter ${encounter_discharge}"
exit 1
fi
wait_for_outbox_processed "${discharge_outbox_id}"
object_key="discharge-summaries/${encounter_discharge}/summary.pdf"
elapsed=0
while (( elapsed < MINIO_WAIT_SECS )); do
if minio_object_exists "${object_key}"; then
break
fi
sleep 1
elapsed=$((elapsed + 1))
done
if ! minio_object_exists "${object_key}"; then
echo "MinIO object not found: ${MINIO_BUCKET}/${object_key}"
echo "Hint: confirm DischargeSummaryWorkerService is running and notifications.discharge.queue is draining."
exit 1
fi
echo "OK: discharge summary object exists at ${MINIO_BUCKET}/${object_key}"
echo ""
echo "[7/${TOTAL_STEPS}] Verifying discharge queue drained after summary upload"
elapsed=0
while (( elapsed < MINIO_WAIT_SECS )); do
discharge_ready="$(queue_field "notifications.discharge.queue" "messages_ready")"
discharge_unacked="$(queue_field "notifications.discharge.queue" "messages_unacknowledged")"
if [[ "${discharge_ready}" == "0" && "${discharge_unacked}" == "0" ]]; then
break
fi
sleep 1
elapsed=$((elapsed + 1))
done
discharge_ready="$(queue_field "notifications.discharge.queue" "messages_ready")"
discharge_unacked="$(queue_field "notifications.discharge.queue" "messages_unacknowledged")"
if [[ "${discharge_ready}" != "0" || "${discharge_unacked}" != "0" ]]; then
echo "Expected notifications.discharge.queue to be empty (ready=${discharge_ready}, unacked=${discharge_unacked})"
exit 1
fi
echo "OK: notifications.discharge.queue is drained"
echo ""
echo "All ${TOTAL_STEPS} notification pipeline checks passed."
echo ""
echo "Prerequisites: docker compose up -d && dotnet run --project VigilCareClinicalAPI"
echo "Optional: set PagingAckTimeoutMs=5000 in appsettings.json (restart API) for ~15s escalation instead of ~10 min."
echo "Optional: ESCALATION_WAIT_SECS=330 when using the production 300000ms DLQ TTL."