#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}" BASE_URL="${BASE_URL:-http://localhost:5270}" RABBITMQ_MGMT_URL="${RABBITMQ_MGMT_URL:-http://localhost:15674}" RABBITMQ_USER="${RABBITMQ_USER:-guest}" RABBITMQ_PASS="${RABBITMQ_PASS:-guest}" MINIO_ENDPOINT="${MINIO_ENDPOINT:-localhost:9005}" MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-minioadmin}" MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-minioadmin}" MINIO_BUCKET="${MINIO_BUCKET:-vigilcare}" PGHOST="${PGHOST:-localhost}" PGPORT="${PGPORT:-5436}" PGDATABASE="${PGDATABASE:-vigilcare}" PGUSER="${PGUSER:-postgres}" PGPASSWORD="${PGPASSWORD:-password}" RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}" KAFKA_READY_WAIT_SECS="${KAFKA_READY_WAIT_SECS:-30}" PAGING_PIPELINE_WAIT_SECS="${PAGING_PIPELINE_WAIT_SECS:-30}" MINIO_WAIT_SECS="${MINIO_WAIT_SECS:-15}" ACK_PIPELINE_WAIT_SECS="${ACK_PIPELINE_WAIT_SECS:-10}" SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")" RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" EXPECTED_QUEUES=( "alerts.paging.queue" "alerts.paging.dlq" "alerts.escalation.queue" "notifications.discharge.queue" "notifications.appointment.queue" ) TMP_FILES=() cleanup() { local f for f in "${TMP_FILES[@]}"; do rm -f "${f}" "${f}.status" 2>/dev/null || true done } trap cleanup EXIT if ! command -v curl >/dev/null 2>&1; then echo "Missing dependency: curl" exit 1 fi if ! command -v jq >/dev/null 2>&1; then echo "Missing dependency: jq" exit 1 fi if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then echo "Missing dependency: docker compose (${COMPOSE_FILE})" exit 1 fi compose() { docker compose -f "${COMPOSE_FILE}" "$@" } kafka_exec() { compose exec -T kafka "$@" } psql_cmd() { local sql="$1" if command -v psql >/dev/null 2>&1; then PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" else compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" fi } request() { local method="$1" local url="$2" local body="${3:-}" local tmp_body tmp_body="$(mktemp)" TMP_FILES+=("${tmp_body}") local status if [[ -n "${body}" ]]; then status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}" \ -H "Content-Type: application/json" -d "${body}")" else status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}")" fi echo "${status}" > "${tmp_body}.status" echo "${tmp_body}" } assert_status() { local expected="$1" local body_file="$2" local status status="$(cat "${body_file}.status")" if [[ "${status}" != "${expected}" ]]; then echo "Expected HTTP ${expected}, got ${status}" echo "Response body:" cat "${body_file}" echo return 1 fi } rabbit_api() { curl -sS -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" "${RABBITMQ_MGMT_URL}/api/${1}" } queue_field() { local queue="$1" local field="$2" rabbit_api "queues/%2F/${queue}" | jq -r ".${field} // 0" } wait_for_kafka() { local elapsed=0 while (( elapsed < KAFKA_READY_WAIT_SECS )); do if kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1; then return 0 fi sleep 2 elapsed=$((elapsed + 2)) done echo "Kafka did not become ready within ${KAFKA_READY_WAIT_SECS}s" return 1 } wait_for_outbox_processed() { local outbox_id="$1" local elapsed=0 while (( elapsed < RELAY_WAIT_SECS )); do local processed processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")" if [[ "${processed}" == "t" ]]; then return 0 fi sleep 1 elapsed=$((elapsed + 1)) done echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s" return 1 } wait_for_alert_outbox_relayed() { local encounter_id="$1" local elapsed=0 while (( elapsed < RELAY_WAIT_SECS )); do local outbox_id outbox_id="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = 'alert.generated' AND partition_key = '${encounter_id}' ORDER BY created_at DESC LIMIT 1")" if [[ -n "${outbox_id}" ]]; then local processed processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")" if [[ "${processed}" == "t" ]]; then return 0 fi fi sleep 1 elapsed=$((elapsed + 1)) done echo "alert.generated outbox for encounter ${encounter_id} was not relayed within ${RELAY_WAIT_SECS}s" return 1 } wait_for_paging_activity() { local elapsed=0 while (( elapsed < PAGING_PIPELINE_WAIT_SECS )); do local unacked ready total unacked="$(queue_field "alerts.paging.queue" "messages_unacknowledged")" ready="$(queue_field "alerts.paging.queue" "messages_ready")" total="$(queue_field "alerts.paging.queue" "messages")" if [[ "${unacked}" -ge 1 || "${ready}" -ge 1 || "${total}" -ge 1 ]]; then return 0 fi sleep 1 elapsed=$((elapsed + 1)) done echo "No paging activity on alerts.paging.queue within ${PAGING_PIPELINE_WAIT_SECS}s" return 1 } wait_for_alert_status() { local alert_id="$1" local expected="$2" local timeout_secs="$3" local elapsed=0 while (( elapsed < timeout_secs )); do local status status="$(psql_cmd "SELECT status FROM clinical_alerts WHERE id = '${alert_id}'")" if [[ "${status}" == "${expected}" ]]; then return 0 fi sleep 1 elapsed=$((elapsed + 1)) done local final final="$(psql_cmd "SELECT status FROM clinical_alerts WHERE id = '${alert_id}'")" echo "Alert ${alert_id} did not reach status ${expected} within ${timeout_secs}s (last status=${final:-unknown})" return 1 } minio_object_exists() { local object_key="$1" if command -v mc >/dev/null 2>&1; then mc alias set local "http://${MINIO_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" --api S3v4 >/dev/null 2>&1 || true mc stat "local/${MINIO_BUCKET}/${object_key}" >/dev/null 2>&1 return $? fi docker run --rm --network host --entrypoint /bin/sh minio/mc:latest \ -c "mc alias set local http://${MINIO_ENDPOINT} ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} >/dev/null 2>&1 && mc stat local/${MINIO_BUCKET}/${object_key}" \ >/dev/null 2>&1 } create_patient() { local payload='{"firstName":"Notify","lastName":"Verifier","dateOfBirth":"1972-03-18","gender":"Female"}' local resp resp="$(request POST "${BASE_URL}/api/v1/patients" "${payload}")" assert_status "201" "${resp}" jq -r '.data.id' "${resp}" } create_encounter() { local patient_id="$1" local suffix="$2" local payload payload="$(jq -nc \ --arg physician "Dr. Notify ${suffix}" \ '{encounterType:"Inpatient",department:"ICU",attendingPhysician:$physician}')" local resp resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${payload}")" assert_status "201" "${resp}" jq -r '.data.id' "${resp}" } ingest_critical_potassium() { local encounter_id="$1" local idempotency_key="$2" local payload payload="$(jq -nc \ --arg recordedAt "${RECORDED_AT}" \ --arg key "${idempotency_key}" \ '{observations:[{observationCode:"POTASSIUM_MEQ_L",value:2.1,unit:"mEq/L",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')" local resp resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")" assert_status "201" "${resp}" if [[ "$(jq -r '.data.alertGenerated' "${resp}")" != "true" ]]; then echo "Expected critical potassium ingest to generate an alert" exit 1 fi jq -r '.data.alertId' "${resp}" } TOTAL_STEPS=8 echo "Running RabbitMQ + MinIO notification pipeline verification against ${BASE_URL}" echo "Script run id: ${SCRIPT_RUN_ID}" echo "" echo "[0/${TOTAL_STEPS}] Preflight — API, Postgres, Kafka, RabbitMQ, and MinIO reachable" preflight_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" if [[ "${preflight_status}" != "200" ]]; then echo "API not reachable at ${BASE_URL} (HTTP ${preflight_status})." echo "Start the API with: dotnet run --project VigilCareClinicalAPI" exit 1 fi if ! psql_cmd "SELECT 1" >/dev/null 2>&1; then echo "Postgres not reachable on ${PGHOST}:${PGPORT}." echo "Start the stack with: docker compose up -d" exit 1 fi if ! wait_for_kafka; then exit 1 fi rabbit_health="$(rabbit_api "health/checks/alarms" 2>/dev/null | jq -r '.status // empty' || true)" if [[ -z "${rabbit_health}" ]]; then echo "RabbitMQ management API not reachable at ${RABBITMQ_MGMT_URL}." echo "Start RabbitMQ with: docker compose up -d rabbitmq" exit 1 fi if ! curl -sS -o /dev/null -w "%{http_code}" "http://${MINIO_ENDPOINT}/minio/health/live" | grep -q '^200$'; then echo "MinIO not reachable at http://${MINIO_ENDPOINT}." echo "Start MinIO with: docker compose up -d minio" exit 1 fi echo "OK: API, Postgres, Kafka, RabbitMQ, and MinIO are up" echo "" echo "[1/${TOTAL_STEPS}] Verifying RabbitMQ topology (exchange, five queues, DLQ arguments)" exchange_name="$(rabbit_api "exchanges/%2F/clinical.notifications.exchange" | jq -r '.name // empty')" if [[ "${exchange_name}" != "clinical.notifications.exchange" ]]; then echo "Exchange clinical.notifications.exchange not found. Start the API so RabbitMqTopologyProvisioner runs." exit 1 fi for queue in "${EXPECTED_QUEUES[@]}"; do queue_name="$(rabbit_api "queues/%2F/${queue}" | jq -r '.name // empty')" if [[ "${queue_name}" != "${queue}" ]]; then echo "Queue ${queue} not found (got '${queue_name}')." exit 1 fi done dlq_ttl_ms="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-message-ttl"] // empty')" dlq_dlx="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-dead-letter-exchange"] // empty')" dlq_dlk="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-dead-letter-routing-key"] // empty')" paging_dlk="$(rabbit_api "queues/%2F/alerts.paging.queue" | jq -r '.arguments["x-dead-letter-routing-key"] // empty')" if [[ -z "${dlq_ttl_ms}" ]]; then echo "alerts.paging.dlq is missing x-message-ttl" exit 1 fi if [[ "${dlq_dlx}" != "clinical.notifications.exchange" ]]; then echo "Expected DLQ x-dead-letter-exchange=clinical.notifications.exchange, got '${dlq_dlx}'" exit 1 fi if [[ "${dlq_dlk}" != "alerts.escalation" ]]; then echo "Expected DLQ x-dead-letter-routing-key=alerts.escalation, got '${dlq_dlk}'" exit 1 fi if [[ "${paging_dlk}" != "alerts.paging.dlq" ]]; then echo "Expected paging queue DLK=alerts.paging.dlq, got '${paging_dlk}'" exit 1 fi ttl_sec=$(( dlq_ttl_ms / 1000 )) ESCALATION_WAIT_SECS="${ESCALATION_WAIT_SECS:-$(( ttl_sec * 2 + 25 ))}" echo "OK: exchange + 5 queues provisioned; DLQ TTL=${dlq_ttl_ms}ms (escalation wait budget=${ESCALATION_WAIT_SECS}s)" echo "" echo "[2/${TOTAL_STEPS}] Verifying non-alert observation does not enqueue paging jobs" patient_no_page="$(create_patient)" encounter_no_page="$(create_encounter "${patient_no_page}" "NoPage")" paging_before="$(queue_field "alerts.paging.queue" "messages")" normal_payload="$(jq -nc \ --arg recordedAt "${RECORDED_AT}" \ --arg key "notify-normal-${SCRIPT_RUN_ID}" \ '{observations:[{observationCode:"POTASSIUM_MEQ_L",value:3.8,unit:"mEq/L",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')" resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_no_page}/observations" "${normal_payload}")" assert_status "201" "${resp}" if [[ "$(jq -r '.data.alertGenerated' "${resp}")" != "false" ]]; then echo "Expected normal potassium ingest to skip alert generation" exit 1 fi sleep 3 paging_after="$(queue_field "alerts.paging.queue" "messages")" if [[ "${paging_after}" -gt "${paging_before}" ]]; then echo "Paging queue grew after non-alert ingest (${paging_before} -> ${paging_after})" exit 1 fi echo "OK: normal observation produced no paging queue traffic" echo "" echo "[3/${TOTAL_STEPS}] Verifying critical alert reaches alerts.paging.queue via Kafka bridge" patient_page="$(create_patient)" encounter_page="$(create_encounter "${patient_page}" "Page")" alert_page_id="$(ingest_critical_potassium "${encounter_page}" "notify-critical-page-${SCRIPT_RUN_ID}")" if ! wait_for_alert_outbox_relayed "${encounter_page}"; then exit 1 fi if ! wait_for_paging_activity; then echo "Hint: confirm NotificationPublisherService is running and alert.generated severity is Critical." exit 1 fi echo "OK: critical alert ${alert_page_id} reached alerts.paging.queue (worker holding or ready message present)" echo "" echo "[4/${TOTAL_STEPS}] Verifying acknowledge-before-timeout path (no escalation)" ack_payload='{"clinicianId":"DR-SCRIPT","note":"Acknowledged from notification verification script."}' resp="$(request POST "${BASE_URL}/api/v1/alerts/${alert_page_id}/acknowledge" "${ack_payload}")" assert_status "200" "${resp}" sleep "${ACK_PIPELINE_WAIT_SECS}" if ! wait_for_alert_status "${alert_page_id}" "ACKNOWLEDGED" 5; then exit 1 fi echo "OK: alert ${alert_page_id} is ACKNOWLEDGED (not escalated)" echo "" echo "[5/${TOTAL_STEPS}] Verifying full DLQ escalation path (unacknowledged critical alert)" patient_escalate="$(create_patient)" encounter_escalate="$(create_encounter "${patient_escalate}" "Escalate")" alert_escalate_id="$(ingest_critical_potassium "${encounter_escalate}" "notify-critical-escalate-${SCRIPT_RUN_ID}")" if ! wait_for_alert_outbox_relayed "${encounter_escalate}"; then exit 1 fi if ! wait_for_paging_activity; then exit 1 fi echo "Waiting up to ${ESCALATION_WAIT_SECS}s for paging timeout + DLQ TTL + escalation worker..." if ! wait_for_alert_status "${alert_escalate_id}" "ESCALATED" "${ESCALATION_WAIT_SECS}"; then echo "Hint: default DLQ TTL is 300000ms (~10 min total). Set PagingAckTimeoutMs=5000 in appsettings.json and restart the API for a faster run, or raise ESCALATION_WAIT_SECS." exit 1 fi echo "OK: alert ${alert_escalate_id} escalated to ESCALATED in PostgreSQL" echo "" echo "[6/${TOTAL_STEPS}] Verifying discharge summary upload to MinIO" patient_discharge="$(create_patient)" encounter_discharge="$(create_encounter "${patient_discharge}" "Discharge")" discharge_payload='{"status":"Discharged"}' resp="$(request PATCH "${BASE_URL}/api/v1/encounters/${encounter_discharge}/status" "${discharge_payload}")" assert_status "200" "${resp}" discharge_outbox_id="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = 'encounter.status.changed' AND partition_key = '${encounter_discharge}' ORDER BY created_at DESC LIMIT 1")" if [[ -z "${discharge_outbox_id}" ]]; then echo "No encounter.status.changed outbox row for encounter ${encounter_discharge}" exit 1 fi wait_for_outbox_processed "${discharge_outbox_id}" object_key="discharge-summaries/${encounter_discharge}/summary.pdf" elapsed=0 while (( elapsed < MINIO_WAIT_SECS )); do if minio_object_exists "${object_key}"; then break fi sleep 1 elapsed=$((elapsed + 1)) done if ! minio_object_exists "${object_key}"; then echo "MinIO object not found: ${MINIO_BUCKET}/${object_key}" echo "Hint: confirm DischargeSummaryWorkerService is running and notifications.discharge.queue is draining." exit 1 fi echo "OK: discharge summary object exists at ${MINIO_BUCKET}/${object_key}" echo "" echo "[7/${TOTAL_STEPS}] Verifying discharge queue drained after summary upload" elapsed=0 while (( elapsed < MINIO_WAIT_SECS )); do discharge_ready="$(queue_field "notifications.discharge.queue" "messages_ready")" discharge_unacked="$(queue_field "notifications.discharge.queue" "messages_unacknowledged")" if [[ "${discharge_ready}" == "0" && "${discharge_unacked}" == "0" ]]; then break fi sleep 1 elapsed=$((elapsed + 1)) done discharge_ready="$(queue_field "notifications.discharge.queue" "messages_ready")" discharge_unacked="$(queue_field "notifications.discharge.queue" "messages_unacknowledged")" if [[ "${discharge_ready}" != "0" || "${discharge_unacked}" != "0" ]]; then echo "Expected notifications.discharge.queue to be empty (ready=${discharge_ready}, unacked=${discharge_unacked})" exit 1 fi echo "OK: notifications.discharge.queue is drained" echo "" echo "All ${TOTAL_STEPS} notification pipeline checks passed." echo "" echo "Prerequisites: docker compose up -d && dotnet run --project VigilCareClinicalAPI" echo "Optional: set PagingAckTimeoutMs=5000 in appsettings.json (restart API) for ~15s escalation instead of ~10 min." echo "Optional: ESCALATION_WAIT_SECS=330 when using the production 300000ms DLQ TTL."