feature: RabbitMQ Notification Workers and DLQ Escalation

2026-06-17 02:37:30 +08:00
parent ec780144c6
commit 0246143815
17 changed files with 1410 additions and 2 deletions
@@ -0,0 +1,474 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}"
+
+BASE_URL="${BASE_URL:-http://localhost:5270}"
+RABBITMQ_MGMT_URL="${RABBITMQ_MGMT_URL:-http://localhost:15674}"
+RABBITMQ_USER="${RABBITMQ_USER:-guest}"
+RABBITMQ_PASS="${RABBITMQ_PASS:-guest}"
+MINIO_ENDPOINT="${MINIO_ENDPOINT:-localhost:9005}"
+MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-minioadmin}"
+MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-minioadmin}"
+MINIO_BUCKET="${MINIO_BUCKET:-vigilcare}"
+
+PGHOST="${PGHOST:-localhost}"
+PGPORT="${PGPORT:-5436}"
+PGDATABASE="${PGDATABASE:-vigilcare}"
+PGUSER="${PGUSER:-postgres}"
+PGPASSWORD="${PGPASSWORD:-password}"
+
+RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}"
+KAFKA_READY_WAIT_SECS="${KAFKA_READY_WAIT_SECS:-30}"
+PAGING_PIPELINE_WAIT_SECS="${PAGING_PIPELINE_WAIT_SECS:-30}"
+MINIO_WAIT_SECS="${MINIO_WAIT_SECS:-15}"
+ACK_PIPELINE_WAIT_SECS="${ACK_PIPELINE_WAIT_SECS:-10}"
+
+SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")"
+RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+
+EXPECTED_QUEUES=(
+  "alerts.paging.queue"
+  "alerts.paging.dlq"
+  "alerts.escalation.queue"
+  "notifications.discharge.queue"
+  "notifications.appointment.queue"
+)
+
+TMP_FILES=()
+
+cleanup() {
+  local f
+  for f in "${TMP_FILES[@]}"; do
+    rm -f "${f}" "${f}.status" 2>/dev/null || true
+  done
+}
+trap cleanup EXIT
+
+if ! command -v curl >/dev/null 2>&1; then
+  echo "Missing dependency: curl"
+  exit 1
+fi
+
+if ! command -v jq >/dev/null 2>&1; then
+  echo "Missing dependency: jq"
+  exit 1
+fi
+
+if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then
+  echo "Missing dependency: docker compose (${COMPOSE_FILE})"
+  exit 1
+fi
+
+compose() {
+  docker compose -f "${COMPOSE_FILE}" "$@"
+}
+
+kafka_exec() {
+  compose exec -T kafka "$@"
+}
+
+psql_cmd() {
+  local sql="$1"
+  if command -v psql >/dev/null 2>&1; then
+    PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
+  else
+    compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
+  fi
+}
+
+request() {
+  local method="$1"
+  local url="$2"
+  local body="${3:-}"
+  local tmp_body
+  tmp_body="$(mktemp)"
+  TMP_FILES+=("${tmp_body}")
+  local status
+
+  if [[ -n "${body}" ]]; then
+    status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}" \
+      -H "Content-Type: application/json" -d "${body}")"
+  else
+    status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}")"
+  fi
+
+  echo "${status}" > "${tmp_body}.status"
+  echo "${tmp_body}"
+}
+
+assert_status() {
+  local expected="$1"
+  local body_file="$2"
+  local status
+  status="$(cat "${body_file}.status")"
+  if [[ "${status}" != "${expected}" ]]; then
+    echo "Expected HTTP ${expected}, got ${status}"
+    echo "Response body:"
+    cat "${body_file}"
+    echo
+    return 1
+  fi
+}
+
+rabbit_api() {
+  curl -sS -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" "${RABBITMQ_MGMT_URL}/api/${1}"
+}
+
+queue_field() {
+  local queue="$1"
+  local field="$2"
+  rabbit_api "queues/%2F/${queue}" | jq -r ".${field} // 0"
+}
+
+wait_for_kafka() {
+  local elapsed=0
+  while (( elapsed < KAFKA_READY_WAIT_SECS )); do
+    if kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 2
+    elapsed=$((elapsed + 2))
+  done
+  echo "Kafka did not become ready within ${KAFKA_READY_WAIT_SECS}s"
+  return 1
+}
+
+wait_for_outbox_processed() {
+  local outbox_id="$1"
+  local elapsed=0
+  while (( elapsed < RELAY_WAIT_SECS )); do
+    local processed
+    processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")"
+    if [[ "${processed}" == "t" ]]; then
+      return 0
+    fi
+    sleep 1
+    elapsed=$((elapsed + 1))
+  done
+  echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s"
+  return 1
+}
+
+wait_for_alert_outbox_relayed() {
+  local encounter_id="$1"
+  local elapsed=0
+  while (( elapsed < RELAY_WAIT_SECS )); do
+    local outbox_id
+    outbox_id="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = 'alert.generated' AND partition_key = '${encounter_id}' ORDER BY created_at DESC LIMIT 1")"
+    if [[ -n "${outbox_id}" ]]; then
+      local processed
+      processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")"
+      if [[ "${processed}" == "t" ]]; then
+        return 0
+      fi
+    fi
+    sleep 1
+    elapsed=$((elapsed + 1))
+  done
+  echo "alert.generated outbox for encounter ${encounter_id} was not relayed within ${RELAY_WAIT_SECS}s"
+  return 1
+}
+
+wait_for_paging_activity() {
+  local elapsed=0
+  while (( elapsed < PAGING_PIPELINE_WAIT_SECS )); do
+    local unacked ready total
+    unacked="$(queue_field "alerts.paging.queue" "messages_unacknowledged")"
+    ready="$(queue_field "alerts.paging.queue" "messages_ready")"
+    total="$(queue_field "alerts.paging.queue" "messages")"
+    if [[ "${unacked}" -ge 1 || "${ready}" -ge 1 || "${total}" -ge 1 ]]; then
+      return 0
+    fi
+    sleep 1
+    elapsed=$((elapsed + 1))
+  done
+  echo "No paging activity on alerts.paging.queue within ${PAGING_PIPELINE_WAIT_SECS}s"
+  return 1
+}
+
+wait_for_alert_status() {
+  local alert_id="$1"
+  local expected="$2"
+  local timeout_secs="$3"
+  local elapsed=0
+  while (( elapsed < timeout_secs )); do
+    local status
+    status="$(psql_cmd "SELECT status FROM clinical_alerts WHERE id = '${alert_id}'")"
+    if [[ "${status}" == "${expected}" ]]; then
+      return 0
+    fi
+    sleep 1
+    elapsed=$((elapsed + 1))
+  done
+  local final
+  final="$(psql_cmd "SELECT status FROM clinical_alerts WHERE id = '${alert_id}'")"
+  echo "Alert ${alert_id} did not reach status ${expected} within ${timeout_secs}s (last status=${final:-unknown})"
+  return 1
+}
+
+minio_object_exists() {
+  local object_key="$1"
+  if command -v mc >/dev/null 2>&1; then
+    mc alias set local "http://${MINIO_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" --api S3v4 >/dev/null 2>&1 || true
+    mc stat "local/${MINIO_BUCKET}/${object_key}" >/dev/null 2>&1
+    return $?
+  fi
+
+  docker run --rm --network host --entrypoint /bin/sh minio/mc:latest \
+    -c "mc alias set local http://${MINIO_ENDPOINT} ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} >/dev/null 2>&1 && mc stat local/${MINIO_BUCKET}/${object_key}" \
+    >/dev/null 2>&1
+}
+
+create_patient() {
+  local payload='{"firstName":"Notify","lastName":"Verifier","dateOfBirth":"1972-03-18","gender":"Female"}'
+  local resp
+  resp="$(request POST "${BASE_URL}/api/v1/patients" "${payload}")"
+  assert_status "201" "${resp}"
+  jq -r '.data.id' "${resp}"
+}
+
+create_encounter() {
+  local patient_id="$1"
+  local suffix="$2"
+  local payload
+  payload="$(jq -nc \
+    --arg physician "Dr. Notify ${suffix}" \
+    '{encounterType:"Inpatient",department:"ICU",attendingPhysician:$physician}')"
+  local resp
+  resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${payload}")"
+  assert_status "201" "${resp}"
+  jq -r '.data.id' "${resp}"
+}
+
+ingest_critical_potassium() {
+  local encounter_id="$1"
+  local idempotency_key="$2"
+  local payload
+  payload="$(jq -nc \
+    --arg recordedAt "${RECORDED_AT}" \
+    --arg key "${idempotency_key}" \
+    '{observations:[{observationCode:"POTASSIUM_MEQ_L",value:2.1,unit:"mEq/L",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')"
+  local resp
+  resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")"
+  assert_status "201" "${resp}"
+  if [[ "$(jq -r '.data.alertGenerated' "${resp}")" != "true" ]]; then
+    echo "Expected critical potassium ingest to generate an alert"
+    exit 1
+  fi
+  jq -r '.data.alertId' "${resp}"
+}
+
+TOTAL_STEPS=8
+
+echo "Running RabbitMQ + MinIO notification pipeline verification against ${BASE_URL}"
+echo "Script run id: ${SCRIPT_RUN_ID}"
+
+echo ""
+echo "[0/${TOTAL_STEPS}] Preflight — API, Postgres, Kafka, RabbitMQ, and MinIO reachable"
+preflight_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
+if [[ "${preflight_status}" != "200" ]]; then
+  echo "API not reachable at ${BASE_URL} (HTTP ${preflight_status})."
+  echo "Start the API with: dotnet run --project VigilCareClinicalAPI"
+  exit 1
+fi
+
+if ! psql_cmd "SELECT 1" >/dev/null 2>&1; then
+  echo "Postgres not reachable on ${PGHOST}:${PGPORT}."
+  echo "Start the stack with: docker compose up -d"
+  exit 1
+fi
+
+if ! wait_for_kafka; then
+  exit 1
+fi
+
+rabbit_health="$(rabbit_api "health/checks/alarms" 2>/dev/null | jq -r '.status // empty' || true)"
+if [[ -z "${rabbit_health}" ]]; then
+  echo "RabbitMQ management API not reachable at ${RABBITMQ_MGMT_URL}."
+  echo "Start RabbitMQ with: docker compose up -d rabbitmq"
+  exit 1
+fi
+
+if ! curl -sS -o /dev/null -w "%{http_code}" "http://${MINIO_ENDPOINT}/minio/health/live" | grep -q '^200$'; then
+  echo "MinIO not reachable at http://${MINIO_ENDPOINT}."
+  echo "Start MinIO with: docker compose up -d minio"
+  exit 1
+fi
+echo "OK: API, Postgres, Kafka, RabbitMQ, and MinIO are up"
+
+echo ""
+echo "[1/${TOTAL_STEPS}] Verifying RabbitMQ topology (exchange, five queues, DLQ arguments)"
+exchange_name="$(rabbit_api "exchanges/%2F/clinical.notifications.exchange" | jq -r '.name // empty')"
+if [[ "${exchange_name}" != "clinical.notifications.exchange" ]]; then
+  echo "Exchange clinical.notifications.exchange not found. Start the API so RabbitMqTopologyProvisioner runs."
+  exit 1
+fi
+
+for queue in "${EXPECTED_QUEUES[@]}"; do
+  queue_name="$(rabbit_api "queues/%2F/${queue}" | jq -r '.name // empty')"
+  if [[ "${queue_name}" != "${queue}" ]]; then
+    echo "Queue ${queue} not found (got '${queue_name}')."
+    exit 1
+  fi
+done
+
+dlq_ttl_ms="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-message-ttl"] // empty')"
+dlq_dlx="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-dead-letter-exchange"] // empty')"
+dlq_dlk="$(rabbit_api "queues/%2F/alerts.paging.dlq" | jq -r '.arguments["x-dead-letter-routing-key"] // empty')"
+paging_dlk="$(rabbit_api "queues/%2F/alerts.paging.queue" | jq -r '.arguments["x-dead-letter-routing-key"] // empty')"
+
+if [[ -z "${dlq_ttl_ms}" ]]; then
+  echo "alerts.paging.dlq is missing x-message-ttl"
+  exit 1
+fi
+if [[ "${dlq_dlx}" != "clinical.notifications.exchange" ]]; then
+  echo "Expected DLQ x-dead-letter-exchange=clinical.notifications.exchange, got '${dlq_dlx}'"
+  exit 1
+fi
+if [[ "${dlq_dlk}" != "alerts.escalation" ]]; then
+  echo "Expected DLQ x-dead-letter-routing-key=alerts.escalation, got '${dlq_dlk}'"
+  exit 1
+fi
+if [[ "${paging_dlk}" != "alerts.paging.dlq" ]]; then
+  echo "Expected paging queue DLK=alerts.paging.dlq, got '${paging_dlk}'"
+  exit 1
+fi
+
+ttl_sec=$(( dlq_ttl_ms / 1000 ))
+ESCALATION_WAIT_SECS="${ESCALATION_WAIT_SECS:-$(( ttl_sec * 2 + 25 ))}"
+echo "OK: exchange + 5 queues provisioned; DLQ TTL=${dlq_ttl_ms}ms (escalation wait budget=${ESCALATION_WAIT_SECS}s)"
+
+echo ""
+echo "[2/${TOTAL_STEPS}] Verifying non-alert observation does not enqueue paging jobs"
+patient_no_page="$(create_patient)"
+encounter_no_page="$(create_encounter "${patient_no_page}" "NoPage")"
+
+paging_before="$(queue_field "alerts.paging.queue" "messages")"
+normal_payload="$(jq -nc \
+  --arg recordedAt "${RECORDED_AT}" \
+  --arg key "notify-normal-${SCRIPT_RUN_ID}" \
+  '{observations:[{observationCode:"POTASSIUM_MEQ_L",value:3.8,unit:"mEq/L",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')"
+resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_no_page}/observations" "${normal_payload}")"
+assert_status "201" "${resp}"
+if [[ "$(jq -r '.data.alertGenerated' "${resp}")" != "false" ]]; then
+  echo "Expected normal potassium ingest to skip alert generation"
+  exit 1
+fi
+
+sleep 3
+paging_after="$(queue_field "alerts.paging.queue" "messages")"
+if [[ "${paging_after}" -gt "${paging_before}" ]]; then
+  echo "Paging queue grew after non-alert ingest (${paging_before} -> ${paging_after})"
+  exit 1
+fi
+echo "OK: normal observation produced no paging queue traffic"
+
+echo ""
+echo "[3/${TOTAL_STEPS}] Verifying critical alert reaches alerts.paging.queue via Kafka bridge"
+patient_page="$(create_patient)"
+encounter_page="$(create_encounter "${patient_page}" "Page")"
+alert_page_id="$(ingest_critical_potassium "${encounter_page}" "notify-critical-page-${SCRIPT_RUN_ID}")"
+
+if ! wait_for_alert_outbox_relayed "${encounter_page}"; then
+  exit 1
+fi
+if ! wait_for_paging_activity; then
+  echo "Hint: confirm NotificationPublisherService is running and alert.generated severity is Critical."
+  exit 1
+fi
+echo "OK: critical alert ${alert_page_id} reached alerts.paging.queue (worker holding or ready message present)"
+
+echo ""
+echo "[4/${TOTAL_STEPS}] Verifying acknowledge-before-timeout path (no escalation)"
+ack_payload='{"clinicianId":"DR-SCRIPT","note":"Acknowledged from notification verification script."}'
+resp="$(request POST "${BASE_URL}/api/v1/alerts/${alert_page_id}/acknowledge" "${ack_payload}")"
+assert_status "200" "${resp}"
+
+sleep "${ACK_PIPELINE_WAIT_SECS}"
+if ! wait_for_alert_status "${alert_page_id}" "ACKNOWLEDGED" 5; then
+  exit 1
+fi
+echo "OK: alert ${alert_page_id} is ACKNOWLEDGED (not escalated)"
+
+echo ""
+echo "[5/${TOTAL_STEPS}] Verifying full DLQ escalation path (unacknowledged critical alert)"
+patient_escalate="$(create_patient)"
+encounter_escalate="$(create_encounter "${patient_escalate}" "Escalate")"
+alert_escalate_id="$(ingest_critical_potassium "${encounter_escalate}" "notify-critical-escalate-${SCRIPT_RUN_ID}")"
+
+if ! wait_for_alert_outbox_relayed "${encounter_escalate}"; then
+  exit 1
+fi
+if ! wait_for_paging_activity; then
+  exit 1
+fi
+
+echo "Waiting up to ${ESCALATION_WAIT_SECS}s for paging timeout + DLQ TTL + escalation worker..."
+if ! wait_for_alert_status "${alert_escalate_id}" "ESCALATED" "${ESCALATION_WAIT_SECS}"; then
+  echo "Hint: default DLQ TTL is 300000ms (~10 min total). Set PagingAckTimeoutMs=5000 in appsettings.json and restart the API for a faster run, or raise ESCALATION_WAIT_SECS."
+  exit 1
+fi
+echo "OK: alert ${alert_escalate_id} escalated to ESCALATED in PostgreSQL"
+
+echo ""
+echo "[6/${TOTAL_STEPS}] Verifying discharge summary upload to MinIO"
+patient_discharge="$(create_patient)"
+encounter_discharge="$(create_encounter "${patient_discharge}" "Discharge")"
+discharge_payload='{"status":"Discharged"}'
+resp="$(request PATCH "${BASE_URL}/api/v1/encounters/${encounter_discharge}/status" "${discharge_payload}")"
+assert_status "200" "${resp}"
+
+discharge_outbox_id="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = 'encounter.status.changed' AND partition_key = '${encounter_discharge}' ORDER BY created_at DESC LIMIT 1")"
+if [[ -z "${discharge_outbox_id}" ]]; then
+  echo "No encounter.status.changed outbox row for encounter ${encounter_discharge}"
+  exit 1
+fi
+wait_for_outbox_processed "${discharge_outbox_id}"
+
+object_key="discharge-summaries/${encounter_discharge}/summary.pdf"
+elapsed=0
+while (( elapsed < MINIO_WAIT_SECS )); do
+  if minio_object_exists "${object_key}"; then
+    break
+  fi
+  sleep 1
+  elapsed=$((elapsed + 1))
+done
+
+if ! minio_object_exists "${object_key}"; then
+  echo "MinIO object not found: ${MINIO_BUCKET}/${object_key}"
+  echo "Hint: confirm DischargeSummaryWorkerService is running and notifications.discharge.queue is draining."
+  exit 1
+fi
+echo "OK: discharge summary object exists at ${MINIO_BUCKET}/${object_key}"
+
+echo ""
+echo "[7/${TOTAL_STEPS}] Verifying discharge queue drained after summary upload"
+elapsed=0
+while (( elapsed < MINIO_WAIT_SECS )); do
+  discharge_ready="$(queue_field "notifications.discharge.queue" "messages_ready")"
+  discharge_unacked="$(queue_field "notifications.discharge.queue" "messages_unacknowledged")"
+  if [[ "${discharge_ready}" == "0" && "${discharge_unacked}" == "0" ]]; then
+    break
+  fi
+  sleep 1
+  elapsed=$((elapsed + 1))
+done
+
+discharge_ready="$(queue_field "notifications.discharge.queue" "messages_ready")"
+discharge_unacked="$(queue_field "notifications.discharge.queue" "messages_unacknowledged")"
+if [[ "${discharge_ready}" != "0" || "${discharge_unacked}" != "0" ]]; then
+  echo "Expected notifications.discharge.queue to be empty (ready=${discharge_ready}, unacked=${discharge_unacked})"
+  exit 1
+fi
+echo "OK: notifications.discharge.queue is drained"
+
+echo ""
+echo "All ${TOTAL_STEPS} notification pipeline checks passed."
+echo ""
+echo "Prerequisites: docker compose up -d && dotnet run --project VigilCareClinicalAPI"
+echo "Optional: set PagingAckTimeoutMs=5000 in appsettings.json (restart API) for ~15s escalation instead of ~10 min."
+echo "Optional: ESCALATION_WAIT_SECS=330 when using the production 300000ms DLQ TTL."