#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}" BASE_URL="${BASE_URL:-http://localhost:5270}" ES_URL="${ES_URL:-http://localhost:9200}" PGHOST="${PGHOST:-localhost}" PGPORT="${PGPORT:-5436}" PGDATABASE="${PGDATABASE:-vigilcare}" PGUSER="${PGUSER:-postgres}" PGPASSWORD="${PGPASSWORD:-password}" ES_CONSUMER_GROUP="${ES_CONSUMER_GROUP:-es-indexer}" INDEX_WAIT_SECS="${INDEX_WAIT_SECS:-60}" RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}" KAFKA_READY_WAIT_SECS="${KAFKA_READY_WAIT_SECS:-30}" SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")" # Per-run time windows so analytics assertions are isolated from prior script runs. _pop_month="$(echo "$SCRIPT_RUN_ID" | cut -c5-6)" _pop_day="$(echo "$SCRIPT_RUN_ID" | cut -c7-8)" _pop_hour="$(echo "$SCRIPT_RUN_ID" | cut -c9-10)" _pop_min="$(echo "$SCRIPT_RUN_ID" | cut -c11-12)" _pop_sec="$(echo "$SCRIPT_RUN_ID" | cut -c13-14)" POPULATION_AT="2099-${_pop_month}-${_pop_day}T${_pop_hour}:${_pop_min}:${_pop_sec}Z" POPULATION_FROM="${POPULATION_AT}" POPULATION_TO="${POPULATION_AT}" TREND_DAY="2099-${_pop_month}-${_pop_day}" TREND_FROM="${TREND_DAY}T00:00:00Z" TREND_TO="${TREND_DAY}T12:00:00Z" TREND_AT_HOUR_1="${TREND_DAY}T01:30:00Z" TREND_AT_HOUR_3="${TREND_DAY}T03:45:00Z" ALERT_AT="2099-${_pop_month}-${_pop_day}T${_pop_hour}:${_pop_min}:${_pop_sec}Z" TMP_FILES=() cleanup() { local f for f in "${TMP_FILES[@]}"; do rm -f "${f}" "${f}.status" 2>/dev/null || true done } trap cleanup EXIT if ! command -v curl >/dev/null 2>&1; then echo "Missing dependency: curl" exit 1 fi if ! command -v jq >/dev/null 2>&1; then echo "Missing dependency: jq" exit 1 fi if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then echo "Missing dependency: docker compose (${COMPOSE_FILE})" exit 1 fi compose() { docker compose -f "${COMPOSE_FILE}" "$@" } kafka_exec() { compose exec -T kafka "$@" } psql_cmd() { local sql="$1" if command -v psql >/dev/null 2>&1; then PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" else compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" fi } request() { local method="$1" local url="$2" local body="${3:-}" local tmp_body tmp_body="$(mktemp)" TMP_FILES+=("${tmp_body}") local status if [[ -n "${body}" ]]; then status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}" \ -H "Content-Type: application/json" -d "${body}")" else status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}")" fi echo "${status}" > "${tmp_body}.status" echo "${tmp_body}" } assert_status() { local expected="$1" local body_file="$2" local status status="$(cat "${body_file}.status")" if [[ "${status}" != "${expected}" ]]; then echo "Expected HTTP ${expected}, got ${status}" echo "Response body:" cat "${body_file}" echo return 1 fi } es_count() { local index="$1" curl -sS "${ES_URL}/${index}/_count" | jq -r '.count' } es_encounter_source() { local encounter_id="$1" curl -sS "${ES_URL}/patient_encounters/_source/${encounter_id}" } wait_for_kafka() { local elapsed=0 while (( elapsed < KAFKA_READY_WAIT_SECS )); do if kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1; then return 0 fi sleep 2 elapsed=$((elapsed + 2)) done echo "Kafka did not become ready within ${KAFKA_READY_WAIT_SECS}s" return 1 } indexer_lag() { kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \ --bootstrap-server localhost:9092 \ --describe \ --group "${ES_CONSUMER_GROUP}" 2>/dev/null | \ awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }' } wait_for_indexer_lag_zero() { local elapsed=0 while (( elapsed < INDEX_WAIT_SECS )); do local lag lag="$(indexer_lag)" if [[ "${lag}" == "0" ]]; then return 0 fi sleep 2 elapsed=$((elapsed + 2)) done echo "es-indexer lag did not reach zero within ${INDEX_WAIT_SECS}s (lag=${lag:-unknown})" return 1 } wait_for_outbox_processed() { local outbox_id="$1" local elapsed=0 while (( elapsed < RELAY_WAIT_SECS )); do local processed processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")" if [[ "${processed}" == "t" ]]; then return 0 fi sleep 1 elapsed=$((elapsed + 1)) done echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s" return 1 } assert_projection_counts_match() { local relayed_obs relayed_enc relayed_alerts es_obs es_enc es_alerts relayed_obs="$(psql_cmd "SELECT COUNT(*) FROM outbox_events WHERE topic = 'observation.recorded' AND processed_at IS NOT NULL")" relayed_enc="$(psql_cmd "SELECT COUNT(*) FROM outbox_events WHERE topic = 'encounter.status.changed' AND processed_at IS NOT NULL")" relayed_alerts="$(psql_cmd "SELECT COUNT(*) FROM outbox_events WHERE topic = 'alert.generated' AND processed_at IS NOT NULL")" es_obs="$(es_count observations)" es_enc="$(es_count patient_encounters)" es_alerts="$(es_count clinical_alerts)" if [[ "${relayed_obs}" != "${es_obs}" ]]; then echo "Observation projection mismatch: relayed=${relayed_obs}, Elasticsearch=${es_obs}" return 1 fi if [[ "${relayed_enc}" != "${es_enc}" ]]; then echo "Encounter projection mismatch: relayed=${relayed_enc}, Elasticsearch=${es_enc}" return 1 fi if [[ "${relayed_alerts}" != "${es_alerts}" ]]; then echo "Clinical alert projection mismatch: relayed=${relayed_alerts}, Elasticsearch=${es_alerts}" return 1 fi echo "OK: Elasticsearch counts match relayed outbox events (obs=${es_obs}, enc=${es_enc}, alerts=${es_alerts})" } ingest_observation() { local encounter_id="$1" local code="$2" local value="$3" local unit="$4" local source="$5" local recorded_at="$6" local idempotency_key="$7" local payload payload="$(jq -nc \ --arg code "${code}" \ --argjson value "${value}" \ --arg unit "${unit}" \ --arg source "${source}" \ --arg recordedAt "${recorded_at}" \ --arg key "${idempotency_key}" \ '{observations:[{observationCode:$code,value:$value,unit:$unit,source:$source,recordedAt:$recordedAt,idempotencyKey:$key}]}')" local resp resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")" assert_status "201" "${resp}" } TOTAL_STEPS=17 echo "Running Elasticsearch + analytics verification against ${BASE_URL}" echo "Script run id: ${SCRIPT_RUN_ID}" echo "" echo "[0/${TOTAL_STEPS}] Preflight — API, Postgres, Kafka, and Elasticsearch reachable" preflight_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" if [[ "${preflight_status}" != "200" ]]; then echo "API not reachable at ${BASE_URL} (HTTP ${preflight_status})." echo "Start the API with: dotnet run --project VigilCareClinicalAPI" exit 1 fi if ! psql_cmd "SELECT 1" >/dev/null 2>&1; then echo "Postgres not reachable on ${PGHOST}:${PGPORT}." echo "Start the stack with: docker compose up -d" exit 1 fi if ! wait_for_kafka; then exit 1 fi es_health_status="$(curl -sS "${ES_URL}/_cluster/health" | jq -r '.status' || true)" if [[ "${es_health_status}" != "green" && "${es_health_status}" != "yellow" ]]; then echo "Elasticsearch cluster health is '${es_health_status}' (expected green or yellow)." echo "Start the stack with: docker compose up -d" exit 1 fi echo "OK: API, Postgres, Kafka, and Elasticsearch are up (cluster=${es_health_status})" echo "" echo "[1/${TOTAL_STEPS}] Verifying Elasticsearch indices exist" for index in patient_encounters observations clinical_alerts; do exists="$(curl -sS -o /dev/null -w "%{http_code}" "${ES_URL}/${index}" || true)" if [[ "${exists}" != "200" ]]; then echo "Missing Elasticsearch index: ${index} (HTTP ${exists})" exit 1 fi done echo "OK: patient_encounters, observations, clinical_alerts indices exist" echo "" echo "[2/${TOTAL_STEPS}] Waiting for es-indexer consumer lag to reach zero" if ! wait_for_indexer_lag_zero; then exit 1 fi echo "OK: es-indexer lag is zero" echo "" echo "[3/${TOTAL_STEPS}] Verifying Elasticsearch document counts match relayed outbox events" assert_projection_counts_match echo "" echo "[4/${TOTAL_STEPS}] Creating patient and two ICU encounters for analytics fixtures" patient_payload='{"firstName":"Verify","lastName":"Smith","dateOfBirth":"1985-03-20","gender":"F"}' resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")" assert_status "201" "${resp}" patient_id="$(jq -r '.data.id' "${resp}")" patient_mrn="$(jq -r '.data.mrn' "${resp}")" if [[ -z "${patient_mrn}" || "${patient_mrn}" == "null" ]]; then echo "Could not parse patient MRN from create response." exit 1 fi enc_payload_a='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Elastic"}' resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload_a}")" assert_status "201" "${resp}" encounter_a="$(jq -r '.data.id' "${resp}")" enc_payload_b='{"encounterType":"Outpatient","department":"ICU","attendingPhysician":"Dr. Elastic"}' resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload_b}")" assert_status "201" "${resp}" encounter_b="$(jq -r '.data.id' "${resp}")" echo "OK: patient ${patient_id} (mrn=${patient_mrn}), encounters ${encounter_a}, ${encounter_b}" echo "" echo "[5/${TOTAL_STEPS}] Waiting for encounter documents to appear in Elasticsearch" elapsed=0 while (( elapsed < INDEX_WAIT_SECS )); do source="$(es_encounter_source "${encounter_a}" 2>/dev/null || true)" if jq -e '.encounterId' >/dev/null 2>&1 <<< "${source}"; then break fi sleep 2 elapsed=$((elapsed + 2)) done if ! jq -e '.encounterId' >/dev/null 2>&1 <<< "$(es_encounter_source "${encounter_a}")"; then echo "patient_encounters document not found for encounter ${encounter_a}" exit 1 fi echo "OK: patient_encounters documents indexed" echo "" echo "[6/${TOTAL_STEPS}] Verifying openAlertCount is zero before any alert" open_before="$(es_encounter_source "${encounter_a}" | jq -r '.openAlertCount')" if [[ "${open_before}" != "0" ]]; then echo "Expected openAlertCount=0 before alert, got ${open_before}" exit 1 fi echo "OK: openAlertCount=0 before critical ingest" echo "" echo "[7/${TOTAL_STEPS}] Ingesting population, trend, and critical observations" ingest_observation "${encounter_a}" "HEART_RATE" 104 "bpm" "DEVICE" "${POPULATION_AT}" \ "es-pop-a1-${SCRIPT_RUN_ID}" ingest_observation "${encounter_a}" "HEART_RATE" 112 "bpm" "DEVICE" "${POPULATION_AT}" \ "es-pop-a2-${SCRIPT_RUN_ID}" ingest_observation "${encounter_b}" "HEART_RATE" 82 "bpm" "DEVICE" "${POPULATION_AT}" \ "es-pop-b1-${SCRIPT_RUN_ID}" ingest_observation "${encounter_a}" "HEART_RATE" 90 "bpm" "DEVICE" "${TREND_AT_HOUR_1}" \ "es-trend-h1-${SCRIPT_RUN_ID}" ingest_observation "${encounter_a}" "HEART_RATE" 95 "bpm" "DEVICE" "${TREND_AT_HOUR_3}" \ "es-trend-h3-${SCRIPT_RUN_ID}" critical_payload="$(jq -nc \ --arg recordedAt "${ALERT_AT}" \ --arg key "es-critical-${SCRIPT_RUN_ID}" \ '{observations:[{observationCode:"POTASSIUM_MEQ_L",value:2.1,unit:"mEq/L",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')" resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_a}/observations" "${critical_payload}")" assert_status "201" "${resp}" if [[ "$(jq -r '.data.alertGenerated' "${resp}")" != "true" ]]; then echo "Expected critical potassium ingest to generate an alert" exit 1 fi echo "OK: five observations and one critical alert ingested" echo "" echo "[8/${TOTAL_STEPS}] Waiting for outbox relay and es-indexer to catch up" latest_outbox="$(psql_cmd "SELECT id FROM outbox_events ORDER BY created_at DESC LIMIT 1")" if [[ -n "${latest_outbox}" ]]; then wait_for_outbox_processed "${latest_outbox}" fi if ! wait_for_indexer_lag_zero; then exit 1 fi echo "OK: relay and es-indexer caught up" echo "" echo "[9/${TOTAL_STEPS}] Re-verifying projection counts after ingest" assert_projection_counts_match echo "" echo "[10/${TOTAL_STEPS}] Verifying population query cardinality (uniquePatientCount = 1)" population_url="${BASE_URL}/api/v1/analytics/population?code=HEART_RATE&threshold=100&direction=above&from=${POPULATION_FROM}&to=${POPULATION_TO}" resp="$(request GET "${population_url}")" assert_status "200" "${resp}" if ! jq -e '.data.uniquePatientCount == 1' "${resp}" >/dev/null; then echo "Expected uniquePatientCount=1 for two above-threshold observations from one patient" echo "Response body:" cat "${resp}" echo exit 1 fi echo "OK: population query returned uniquePatientCount=1" echo "" echo "[11/${TOTAL_STEPS}] Verifying observation trend hourly buckets" trend_url="${BASE_URL}/api/v1/analytics/observations/trend?encounterId=${encounter_a}&code=HEART_RATE&from=${TREND_FROM}&to=${TREND_TO}" resp="$(request GET "${trend_url}")" assert_status "200" "${resp}" trend_count="$(jq -r '.data.trend | length' "${resp}")" if [[ "${trend_count}" -lt 2 ]]; then echo "Expected at least two hourly trend buckets, got ${trend_count}" echo "Response body:" cat "${resp}" echo exit 1 fi if ! jq -e '.data.trend[] | select(.count >= 1)' "${resp}" >/dev/null; then echo "Expected trend buckets with count >= 1" exit 1 fi echo "OK: trend query returned ${trend_count} hourly bucket(s)" echo "" echo "[12/${TOTAL_STEPS}] Verifying patient search by exact MRN" search_url="${BASE_URL}/api/v1/analytics/patients?q=${patient_mrn}" resp="$(request GET "${search_url}")" assert_status "200" "${resp}" if ! jq -e --arg mrn "${patient_mrn}" '.data.data[] | select(.mrn == $mrn)' "${resp}" >/dev/null; then echo "MRN search did not return patient with mrn=${patient_mrn}" echo "Response body:" cat "${resp}" echo exit 1 fi echo "OK: MRN search returned exact match (${patient_mrn})" echo "" echo "[13/${TOTAL_STEPS}] Verifying patient search by partial name" resp="$(request GET "${BASE_URL}/api/v1/analytics/patients?q=smith")" assert_status "200" "${resp}" if ! jq -e --arg id "${patient_id}" '.data.data[] | select(.patientId == $id)' "${resp}" >/dev/null; then echo "Name search for 'smith' did not return patient ${patient_id}" echo "Response body:" cat "${resp}" echo exit 1 fi echo "OK: partial name search matched Smith" echo "" echo "[14/${TOTAL_STEPS}] Verifying department and status filter (no text query)" resp="$(request GET "${BASE_URL}/api/v1/analytics/patients?department=ICU&status=ACTIVE&pageSize=100")" assert_status "200" "${resp}" if ! jq -e --arg enc "${encounter_a}" '.data.data[] | select(.encounterId == $enc)' "${resp}" >/dev/null; then echo "Department/status filter did not return encounter ${encounter_a}" echo "Response body:" cat "${resp}" echo exit 1 fi echo "OK: ICU + ACTIVE filter returned expected encounter" echo "" echo "[15/${TOTAL_STEPS}] Verifying openAlertCount increments after alert.generated" open_after="$(es_encounter_source "${encounter_a}" | jq -r '.openAlertCount')" if [[ "${open_after}" != "1" ]]; then echo "Expected openAlertCount=1 after critical alert, got ${open_after}" exit 1 fi echo "OK: openAlertCount incremented to 1" echo "" echo "[16/${TOTAL_STEPS}] Verifying alert summary by department" summary_url="${BASE_URL}/api/v1/analytics/alerts/summary?department=ICU" resp="$(request GET "${summary_url}")" assert_status "200" "${resp}" if ! jq -e '.data.summary[] | select(.department == "ICU" and (.total | tonumber) >= 1)' "${resp}" >/dev/null; then echo "Alert summary did not include ICU with total >= 1" echo "Response body:" cat "${resp}" echo exit 1 fi echo "OK: alert summary includes ICU department" echo "" echo "[17/${TOTAL_STEPS}] Verifying population endpoint validates required code" resp="$(request GET "${BASE_URL}/api/v1/analytics/population?threshold=100&direction=above")" assert_status "400" "${resp}" if [[ "$(jq -r '.error.code' "${resp}")" != "MISSING_CODE" ]]; then echo "Expected MISSING_CODE for population without code parameter" exit 1 fi echo "OK: missing code returns 400 MISSING_CODE" echo "" echo "All ${TOTAL_STEPS} Elasticsearch + analytics checks passed." echo "" echo "Note: Full index replay (delete indices, reset es-indexer offsets, restart API) is documented" echo "in docs/plans/phase-4-plan.md Step 7 and must be run manually to complete the replay checklist item."