Files
vigilcare-clinical/scripts/run-elasticsearch-analytics-tests.sh
T
voltsrage 2a3ef62a7d
CI / frontend (push) Failing after 57s
CI / backend (push) Failing after 6m27s
Add deployment
2026-08-05 00:26:20 +08:00

479 lines
16 KiB
Bash

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}"
BASE_URL="${BASE_URL:-http://localhost:5270}"
ES_URL="${ES_URL:-http://localhost:9200}"
PGHOST="${PGHOST:-localhost}"
PGPORT="${PGPORT:-5436}"
PGDATABASE="${PGDATABASE:-vigilcare}"
PGUSER="${PGUSER:-postgres}"
PGPASSWORD="${PGPASSWORD:-password}"
ES_CONSUMER_GROUP="${ES_CONSUMER_GROUP:-es-indexer}"
INDEX_WAIT_SECS="${INDEX_WAIT_SECS:-60}"
RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}"
KAFKA_READY_WAIT_SECS="${KAFKA_READY_WAIT_SECS:-30}"
SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")"
# Per-run time windows so analytics assertions are isolated from prior script runs.
_pop_month="$(echo "$SCRIPT_RUN_ID" | cut -c5-6)"
_pop_day="$(echo "$SCRIPT_RUN_ID" | cut -c7-8)"
_pop_hour="$(echo "$SCRIPT_RUN_ID" | cut -c9-10)"
_pop_min="$(echo "$SCRIPT_RUN_ID" | cut -c11-12)"
_pop_sec="$(echo "$SCRIPT_RUN_ID" | cut -c13-14)"
POPULATION_AT="2099-${_pop_month}-${_pop_day}T${_pop_hour}:${_pop_min}:${_pop_sec}Z"
POPULATION_FROM="${POPULATION_AT}"
POPULATION_TO="${POPULATION_AT}"
TREND_DAY="2099-${_pop_month}-${_pop_day}"
TREND_FROM="${TREND_DAY}T00:00:00Z"
TREND_TO="${TREND_DAY}T12:00:00Z"
TREND_AT_HOUR_1="${TREND_DAY}T01:30:00Z"
TREND_AT_HOUR_3="${TREND_DAY}T03:45:00Z"
ALERT_AT="2099-${_pop_month}-${_pop_day}T${_pop_hour}:${_pop_min}:${_pop_sec}Z"
TMP_FILES=()
cleanup() {
local f
for f in "${TMP_FILES[@]}"; do
rm -f "${f}" "${f}.status" 2>/dev/null || true
done
}
trap cleanup EXIT
if ! command -v curl >/dev/null 2>&1; then
echo "Missing dependency: curl"
exit 1
fi
if ! command -v jq >/dev/null 2>&1; then
echo "Missing dependency: jq"
exit 1
fi
if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then
echo "Missing dependency: docker compose (${COMPOSE_FILE})"
exit 1
fi
compose() {
docker compose -f "${COMPOSE_FILE}" "$@"
}
kafka_exec() {
compose exec -T kafka "$@"
}
psql_cmd() {
local sql="$1"
if command -v psql >/dev/null 2>&1; then
PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
else
compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
fi
}
request() {
local method="$1"
local url="$2"
local body="${3:-}"
local tmp_body
tmp_body="$(mktemp)"
TMP_FILES+=("${tmp_body}")
local status
if [[ -n "${body}" ]]; then
status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}" \
-H "Content-Type: application/json" -d "${body}")"
else
status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}")"
fi
echo "${status}" > "${tmp_body}.status"
echo "${tmp_body}"
}
assert_status() {
local expected="$1"
local body_file="$2"
local status
status="$(cat "${body_file}.status")"
if [[ "${status}" != "${expected}" ]]; then
echo "Expected HTTP ${expected}, got ${status}"
echo "Response body:"
cat "${body_file}"
echo
return 1
fi
}
es_count() {
local index="$1"
curl -sS "${ES_URL}/${index}/_count" | jq -r '.count'
}
es_encounter_source() {
local encounter_id="$1"
curl -sS "${ES_URL}/patient_encounters/_source/${encounter_id}"
}
wait_for_kafka() {
local elapsed=0
while (( elapsed < KAFKA_READY_WAIT_SECS )); do
if kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1; then
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
echo "Kafka did not become ready within ${KAFKA_READY_WAIT_SECS}s"
return 1
}
indexer_lag() {
kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \
--bootstrap-server localhost:9092 \
--describe \
--group "${ES_CONSUMER_GROUP}" 2>/dev/null | \
awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }'
}
wait_for_indexer_lag_zero() {
local elapsed=0
while (( elapsed < INDEX_WAIT_SECS )); do
local lag
lag="$(indexer_lag)"
if [[ "${lag}" == "0" ]]; then
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
echo "es-indexer lag did not reach zero within ${INDEX_WAIT_SECS}s (lag=${lag:-unknown})"
return 1
}
wait_for_outbox_processed() {
local outbox_id="$1"
local elapsed=0
while (( elapsed < RELAY_WAIT_SECS )); do
local processed
processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")"
if [[ "${processed}" == "t" ]]; then
return 0
fi
sleep 1
elapsed=$((elapsed + 1))
done
echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s"
return 1
}
assert_projection_counts_match() {
local relayed_obs relayed_enc relayed_alerts es_obs es_enc es_alerts
relayed_obs="$(psql_cmd "SELECT COUNT(*) FROM outbox_events WHERE topic = 'observation.recorded' AND processed_at IS NOT NULL")"
relayed_enc="$(psql_cmd "SELECT COUNT(*) FROM outbox_events WHERE topic = 'encounter.status.changed' AND processed_at IS NOT NULL")"
relayed_alerts="$(psql_cmd "SELECT COUNT(*) FROM outbox_events WHERE topic = 'alert.generated' AND processed_at IS NOT NULL")"
es_obs="$(es_count observations)"
es_enc="$(es_count patient_encounters)"
es_alerts="$(es_count clinical_alerts)"
if [[ "${relayed_obs}" != "${es_obs}" ]]; then
echo "Observation projection mismatch: relayed=${relayed_obs}, Elasticsearch=${es_obs}"
return 1
fi
if [[ "${relayed_enc}" != "${es_enc}" ]]; then
echo "Encounter projection mismatch: relayed=${relayed_enc}, Elasticsearch=${es_enc}"
return 1
fi
if [[ "${relayed_alerts}" != "${es_alerts}" ]]; then
echo "Clinical alert projection mismatch: relayed=${relayed_alerts}, Elasticsearch=${es_alerts}"
return 1
fi
echo "OK: Elasticsearch counts match relayed outbox events (obs=${es_obs}, enc=${es_enc}, alerts=${es_alerts})"
}
ingest_observation() {
local encounter_id="$1"
local code="$2"
local value="$3"
local unit="$4"
local source="$5"
local recorded_at="$6"
local idempotency_key="$7"
local payload
payload="$(jq -nc \
--arg code "${code}" \
--argjson value "${value}" \
--arg unit "${unit}" \
--arg source "${source}" \
--arg recordedAt "${recorded_at}" \
--arg key "${idempotency_key}" \
'{observations:[{observationCode:$code,value:$value,unit:$unit,source:$source,recordedAt:$recordedAt,idempotencyKey:$key}]}')"
local resp
resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")"
assert_status "201" "${resp}"
}
TOTAL_STEPS=17
echo "Running Elasticsearch + analytics verification against ${BASE_URL}"
echo "Script run id: ${SCRIPT_RUN_ID}"
echo ""
echo "[0/${TOTAL_STEPS}] Preflight — API, Postgres, Kafka, and Elasticsearch reachable"
preflight_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
if [[ "${preflight_status}" != "200" ]]; then
echo "API not reachable at ${BASE_URL} (HTTP ${preflight_status})."
echo "Start the API with: dotnet run --project VigilCareClinicalAPI"
exit 1
fi
if ! psql_cmd "SELECT 1" >/dev/null 2>&1; then
echo "Postgres not reachable on ${PGHOST}:${PGPORT}."
echo "Start the stack with: docker compose up -d"
exit 1
fi
if ! wait_for_kafka; then
exit 1
fi
es_health_status="$(curl -sS "${ES_URL}/_cluster/health" | jq -r '.status' || true)"
if [[ "${es_health_status}" != "green" && "${es_health_status}" != "yellow" ]]; then
echo "Elasticsearch cluster health is '${es_health_status}' (expected green or yellow)."
echo "Start the stack with: docker compose up -d"
exit 1
fi
echo "OK: API, Postgres, Kafka, and Elasticsearch are up (cluster=${es_health_status})"
echo ""
echo "[1/${TOTAL_STEPS}] Verifying Elasticsearch indices exist"
for index in patient_encounters observations clinical_alerts; do
exists="$(curl -sS -o /dev/null -w "%{http_code}" "${ES_URL}/${index}" || true)"
if [[ "${exists}" != "200" ]]; then
echo "Missing Elasticsearch index: ${index} (HTTP ${exists})"
exit 1
fi
done
echo "OK: patient_encounters, observations, clinical_alerts indices exist"
echo ""
echo "[2/${TOTAL_STEPS}] Waiting for es-indexer consumer lag to reach zero"
if ! wait_for_indexer_lag_zero; then
exit 1
fi
echo "OK: es-indexer lag is zero"
echo ""
echo "[3/${TOTAL_STEPS}] Verifying Elasticsearch document counts match relayed outbox events"
assert_projection_counts_match
echo ""
echo "[4/${TOTAL_STEPS}] Creating patient and two ICU encounters for analytics fixtures"
patient_payload='{"firstName":"Verify","lastName":"Smith","dateOfBirth":"1985-03-20","gender":"F"}'
resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")"
assert_status "201" "${resp}"
patient_id="$(jq -r '.data.id' "${resp}")"
patient_mrn="$(jq -r '.data.mrn' "${resp}")"
if [[ -z "${patient_mrn}" || "${patient_mrn}" == "null" ]]; then
echo "Could not parse patient MRN from create response."
exit 1
fi
enc_payload_a='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Elastic"}'
resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload_a}")"
assert_status "201" "${resp}"
encounter_a="$(jq -r '.data.id' "${resp}")"
enc_payload_b='{"encounterType":"Outpatient","department":"ICU","attendingPhysician":"Dr. Elastic"}'
resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload_b}")"
assert_status "201" "${resp}"
encounter_b="$(jq -r '.data.id' "${resp}")"
echo "OK: patient ${patient_id} (mrn=${patient_mrn}), encounters ${encounter_a}, ${encounter_b}"
echo ""
echo "[5/${TOTAL_STEPS}] Waiting for encounter documents to appear in Elasticsearch"
elapsed=0
while (( elapsed < INDEX_WAIT_SECS )); do
source="$(es_encounter_source "${encounter_a}" 2>/dev/null || true)"
if jq -e '.encounterId' >/dev/null 2>&1 <<< "${source}"; then
break
fi
sleep 2
elapsed=$((elapsed + 2))
done
if ! jq -e '.encounterId' >/dev/null 2>&1 <<< "$(es_encounter_source "${encounter_a}")"; then
echo "patient_encounters document not found for encounter ${encounter_a}"
exit 1
fi
echo "OK: patient_encounters documents indexed"
echo ""
echo "[6/${TOTAL_STEPS}] Verifying openAlertCount is zero before any alert"
open_before="$(es_encounter_source "${encounter_a}" | jq -r '.openAlertCount')"
if [[ "${open_before}" != "0" ]]; then
echo "Expected openAlertCount=0 before alert, got ${open_before}"
exit 1
fi
echo "OK: openAlertCount=0 before critical ingest"
echo ""
echo "[7/${TOTAL_STEPS}] Ingesting population, trend, and critical observations"
ingest_observation "${encounter_a}" "HEART_RATE" 104 "bpm" "DEVICE" "${POPULATION_AT}" \
"es-pop-a1-${SCRIPT_RUN_ID}"
ingest_observation "${encounter_a}" "HEART_RATE" 112 "bpm" "DEVICE" "${POPULATION_AT}" \
"es-pop-a2-${SCRIPT_RUN_ID}"
ingest_observation "${encounter_b}" "HEART_RATE" 82 "bpm" "DEVICE" "${POPULATION_AT}" \
"es-pop-b1-${SCRIPT_RUN_ID}"
ingest_observation "${encounter_a}" "HEART_RATE" 90 "bpm" "DEVICE" "${TREND_AT_HOUR_1}" \
"es-trend-h1-${SCRIPT_RUN_ID}"
ingest_observation "${encounter_a}" "HEART_RATE" 95 "bpm" "DEVICE" "${TREND_AT_HOUR_3}" \
"es-trend-h3-${SCRIPT_RUN_ID}"
critical_payload="$(jq -nc \
--arg recordedAt "${ALERT_AT}" \
--arg key "es-critical-${SCRIPT_RUN_ID}" \
'{observations:[{observationCode:"POTASSIUM_MEQ_L",value:2.1,unit:"mEq/L",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')"
resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_a}/observations" "${critical_payload}")"
assert_status "201" "${resp}"
if [[ "$(jq -r '.data.alertGenerated' "${resp}")" != "true" ]]; then
echo "Expected critical potassium ingest to generate an alert"
exit 1
fi
echo "OK: five observations and one critical alert ingested"
echo ""
echo "[8/${TOTAL_STEPS}] Waiting for outbox relay and es-indexer to catch up"
latest_outbox="$(psql_cmd "SELECT id FROM outbox_events ORDER BY created_at DESC LIMIT 1")"
if [[ -n "${latest_outbox}" ]]; then
wait_for_outbox_processed "${latest_outbox}"
fi
if ! wait_for_indexer_lag_zero; then
exit 1
fi
echo "OK: relay and es-indexer caught up"
echo ""
echo "[9/${TOTAL_STEPS}] Re-verifying projection counts after ingest"
assert_projection_counts_match
echo ""
echo "[10/${TOTAL_STEPS}] Verifying population query cardinality (uniquePatientCount = 1)"
population_url="${BASE_URL}/api/v1/analytics/population?code=HEART_RATE&threshold=100&direction=above&from=${POPULATION_FROM}&to=${POPULATION_TO}"
resp="$(request GET "${population_url}")"
assert_status "200" "${resp}"
if ! jq -e '.data.uniquePatientCount == 1' "${resp}" >/dev/null; then
echo "Expected uniquePatientCount=1 for two above-threshold observations from one patient"
echo "Response body:"
cat "${resp}"
echo
exit 1
fi
echo "OK: population query returned uniquePatientCount=1"
echo ""
echo "[11/${TOTAL_STEPS}] Verifying observation trend hourly buckets"
trend_url="${BASE_URL}/api/v1/analytics/observations/trend?encounterId=${encounter_a}&code=HEART_RATE&from=${TREND_FROM}&to=${TREND_TO}"
resp="$(request GET "${trend_url}")"
assert_status "200" "${resp}"
trend_count="$(jq -r '.data.trend | length' "${resp}")"
if [[ "${trend_count}" -lt 2 ]]; then
echo "Expected at least two hourly trend buckets, got ${trend_count}"
echo "Response body:"
cat "${resp}"
echo
exit 1
fi
if ! jq -e '.data.trend[] | select(.count >= 1)' "${resp}" >/dev/null; then
echo "Expected trend buckets with count >= 1"
exit 1
fi
echo "OK: trend query returned ${trend_count} hourly bucket(s)"
echo ""
echo "[12/${TOTAL_STEPS}] Verifying patient search by exact MRN"
search_url="${BASE_URL}/api/v1/analytics/patients?q=${patient_mrn}"
resp="$(request GET "${search_url}")"
assert_status "200" "${resp}"
if ! jq -e --arg mrn "${patient_mrn}" '.data.data[] | select(.mrn == $mrn)' "${resp}" >/dev/null; then
echo "MRN search did not return patient with mrn=${patient_mrn}"
echo "Response body:"
cat "${resp}"
echo
exit 1
fi
echo "OK: MRN search returned exact match (${patient_mrn})"
echo ""
echo "[13/${TOTAL_STEPS}] Verifying patient search by partial name"
resp="$(request GET "${BASE_URL}/api/v1/analytics/patients?q=smith")"
assert_status "200" "${resp}"
if ! jq -e --arg id "${patient_id}" '.data.data[] | select(.patientId == $id)' "${resp}" >/dev/null; then
echo "Name search for 'smith' did not return patient ${patient_id}"
echo "Response body:"
cat "${resp}"
echo
exit 1
fi
echo "OK: partial name search matched Smith"
echo ""
echo "[14/${TOTAL_STEPS}] Verifying department and status filter (no text query)"
resp="$(request GET "${BASE_URL}/api/v1/analytics/patients?department=ICU&status=ACTIVE&pageSize=100")"
assert_status "200" "${resp}"
if ! jq -e --arg enc "${encounter_a}" '.data.data[] | select(.encounterId == $enc)' "${resp}" >/dev/null; then
echo "Department/status filter did not return encounter ${encounter_a}"
echo "Response body:"
cat "${resp}"
echo
exit 1
fi
echo "OK: ICU + ACTIVE filter returned expected encounter"
echo ""
echo "[15/${TOTAL_STEPS}] Verifying openAlertCount increments after alert.generated"
open_after="$(es_encounter_source "${encounter_a}" | jq -r '.openAlertCount')"
if [[ "${open_after}" != "1" ]]; then
echo "Expected openAlertCount=1 after critical alert, got ${open_after}"
exit 1
fi
echo "OK: openAlertCount incremented to 1"
echo ""
echo "[16/${TOTAL_STEPS}] Verifying alert summary by department"
summary_url="${BASE_URL}/api/v1/analytics/alerts/summary?department=ICU"
resp="$(request GET "${summary_url}")"
assert_status "200" "${resp}"
if ! jq -e '.data.summary[] | select(.department == "ICU" and (.total | tonumber) >= 1)' "${resp}" >/dev/null; then
echo "Alert summary did not include ICU with total >= 1"
echo "Response body:"
cat "${resp}"
echo
exit 1
fi
echo "OK: alert summary includes ICU department"
echo ""
echo "[17/${TOTAL_STEPS}] Verifying population endpoint validates required code"
resp="$(request GET "${BASE_URL}/api/v1/analytics/population?threshold=100&direction=above")"
assert_status "400" "${resp}"
if [[ "$(jq -r '.error.code' "${resp}")" != "MISSING_CODE" ]]; then
echo "Expected MISSING_CODE for population without code parameter"
exit 1
fi
echo "OK: missing code returns 400 MISSING_CODE"
echo ""
echo "All ${TOTAL_STEPS} Elasticsearch + analytics checks passed."
echo ""
echo "Note: Full index replay (delete indices, reset es-indexer offsets, restart API) is documented"
echo "in docs/plans/phase-4-plan.md Step 7 and must be run manually to complete the replay checklist item."