#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" BASE_URL="${BASE_URL:-http://localhost:5270}" REDIS_PORT="${REDIS_PORT:-6382}" PGHOST="${PGHOST:-localhost}" PGPORT="${PGPORT:-5436}" PGDATABASE="${PGDATABASE:-vigilcare}" PGUSER="${PGUSER:-postgres}" PGPASSWORD="${PGPASSWORD:-password}" WARNING_CONSUMER_GROUP="${WARNING_CONSUMER_GROUP:-warning-evaluator}" WARNING_WAIT_SECS="${WARNING_WAIT_SECS:-45}" RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}" SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")" RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" TMP_FILES=() cleanup() { local f for f in "${TMP_FILES[@]}"; do rm -f "${f}" "${f}.status" 2>/dev/null || true done } trap cleanup EXIT need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing dependency: $1" exit 1 } } need curl need jq if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then echo "Missing dependency: docker compose (${COMPOSE_FILE})" exit 1 fi compose() { docker compose -f "${COMPOSE_FILE}" "$@" } kafka_exec() { compose exec -T kafka "$@" } redis_cmd() { if command -v redis-cli >/dev/null 2>&1; then redis-cli -p "${REDIS_PORT}" "$@" else compose exec -T redis redis-cli "$@" fi } psql_cmd() { local sql="$1" if command -v psql >/dev/null 2>&1; then PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" else compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" fi } request() { local method="$1" local url="$2" local body="${3:-}" local tmp tmp="$(mktemp)" TMP_FILES+=("${tmp}") local status if [[ -n "${body}" ]]; then status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \ -H "Content-Type: application/json" -d "${body}")" else status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")" fi echo "${status}" > "${tmp}.status" echo "${tmp}" } assert_status() { local expected="$1" local body_file="$2" local status status="$(<"${body_file}.status")" if [[ "${status}" != "${expected}" ]]; then echo "Expected HTTP ${expected}, got ${status}" cat "${body_file}" echo exit 1 fi } ingest_observation() { local encounter_id="$1" local code="$2" local value="$3" local unit="$4" local source="$5" local recorded_at="$6" local idempotency_key="$7" local payload payload="$(jq -nc \ --arg code "${code}" \ --argjson value "${value}" \ --arg unit "${unit}" \ --arg source "${source}" \ --arg recordedAt "${recorded_at}" \ --arg key "${idempotency_key}" \ '{observations:[{observationCode:$code,value:$value,unit:$unit,source:$source,recordedAt:$recordedAt,idempotencyKey:$key}]}')" local resp resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")" assert_status "201" "${resp}" } consumer_group_lag() { local group="$1" kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \ --bootstrap-server localhost:9092 \ --describe \ --group "${group}" 2>/dev/null | \ awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }' } wait_for_consumer_lag_zero() { local group="$1" local max_secs="$2" local elapsed=0 local lag="unknown" while (( elapsed < max_secs )); do lag="$(consumer_group_lag "${group}")" if [[ "${lag}" == "0" ]]; then return 0 fi sleep 2 elapsed=$((elapsed + 2)) done echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})" exit 1 } wait_for_outbox_processed() { local outbox_id="$1" local elapsed=0 while (( elapsed < RELAY_WAIT_SECS )); do local processed processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")" if [[ "${processed}" == "t" ]]; then return 0 fi sleep 1 elapsed=$((elapsed + 1)) done echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s" exit 1 } latest_observation_outbox_id() { local encounter_id="$1" psql_cmd "SELECT id FROM outbox_events WHERE topic = 'observation.recorded' AND partition_key = '${encounter_id}' ORDER BY created_at DESC LIMIT 1" } wait_for_observation_pipeline() { local encounter_id="$1" local outbox_id outbox_id="$(latest_observation_outbox_id "${encounter_id}")" if [[ -z "${outbox_id}" ]]; then echo "No observation.recorded outbox row for encounter ${encounter_id}" exit 1 fi wait_for_outbox_processed "${outbox_id}" wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}" } wait_for_warning_systolic_alert() { local encounter_id="$1" local elapsed=0 while (( elapsed < WARNING_WAIT_SECS )); do local count count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_SYSTOLIC_BP'")" if [[ "${count}" == "1" ]]; then return 0 fi sleep 2 elapsed=$((elapsed + 2)) done echo "WARNING_SYSTOLIC_BP alert not found for encounter ${encounter_id} within ${WARNING_WAIT_SECS}s" psql_cmd "SELECT alert_type, severity, status, details FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true exit 1 } TOTAL_STEPS=7 echo "Phase 15 verification starting..." echo "Repo root: ${ROOT_DIR}" echo "API: ${BASE_URL}" echo "[1/${TOTAL_STEPS}] Infrastructure preflight: API, PostgreSQL, Redis, Kafka, threshold cache" api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" [[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; } redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; } psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; } kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || { echo "Kafka not ready" exit 1 } redis_cmd EXISTS "threshold:SYSTOLIC_BP" | grep -q '^1$' || { echo "Missing Redis threshold:SYSTOLIC_BP — restart API to run ThresholdCacheLoader" exit 1 } echo "OK: infrastructure preflight passed" echo "[2/${TOTAL_STEPS}] Register patient + open active encounter" patient_payload="$(jq -nc \ --arg fn "Phase15" \ --arg ln "Verify${SCRIPT_RUN_ID}" \ '{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')" patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")" assert_status "201" "${patient_resp}" patient_id="$(jq -r '.data.id' "${patient_resp}")" enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase15"}' enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")" assert_status "201" "${enc_resp}" encounter_id="$(jq -r '.data.id' "${enc_resp}")" echo "OK: patient=${patient_id} encounter=${encounter_id}" echo "[3/${TOTAL_STEPS}] POST /encounters/{id}/medications — metoprolol 25mg PO" med_payload='{"drugName":"metoprolol","dose":25,"doseUnit":"mg","route":"PO","administeredBy":"nurse-1"}' med_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/medications" "${med_payload}")" assert_status "201" "${med_resp}" med_id="$(jq -r '.data.id' "${med_resp}")" [[ "$(jq -r '.data.drugName' "${med_resp}")" == "metoprolol" ]] || { echo "Expected drugName metoprolol" cat "${med_resp}" exit 1 } [[ "$(jq -r '.data.dose' "${med_resp}")" == "25" ]] || { echo "Expected dose 25" cat "${med_resp}" exit 1 } echo "OK: medication created (id=${med_id})" echo "[4/${TOTAL_STEPS}] Ingest SBP 85 (warning range) → wait for warning-evaluator lag = 0" ingest_observation "${encounter_id}" "SYSTOLIC_BP" 85 "mmHg" "manual" "${RECORDED_AT}" \ "p15-sbp-${SCRIPT_RUN_ID}" wait_for_observation_pipeline "${encounter_id}" wait_for_warning_systolic_alert "${encounter_id}" echo "OK: WARNING_SYSTOLIC_BP alert created" echo "[5/${TOTAL_STEPS}] Assert WARNING_SYSTOLIC_BP details contain metoprolol and min ago" alert_details="$(psql_cmd "SELECT details FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_SYSTOLIC_BP' ORDER BY triggered_at DESC LIMIT 1")" [[ "${alert_details}" == *"metoprolol"* ]] || { echo "Expected details to contain 'metoprolol', got: ${alert_details}" exit 1 } [[ "${alert_details}" == *"min ago"* ]] || { echo "Expected details to contain 'min ago', got: ${alert_details}" exit 1 } [[ "${alert_details}" == *"note:"* ]] || { echo "Expected details to contain 'note:', got: ${alert_details}" exit 1 } echo "OK: alert details annotated with medication context" echo "[6/${TOTAL_STEPS}] Control encounter: SBP 85 without medication → details have no note:" patient2_payload="$(jq -nc \ --arg fn "Phase15Ctrl" \ --arg ln "Verify${SCRIPT_RUN_ID}" \ '{firstName:$fn,lastName:$ln,dateOfBirth:"1970-03-15",gender:"F"}')" patient2_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient2_payload}")" assert_status "201" "${patient2_resp}" patient2_id="$(jq -r '.data.id' "${patient2_resp}")" enc2_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient2_id}/encounters" "${enc_payload}")" assert_status "201" "${enc2_resp}" encounter2_id="$(jq -r '.data.id' "${enc2_resp}")" control_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" ingest_observation "${encounter2_id}" "SYSTOLIC_BP" 85 "mmHg" "manual" "${control_at}" \ "p15-ctrl-sbp-${SCRIPT_RUN_ID}" wait_for_observation_pipeline "${encounter2_id}" wait_for_warning_systolic_alert "${encounter2_id}" control_details="$(psql_cmd "SELECT details FROM clinical_alerts WHERE encounter_id = '${encounter2_id}' AND alert_type = 'WARNING_SYSTOLIC_BP' LIMIT 1")" [[ "${control_details}" != *"note:"* ]] || { echo "Control encounter details must not contain 'note:', got: ${control_details}" exit 1 } [[ "${control_details}" != *"metoprolol"* ]] || { echo "Control encounter details must not mention metoprolol, got: ${control_details}" exit 1 } echo "OK: control alert has unannotated details (encounter=${encounter2_id})" echo "[7/${TOTAL_STEPS}] Medication API round-trip: list, get by id, validation" list_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/medications")" assert_status "200" "${list_resp}" [[ "$(jq -r '.data.totalCount' "${list_resp}")" -ge 1 ]] || { echo "Expected totalCount >= 1 on medication list" cat "${list_resp}" exit 1 } list_has_metoprolol="$(jq -r --arg id "${med_id}" '[.data.items[] | select(.id == $id and .drugName == "metoprolol")] | length' "${list_resp}")" [[ "${list_has_metoprolol}" == "1" ]] || { echo "Expected metoprolol administration in list response" cat "${list_resp}" exit 1 } get_resp="$(request GET "${BASE_URL}/api/v1/medications/${med_id}")" assert_status "200" "${get_resp}" [[ "$(jq -r '.data.id' "${get_resp}")" == "${med_id}" ]] || { echo "Expected medication id ${med_id} from GET" cat "${get_resp}" exit 1 } [[ "$(jq -r '.data.encounterId' "${get_resp}")" == "${encounter_id}" ]] || { echo "Expected encounterId ${encounter_id} on GET response" cat "${get_resp}" exit 1 } invalid_med='{"drugName":"","dose":25,"doseUnit":"mg","route":"PO","administeredBy":"nurse-1"}' invalid_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/medications" "${invalid_med}")" assert_status "400" "${invalid_resp}" [[ "$(jq -r '.error.code' "${invalid_resp}")" == "VALIDATION_ERROR" ]] || { echo "Expected VALIDATION_ERROR for empty drugName" cat "${invalid_resp}" exit 1 } echo "OK: list, get, and validation checks passed" echo echo "Phase 15 verification checks passed." echo "Medication encounter id: ${encounter_id}" echo "Control encounter id: ${encounter2_id}" echo "Medication id: ${med_id}"