#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" BASE_URL="${BASE_URL:-http://localhost:5270}" REDIS_PORT="${REDIS_PORT:-6382}" PGHOST="${PGHOST:-localhost}" PGPORT="${PGPORT:-5436}" PGDATABASE="${PGDATABASE:-vigilcare}" PGUSER="${PGUSER:-postgres}" PGPASSWORD="${PGPASSWORD:-password}" TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}" TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~Trend|FullyQualifiedName~Suppression}" FULL_TEST="${FULL_TEST:-0}" TREND_CONSUMER_GROUP="${TREND_CONSUMER_GROUP:-trend-analyzer}" WARNING_CONSUMER_GROUP="${WARNING_CONSUMER_GROUP:-warning-evaluator}" TREND_WAIT_SECS="${TREND_WAIT_SECS:-45}" WARNING_WAIT_SECS="${WARNING_WAIT_SECS:-45}" SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")" RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" TMP_FILES=() cleanup() { local f for f in "${TMP_FILES[@]}"; do rm -f "${f}" "${f}.status" 2>/dev/null || true done } trap cleanup EXIT need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing dependency: $1" exit 1 } } need curl need jq need dotnet if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then echo "Missing dependency: docker compose (${COMPOSE_FILE})" exit 1 fi compose() { docker compose -f "${COMPOSE_FILE}" "$@" } kafka_exec() { compose exec -T kafka "$@" } redis_cmd() { if command -v redis-cli >/dev/null 2>&1; then redis-cli -p "${REDIS_PORT}" "$@" else compose exec -T redis redis-cli "$@" fi } psql_cmd() { local sql="$1" if command -v psql >/dev/null 2>&1; then PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" else compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" fi } request() { local method="$1" local url="$2" local body="${3:-}" local tmp tmp="$(mktemp)" TMP_FILES+=("${tmp}") local status if [[ -n "${body}" ]]; then status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \ -H "Content-Type: application/json" -d "${body}")" else status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")" fi echo "${status}" > "${tmp}.status" echo "${tmp}" } assert_status() { local expected="$1" local body_file="$2" local status status="$(<"${body_file}.status")" if [[ "${status}" != "${expected}" ]]; then echo "Expected HTTP ${expected}, got ${status}" cat "${body_file}" echo exit 1 fi } consumer_group_lag() { local group="$1" kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \ --bootstrap-server localhost:9092 \ --describe \ --group "${group}" 2>/dev/null | \ awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }' } wait_for_consumer_lag_zero() { local group="$1" local max_secs="$2" local elapsed=0 local lag="unknown" while (( elapsed < max_secs )); do lag="$(consumer_group_lag "${group}")" if [[ "${lag}" == "0" ]]; then return 0 fi sleep 2 elapsed=$((elapsed + 2)) done echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})" exit 1 } ingest_hr() { local encounter_id="$1" local value="$2" local recorded_at="${3:-${RECORDED_AT}}" jq -nc \ --arg recordedAt "${recorded_at}" \ --argjson value "${value}" \ '{observations:[{"observationCode":"HEART_RATE","value":$value,"unit":"bpm","source":"DEVICE","recordedAt":$recordedAt}]}' } TOTAL_STEPS=10 echo "Phase 13 verification starting..." echo "Repo root: ${ROOT_DIR}" echo "API: ${BASE_URL}" echo "[1/${TOTAL_STEPS}] Infrastructure preflight: API, PostgreSQL, Redis, Kafka, threshold cache" api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" [[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; } redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; } psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; } kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || { echo "Kafka not ready" exit 1 } redis_cmd EXISTS "threshold:HEART_RATE" | grep -q '^1$' || { echo "Missing Redis threshold:HEART_RATE — restart API to run ThresholdCacheLoader" exit 1 } echo "OK: infrastructure preflight passed" echo "[2/${TOTAL_STEPS}] Register patient + open active encounter" patient_payload="$(jq -nc \ --arg fn "Phase13" \ --arg ln "Verify${SCRIPT_RUN_ID}" \ '{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')" patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")" assert_status "201" "${patient_resp}" patient_id="$(jq -r '.data.id' "${patient_resp}")" enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase13"}' enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")" assert_status "201" "${enc_resp}" encounter_id="$(jq -r '.data.id' "${enc_resp}")" echo "OK: patient=${patient_id} encounter=${encounter_id}" echo "[3/${TOTAL_STEPS}] Ingest HR 72, wait 5s, ingest HR 95 → assert RAPID_DETERIORATION" baseline_at="$(date -u -d '10 minutes ago' +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-10M +"%Y-%m-%dT%H:%M:%SZ")" climb_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" hr72_payload="$(ingest_hr "${encounter_id}" 72 "${baseline_at}")" obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr72_payload}")" assert_status "201" "${obs_resp}" sleep 5 hr95_payload="$(ingest_hr "${encounter_id}" 95 "${climb_at}")" obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr95_payload}")" assert_status "201" "${obs_resp}" wait_for_consumer_lag_zero "${TREND_CONSUMER_GROUP}" "${TREND_WAIT_SECS}" trend_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'RAPID_DETERIORATION'")" [[ "${trend_count}" == "1" ]] || { echo "Expected 1 RAPID_DETERIORATION alert, found ${trend_count}" psql_cmd "SELECT alert_type, severity, details FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true exit 1 } echo "OK: RAPID_DETERIORATION alert created" echo "[4/${TOTAL_STEPS}] Re-ingest HR 95 → assert no duplicate trend alert" obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr95_payload}")" assert_status "201" "${obs_resp}" wait_for_consumer_lag_zero "${TREND_CONSUMER_GROUP}" "${TREND_WAIT_SECS}" trend_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'RAPID_DETERIORATION'")" [[ "${trend_count}" == "1" ]] || { echo "Expected idempotent RAPID_DETERIORATION count=1, found ${trend_count}" exit 1 } echo "OK: no duplicate RAPID_DETERIORATION alert" echo "[5/${TOTAL_STEPS}] Ingest HR 105 (warning range) → assert WARNING_HEART_RATE created" hr105_payload="$(ingest_hr "${encounter_id}" 105)" obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr105_payload}")" assert_status "201" "${obs_resp}" wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}" warning_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_HEART_RATE'")" [[ "${warning_count}" == "1" ]] || { echo "Expected 1 WARNING_HEART_RATE alert, found ${warning_count}" psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true exit 1 } echo "OK: WARNING_HEART_RATE alert created" echo "[6/${TOTAL_STEPS}] Acknowledge warning alert → assert Redis suppress key exists" alert_id="$(psql_cmd "SELECT id FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_HEART_RATE' LIMIT 1")" ack_resp="$(request POST "${BASE_URL}/api/v1/alerts/${alert_id}/acknowledge" \ '{"clinicianId":"nurse-1","note":"monitoring"}')" assert_status "200" "${ack_resp}" suppress_key="suppress:${encounter_id}:WARNING_HEART_RATE" redis_cmd EXISTS "${suppress_key}" | grep -q '^1$' || { echo "Expected Redis key ${suppress_key}" redis_cmd KEYS "suppress:${encounter_id}:*" || true exit 1 } echo "OK: suppression key ${suppress_key} exists" echo "[7/${TOTAL_STEPS}] Re-ingest HR 105 → assert no new WARNING_HEART_RATE (suppressed)" obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr105_payload}")" assert_status "201" "${obs_resp}" wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}" warning_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_HEART_RATE'")" [[ "${warning_count}" == "1" ]] || { echo "Expected suppressed WARNING_HEART_RATE count=1, found ${warning_count}" exit 1 } echo "OK: no new WARNING_HEART_RATE while suppressed" echo "[8/${TOTAL_STEPS}] Ingest HR 160 (critical) → assert CRITICAL_HEART_RATE fires despite suppression" hr160_payload="$(ingest_hr "${encounter_id}" 160)" obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr160_payload}")" assert_status "201" "${obs_resp}" critical_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'CRITICAL_HEART_RATE'")" [[ "${critical_count}" == "1" ]] || { echo "Expected 1 CRITICAL_HEART_RATE alert despite active WARNING suppression, found ${critical_count}" psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true exit 1 } echo "OK: CRITICAL_HEART_RATE created despite suppression window" echo "[9/${TOTAL_STEPS}] Consumer lag = 0 for trend-analyzer and warning-evaluator" wait_for_consumer_lag_zero "${TREND_CONSUMER_GROUP}" "${TREND_WAIT_SECS}" wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}" echo "OK: trend-analyzer and warning-evaluator lag = 0" echo "[10/${TOTAL_STEPS}] Run Phase 13 unit/integration tests" dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}" if [[ "${FULL_TEST}" == "1" ]]; then echo "Running full test suite (FULL_TEST=1)" dotnet test "${ROOT_DIR}/VigilCareClinicalAPI.sln" 2>/dev/null || dotnet test "${ROOT_DIR}" fi echo echo "Phase 13 verification checks passed." echo "Encounter id: ${encounter_id}"