#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}" BASE_URL="${BASE_URL:-http://localhost:5270}" PROM_URL="${PROM_URL:-http://localhost:9101}" SEQ_URL="${SEQ_URL:-http://localhost:5345}" PGHOST="${PGHOST:-localhost}" PGPORT="${PGPORT:-5436}" PGDATABASE="${PGDATABASE:-vigilcare}" PGUSER="${PGUSER:-postgres}" PGPASSWORD="${PGPASSWORD:-password}" COLLECTOR_WAIT_SECS="${COLLECTOR_WAIT_SECS:-35}" TMP_FILES=() cleanup() { local f for f in "${TMP_FILES[@]}"; do rm -f "$f" "$f.status" 2>/dev/null || true done } trap cleanup EXIT need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing dependency: $1" exit 1 } } need curl need jq request() { local method="$1" local url="$2" local body="${3:-}" local tmp tmp="$(mktemp)" TMP_FILES+=("$tmp") local status if [[ -n "$body" ]]; then status="$(curl -sS -o "$tmp" -w "%{http_code}" -X "$method" "$url" \ -H "Content-Type: application/json" -d "$body")" else status="$(curl -sS -o "$tmp" -w "%{http_code}" -X "$method" "$url")" fi echo "$status" > "$tmp.status" echo "$tmp" } assert_status() { local expected="$1" local body_file="$2" local status status="$(<"$body_file.status")" if [[ "$status" != "$expected" ]]; then echo "Expected HTTP $expected, got $status" cat "$body_file" echo exit 1 fi } psql_cmd() { local sql="$1" if command -v psql >/dev/null 2>&1; then PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -tAc "$sql" elif command -v docker >/dev/null 2>&1 && [[ -f "${COMPOSE_FILE}" ]]; then docker compose -f "${COMPOSE_FILE}" exec -T postgres \ psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" else echo "Missing dependency: psql (or docker compose with postgres service)" exit 1 fi } metric_value() { local name="$1" local body="$2" awk -v m="$name" ' $0 ~ "^"m"([ \t]|\\{|$)" { n=split($0, a, /[ \t]+/); if (n >= 2) { print a[n]; exit 0; } } ' <<< "$body" } echo "Phase 8 verification against ${BASE_URL}" echo "[1/9] Preflight API and Prometheus" api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" prom_status="$(curl -sS -o /dev/null -w "%{http_code}" "${PROM_URL}/-/healthy" || true)" [[ "$api_status" == "200" ]] || { echo "API not ready (${api_status})"; exit 1; } [[ "$prom_status" == "200" ]] || { echo "Prometheus not ready (${prom_status})"; exit 1; } echo "[2/9] Ensure Prometheus target vigilcare_api is UP" target_json="$(curl -sS "${PROM_URL}/api/v1/query?query=up%7Bjob%3D%22vigilcare_api%22%7D")" up_val="$(jq -r '.data.result[0].value[1] // "0"' <<< "$target_json")" [[ "$up_val" == "1" ]] || { echo "vigilcare_api target not UP"; exit 1; } echo "[3/9] Validate /metrics contains all eight metric families" metrics_body="$(curl -sS "${BASE_URL}/metrics")" required=( observations_ingested_total observation_ingest_duration_seconds clinical_alerts_total alerts_unacknowledged_gauge kafka_consumer_lag outbox_pending_events sirs_detections_total escalations_total ) for m in "${required[@]}"; do grep -q "$m" <<< "$metrics_body" || { echo "Missing metric: $m"; exit 1; } done echo "[4/9] Verify correlation header on API response" corr="$(curl -sSI "${BASE_URL}/api/v1/alert-thresholds" | awk -F': ' 'tolower($1)=="x-correlation-id"{print $2}' | tr -d '\r')" [[ -n "$corr" ]] || { echo "Missing X-Correlation-Id header"; exit 1; } echo "[5/9] Create patient and active encounter" patient_resp="$(request POST "${BASE_URL}/api/v1/patients" '{"firstName":"Phase8","lastName":"Verify","dateOfBirth":"1988-01-10","gender":"F"}')" assert_status "201" "$patient_resp" patient_id="$(jq -r '.data.id' "$patient_resp")" enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" '{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Verify"}')" assert_status "201" "$enc_resp" encounter_id="$(jq -r '.data.id' "$enc_resp")" transition_resp="$(request PATCH "${BASE_URL}/api/v1/encounters/${encounter_id}/status" '{"status":"Active"}')" transition_status="$(<"$transition_resp.status")" if [[ "$transition_status" != "200" && "$transition_status" != "409" ]]; then echo "Unexpected status transitioning encounter to active: ${transition_status}" cat "$transition_resp" exit 1 fi echo "[6/9] Create critical alert, backdate 6 minutes" baseline_metrics="$(curl -sS "${BASE_URL}/metrics")" baseline_gauge="$(metric_value "alerts_unacknowledged_gauge" "$baseline_metrics")" baseline_gauge_int="${baseline_gauge%.*}" baseline_gauge_int="${baseline_gauge_int:-0}" recorded_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" obs_payload="$(jq -nc --arg ts "$recorded_at" \ '{observations:[{observationCode:"POTASSIUM_MEQ_L",value:2.1,unit:"mEq/L",source:"LAB",recordedAt:$ts}]}' )" obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "$obs_payload")" assert_status "201" "$obs_resp" alert_id="$(jq -r '.data.alertId // empty' "$obs_resp")" [[ -n "$alert_id" ]] || { echo "Critical observation did not return alertId"; cat "$obs_resp"; exit 1; } psql_cmd "UPDATE clinical_alerts SET triggered_at = NOW() - INTERVAL '6 minutes' WHERE id = '${alert_id}';" >/dev/null echo "[7/9] Wait collector and verify alerts_unacknowledged_gauge increased" sleep "$COLLECTOR_WAIT_SECS" metrics_body="$(curl -sS "${BASE_URL}/metrics")" gauge="$(metric_value "alerts_unacknowledged_gauge" "$metrics_body")" gauge_int="${gauge%.*}" gauge_int="${gauge_int:-0}" expected_min=$((baseline_gauge_int + 1)) [[ -n "$gauge" && "$gauge_int" -ge "$expected_min" ]] || { echo "alerts_unacknowledged_gauge did not increase (baseline=${baseline_gauge_int}, value=${gauge:-missing})" exit 1 } echo "[8/9] Acknowledge alert and verify gauge returned to baseline" ack_resp="$(request POST "${BASE_URL}/api/v1/alerts/${alert_id}/acknowledge" '{"clinicianId":"DR-VERIFY","note":"phase8 verification"}')" assert_status "200" "$ack_resp" sleep "$COLLECTOR_WAIT_SECS" metrics_body="$(curl -sS "${BASE_URL}/metrics")" gauge="$(metric_value "alerts_unacknowledged_gauge" "$metrics_body")" gauge_int="${gauge%.*}" gauge_int="${gauge_int:-0}" [[ "$gauge_int" -le "$baseline_gauge_int" ]] || { echo "alerts_unacknowledged_gauge did not return to baseline (baseline=${baseline_gauge_int}, value=${gauge:-missing})" exit 1 } echo "[9/9] Manual Seq check" echo "Open ${SEQ_URL} and filter: EncounterId IS NOT NULL" echo "Confirm logs include EncounterId, PatientId, CorrelationId." echo echo "Phase 8 verification checks passed."