vigilcare-clinical/scripts/run-phase15-verification.sh

#!/usr/bin/env bash

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}"

BASE_URL="${BASE_URL:-http://localhost:5270}"
REDIS_PORT="${REDIS_PORT:-6382}"

PGHOST="${PGHOST:-localhost}"
PGPORT="${PGPORT:-5436}"
PGDATABASE="${PGDATABASE:-vigilcare}"
PGUSER="${PGUSER:-postgres}"
PGPASSWORD="${PGPASSWORD:-password}"

WARNING_CONSUMER_GROUP="${WARNING_CONSUMER_GROUP:-warning-evaluator}"
WARNING_WAIT_SECS="${WARNING_WAIT_SECS:-45}"
RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}"

SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")"
RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"

TMP_FILES=()
cleanup() {
  local f
  for f in "${TMP_FILES[@]}"; do
    rm -f "${f}" "${f}.status" 2>/dev/null || true
  done
}
trap cleanup EXIT

need() {
  command -v "$1" >/dev/null 2>&1 || {
    echo "Missing dependency: $1"
    exit 1
  }
}

need curl
need jq

if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then
  echo "Missing dependency: docker compose (${COMPOSE_FILE})"
  exit 1
fi

compose() {
  docker compose -f "${COMPOSE_FILE}" "$@"
}

kafka_exec() {
  compose exec -T kafka "$@"
}

redis_cmd() {
  if command -v redis-cli >/dev/null 2>&1; then
    redis-cli -p "${REDIS_PORT}" "$@"
  else
    compose exec -T redis redis-cli "$@"
  fi
}

psql_cmd() {
  local sql="$1"
  if command -v psql >/dev/null 2>&1; then
    PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
  else
    compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
  fi
}

request() {
  local method="$1"
  local url="$2"
  local body="${3:-}"
  local tmp
  tmp="$(mktemp)"
  TMP_FILES+=("${tmp}")
  local status

  if [[ -n "${body}" ]]; then
    status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \
      -H "Content-Type: application/json" -d "${body}")"
  else
    status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")"
  fi

  echo "${status}" > "${tmp}.status"
  echo "${tmp}"
}

assert_status() {
  local expected="$1"
  local body_file="$2"
  local status
  status="$(<"${body_file}.status")"
  if [[ "${status}" != "${expected}" ]]; then
    echo "Expected HTTP ${expected}, got ${status}"
    cat "${body_file}"
    echo
    exit 1
  fi
}

ingest_observation() {
  local encounter_id="$1"
  local code="$2"
  local value="$3"
  local unit="$4"
  local source="$5"
  local recorded_at="$6"
  local idempotency_key="$7"

  local payload
  payload="$(jq -nc \
    --arg code "${code}" \
    --argjson value "${value}" \
    --arg unit "${unit}" \
    --arg source "${source}" \
    --arg recordedAt "${recorded_at}" \
    --arg key "${idempotency_key}" \
    '{observations:[{observationCode:$code,value:$value,unit:$unit,source:$source,recordedAt:$recordedAt,idempotencyKey:$key}]}')"

  local resp
  resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")"
  assert_status "201" "${resp}"
}

consumer_group_lag() {
  local group="$1"
  kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \
    --bootstrap-server localhost:9092 \
    --describe \
    --group "${group}" 2>/dev/null | \
    awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }'
}

wait_for_consumer_lag_zero() {
  local group="$1"
  local max_secs="$2"
  local elapsed=0
  local lag="unknown"
  while (( elapsed < max_secs )); do
    lag="$(consumer_group_lag "${group}")"
    if [[ "${lag}" == "0" ]]; then
      return 0
    fi
    sleep 2
    elapsed=$((elapsed + 2))
  done
  echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})"
  exit 1
}

wait_for_outbox_processed() {
  local outbox_id="$1"
  local elapsed=0
  while (( elapsed < RELAY_WAIT_SECS )); do
    local processed
    processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")"
    if [[ "${processed}" == "t" ]]; then
      return 0
    fi
    sleep 1
    elapsed=$((elapsed + 1))
  done
  echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s"
  exit 1
}

latest_observation_outbox_id() {
  local encounter_id="$1"
  psql_cmd "SELECT id FROM outbox_events WHERE topic = 'observation.recorded' AND partition_key = '${encounter_id}' ORDER BY created_at DESC LIMIT 1"
}

wait_for_observation_pipeline() {
  local encounter_id="$1"
  local outbox_id
  outbox_id="$(latest_observation_outbox_id "${encounter_id}")"
  if [[ -z "${outbox_id}" ]]; then
    echo "No observation.recorded outbox row for encounter ${encounter_id}"
    exit 1
  fi
  wait_for_outbox_processed "${outbox_id}"
  wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}"
}

wait_for_warning_systolic_alert() {
  local encounter_id="$1"
  local elapsed=0
  while (( elapsed < WARNING_WAIT_SECS )); do
    local count
    count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_SYSTOLIC_BP'")"
    if [[ "${count}" == "1" ]]; then
      return 0
    fi
    sleep 2
    elapsed=$((elapsed + 2))
  done
  echo "WARNING_SYSTOLIC_BP alert not found for encounter ${encounter_id} within ${WARNING_WAIT_SECS}s"
  psql_cmd "SELECT alert_type, severity, status, details FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true
  exit 1
}

TOTAL_STEPS=7
echo "Phase 15 verification starting..."
echo "Repo root: ${ROOT_DIR}"
echo "API: ${BASE_URL}"

echo "[1/${TOTAL_STEPS}] Infrastructure preflight: API, PostgreSQL, Redis, Kafka, threshold cache"
api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
[[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; }
redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; }
psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; }
kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || {
  echo "Kafka not ready"
  exit 1
}
redis_cmd EXISTS "threshold:SYSTOLIC_BP" | grep -q '^1$' || {
  echo "Missing Redis threshold:SYSTOLIC_BP — restart API to run ThresholdCacheLoader"
  exit 1
}
echo "OK: infrastructure preflight passed"

echo "[2/${TOTAL_STEPS}] Register patient + open active encounter"
patient_payload="$(jq -nc \
  --arg fn "Phase15" \
  --arg ln "Verify${SCRIPT_RUN_ID}" \
  '{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')"
patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")"
assert_status "201" "${patient_resp}"
patient_id="$(jq -r '.data.id' "${patient_resp}")"

enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase15"}'
enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")"
assert_status "201" "${enc_resp}"
encounter_id="$(jq -r '.data.id' "${enc_resp}")"
echo "OK: patient=${patient_id} encounter=${encounter_id}"

echo "[3/${TOTAL_STEPS}] POST /encounters/{id}/medications — metoprolol 25mg PO"
med_payload='{"drugName":"metoprolol","dose":25,"doseUnit":"mg","route":"PO","administeredBy":"nurse-1"}'
med_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/medications" "${med_payload}")"
assert_status "201" "${med_resp}"
med_id="$(jq -r '.data.id' "${med_resp}")"
[[ "$(jq -r '.data.drugName' "${med_resp}")" == "metoprolol" ]] || {
  echo "Expected drugName metoprolol"
  cat "${med_resp}"
  exit 1
}
[[ "$(jq -r '.data.dose' "${med_resp}")" == "25" ]] || {
  echo "Expected dose 25"
  cat "${med_resp}"
  exit 1
}
echo "OK: medication created (id=${med_id})"

echo "[4/${TOTAL_STEPS}] Ingest SBP 85 (warning range) → wait for warning-evaluator lag = 0"
ingest_observation "${encounter_id}" "SYSTOLIC_BP" 85 "mmHg" "manual" "${RECORDED_AT}" \
  "p15-sbp-${SCRIPT_RUN_ID}"
wait_for_observation_pipeline "${encounter_id}"
wait_for_warning_systolic_alert "${encounter_id}"
echo "OK: WARNING_SYSTOLIC_BP alert created"

echo "[5/${TOTAL_STEPS}] Assert WARNING_SYSTOLIC_BP details contain metoprolol and min ago"
alert_details="$(psql_cmd "SELECT details FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_SYSTOLIC_BP' ORDER BY triggered_at DESC LIMIT 1")"
[[ "${alert_details}" == *"metoprolol"* ]] || {
  echo "Expected details to contain 'metoprolol', got: ${alert_details}"
  exit 1
}
[[ "${alert_details}" == *"min ago"* ]] || {
  echo "Expected details to contain 'min ago', got: ${alert_details}"
  exit 1
}
[[ "${alert_details}" == *"note:"* ]] || {
  echo "Expected details to contain 'note:', got: ${alert_details}"
  exit 1
}
echo "OK: alert details annotated with medication context"

echo "[6/${TOTAL_STEPS}] Control encounter: SBP 85 without medication → details have no note:"
patient2_payload="$(jq -nc \
  --arg fn "Phase15Ctrl" \
  --arg ln "Verify${SCRIPT_RUN_ID}" \
  '{firstName:$fn,lastName:$ln,dateOfBirth:"1970-03-15",gender:"F"}')"
patient2_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient2_payload}")"
assert_status "201" "${patient2_resp}"
patient2_id="$(jq -r '.data.id' "${patient2_resp}")"

enc2_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient2_id}/encounters" "${enc_payload}")"
assert_status "201" "${enc2_resp}"
encounter2_id="$(jq -r '.data.id' "${enc2_resp}")"

control_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
ingest_observation "${encounter2_id}" "SYSTOLIC_BP" 85 "mmHg" "manual" "${control_at}" \
  "p15-ctrl-sbp-${SCRIPT_RUN_ID}"
wait_for_observation_pipeline "${encounter2_id}"
wait_for_warning_systolic_alert "${encounter2_id}"

control_details="$(psql_cmd "SELECT details FROM clinical_alerts WHERE encounter_id = '${encounter2_id}' AND alert_type = 'WARNING_SYSTOLIC_BP' LIMIT 1")"
[[ "${control_details}" != *"note:"* ]] || {
  echo "Control encounter details must not contain 'note:', got: ${control_details}"
  exit 1
}
[[ "${control_details}" != *"metoprolol"* ]] || {
  echo "Control encounter details must not mention metoprolol, got: ${control_details}"
  exit 1
}
echo "OK: control alert has unannotated details (encounter=${encounter2_id})"

echo "[7/${TOTAL_STEPS}] Medication API round-trip: list, get by id, validation"
list_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/medications")"
assert_status "200" "${list_resp}"
[[ "$(jq -r '.data.totalCount' "${list_resp}")" -ge 1 ]] || {
  echo "Expected totalCount >= 1 on medication list"
  cat "${list_resp}"
  exit 1
}
list_has_metoprolol="$(jq -r --arg id "${med_id}" '[.data.items[] | select(.id == $id and .drugName == "metoprolol")] | length' "${list_resp}")"
[[ "${list_has_metoprolol}" == "1" ]] || {
  echo "Expected metoprolol administration in list response"
  cat "${list_resp}"
  exit 1
}

get_resp="$(request GET "${BASE_URL}/api/v1/medications/${med_id}")"
assert_status "200" "${get_resp}"
[[ "$(jq -r '.data.id' "${get_resp}")" == "${med_id}" ]] || {
  echo "Expected medication id ${med_id} from GET"
  cat "${get_resp}"
  exit 1
}
[[ "$(jq -r '.data.encounterId' "${get_resp}")" == "${encounter_id}" ]] || {
  echo "Expected encounterId ${encounter_id} on GET response"
  cat "${get_resp}"
  exit 1
}

invalid_med='{"drugName":"","dose":25,"doseUnit":"mg","route":"PO","administeredBy":"nurse-1"}'
invalid_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/medications" "${invalid_med}")"
assert_status "400" "${invalid_resp}"
[[ "$(jq -r '.error.code' "${invalid_resp}")" == "VALIDATION_ERROR" ]] || {
  echo "Expected VALIDATION_ERROR for empty drugName"
  cat "${invalid_resp}"
  exit 1
}
echo "OK: list, get, and validation checks passed"

echo
echo "Phase 15 verification checks passed."
echo "Medication encounter id: ${encounter_id}"
echo "Control encounter id: ${encounter2_id}"
echo "Medication id: ${med_id}"