vigilcare-clinical/scripts/run-phase10-verification.sh

#!/usr/bin/env bash

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}"

BASE_URL="${BASE_URL:-http://localhost:5270}"
ES_URL="${ES_URL:-http://localhost:9200}"
REDIS_PORT="${REDIS_PORT:-6382}"
TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}"
TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~ClinicalDemographicsAndObservationTests}"
ES_CONSUMER_GROUP="${ES_CONSUMER_GROUP:-es-indexer}"
INDEX_WAIT_SECS="${INDEX_WAIT_SECS:-60}"
PIPELINE_WAIT_SECS="${PIPELINE_WAIT_SECS:-45}"

EXPECTED_THRESHOLD_CODES=(
  HEART_RATE
  TEMP_C
  POTASSIUM_MEQ_L
  SPO2
  RESP_RATE
  WBC_K_UL
  SYSTOLIC_BP
  DIASTOLIC_BP
  LACTATE_MMOL_L
  AVPU
  SUPPLEMENTAL_O2
  GLUCOSE_MG_DL
)

NEW_OBSERVATION_CODES=(
  SYSTOLIC_BP
  DIASTOLIC_BP
  LACTATE_MMOL_L
  AVPU
  SUPPLEMENTAL_O2
)

TMP_FILES=()
cleanup() {
  local f
  for f in "${TMP_FILES[@]}"; do
    rm -f "${f}" "${f}.status" 2>/dev/null || true
  done
}
trap cleanup EXIT

need() {
  command -v "$1" >/dev/null 2>&1 || {
    echo "Missing dependency: $1"
    exit 1
  }
}

need curl
need jq
need dotnet

if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then
  echo "Missing dependency: docker compose (${COMPOSE_FILE})"
  exit 1
fi

compose() {
  docker compose -f "${COMPOSE_FILE}" "$@"
}

redis_cmd() {
  if command -v redis-cli >/dev/null 2>&1; then
    redis-cli -p "${REDIS_PORT}" "$@"
  else
    compose exec -T redis redis-cli "$@"
  fi
}

request() {
  local method="$1"
  local url="$2"
  local body="${3:-}"
  local tmp
  tmp="$(mktemp)"
  TMP_FILES+=("${tmp}")
  local status

  if [[ -n "${body}" ]]; then
    status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \
      -H "Content-Type: application/json" -d "${body}")"
  else
    status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")"
  fi

  echo "${status}" > "${tmp}.status"
  echo "${tmp}"
}

assert_status() {
  local expected="$1"
  local body_file="$2"
  local status
  status="$(<"${body_file}.status")"
  if [[ "${status}" != "${expected}" ]]; then
    echo "Expected HTTP ${expected}, got ${status}"
    cat "${body_file}"
    echo
    exit 1
  fi
}

indexer_lag() {
  compose exec -T kafka /opt/kafka/bin/kafka-consumer-groups.sh \
    --bootstrap-server localhost:9092 \
    --describe \
    --group "${ES_CONSUMER_GROUP}" 2>/dev/null | \
    awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }'
}

wait_for_indexer_lag_zero() {
  local elapsed=0
  local lag="unknown"
  while (( elapsed < INDEX_WAIT_SECS )); do
    lag="$(indexer_lag)"
    if [[ "${lag}" == "0" ]]; then
      return 0
    fi
    sleep 2
    elapsed=$((elapsed + 2))
  done
  echo "es-indexer lag did not reach zero within ${INDEX_WAIT_SECS}s (lag=${lag})"
  exit 1
}

es_observation_hits() {
  local code="$1"
  local encounter_id="$2"
  local payload
  payload="$(jq -nc \
    --arg code "${code}" \
    --arg enc "${encounter_id}" \
    '{query:{bool:{must:[{term:{observationCode:$code}},{term:{encounterId:$enc}}]}},size:0,track_total_hits:true}')"
  local body
  body="$(curl -sS "${ES_URL}/observations/_search" \
    -H "Content-Type: application/json" \
    -d "${payload}")"
  jq -r '.hits.total.value // .hits.total // 0' <<< "${body}"
}

echo "Phase 10 verification starting..."
echo "Repo root: ${ROOT_DIR}"
echo "API: ${BASE_URL}"

echo "[1/9] Preflight API, Elasticsearch, and Redis"
api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
es_status="$(curl -sS -o /dev/null -w "%{http_code}" "${ES_URL}/_cluster/health" || true)"
[[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status})"; exit 1; }
[[ "${es_status}" == "200" ]] || { echo "Elasticsearch not ready (${es_status})"; exit 1; }
redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; }

echo "[2/9] Verify Redis threshold cache has 12 keys"
mapfile -t redis_keys < <(redis_cmd KEYS 'threshold:*')
threshold_count="${#redis_keys[@]}"
[[ "${threshold_count}" -eq 12 ]] || {
  echo "Expected 12 Redis threshold keys, found ${threshold_count}"
  printf '  %s\n' "${redis_keys[@]}"
  exit 1
}
for code in "${EXPECTED_THRESHOLD_CODES[@]}"; do
  if ! redis_cmd EXISTS "threshold:${code}" | grep -q '^1$'; then
    echo "Missing Redis threshold key: threshold:${code}"
    exit 1
  fi
done
echo "OK: all 12 threshold keys present in Redis"

echo "[3/9] Verify seeded patient clinical demographics"
seed_resp="$(request GET "${BASE_URL}/api/v1/patients?q=MRN-000001")"
assert_status "200" "${seed_resp}"
seed_blood="$(jq -r '.data.items[0].bloodType // empty' "${seed_resp}")"
seed_allergies="$(jq -r '.data.items[0].allergies // empty' "${seed_resp}")"
[[ "${seed_blood}" == "A+" ]] || { echo "Expected seeded bloodType A+, got '${seed_blood}'"; exit 1; }
[[ "${seed_allergies}" == "Penicillin" ]] || { echo "Expected seeded allergies Penicillin, got '${seed_allergies}'"; exit 1; }
echo "OK: seeded patient has bloodType and allergies"

echo "[4/9] Register patient with clinical enrichment fields"
patient_payload='{"firstName":"Phase10","lastName":"Verify","dateOfBirth":"1980-01-01","gender":"M","bloodType":"O+","allergies":"Penicillin","emergencyContactName":"Jane Doe","emergencyContactPhone":"555-0000"}'
patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")"
assert_status "201" "${patient_resp}"
patient_id="$(jq -r '.data.id' "${patient_resp}")"
[[ "$(jq -r '.data.bloodType' "${patient_resp}")" == "O+" ]] || { echo "bloodType not returned as O+"; exit 1; }
[[ "$(jq -r '.data.allergies' "${patient_resp}")" == "Penicillin" ]] || { echo "allergies not round-tripped"; exit 1; }
echo "OK: patient id = ${patient_id}"

echo "[5/9] Open encounter with roomBed and admissionReason"
enc_payload='{"encounterType":"INPATIENT","department":"ICU","attendingPhysician":"Dr. Phase10","roomBed":"ICU-1A","admissionReason":"Chest pain"}'
enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")"
assert_status "201" "${enc_resp}"
encounter_id="$(jq -r '.data.id' "${enc_resp}")"
[[ "$(jq -r '.data.roomBed' "${enc_resp}")" == "ICU-1A" ]] || { echo "roomBed not round-tripped"; exit 1; }
[[ "$(jq -r '.data.admissionReason' "${enc_resp}")" == "Chest pain" ]] || { echo "admissionReason not round-tripped"; exit 1; }
echo "OK: encounter id = ${encounter_id}"

recorded_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"

echo "[6/9] Ingest expanded observation codes"
for code in "${NEW_OBSERVATION_CODES[@]}"; do
  case "${code}" in
    SYSTOLIC_BP)
      value=128; unit="mmHg"; source="DEVICE" ;;
    DIASTOLIC_BP)
      value=82; unit="mmHg"; source="DEVICE" ;;
    LACTATE_MMOL_L)
      value=1.2; unit="mmol/L"; source="LAB" ;;
    AVPU)
      value=0; unit="score"; source="MANUAL" ;;
    SUPPLEMENTAL_O2)
      value=0; unit="flag"; source="MANUAL" ;;
    *)
      echo "Unhandled observation code ${code}"
      exit 1
      ;;
  esac

  obs_payload="$(jq -nc \
    --arg code "${code}" \
    --argjson value "${value}" \
    --arg unit "${unit}" \
    --arg source "${source}" \
    --arg recordedAt "${recorded_at}" \
    --arg key "phase10-${code}-${encounter_id}" \
    '{observations:[{observationCode:$code,value:$value,unit:$unit,source:$source,recordedAt:$recordedAt,idempotencyKey:$key}]}')"
  obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${obs_payload}")"
  assert_status "201" "${obs_resp}"
  echo "  OK: ingested ${code}"
done

echo "[7/9] Wait for Kafka relay and Elasticsearch indexer, then verify projections"
sleep "${PIPELINE_WAIT_SECS}"
wait_for_indexer_lag_zero

for code in "${NEW_OBSERVATION_CODES[@]}"; do
  hits="$(es_observation_hits "${code}" "${encounter_id}")"
  [[ "${hits}" -ge 1 ]] || {
    echo "Expected Elasticsearch observation for ${code} on encounter ${encounter_id}, found ${hits}"
    exit 1
  }
  echo "  OK: Elasticsearch indexed ${code}"
done

enc_doc="$(curl -sS "${ES_URL}/patient_encounters/_source/${encounter_id}")"
[[ "$(jq -r '.roomBed // empty' <<< "${enc_doc}")" == "ICU-1A" ]] || {
  echo "patient_encounters document missing roomBed"
  echo "${enc_doc}"
  exit 1
}
[[ "$(jq -r '.admissionReason // empty' <<< "${enc_doc}")" == "Chest pain" ]] || {
  echo "patient_encounters document missing admissionReason"
  echo "${enc_doc}"
  exit 1
}
echo "OK: patient_encounters document has roomBed and admissionReason"

echo "[8/9] Verify GLUCOSE_MG_DL critical alert and SUPPLEMENTAL_O2 no-alert behavior"
glucose_payload="$(jq -nc \
  --arg recordedAt "${recorded_at}" \
  --arg key "phase10-glucose-${encounter_id}" \
  '{observations:[{observationCode:"GLUCOSE_MG_DL",value:35,unit:"mg/dL",source:"LAB",recordedAt:$recordedAt,idempotencyKey:$key}]}')"
glucose_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${glucose_payload}")"
assert_status "201" "${glucose_resp}"
[[ "$(jq -r '.data.alertGenerated' "${glucose_resp}")" == "true" ]] || {
  echo "Expected alertGenerated=true for critical glucose"
  cat "${glucose_resp}"
  exit 1
}
echo "OK: critical glucose generated alert"

o2_payload="$(jq -nc \
  --arg recordedAt "${recorded_at}" \
  --arg key "phase10-o2-flag-${encounter_id}" \
  '{observations:[{observationCode:"SUPPLEMENTAL_O2",value:1,unit:"flag",source:"MANUAL",recordedAt:$recordedAt,idempotencyKey:$key}]}')"
o2_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${o2_payload}")"
assert_status "201" "${o2_resp}"
[[ "$(jq -r '.data.alertGenerated' "${o2_resp}")" == "false" ]] || {
  echo "Expected no alert for SUPPLEMENTAL_O2 value 1"
  cat "${o2_resp}"
  exit 1
}
echo "OK: supplemental O2 flag did not generate alert"

echo "[9/9] Run Phase 10 integration tests"
dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}"

echo
echo "Phase 10 verification checks passed."