From 835311dffc8ffd1826850a3c4c144b46383dc1ae Mon Sep 17 00:00:00 2001 From: voltsrage Date: Fri, 19 Jun 2026 12:38:49 +0800 Subject: [PATCH] test: create verification scripts and for phase 14 or 15 --- scripts/run-phase14-verification.sh | 423 ++++++++++++++++++++++++++++ scripts/run-phase15-verification.sh | 354 +++++++++++++++++++++++ 2 files changed, 777 insertions(+) create mode 100755 scripts/run-phase14-verification.sh create mode 100755 scripts/run-phase15-verification.sh diff --git a/scripts/run-phase14-verification.sh b/scripts/run-phase14-verification.sh new file mode 100755 index 0000000..0b6f5bf --- /dev/null +++ b/scripts/run-phase14-verification.sh @@ -0,0 +1,423 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" + +BASE_URL="${BASE_URL:-http://localhost:5270}" +REDIS_PORT="${REDIS_PORT:-6382}" + +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5436}" +PGDATABASE="${PGDATABASE:-vigilcare}" +PGUSER="${PGUSER:-postgres}" +PGPASSWORD="${PGPASSWORD:-password}" + +TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}" +TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~Qsofa|FullyQualifiedName~SepsisBundle}" +FULL_TEST="${FULL_TEST:-0}" + +SEPSIS_CONSUMER_GROUP="${SEPSIS_CONSUMER_GROUP:-sepsis-engine}" +SEPSIS_WAIT_SECS="${SEPSIS_WAIT_SECS:-45}" +RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}" + +SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")" +RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + +TMP_FILES=() +cleanup() { + local f + for f in "${TMP_FILES[@]}"; do + rm -f "${f}" "${f}.status" 2>/dev/null || true + done +} +trap cleanup EXIT + +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing dependency: $1" + exit 1 + } +} + +need curl +need jq +need dotnet + +if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then + echo "Missing dependency: docker compose (${COMPOSE_FILE})" + exit 1 +fi + +compose() { + docker compose -f "${COMPOSE_FILE}" "$@" +} + +kafka_exec() { + compose exec -T kafka "$@" +} + +redis_cmd() { + if command -v redis-cli >/dev/null 2>&1; then + redis-cli -p "${REDIS_PORT}" "$@" + else + compose exec -T redis redis-cli "$@" + fi +} + +psql_cmd() { + local sql="$1" + if command -v psql >/dev/null 2>&1; then + PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" + else + compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" + fi +} + +request() { + local method="$1" + local url="$2" + local body="${3:-}" + local tmp + tmp="$(mktemp)" + TMP_FILES+=("${tmp}") + local status + + if [[ -n "${body}" ]]; then + status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \ + -H "Content-Type: application/json" -d "${body}")" + else + status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")" + fi + + echo "${status}" > "${tmp}.status" + echo "${tmp}" +} + +assert_status() { + local expected="$1" + local body_file="$2" + local status + status="$(<"${body_file}.status")" + if [[ "${status}" != "${expected}" ]]; then + echo "Expected HTTP ${expected}, got ${status}" + cat "${body_file}" + echo + exit 1 + fi +} + +ingest_observation() { + local encounter_id="$1" + local code="$2" + local value="$3" + local unit="$4" + local source="$5" + local recorded_at="$6" + local idempotency_key="$7" + + local payload + payload="$(jq -nc \ + --arg code "${code}" \ + --argjson value "${value}" \ + --arg unit "${unit}" \ + --arg source "${source}" \ + --arg recordedAt "${recorded_at}" \ + --arg key "${idempotency_key}" \ + '{observations:[{observationCode:$code,value:$value,unit:$unit,source:$source,recordedAt:$recordedAt,idempotencyKey:$key}]}')" + + local resp + resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")" + assert_status "201" "${resp}" +} + +consumer_group_lag() { + local group="$1" + kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \ + --bootstrap-server localhost:9092 \ + --describe \ + --group "${group}" 2>/dev/null | \ + awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }' +} + +wait_for_consumer_lag_zero() { + local group="$1" + local max_secs="$2" + local elapsed=0 + local lag="unknown" + while (( elapsed < max_secs )); do + lag="$(consumer_group_lag "${group}")" + if [[ "${lag}" == "0" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})" + exit 1 +} + +wait_for_outbox_processed() { + local outbox_id="$1" + local elapsed=0 + while (( elapsed < RELAY_WAIT_SECS )); do + local processed + processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")" + if [[ "${processed}" == "t" ]]; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s" + exit 1 +} + +latest_observation_outbox_id() { + local encounter_id="$1" + psql_cmd "SELECT id FROM outbox_events WHERE topic = 'observation.recorded' AND partition_key = '${encounter_id}' ORDER BY created_at DESC LIMIT 1" +} + +wait_for_observation_pipeline() { + local encounter_id="$1" + local outbox_id + outbox_id="$(latest_observation_outbox_id "${encounter_id}")" + if [[ -z "${outbox_id}" ]]; then + echo "No observation.recorded outbox row for encounter ${encounter_id}" + exit 1 + fi + wait_for_outbox_processed "${outbox_id}" + wait_for_consumer_lag_zero "${SEPSIS_CONSUMER_GROUP}" "${SEPSIS_WAIT_SECS}" +} + +wait_for_qsofa_alert() { + local encounter_id="$1" + local elapsed=0 + while (( elapsed < SEPSIS_WAIT_SECS )); do + local count + count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'QSOFA_WARNING'")" + if [[ "${count}" == "1" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "QSOFA_WARNING alert not found for encounter ${encounter_id} within ${SEPSIS_WAIT_SECS}s" + psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true + exit 1 +} + +wait_for_sepsis_alert() { + local encounter_id="$1" + local elapsed=0 + while (( elapsed < SEPSIS_WAIT_SECS )); do + local count + count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'SEPSIS_WARNING'")" + if [[ "${count}" == "1" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "SEPSIS_WARNING alert not found for encounter ${encounter_id} within ${SEPSIS_WAIT_SECS}s" + psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true + exit 1 +} + +TOTAL_STEPS=9 +echo "Phase 14 verification starting..." +echo "Repo root: ${ROOT_DIR}" +echo "API: ${BASE_URL}" + +echo "[1/${TOTAL_STEPS}] Infrastructure preflight: API, PostgreSQL, Redis, Kafka, threshold cache" +api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" +[[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; } +redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; } +psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; } +kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || { + echo "Kafka not ready" + exit 1 +} +redis_cmd EXISTS "threshold:HEART_RATE" | grep -q '^1$' || { + echo "Missing Redis threshold:HEART_RATE — restart API to run ThresholdCacheLoader" + exit 1 +} +echo "OK: infrastructure preflight passed" + +echo "[2/${TOTAL_STEPS}] Register patient + open active encounter" +patient_payload="$(jq -nc \ + --arg fn "Phase14" \ + --arg ln "Verify${SCRIPT_RUN_ID}" \ + '{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')" +patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")" +assert_status "201" "${patient_resp}" +patient_id="$(jq -r '.data.id' "${patient_resp}")" + +enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase14"}' +enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")" +assert_status "201" "${enc_resp}" +encounter_id="$(jq -r '.data.id' "${enc_resp}")" +echo "OK: patient=${patient_id} encounter=${encounter_id}" + +echo "[3/${TOTAL_STEPS}] Ingest RR 24, SBP 95, AVPU 1 → assert QSOFA_WARNING alert" +ingest_observation "${encounter_id}" "RESP_RATE" 24 "/min" "manual" "${RECORDED_AT}" \ + "p14-rr-${SCRIPT_RUN_ID}" +wait_for_observation_pipeline "${encounter_id}" + +ingest_observation "${encounter_id}" "SYSTOLIC_BP" 95 "mmHg" "manual" "${RECORDED_AT}" \ + "p14-sbp-${SCRIPT_RUN_ID}" +wait_for_observation_pipeline "${encounter_id}" + +ingest_observation "${encounter_id}" "AVPU" 1 "score" "manual" "${RECORDED_AT}" \ + "p14-avpu-${SCRIPT_RUN_ID}" +wait_for_observation_pipeline "${encounter_id}" + +wait_for_qsofa_alert "${encounter_id}" + +alert_row="$(psql_cmd "SELECT alert_type || '|' || severity FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'QSOFA_WARNING' LIMIT 1")" +[[ "${alert_row}" == "QSOFA_WARNING|CRITICAL" ]] || { + echo "Unexpected QSOFA_WARNING row: '${alert_row}'" + exit 1 +} +echo "OK: QSOFA_WARNING alert created (severity=CRITICAL)" + +echo "[4/${TOTAL_STEPS}] Assert sepsis_bundles row with 4 elements and 4 pending orders" +bundle_count="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundles WHERE encounter_id = '${encounter_id}'")" +[[ "${bundle_count}" == "1" ]] || { + echo "Expected 1 sepsis_bundles row, found ${bundle_count}" + exit 1 +} + +element_count="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}')")" +[[ "${element_count}" == "4" ]] || { + echo "Expected 4 bundle elements, found ${element_count}" + exit 1 +} + +pending_elements="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}') AND status = 'PENDING'")" +[[ "${pending_elements}" == "4" ]] || { + echo "Expected 4 pending elements, found ${pending_elements}" + exit 1 +} + +compliance_status="$(psql_cmd "SELECT compliance_status FROM sepsis_bundles WHERE encounter_id = '${encounter_id}'")" +[[ "${compliance_status}" == "IN_PROGRESS" ]] || { + echo "Expected compliance_status IN_PROGRESS, got '${compliance_status}'" + exit 1 +} + +orders_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/orders?status=PENDING")" +assert_status "200" "${orders_resp}" +pending_order_count="$(jq -r '.data.totalCount' "${orders_resp}")" +[[ "${pending_order_count}" == "4" ]] || { + echo "Expected 4 pending orders, found ${pending_order_count}" + cat "${orders_resp}" + exit 1 +} + +bundle_api_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/sepsis-bundle/current")" +assert_status "200" "${bundle_api_resp}" +[[ "$(jq -r '.data.elements | length' "${bundle_api_resp}")" == "4" ]] || { + echo "Expected 4 elements on sepsis-bundle/current API" + cat "${bundle_api_resp}" + exit 1 +} +echo "OK: bundle with 4 pending elements and 4 pending orders" + +echo "[5/${TOTAL_STEPS}] Result each order → assert elements completed" +order_ids="$(psql_cmd "SELECT order_id FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}') ORDER BY element_type")" +while IFS= read -r order_id; do + [[ -n "${order_id}" ]] || continue + result_payload='{"resultSummary":"Phase 14 verification — result recorded"}' + result_resp="$(request PATCH "${BASE_URL}/api/v1/orders/${order_id}/result" "${result_payload}")" + assert_status "200" "${result_resp}" + [[ "$(jq -r '.data.status' "${result_resp}")" == "Resulted" ]] || { + echo "Expected order ${order_id} status Resulted" + cat "${result_resp}" + exit 1 + } +done <<< "${order_ids}" + +completed_elements="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}') AND status = 'COMPLETED'")" +[[ "${completed_elements}" == "4" ]] || { + echo "Expected 4 completed elements, found ${completed_elements}" + psql_cmd "SELECT element_type, status FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}')" || true + exit 1 +} +echo "OK: all 4 bundle elements completed" + +echo "[6/${TOTAL_STEPS}] Assert bundle compliance_status = COMPLIANT" +compliance_status="$(psql_cmd "SELECT compliance_status FROM sepsis_bundles WHERE encounter_id = '${encounter_id}'")" +[[ "${compliance_status}" == "COMPLIANT" ]] || { + echo "Expected compliance_status COMPLIANT, got '${compliance_status}'" + exit 1 +} + +completed_at_set="$(psql_cmd "SELECT completed_at IS NOT NULL FROM sepsis_bundles WHERE encounter_id = '${encounter_id}'")" +[[ "${completed_at_set}" == "t" ]] || { + echo "Expected completed_at to be set on compliant bundle" + exit 1 +} +echo "OK: bundle marked COMPLIANT" + +echo "[7/${TOTAL_STEPS}] Second encounter: trigger SIRS (temp + HR) → assert bundle created" +patient2_payload="$(jq -nc \ + --arg fn "Phase14SIRS" \ + --arg ln "Verify${SCRIPT_RUN_ID}" \ + '{firstName:$fn,lastName:$ln,dateOfBirth:"1970-03-15",gender:"F"}')" +patient2_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient2_payload}")" +assert_status "201" "${patient2_resp}" +patient2_id="$(jq -r '.data.id' "${patient2_resp}")" + +enc2_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient2_id}/encounters" "${enc_payload}")" +assert_status "201" "${enc2_resp}" +encounter2_id="$(jq -r '.data.id' "${enc2_resp}")" + +sirs_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +ingest_observation "${encounter2_id}" "HEART_RATE" 95 "bpm" "DEVICE" "${sirs_at}" \ + "p14-sirs-hr-${SCRIPT_RUN_ID}" +wait_for_observation_pipeline "${encounter2_id}" + +ingest_observation "${encounter2_id}" "TEMP_C" 38.5 "°C" "DEVICE" "${sirs_at}" \ + "p14-sirs-temp-${SCRIPT_RUN_ID}" +wait_for_observation_pipeline "${encounter2_id}" + +wait_for_sepsis_alert "${encounter2_id}" + +bundle2_count="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundles WHERE encounter_id = '${encounter2_id}'")" +[[ "${bundle2_count}" == "1" ]] || { + echo "Expected 1 sepsis_bundles row for SIRS encounter, found ${bundle2_count}" + exit 1 +} + +trigger_type="$(psql_cmd "SELECT triggering_alert_type FROM sepsis_bundles WHERE encounter_id = '${encounter2_id}'")" +[[ "${trigger_type}" == "SEPSIS_WARNING" ]] || { + echo "Expected triggering_alert_type SEPSIS_WARNING, got '${trigger_type}'" + exit 1 +} + +element2_count="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter2_id}')")" +[[ "${element2_count}" == "4" ]] || { + echo "Expected 4 bundle elements for SIRS encounter, found ${element2_count}" + exit 1 +} +echo "OK: SIRS encounter bundle created (encounter=${encounter2_id})" + +echo "[8/${TOTAL_STEPS}] Consumer lag = 0 for sepsis-engine" +wait_for_consumer_lag_zero "${SEPSIS_CONSUMER_GROUP}" "${SEPSIS_WAIT_SECS}" +echo "OK: sepsis-engine lag = 0" + +echo "[9/${TOTAL_STEPS}] Run Phase 14 unit/integration tests" +dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}" + +if [[ "${FULL_TEST}" == "1" ]]; then + echo "Running full test suite (FULL_TEST=1)" + dotnet test "${ROOT_DIR}/VigilCareClinicalAPI.sln" 2>/dev/null || dotnet test "${ROOT_DIR}" +fi + +echo +echo "Phase 14 verification checks passed." +echo "qSOFA encounter id: ${encounter_id}" +echo "SIRS encounter id: ${encounter2_id}" diff --git a/scripts/run-phase15-verification.sh b/scripts/run-phase15-verification.sh new file mode 100755 index 0000000..4723c51 --- /dev/null +++ b/scripts/run-phase15-verification.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" + +BASE_URL="${BASE_URL:-http://localhost:5270}" +REDIS_PORT="${REDIS_PORT:-6382}" + +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5436}" +PGDATABASE="${PGDATABASE:-vigilcare}" +PGUSER="${PGUSER:-postgres}" +PGPASSWORD="${PGPASSWORD:-password}" + +WARNING_CONSUMER_GROUP="${WARNING_CONSUMER_GROUP:-warning-evaluator}" +WARNING_WAIT_SECS="${WARNING_WAIT_SECS:-45}" +RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}" + +SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")" +RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + +TMP_FILES=() +cleanup() { + local f + for f in "${TMP_FILES[@]}"; do + rm -f "${f}" "${f}.status" 2>/dev/null || true + done +} +trap cleanup EXIT + +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing dependency: $1" + exit 1 + } +} + +need curl +need jq + +if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then + echo "Missing dependency: docker compose (${COMPOSE_FILE})" + exit 1 +fi + +compose() { + docker compose -f "${COMPOSE_FILE}" "$@" +} + +kafka_exec() { + compose exec -T kafka "$@" +} + +redis_cmd() { + if command -v redis-cli >/dev/null 2>&1; then + redis-cli -p "${REDIS_PORT}" "$@" + else + compose exec -T redis redis-cli "$@" + fi +} + +psql_cmd() { + local sql="$1" + if command -v psql >/dev/null 2>&1; then + PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" + else + compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" + fi +} + +request() { + local method="$1" + local url="$2" + local body="${3:-}" + local tmp + tmp="$(mktemp)" + TMP_FILES+=("${tmp}") + local status + + if [[ -n "${body}" ]]; then + status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \ + -H "Content-Type: application/json" -d "${body}")" + else + status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")" + fi + + echo "${status}" > "${tmp}.status" + echo "${tmp}" +} + +assert_status() { + local expected="$1" + local body_file="$2" + local status + status="$(<"${body_file}.status")" + if [[ "${status}" != "${expected}" ]]; then + echo "Expected HTTP ${expected}, got ${status}" + cat "${body_file}" + echo + exit 1 + fi +} + +ingest_observation() { + local encounter_id="$1" + local code="$2" + local value="$3" + local unit="$4" + local source="$5" + local recorded_at="$6" + local idempotency_key="$7" + + local payload + payload="$(jq -nc \ + --arg code "${code}" \ + --argjson value "${value}" \ + --arg unit "${unit}" \ + --arg source "${source}" \ + --arg recordedAt "${recorded_at}" \ + --arg key "${idempotency_key}" \ + '{observations:[{observationCode:$code,value:$value,unit:$unit,source:$source,recordedAt:$recordedAt,idempotencyKey:$key}]}')" + + local resp + resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")" + assert_status "201" "${resp}" +} + +consumer_group_lag() { + local group="$1" + kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \ + --bootstrap-server localhost:9092 \ + --describe \ + --group "${group}" 2>/dev/null | \ + awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }' +} + +wait_for_consumer_lag_zero() { + local group="$1" + local max_secs="$2" + local elapsed=0 + local lag="unknown" + while (( elapsed < max_secs )); do + lag="$(consumer_group_lag "${group}")" + if [[ "${lag}" == "0" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})" + exit 1 +} + +wait_for_outbox_processed() { + local outbox_id="$1" + local elapsed=0 + while (( elapsed < RELAY_WAIT_SECS )); do + local processed + processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")" + if [[ "${processed}" == "t" ]]; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s" + exit 1 +} + +latest_observation_outbox_id() { + local encounter_id="$1" + psql_cmd "SELECT id FROM outbox_events WHERE topic = 'observation.recorded' AND partition_key = '${encounter_id}' ORDER BY created_at DESC LIMIT 1" +} + +wait_for_observation_pipeline() { + local encounter_id="$1" + local outbox_id + outbox_id="$(latest_observation_outbox_id "${encounter_id}")" + if [[ -z "${outbox_id}" ]]; then + echo "No observation.recorded outbox row for encounter ${encounter_id}" + exit 1 + fi + wait_for_outbox_processed "${outbox_id}" + wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}" +} + +wait_for_warning_systolic_alert() { + local encounter_id="$1" + local elapsed=0 + while (( elapsed < WARNING_WAIT_SECS )); do + local count + count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_SYSTOLIC_BP'")" + if [[ "${count}" == "1" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "WARNING_SYSTOLIC_BP alert not found for encounter ${encounter_id} within ${WARNING_WAIT_SECS}s" + psql_cmd "SELECT alert_type, severity, status, details FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true + exit 1 +} + +TOTAL_STEPS=7 +echo "Phase 15 verification starting..." +echo "Repo root: ${ROOT_DIR}" +echo "API: ${BASE_URL}" + +echo "[1/${TOTAL_STEPS}] Infrastructure preflight: API, PostgreSQL, Redis, Kafka, threshold cache" +api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" +[[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; } +redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; } +psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; } +kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || { + echo "Kafka not ready" + exit 1 +} +redis_cmd EXISTS "threshold:SYSTOLIC_BP" | grep -q '^1$' || { + echo "Missing Redis threshold:SYSTOLIC_BP — restart API to run ThresholdCacheLoader" + exit 1 +} +echo "OK: infrastructure preflight passed" + +echo "[2/${TOTAL_STEPS}] Register patient + open active encounter" +patient_payload="$(jq -nc \ + --arg fn "Phase15" \ + --arg ln "Verify${SCRIPT_RUN_ID}" \ + '{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')" +patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")" +assert_status "201" "${patient_resp}" +patient_id="$(jq -r '.data.id' "${patient_resp}")" + +enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase15"}' +enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")" +assert_status "201" "${enc_resp}" +encounter_id="$(jq -r '.data.id' "${enc_resp}")" +echo "OK: patient=${patient_id} encounter=${encounter_id}" + +echo "[3/${TOTAL_STEPS}] POST /encounters/{id}/medications — metoprolol 25mg PO" +med_payload='{"drugName":"metoprolol","dose":25,"doseUnit":"mg","route":"PO","administeredBy":"nurse-1"}' +med_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/medications" "${med_payload}")" +assert_status "201" "${med_resp}" +med_id="$(jq -r '.data.id' "${med_resp}")" +[[ "$(jq -r '.data.drugName' "${med_resp}")" == "metoprolol" ]] || { + echo "Expected drugName metoprolol" + cat "${med_resp}" + exit 1 +} +[[ "$(jq -r '.data.dose' "${med_resp}")" == "25" ]] || { + echo "Expected dose 25" + cat "${med_resp}" + exit 1 +} +echo "OK: medication created (id=${med_id})" + +echo "[4/${TOTAL_STEPS}] Ingest SBP 85 (warning range) → wait for warning-evaluator lag = 0" +ingest_observation "${encounter_id}" "SYSTOLIC_BP" 85 "mmHg" "manual" "${RECORDED_AT}" \ + "p15-sbp-${SCRIPT_RUN_ID}" +wait_for_observation_pipeline "${encounter_id}" +wait_for_warning_systolic_alert "${encounter_id}" +echo "OK: WARNING_SYSTOLIC_BP alert created" + +echo "[5/${TOTAL_STEPS}] Assert WARNING_SYSTOLIC_BP details contain metoprolol and min ago" +alert_details="$(psql_cmd "SELECT details FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_SYSTOLIC_BP' ORDER BY triggered_at DESC LIMIT 1")" +[[ "${alert_details}" == *"metoprolol"* ]] || { + echo "Expected details to contain 'metoprolol', got: ${alert_details}" + exit 1 +} +[[ "${alert_details}" == *"min ago"* ]] || { + echo "Expected details to contain 'min ago', got: ${alert_details}" + exit 1 +} +[[ "${alert_details}" == *"note:"* ]] || { + echo "Expected details to contain 'note:', got: ${alert_details}" + exit 1 +} +echo "OK: alert details annotated with medication context" + +echo "[6/${TOTAL_STEPS}] Control encounter: SBP 85 without medication → details have no note:" +patient2_payload="$(jq -nc \ + --arg fn "Phase15Ctrl" \ + --arg ln "Verify${SCRIPT_RUN_ID}" \ + '{firstName:$fn,lastName:$ln,dateOfBirth:"1970-03-15",gender:"F"}')" +patient2_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient2_payload}")" +assert_status "201" "${patient2_resp}" +patient2_id="$(jq -r '.data.id' "${patient2_resp}")" + +enc2_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient2_id}/encounters" "${enc_payload}")" +assert_status "201" "${enc2_resp}" +encounter2_id="$(jq -r '.data.id' "${enc2_resp}")" + +control_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +ingest_observation "${encounter2_id}" "SYSTOLIC_BP" 85 "mmHg" "manual" "${control_at}" \ + "p15-ctrl-sbp-${SCRIPT_RUN_ID}" +wait_for_observation_pipeline "${encounter2_id}" +wait_for_warning_systolic_alert "${encounter2_id}" + +control_details="$(psql_cmd "SELECT details FROM clinical_alerts WHERE encounter_id = '${encounter2_id}' AND alert_type = 'WARNING_SYSTOLIC_BP' LIMIT 1")" +[[ "${control_details}" != *"note:"* ]] || { + echo "Control encounter details must not contain 'note:', got: ${control_details}" + exit 1 +} +[[ "${control_details}" != *"metoprolol"* ]] || { + echo "Control encounter details must not mention metoprolol, got: ${control_details}" + exit 1 +} +echo "OK: control alert has unannotated details (encounter=${encounter2_id})" + +echo "[7/${TOTAL_STEPS}] Medication API round-trip: list, get by id, validation" +list_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/medications")" +assert_status "200" "${list_resp}" +[[ "$(jq -r '.data.totalCount' "${list_resp}")" -ge 1 ]] || { + echo "Expected totalCount >= 1 on medication list" + cat "${list_resp}" + exit 1 +} +list_has_metoprolol="$(jq -r --arg id "${med_id}" '[.data.items[] | select(.id == $id and .drugName == "metoprolol")] | length' "${list_resp}")" +[[ "${list_has_metoprolol}" == "1" ]] || { + echo "Expected metoprolol administration in list response" + cat "${list_resp}" + exit 1 +} + +get_resp="$(request GET "${BASE_URL}/api/v1/medications/${med_id}")" +assert_status "200" "${get_resp}" +[[ "$(jq -r '.data.id' "${get_resp}")" == "${med_id}" ]] || { + echo "Expected medication id ${med_id} from GET" + cat "${get_resp}" + exit 1 +} +[[ "$(jq -r '.data.encounterId' "${get_resp}")" == "${encounter_id}" ]] || { + echo "Expected encounterId ${encounter_id} on GET response" + cat "${get_resp}" + exit 1 +} + +invalid_med='{"drugName":"","dose":25,"doseUnit":"mg","route":"PO","administeredBy":"nurse-1"}' +invalid_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/medications" "${invalid_med}")" +assert_status "400" "${invalid_resp}" +[[ "$(jq -r '.error.code' "${invalid_resp}")" == "VALIDATION_ERROR" ]] || { + echo "Expected VALIDATION_ERROR for empty drugName" + cat "${invalid_resp}" + exit 1 +} +echo "OK: list, get, and validation checks passed" + +echo +echo "Phase 15 verification checks passed." +echo "Medication encounter id: ${encounter_id}" +echo "Control encounter id: ${encounter2_id}" +echo "Medication id: ${med_id}"