424 lines
15 KiB
Bash
Executable File
424 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}"
|
|
|
|
BASE_URL="${BASE_URL:-http://localhost:5270}"
|
|
REDIS_PORT="${REDIS_PORT:-6382}"
|
|
|
|
PGHOST="${PGHOST:-localhost}"
|
|
PGPORT="${PGPORT:-5436}"
|
|
PGDATABASE="${PGDATABASE:-vigilcare}"
|
|
PGUSER="${PGUSER:-postgres}"
|
|
PGPASSWORD="${PGPASSWORD:-password}"
|
|
|
|
TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}"
|
|
TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~Qsofa|FullyQualifiedName~SepsisBundle}"
|
|
FULL_TEST="${FULL_TEST:-0}"
|
|
|
|
SEPSIS_CONSUMER_GROUP="${SEPSIS_CONSUMER_GROUP:-sepsis-engine}"
|
|
SEPSIS_WAIT_SECS="${SEPSIS_WAIT_SECS:-45}"
|
|
RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}"
|
|
|
|
SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")"
|
|
RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
|
|
|
TMP_FILES=()
|
|
cleanup() {
|
|
local f
|
|
for f in "${TMP_FILES[@]}"; do
|
|
rm -f "${f}" "${f}.status" 2>/dev/null || true
|
|
done
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
need() {
|
|
command -v "$1" >/dev/null 2>&1 || {
|
|
echo "Missing dependency: $1"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
need curl
|
|
need jq
|
|
need dotnet
|
|
|
|
if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then
|
|
echo "Missing dependency: docker compose (${COMPOSE_FILE})"
|
|
exit 1
|
|
fi
|
|
|
|
compose() {
|
|
docker compose -f "${COMPOSE_FILE}" "$@"
|
|
}
|
|
|
|
kafka_exec() {
|
|
compose exec -T kafka "$@"
|
|
}
|
|
|
|
redis_cmd() {
|
|
if command -v redis-cli >/dev/null 2>&1; then
|
|
redis-cli -p "${REDIS_PORT}" "$@"
|
|
else
|
|
compose exec -T redis redis-cli "$@"
|
|
fi
|
|
}
|
|
|
|
psql_cmd() {
|
|
local sql="$1"
|
|
if command -v psql >/dev/null 2>&1; then
|
|
PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
|
|
else
|
|
compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
|
|
fi
|
|
}
|
|
|
|
request() {
|
|
local method="$1"
|
|
local url="$2"
|
|
local body="${3:-}"
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
TMP_FILES+=("${tmp}")
|
|
local status
|
|
|
|
if [[ -n "${body}" ]]; then
|
|
status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \
|
|
-H "Content-Type: application/json" -d "${body}")"
|
|
else
|
|
status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")"
|
|
fi
|
|
|
|
echo "${status}" > "${tmp}.status"
|
|
echo "${tmp}"
|
|
}
|
|
|
|
assert_status() {
|
|
local expected="$1"
|
|
local body_file="$2"
|
|
local status
|
|
status="$(<"${body_file}.status")"
|
|
if [[ "${status}" != "${expected}" ]]; then
|
|
echo "Expected HTTP ${expected}, got ${status}"
|
|
cat "${body_file}"
|
|
echo
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
ingest_observation() {
|
|
local encounter_id="$1"
|
|
local code="$2"
|
|
local value="$3"
|
|
local unit="$4"
|
|
local source="$5"
|
|
local recorded_at="$6"
|
|
local idempotency_key="$7"
|
|
|
|
local payload
|
|
payload="$(jq -nc \
|
|
--arg code "${code}" \
|
|
--argjson value "${value}" \
|
|
--arg unit "${unit}" \
|
|
--arg source "${source}" \
|
|
--arg recordedAt "${recorded_at}" \
|
|
--arg key "${idempotency_key}" \
|
|
'{observations:[{observationCode:$code,value:$value,unit:$unit,source:$source,recordedAt:$recordedAt,idempotencyKey:$key}]}')"
|
|
|
|
local resp
|
|
resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${payload}")"
|
|
assert_status "201" "${resp}"
|
|
}
|
|
|
|
consumer_group_lag() {
|
|
local group="$1"
|
|
kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \
|
|
--bootstrap-server localhost:9092 \
|
|
--describe \
|
|
--group "${group}" 2>/dev/null | \
|
|
awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }'
|
|
}
|
|
|
|
wait_for_consumer_lag_zero() {
|
|
local group="$1"
|
|
local max_secs="$2"
|
|
local elapsed=0
|
|
local lag="unknown"
|
|
while (( elapsed < max_secs )); do
|
|
lag="$(consumer_group_lag "${group}")"
|
|
if [[ "${lag}" == "0" ]]; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})"
|
|
exit 1
|
|
}
|
|
|
|
wait_for_outbox_processed() {
|
|
local outbox_id="$1"
|
|
local elapsed=0
|
|
while (( elapsed < RELAY_WAIT_SECS )); do
|
|
local processed
|
|
processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")"
|
|
if [[ "${processed}" == "t" ]]; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
elapsed=$((elapsed + 1))
|
|
done
|
|
echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s"
|
|
exit 1
|
|
}
|
|
|
|
latest_observation_outbox_id() {
|
|
local encounter_id="$1"
|
|
psql_cmd "SELECT id FROM outbox_events WHERE topic = 'observation.recorded' AND partition_key = '${encounter_id}' ORDER BY created_at DESC LIMIT 1"
|
|
}
|
|
|
|
wait_for_observation_pipeline() {
|
|
local encounter_id="$1"
|
|
local outbox_id
|
|
outbox_id="$(latest_observation_outbox_id "${encounter_id}")"
|
|
if [[ -z "${outbox_id}" ]]; then
|
|
echo "No observation.recorded outbox row for encounter ${encounter_id}"
|
|
exit 1
|
|
fi
|
|
wait_for_outbox_processed "${outbox_id}"
|
|
wait_for_consumer_lag_zero "${SEPSIS_CONSUMER_GROUP}" "${SEPSIS_WAIT_SECS}"
|
|
}
|
|
|
|
wait_for_qsofa_alert() {
|
|
local encounter_id="$1"
|
|
local elapsed=0
|
|
while (( elapsed < SEPSIS_WAIT_SECS )); do
|
|
local count
|
|
count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'QSOFA_WARNING'")"
|
|
if [[ "${count}" == "1" ]]; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
echo "QSOFA_WARNING alert not found for encounter ${encounter_id} within ${SEPSIS_WAIT_SECS}s"
|
|
psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true
|
|
exit 1
|
|
}
|
|
|
|
wait_for_sepsis_alert() {
|
|
local encounter_id="$1"
|
|
local elapsed=0
|
|
while (( elapsed < SEPSIS_WAIT_SECS )); do
|
|
local count
|
|
count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'SEPSIS_WARNING'")"
|
|
if [[ "${count}" == "1" ]]; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
echo "SEPSIS_WARNING alert not found for encounter ${encounter_id} within ${SEPSIS_WAIT_SECS}s"
|
|
psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true
|
|
exit 1
|
|
}
|
|
|
|
TOTAL_STEPS=9
|
|
echo "Phase 14 verification starting..."
|
|
echo "Repo root: ${ROOT_DIR}"
|
|
echo "API: ${BASE_URL}"
|
|
|
|
echo "[1/${TOTAL_STEPS}] Infrastructure preflight: API, PostgreSQL, Redis, Kafka, threshold cache"
|
|
api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
|
|
[[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; }
|
|
redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; }
|
|
psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; }
|
|
kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || {
|
|
echo "Kafka not ready"
|
|
exit 1
|
|
}
|
|
redis_cmd EXISTS "threshold:HEART_RATE" | grep -q '^1$' || {
|
|
echo "Missing Redis threshold:HEART_RATE — restart API to run ThresholdCacheLoader"
|
|
exit 1
|
|
}
|
|
echo "OK: infrastructure preflight passed"
|
|
|
|
echo "[2/${TOTAL_STEPS}] Register patient + open active encounter"
|
|
patient_payload="$(jq -nc \
|
|
--arg fn "Phase14" \
|
|
--arg ln "Verify${SCRIPT_RUN_ID}" \
|
|
'{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')"
|
|
patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")"
|
|
assert_status "201" "${patient_resp}"
|
|
patient_id="$(jq -r '.data.id' "${patient_resp}")"
|
|
|
|
enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase14"}'
|
|
enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")"
|
|
assert_status "201" "${enc_resp}"
|
|
encounter_id="$(jq -r '.data.id' "${enc_resp}")"
|
|
echo "OK: patient=${patient_id} encounter=${encounter_id}"
|
|
|
|
echo "[3/${TOTAL_STEPS}] Ingest RR 24, SBP 95, AVPU 1 → assert QSOFA_WARNING alert"
|
|
ingest_observation "${encounter_id}" "RESP_RATE" 24 "/min" "manual" "${RECORDED_AT}" \
|
|
"p14-rr-${SCRIPT_RUN_ID}"
|
|
wait_for_observation_pipeline "${encounter_id}"
|
|
|
|
ingest_observation "${encounter_id}" "SYSTOLIC_BP" 95 "mmHg" "manual" "${RECORDED_AT}" \
|
|
"p14-sbp-${SCRIPT_RUN_ID}"
|
|
wait_for_observation_pipeline "${encounter_id}"
|
|
|
|
ingest_observation "${encounter_id}" "AVPU" 1 "score" "manual" "${RECORDED_AT}" \
|
|
"p14-avpu-${SCRIPT_RUN_ID}"
|
|
wait_for_observation_pipeline "${encounter_id}"
|
|
|
|
wait_for_qsofa_alert "${encounter_id}"
|
|
|
|
alert_row="$(psql_cmd "SELECT alert_type || '|' || severity FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'QSOFA_WARNING' LIMIT 1")"
|
|
[[ "${alert_row}" == "QSOFA_WARNING|CRITICAL" ]] || {
|
|
echo "Unexpected QSOFA_WARNING row: '${alert_row}'"
|
|
exit 1
|
|
}
|
|
echo "OK: QSOFA_WARNING alert created (severity=CRITICAL)"
|
|
|
|
echo "[4/${TOTAL_STEPS}] Assert sepsis_bundles row with 4 elements and 4 pending orders"
|
|
bundle_count="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundles WHERE encounter_id = '${encounter_id}'")"
|
|
[[ "${bundle_count}" == "1" ]] || {
|
|
echo "Expected 1 sepsis_bundles row, found ${bundle_count}"
|
|
exit 1
|
|
}
|
|
|
|
element_count="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}')")"
|
|
[[ "${element_count}" == "4" ]] || {
|
|
echo "Expected 4 bundle elements, found ${element_count}"
|
|
exit 1
|
|
}
|
|
|
|
pending_elements="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}') AND status = 'PENDING'")"
|
|
[[ "${pending_elements}" == "4" ]] || {
|
|
echo "Expected 4 pending elements, found ${pending_elements}"
|
|
exit 1
|
|
}
|
|
|
|
compliance_status="$(psql_cmd "SELECT compliance_status FROM sepsis_bundles WHERE encounter_id = '${encounter_id}'")"
|
|
[[ "${compliance_status}" == "IN_PROGRESS" ]] || {
|
|
echo "Expected compliance_status IN_PROGRESS, got '${compliance_status}'"
|
|
exit 1
|
|
}
|
|
|
|
orders_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/orders?status=PENDING")"
|
|
assert_status "200" "${orders_resp}"
|
|
pending_order_count="$(jq -r '.data.totalCount' "${orders_resp}")"
|
|
[[ "${pending_order_count}" == "4" ]] || {
|
|
echo "Expected 4 pending orders, found ${pending_order_count}"
|
|
cat "${orders_resp}"
|
|
exit 1
|
|
}
|
|
|
|
bundle_api_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/sepsis-bundle/current")"
|
|
assert_status "200" "${bundle_api_resp}"
|
|
[[ "$(jq -r '.data.elements | length' "${bundle_api_resp}")" == "4" ]] || {
|
|
echo "Expected 4 elements on sepsis-bundle/current API"
|
|
cat "${bundle_api_resp}"
|
|
exit 1
|
|
}
|
|
echo "OK: bundle with 4 pending elements and 4 pending orders"
|
|
|
|
echo "[5/${TOTAL_STEPS}] Result each order → assert elements completed"
|
|
order_ids="$(psql_cmd "SELECT order_id FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}') ORDER BY element_type")"
|
|
while IFS= read -r order_id; do
|
|
[[ -n "${order_id}" ]] || continue
|
|
result_payload='{"resultSummary":"Phase 14 verification — result recorded"}'
|
|
result_resp="$(request PATCH "${BASE_URL}/api/v1/orders/${order_id}/result" "${result_payload}")"
|
|
assert_status "200" "${result_resp}"
|
|
[[ "$(jq -r '.data.status' "${result_resp}")" == "Resulted" ]] || {
|
|
echo "Expected order ${order_id} status Resulted"
|
|
cat "${result_resp}"
|
|
exit 1
|
|
}
|
|
done <<< "${order_ids}"
|
|
|
|
completed_elements="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}') AND status = 'COMPLETED'")"
|
|
[[ "${completed_elements}" == "4" ]] || {
|
|
echo "Expected 4 completed elements, found ${completed_elements}"
|
|
psql_cmd "SELECT element_type, status FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter_id}')" || true
|
|
exit 1
|
|
}
|
|
echo "OK: all 4 bundle elements completed"
|
|
|
|
echo "[6/${TOTAL_STEPS}] Assert bundle compliance_status = COMPLIANT"
|
|
compliance_status="$(psql_cmd "SELECT compliance_status FROM sepsis_bundles WHERE encounter_id = '${encounter_id}'")"
|
|
[[ "${compliance_status}" == "COMPLIANT" ]] || {
|
|
echo "Expected compliance_status COMPLIANT, got '${compliance_status}'"
|
|
exit 1
|
|
}
|
|
|
|
completed_at_set="$(psql_cmd "SELECT completed_at IS NOT NULL FROM sepsis_bundles WHERE encounter_id = '${encounter_id}'")"
|
|
[[ "${completed_at_set}" == "t" ]] || {
|
|
echo "Expected completed_at to be set on compliant bundle"
|
|
exit 1
|
|
}
|
|
echo "OK: bundle marked COMPLIANT"
|
|
|
|
echo "[7/${TOTAL_STEPS}] Second encounter: trigger SIRS (temp + HR) → assert bundle created"
|
|
patient2_payload="$(jq -nc \
|
|
--arg fn "Phase14SIRS" \
|
|
--arg ln "Verify${SCRIPT_RUN_ID}" \
|
|
'{firstName:$fn,lastName:$ln,dateOfBirth:"1970-03-15",gender:"F"}')"
|
|
patient2_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient2_payload}")"
|
|
assert_status "201" "${patient2_resp}"
|
|
patient2_id="$(jq -r '.data.id' "${patient2_resp}")"
|
|
|
|
enc2_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient2_id}/encounters" "${enc_payload}")"
|
|
assert_status "201" "${enc2_resp}"
|
|
encounter2_id="$(jq -r '.data.id' "${enc2_resp}")"
|
|
|
|
sirs_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
|
ingest_observation "${encounter2_id}" "HEART_RATE" 95 "bpm" "DEVICE" "${sirs_at}" \
|
|
"p14-sirs-hr-${SCRIPT_RUN_ID}"
|
|
wait_for_observation_pipeline "${encounter2_id}"
|
|
|
|
ingest_observation "${encounter2_id}" "TEMP_C" 38.5 "°C" "DEVICE" "${sirs_at}" \
|
|
"p14-sirs-temp-${SCRIPT_RUN_ID}"
|
|
wait_for_observation_pipeline "${encounter2_id}"
|
|
|
|
wait_for_sepsis_alert "${encounter2_id}"
|
|
|
|
bundle2_count="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundles WHERE encounter_id = '${encounter2_id}'")"
|
|
[[ "${bundle2_count}" == "1" ]] || {
|
|
echo "Expected 1 sepsis_bundles row for SIRS encounter, found ${bundle2_count}"
|
|
exit 1
|
|
}
|
|
|
|
trigger_type="$(psql_cmd "SELECT triggering_alert_type FROM sepsis_bundles WHERE encounter_id = '${encounter2_id}'")"
|
|
[[ "${trigger_type}" == "SEPSIS_WARNING" ]] || {
|
|
echo "Expected triggering_alert_type SEPSIS_WARNING, got '${trigger_type}'"
|
|
exit 1
|
|
}
|
|
|
|
element2_count="$(psql_cmd "SELECT COUNT(*) FROM sepsis_bundle_elements WHERE bundle_id = (SELECT id FROM sepsis_bundles WHERE encounter_id = '${encounter2_id}')")"
|
|
[[ "${element2_count}" == "4" ]] || {
|
|
echo "Expected 4 bundle elements for SIRS encounter, found ${element2_count}"
|
|
exit 1
|
|
}
|
|
echo "OK: SIRS encounter bundle created (encounter=${encounter2_id})"
|
|
|
|
echo "[8/${TOTAL_STEPS}] Consumer lag = 0 for sepsis-engine"
|
|
wait_for_consumer_lag_zero "${SEPSIS_CONSUMER_GROUP}" "${SEPSIS_WAIT_SECS}"
|
|
echo "OK: sepsis-engine lag = 0"
|
|
|
|
echo "[9/${TOTAL_STEPS}] Run Phase 14 unit/integration tests"
|
|
dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}"
|
|
|
|
if [[ "${FULL_TEST}" == "1" ]]; then
|
|
echo "Running full test suite (FULL_TEST=1)"
|
|
dotnet test "${ROOT_DIR}/VigilCareClinicalAPI.sln" 2>/dev/null || dotnet test "${ROOT_DIR}"
|
|
fi
|
|
|
|
echo
|
|
echo "Phase 14 verification checks passed."
|
|
echo "qSOFA encounter id: ${encounter_id}"
|
|
echo "SIRS encounter id: ${encounter2_id}"
|