feature: Trend Detection & Alert Suppression Windows
This commit is contained in:
voltsrage
Executable
+281
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}"
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:5270}"
|
||||
REDIS_PORT="${REDIS_PORT:-6382}"
|
||||
|
||||
PGHOST="${PGHOST:-localhost}"
|
||||
PGPORT="${PGPORT:-5436}"
|
||||
PGDATABASE="${PGDATABASE:-vigilcare}"
|
||||
PGUSER="${PGUSER:-postgres}"
|
||||
PGPASSWORD="${PGPASSWORD:-password}"
|
||||
|
||||
TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}"
|
||||
TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~Trend|FullyQualifiedName~Suppression}"
|
||||
FULL_TEST="${FULL_TEST:-0}"
|
||||
|
||||
TREND_CONSUMER_GROUP="${TREND_CONSUMER_GROUP:-trend-analyzer}"
|
||||
WARNING_CONSUMER_GROUP="${WARNING_CONSUMER_GROUP:-warning-evaluator}"
|
||||
TREND_WAIT_SECS="${TREND_WAIT_SECS:-45}"
|
||||
WARNING_WAIT_SECS="${WARNING_WAIT_SECS:-45}"
|
||||
|
||||
SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")"
|
||||
RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
|
||||
TMP_FILES=()
|
||||
cleanup() {
|
||||
local f
|
||||
for f in "${TMP_FILES[@]}"; do
|
||||
rm -f "${f}" "${f}.status" 2>/dev/null || true
|
||||
done
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
need() {
|
||||
command -v "$1" >/dev/null 2>&1 || {
|
||||
echo "Missing dependency: $1"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
need curl
|
||||
need jq
|
||||
need dotnet
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then
|
||||
echo "Missing dependency: docker compose (${COMPOSE_FILE})"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compose() {
|
||||
docker compose -f "${COMPOSE_FILE}" "$@"
|
||||
}
|
||||
|
||||
kafka_exec() {
|
||||
compose exec -T kafka "$@"
|
||||
}
|
||||
|
||||
redis_cmd() {
|
||||
if command -v redis-cli >/dev/null 2>&1; then
|
||||
redis-cli -p "${REDIS_PORT}" "$@"
|
||||
else
|
||||
compose exec -T redis redis-cli "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
psql_cmd() {
|
||||
local sql="$1"
|
||||
if command -v psql >/dev/null 2>&1; then
|
||||
PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
|
||||
else
|
||||
compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
|
||||
fi
|
||||
}
|
||||
|
||||
request() {
|
||||
local method="$1"
|
||||
local url="$2"
|
||||
local body="${3:-}"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
TMP_FILES+=("${tmp}")
|
||||
local status
|
||||
|
||||
if [[ -n "${body}" ]]; then
|
||||
status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \
|
||||
-H "Content-Type: application/json" -d "${body}")"
|
||||
else
|
||||
status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")"
|
||||
fi
|
||||
|
||||
echo "${status}" > "${tmp}.status"
|
||||
echo "${tmp}"
|
||||
}
|
||||
|
||||
assert_status() {
|
||||
local expected="$1"
|
||||
local body_file="$2"
|
||||
local status
|
||||
status="$(<"${body_file}.status")"
|
||||
if [[ "${status}" != "${expected}" ]]; then
|
||||
echo "Expected HTTP ${expected}, got ${status}"
|
||||
cat "${body_file}"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
consumer_group_lag() {
|
||||
local group="$1"
|
||||
kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \
|
||||
--bootstrap-server localhost:9092 \
|
||||
--describe \
|
||||
--group "${group}" 2>/dev/null | \
|
||||
awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }'
|
||||
}
|
||||
|
||||
wait_for_consumer_lag_zero() {
|
||||
local group="$1"
|
||||
local max_secs="$2"
|
||||
local elapsed=0
|
||||
local lag="unknown"
|
||||
while (( elapsed < max_secs )); do
|
||||
lag="$(consumer_group_lag "${group}")"
|
||||
if [[ "${lag}" == "0" ]]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
done
|
||||
echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})"
|
||||
exit 1
|
||||
}
|
||||
|
||||
ingest_hr() {
|
||||
local encounter_id="$1"
|
||||
local value="$2"
|
||||
local recorded_at="${3:-${RECORDED_AT}}"
|
||||
jq -nc \
|
||||
--arg recordedAt "${recorded_at}" \
|
||||
--argjson value "${value}" \
|
||||
'{observations:[{"observationCode":"HEART_RATE","value":$value,"unit":"bpm","source":"DEVICE","recordedAt":$recordedAt}]}'
|
||||
}
|
||||
|
||||
TOTAL_STEPS=10
|
||||
echo "Phase 13 verification starting..."
|
||||
echo "Repo root: ${ROOT_DIR}"
|
||||
echo "API: ${BASE_URL}"
|
||||
|
||||
echo "[1/${TOTAL_STEPS}] Infrastructure preflight: API, PostgreSQL, Redis, Kafka, threshold cache"
|
||||
api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
|
||||
[[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; }
|
||||
redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; }
|
||||
psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; }
|
||||
kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || {
|
||||
echo "Kafka not ready"
|
||||
exit 1
|
||||
}
|
||||
redis_cmd EXISTS "threshold:HEART_RATE" | grep -q '^1$' || {
|
||||
echo "Missing Redis threshold:HEART_RATE — restart API to run ThresholdCacheLoader"
|
||||
exit 1
|
||||
}
|
||||
echo "OK: infrastructure preflight passed"
|
||||
|
||||
echo "[2/${TOTAL_STEPS}] Register patient + open active encounter"
|
||||
patient_payload="$(jq -nc \
|
||||
--arg fn "Phase13" \
|
||||
--arg ln "Verify${SCRIPT_RUN_ID}" \
|
||||
'{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')"
|
||||
patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")"
|
||||
assert_status "201" "${patient_resp}"
|
||||
patient_id="$(jq -r '.data.id' "${patient_resp}")"
|
||||
|
||||
enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase13"}'
|
||||
enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")"
|
||||
assert_status "201" "${enc_resp}"
|
||||
encounter_id="$(jq -r '.data.id' "${enc_resp}")"
|
||||
echo "OK: patient=${patient_id} encounter=${encounter_id}"
|
||||
|
||||
echo "[3/${TOTAL_STEPS}] Ingest HR 72, wait 5s, ingest HR 95 → assert RAPID_DETERIORATION"
|
||||
baseline_at="$(date -u -d '10 minutes ago' +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-10M +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
climb_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
hr72_payload="$(ingest_hr "${encounter_id}" 72 "${baseline_at}")"
|
||||
obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr72_payload}")"
|
||||
assert_status "201" "${obs_resp}"
|
||||
sleep 5
|
||||
hr95_payload="$(ingest_hr "${encounter_id}" 95 "${climb_at}")"
|
||||
obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr95_payload}")"
|
||||
assert_status "201" "${obs_resp}"
|
||||
wait_for_consumer_lag_zero "${TREND_CONSUMER_GROUP}" "${TREND_WAIT_SECS}"
|
||||
|
||||
trend_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'RAPID_DETERIORATION'")"
|
||||
[[ "${trend_count}" == "1" ]] || {
|
||||
echo "Expected 1 RAPID_DETERIORATION alert, found ${trend_count}"
|
||||
psql_cmd "SELECT alert_type, severity, details FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true
|
||||
exit 1
|
||||
}
|
||||
echo "OK: RAPID_DETERIORATION alert created"
|
||||
|
||||
echo "[4/${TOTAL_STEPS}] Re-ingest HR 95 → assert no duplicate trend alert"
|
||||
obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr95_payload}")"
|
||||
assert_status "201" "${obs_resp}"
|
||||
wait_for_consumer_lag_zero "${TREND_CONSUMER_GROUP}" "${TREND_WAIT_SECS}"
|
||||
trend_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'RAPID_DETERIORATION'")"
|
||||
[[ "${trend_count}" == "1" ]] || {
|
||||
echo "Expected idempotent RAPID_DETERIORATION count=1, found ${trend_count}"
|
||||
exit 1
|
||||
}
|
||||
echo "OK: no duplicate RAPID_DETERIORATION alert"
|
||||
|
||||
echo "[5/${TOTAL_STEPS}] Ingest HR 105 (warning range) → assert WARNING_HEART_RATE created"
|
||||
hr105_payload="$(ingest_hr "${encounter_id}" 105)"
|
||||
obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr105_payload}")"
|
||||
assert_status "201" "${obs_resp}"
|
||||
wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}"
|
||||
|
||||
warning_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_HEART_RATE'")"
|
||||
[[ "${warning_count}" == "1" ]] || {
|
||||
echo "Expected 1 WARNING_HEART_RATE alert, found ${warning_count}"
|
||||
psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true
|
||||
exit 1
|
||||
}
|
||||
echo "OK: WARNING_HEART_RATE alert created"
|
||||
|
||||
echo "[6/${TOTAL_STEPS}] Acknowledge warning alert → assert Redis suppress key exists"
|
||||
alert_id="$(psql_cmd "SELECT id FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_HEART_RATE' LIMIT 1")"
|
||||
ack_resp="$(request POST "${BASE_URL}/api/v1/alerts/${alert_id}/acknowledge" \
|
||||
'{"clinicianId":"nurse-1","note":"monitoring"}')"
|
||||
assert_status "200" "${ack_resp}"
|
||||
|
||||
suppress_key="suppress:${encounter_id}:WARNING_HEART_RATE"
|
||||
redis_cmd EXISTS "${suppress_key}" | grep -q '^1$' || {
|
||||
echo "Expected Redis key ${suppress_key}"
|
||||
redis_cmd KEYS "suppress:${encounter_id}:*" || true
|
||||
exit 1
|
||||
}
|
||||
echo "OK: suppression key ${suppress_key} exists"
|
||||
|
||||
echo "[7/${TOTAL_STEPS}] Re-ingest HR 105 → assert no new WARNING_HEART_RATE (suppressed)"
|
||||
obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr105_payload}")"
|
||||
assert_status "201" "${obs_resp}"
|
||||
wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}"
|
||||
warning_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'WARNING_HEART_RATE'")"
|
||||
[[ "${warning_count}" == "1" ]] || {
|
||||
echo "Expected suppressed WARNING_HEART_RATE count=1, found ${warning_count}"
|
||||
exit 1
|
||||
}
|
||||
echo "OK: no new WARNING_HEART_RATE while suppressed"
|
||||
|
||||
echo "[8/${TOTAL_STEPS}] Ingest HR 160 (critical) → assert CRITICAL_HEART_RATE fires despite suppression"
|
||||
hr160_payload="$(ingest_hr "${encounter_id}" 160)"
|
||||
obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr160_payload}")"
|
||||
assert_status "201" "${obs_resp}"
|
||||
critical_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND alert_type = 'CRITICAL_HEART_RATE'")"
|
||||
[[ "${critical_count}" == "1" ]] || {
|
||||
echo "Expected 1 CRITICAL_HEART_RATE alert despite active WARNING suppression, found ${critical_count}"
|
||||
psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true
|
||||
exit 1
|
||||
}
|
||||
echo "OK: CRITICAL_HEART_RATE created despite suppression window"
|
||||
|
||||
echo "[9/${TOTAL_STEPS}] Consumer lag = 0 for trend-analyzer and warning-evaluator"
|
||||
wait_for_consumer_lag_zero "${TREND_CONSUMER_GROUP}" "${TREND_WAIT_SECS}"
|
||||
wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}"
|
||||
echo "OK: trend-analyzer and warning-evaluator lag = 0"
|
||||
|
||||
echo "[10/${TOTAL_STEPS}] Run Phase 13 unit/integration tests"
|
||||
dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}"
|
||||
|
||||
if [[ "${FULL_TEST}" == "1" ]]; then
|
||||
echo "Running full test suite (FULL_TEST=1)"
|
||||
dotnet test "${ROOT_DIR}/VigilCareClinicalAPI.sln" 2>/dev/null || dotnet test "${ROOT_DIR}"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Phase 13 verification checks passed."
|
||||
echo "Encounter id: ${encounter_id}"
|
||||
Reference in New Issue
Block a user