193 lines
6.6 KiB
Bash
193 lines
6.6 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}"
|
|
|
|
BASE_URL="${BASE_URL:-http://localhost:5270}"
|
|
PROM_URL="${PROM_URL:-http://localhost:9101}"
|
|
SEQ_URL="${SEQ_URL:-http://localhost:5345}"
|
|
|
|
PGHOST="${PGHOST:-localhost}"
|
|
PGPORT="${PGPORT:-5436}"
|
|
PGDATABASE="${PGDATABASE:-vigilcare}"
|
|
PGUSER="${PGUSER:-postgres}"
|
|
PGPASSWORD="${PGPASSWORD:-password}"
|
|
|
|
COLLECTOR_WAIT_SECS="${COLLECTOR_WAIT_SECS:-35}"
|
|
|
|
TMP_FILES=()
|
|
cleanup() {
|
|
local f
|
|
for f in "${TMP_FILES[@]}"; do
|
|
rm -f "$f" "$f.status" 2>/dev/null || true
|
|
done
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
need() {
|
|
command -v "$1" >/dev/null 2>&1 || {
|
|
echo "Missing dependency: $1"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
need curl
|
|
need jq
|
|
|
|
request() {
|
|
local method="$1"
|
|
local url="$2"
|
|
local body="${3:-}"
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
TMP_FILES+=("$tmp")
|
|
local status
|
|
|
|
if [[ -n "$body" ]]; then
|
|
status="$(curl -sS -o "$tmp" -w "%{http_code}" -X "$method" "$url" \
|
|
-H "Content-Type: application/json" -d "$body")"
|
|
else
|
|
status="$(curl -sS -o "$tmp" -w "%{http_code}" -X "$method" "$url")"
|
|
fi
|
|
|
|
echo "$status" > "$tmp.status"
|
|
echo "$tmp"
|
|
}
|
|
|
|
assert_status() {
|
|
local expected="$1"
|
|
local body_file="$2"
|
|
local status
|
|
status="$(<"$body_file.status")"
|
|
if [[ "$status" != "$expected" ]]; then
|
|
echo "Expected HTTP $expected, got $status"
|
|
cat "$body_file"
|
|
echo
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
psql_cmd() {
|
|
local sql="$1"
|
|
if command -v psql >/dev/null 2>&1; then
|
|
PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -tAc "$sql"
|
|
elif command -v docker >/dev/null 2>&1 && [[ -f "${COMPOSE_FILE}" ]]; then
|
|
docker compose -f "${COMPOSE_FILE}" exec -T postgres \
|
|
psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
|
|
else
|
|
echo "Missing dependency: psql (or docker compose with postgres service)"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
metric_value() {
|
|
local name="$1"
|
|
local body="$2"
|
|
awk -v m="$name" '
|
|
$0 ~ "^"m"([ \t]|\\{|$)" {
|
|
n=split($0, a, /[ \t]+/);
|
|
if (n >= 2) { print a[n]; exit 0; }
|
|
}
|
|
' <<< "$body"
|
|
}
|
|
|
|
echo "Phase 8 verification against ${BASE_URL}"
|
|
|
|
echo "[1/9] Preflight API and Prometheus"
|
|
api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
|
|
prom_status="$(curl -sS -o /dev/null -w "%{http_code}" "${PROM_URL}/-/healthy" || true)"
|
|
[[ "$api_status" == "200" ]] || { echo "API not ready (${api_status})"; exit 1; }
|
|
[[ "$prom_status" == "200" ]] || { echo "Prometheus not ready (${prom_status})"; exit 1; }
|
|
|
|
echo "[2/9] Ensure Prometheus target vigilcare_api is UP"
|
|
target_json="$(curl -sS "${PROM_URL}/api/v1/query?query=up%7Bjob%3D%22vigilcare_api%22%7D")"
|
|
up_val="$(jq -r '.data.result[0].value[1] // "0"' <<< "$target_json")"
|
|
[[ "$up_val" == "1" ]] || { echo "vigilcare_api target not UP"; exit 1; }
|
|
|
|
echo "[3/9] Validate /metrics contains all eight metric families"
|
|
metrics_body="$(curl -sS "${BASE_URL}/metrics")"
|
|
required=(
|
|
observations_ingested_total
|
|
observation_ingest_duration_seconds
|
|
clinical_alerts_total
|
|
alerts_unacknowledged_gauge
|
|
kafka_consumer_lag
|
|
outbox_pending_events
|
|
sirs_detections_total
|
|
escalations_total
|
|
)
|
|
for m in "${required[@]}"; do
|
|
grep -q "$m" <<< "$metrics_body" || { echo "Missing metric: $m"; exit 1; }
|
|
done
|
|
|
|
echo "[4/9] Verify correlation header on API response"
|
|
corr="$(curl -sSI "${BASE_URL}/api/v1/alert-thresholds" | awk -F': ' 'tolower($1)=="x-correlation-id"{print $2}' | tr -d '\r')"
|
|
[[ -n "$corr" ]] || { echo "Missing X-Correlation-Id header"; exit 1; }
|
|
|
|
echo "[5/9] Create patient and active encounter"
|
|
patient_resp="$(request POST "${BASE_URL}/api/v1/patients" '{"firstName":"Phase8","lastName":"Verify","dateOfBirth":"1988-01-10","gender":"F"}')"
|
|
assert_status "201" "$patient_resp"
|
|
patient_id="$(jq -r '.data.id' "$patient_resp")"
|
|
|
|
enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" '{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Verify"}')"
|
|
assert_status "201" "$enc_resp"
|
|
encounter_id="$(jq -r '.data.id' "$enc_resp")"
|
|
|
|
transition_resp="$(request PATCH "${BASE_URL}/api/v1/encounters/${encounter_id}/status" '{"status":"Active"}')"
|
|
transition_status="$(<"$transition_resp.status")"
|
|
if [[ "$transition_status" != "200" && "$transition_status" != "409" ]]; then
|
|
echo "Unexpected status transitioning encounter to active: ${transition_status}"
|
|
cat "$transition_resp"
|
|
exit 1
|
|
fi
|
|
|
|
echo "[6/9] Create critical alert, backdate 6 minutes"
|
|
baseline_metrics="$(curl -sS "${BASE_URL}/metrics")"
|
|
baseline_gauge="$(metric_value "alerts_unacknowledged_gauge" "$baseline_metrics")"
|
|
baseline_gauge_int="${baseline_gauge%.*}"
|
|
baseline_gauge_int="${baseline_gauge_int:-0}"
|
|
|
|
recorded_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
|
obs_payload="$(jq -nc --arg ts "$recorded_at" \
|
|
'{observations:[{observationCode:"POTASSIUM_MEQ_L",value:2.1,unit:"mEq/L",source:"LAB",recordedAt:$ts}]}' )"
|
|
obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "$obs_payload")"
|
|
assert_status "201" "$obs_resp"
|
|
alert_id="$(jq -r '.data.alertId // empty' "$obs_resp")"
|
|
[[ -n "$alert_id" ]] || { echo "Critical observation did not return alertId"; cat "$obs_resp"; exit 1; }
|
|
|
|
psql_cmd "UPDATE clinical_alerts SET triggered_at = NOW() - INTERVAL '6 minutes' WHERE id = '${alert_id}';" >/dev/null
|
|
|
|
echo "[7/9] Wait collector and verify alerts_unacknowledged_gauge increased"
|
|
sleep "$COLLECTOR_WAIT_SECS"
|
|
metrics_body="$(curl -sS "${BASE_URL}/metrics")"
|
|
gauge="$(metric_value "alerts_unacknowledged_gauge" "$metrics_body")"
|
|
gauge_int="${gauge%.*}"
|
|
gauge_int="${gauge_int:-0}"
|
|
expected_min=$((baseline_gauge_int + 1))
|
|
[[ -n "$gauge" && "$gauge_int" -ge "$expected_min" ]] || {
|
|
echo "alerts_unacknowledged_gauge did not increase (baseline=${baseline_gauge_int}, value=${gauge:-missing})"
|
|
exit 1
|
|
}
|
|
|
|
echo "[8/9] Acknowledge alert and verify gauge returned to baseline"
|
|
ack_resp="$(request POST "${BASE_URL}/api/v1/alerts/${alert_id}/acknowledge" '{"clinicianId":"DR-VERIFY","note":"phase8 verification"}')"
|
|
assert_status "200" "$ack_resp"
|
|
sleep "$COLLECTOR_WAIT_SECS"
|
|
metrics_body="$(curl -sS "${BASE_URL}/metrics")"
|
|
gauge="$(metric_value "alerts_unacknowledged_gauge" "$metrics_body")"
|
|
gauge_int="${gauge%.*}"
|
|
gauge_int="${gauge_int:-0}"
|
|
[[ "$gauge_int" -le "$baseline_gauge_int" ]] || {
|
|
echo "alerts_unacknowledged_gauge did not return to baseline (baseline=${baseline_gauge_int}, value=${gauge:-missing})"
|
|
exit 1
|
|
}
|
|
|
|
echo "[9/9] Manual Seq check"
|
|
echo "Open ${SEQ_URL} and filter: EncounterId IS NOT NULL"
|
|
echo "Confirm logs include EncounterId, PatientId, CorrelationId."
|
|
|
|
echo
|
|
echo "Phase 8 verification checks passed."
|