feature: Observability: Prometheus Metrics and Grafana
This commit is contained in:
voltsrage
Executable
+192
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}"
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:5270}"
|
||||
PROM_URL="${PROM_URL:-http://localhost:9101}"
|
||||
SEQ_URL="${SEQ_URL:-http://localhost:5345}"
|
||||
|
||||
PGHOST="${PGHOST:-localhost}"
|
||||
PGPORT="${PGPORT:-5436}"
|
||||
PGDATABASE="${PGDATABASE:-vigilcare}"
|
||||
PGUSER="${PGUSER:-postgres}"
|
||||
PGPASSWORD="${PGPASSWORD:-password}"
|
||||
|
||||
COLLECTOR_WAIT_SECS="${COLLECTOR_WAIT_SECS:-35}"
|
||||
|
||||
TMP_FILES=()
|
||||
cleanup() {
|
||||
local f
|
||||
for f in "${TMP_FILES[@]}"; do
|
||||
rm -f "$f" "$f.status" 2>/dev/null || true
|
||||
done
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
need() {
|
||||
command -v "$1" >/dev/null 2>&1 || {
|
||||
echo "Missing dependency: $1"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
need curl
|
||||
need jq
|
||||
|
||||
request() {
|
||||
local method="$1"
|
||||
local url="$2"
|
||||
local body="${3:-}"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
TMP_FILES+=("$tmp")
|
||||
local status
|
||||
|
||||
if [[ -n "$body" ]]; then
|
||||
status="$(curl -sS -o "$tmp" -w "%{http_code}" -X "$method" "$url" \
|
||||
-H "Content-Type: application/json" -d "$body")"
|
||||
else
|
||||
status="$(curl -sS -o "$tmp" -w "%{http_code}" -X "$method" "$url")"
|
||||
fi
|
||||
|
||||
echo "$status" > "$tmp.status"
|
||||
echo "$tmp"
|
||||
}
|
||||
|
||||
assert_status() {
|
||||
local expected="$1"
|
||||
local body_file="$2"
|
||||
local status
|
||||
status="$(<"$body_file.status")"
|
||||
if [[ "$status" != "$expected" ]]; then
|
||||
echo "Expected HTTP $expected, got $status"
|
||||
cat "$body_file"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
psql_cmd() {
|
||||
local sql="$1"
|
||||
if command -v psql >/dev/null 2>&1; then
|
||||
PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -tAc "$sql"
|
||||
elif command -v docker >/dev/null 2>&1 && [[ -f "${COMPOSE_FILE}" ]]; then
|
||||
docker compose -f "${COMPOSE_FILE}" exec -T postgres \
|
||||
psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
|
||||
else
|
||||
echo "Missing dependency: psql (or docker compose with postgres service)"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
metric_value() {
|
||||
local name="$1"
|
||||
local body="$2"
|
||||
awk -v m="$name" '
|
||||
$0 ~ "^"m"([ \t]|\\{|$)" {
|
||||
n=split($0, a, /[ \t]+/);
|
||||
if (n >= 2) { print a[n]; exit 0; }
|
||||
}
|
||||
' <<< "$body"
|
||||
}
|
||||
|
||||
echo "Phase 8 verification against ${BASE_URL}"
|
||||
|
||||
echo "[1/9] Preflight API and Prometheus"
|
||||
api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
|
||||
prom_status="$(curl -sS -o /dev/null -w "%{http_code}" "${PROM_URL}/-/healthy" || true)"
|
||||
[[ "$api_status" == "200" ]] || { echo "API not ready (${api_status})"; exit 1; }
|
||||
[[ "$prom_status" == "200" ]] || { echo "Prometheus not ready (${prom_status})"; exit 1; }
|
||||
|
||||
echo "[2/9] Ensure Prometheus target vigilcare_api is UP"
|
||||
target_json="$(curl -sS "${PROM_URL}/api/v1/query?query=up%7Bjob%3D%22vigilcare_api%22%7D")"
|
||||
up_val="$(jq -r '.data.result[0].value[1] // "0"' <<< "$target_json")"
|
||||
[[ "$up_val" == "1" ]] || { echo "vigilcare_api target not UP"; exit 1; }
|
||||
|
||||
echo "[3/9] Validate /metrics contains all eight metric families"
|
||||
metrics_body="$(curl -sS "${BASE_URL}/metrics")"
|
||||
required=(
|
||||
observations_ingested_total
|
||||
observation_ingest_duration_seconds
|
||||
clinical_alerts_total
|
||||
alerts_unacknowledged_gauge
|
||||
kafka_consumer_lag
|
||||
outbox_pending_events
|
||||
sirs_detections_total
|
||||
escalations_total
|
||||
)
|
||||
for m in "${required[@]}"; do
|
||||
grep -q "$m" <<< "$metrics_body" || { echo "Missing metric: $m"; exit 1; }
|
||||
done
|
||||
|
||||
echo "[4/9] Verify correlation header on API response"
|
||||
corr="$(curl -sSI "${BASE_URL}/api/v1/alert-thresholds" | awk -F': ' 'tolower($1)=="x-correlation-id"{print $2}' | tr -d '\r')"
|
||||
[[ -n "$corr" ]] || { echo "Missing X-Correlation-Id header"; exit 1; }
|
||||
|
||||
echo "[5/9] Create patient and active encounter"
|
||||
patient_resp="$(request POST "${BASE_URL}/api/v1/patients" '{"firstName":"Phase8","lastName":"Verify","dateOfBirth":"1988-01-10","gender":"F"}')"
|
||||
assert_status "201" "$patient_resp"
|
||||
patient_id="$(jq -r '.data.id' "$patient_resp")"
|
||||
|
||||
enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" '{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Verify"}')"
|
||||
assert_status "201" "$enc_resp"
|
||||
encounter_id="$(jq -r '.data.id' "$enc_resp")"
|
||||
|
||||
transition_resp="$(request PATCH "${BASE_URL}/api/v1/encounters/${encounter_id}/status" '{"status":"Active"}')"
|
||||
transition_status="$(<"$transition_resp.status")"
|
||||
if [[ "$transition_status" != "200" && "$transition_status" != "409" ]]; then
|
||||
echo "Unexpected status transitioning encounter to active: ${transition_status}"
|
||||
cat "$transition_resp"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[6/9] Create critical alert, backdate 6 minutes"
|
||||
baseline_metrics="$(curl -sS "${BASE_URL}/metrics")"
|
||||
baseline_gauge="$(metric_value "alerts_unacknowledged_gauge" "$baseline_metrics")"
|
||||
baseline_gauge_int="${baseline_gauge%.*}"
|
||||
baseline_gauge_int="${baseline_gauge_int:-0}"
|
||||
|
||||
recorded_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
obs_payload="$(jq -nc --arg ts "$recorded_at" \
|
||||
'{observations:[{observationCode:"POTASSIUM_MEQ_L",value:2.1,unit:"mEq/L",source:"LAB",recordedAt:$ts}]}' )"
|
||||
obs_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "$obs_payload")"
|
||||
assert_status "201" "$obs_resp"
|
||||
alert_id="$(jq -r '.data.alertId // empty' "$obs_resp")"
|
||||
[[ -n "$alert_id" ]] || { echo "Critical observation did not return alertId"; cat "$obs_resp"; exit 1; }
|
||||
|
||||
psql_cmd "UPDATE clinical_alerts SET triggered_at = NOW() - INTERVAL '6 minutes' WHERE id = '${alert_id}';" >/dev/null
|
||||
|
||||
echo "[7/9] Wait collector and verify alerts_unacknowledged_gauge increased"
|
||||
sleep "$COLLECTOR_WAIT_SECS"
|
||||
metrics_body="$(curl -sS "${BASE_URL}/metrics")"
|
||||
gauge="$(metric_value "alerts_unacknowledged_gauge" "$metrics_body")"
|
||||
gauge_int="${gauge%.*}"
|
||||
gauge_int="${gauge_int:-0}"
|
||||
expected_min=$((baseline_gauge_int + 1))
|
||||
[[ -n "$gauge" && "$gauge_int" -ge "$expected_min" ]] || {
|
||||
echo "alerts_unacknowledged_gauge did not increase (baseline=${baseline_gauge_int}, value=${gauge:-missing})"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "[8/9] Acknowledge alert and verify gauge returned to baseline"
|
||||
ack_resp="$(request POST "${BASE_URL}/api/v1/alerts/${alert_id}/acknowledge" '{"clinicianId":"DR-VERIFY","note":"phase8 verification"}')"
|
||||
assert_status "200" "$ack_resp"
|
||||
sleep "$COLLECTOR_WAIT_SECS"
|
||||
metrics_body="$(curl -sS "${BASE_URL}/metrics")"
|
||||
gauge="$(metric_value "alerts_unacknowledged_gauge" "$metrics_body")"
|
||||
gauge_int="${gauge%.*}"
|
||||
gauge_int="${gauge_int:-0}"
|
||||
[[ "$gauge_int" -le "$baseline_gauge_int" ]] || {
|
||||
echo "alerts_unacknowledged_gauge did not return to baseline (baseline=${baseline_gauge_int}, value=${gauge:-missing})"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "[9/9] Manual Seq check"
|
||||
echo "Open ${SEQ_URL} and filter: EncounterId IS NOT NULL"
|
||||
echo "Confirm logs include EncounterId, PatientId, CorrelationId."
|
||||
|
||||
echo
|
||||
echo "Phase 8 verification checks passed."
|
||||
Reference in New Issue
Block a user