Trent/vigilcare-records
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job

Browse Source
This commit is contained in:
voltsrage
2026-06-27 13:29:04 +08:00
parent e22d33b654
commit 04bdb7e85c
31 changed files with 4482 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run-vigilcare-records-phase-8-verification.sh
Executable
+800
View File
@@ -0,0 +1,800 @@
#!/usr/bin/env bash
# Runs Phase 8 verification checks from docs/plans/phase-8-plan.md.
#
# Covers Prometheus metrics, MetricsCollectorService gauges, rejection counter,
# promotion duration histogram, work queue overview, cursor-paginated audit trail,
# promotion retry schema, and the Prometheus/Grafana Docker stack.
#
# Prerequisites:
# docker compose up -d (PostgreSQL, Redis, MinIO, Prometheus, Grafana)
# dotnet ef database update --project VigilCareRecordsAPI
# dotnet run --project VigilCareRecordsAPI
# Phase 1–7 seed data (intake1, entry1, verifier1, approver1, admin1)
#
# Environment overrides:
# VIGILCARE_API_URL default: http://localhost:5217
# VIGILCARE_PROMETHEUS_URL default: http://localhost:9095
# VIGILCARE_GRAFANA_URL default: http://localhost:3013
# VIGILCARE_COMPOSE_FILE default: <repo>/docker-compose.yml
# VIGILCARE_PG_HOST default: localhost
# VIGILCARE_PG_PORT default: 5437
# VIGILCARE_PG_DB default: vigilcare_records
# VIGILCARE_PG_USER default: postgres
# VIGILCARE_PG_PASSWORD default: password
# VIGILCARE_SKIP_DB_CHECKS set to 1 to skip PostgreSQL assertions
# VIGILCARE_SKIP_MONITORING_CHECKS set to 1 to skip Prometheus/Grafana checks
# VIGILCARE_METRICS_COLLECTOR_WAIT default: 35 (seconds; MetricsCollectorService interval is 30s)
# VIGILCARE_RECORDED_AT default: 2026-06-27T10:00:00Z
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
FIXTURE_PDF="$SCRIPT_DIR/fixtures/test-scan.pdf"
API_URL="${VIGILCARE_API_URL:-http://localhost:5217}"
PROMETHEUS_URL="${VIGILCARE_PROMETHEUS_URL:-http://localhost:9095}"
GRAFANA_URL="${VIGILCARE_GRAFANA_URL:-http://localhost:3013}"
COMPOSE_FILE="${VIGILCARE_COMPOSE_FILE:-$REPO_ROOT/docker-compose.yml}"
COMPOSE=(docker compose -f "$COMPOSE_FILE")
PG_HOST="${VIGILCARE_PG_HOST:-localhost}"
PG_PORT="${VIGILCARE_PG_PORT:-5437}"
PG_DB="${VIGILCARE_PG_DB:-vigilcare_records}"
PG_USER="${VIGILCARE_PG_USER:-postgres}"
PG_PASSWORD="${VIGILCARE_PG_PASSWORD:-password}"
SKIP_DB_CHECKS="${VIGILCARE_SKIP_DB_CHECKS:-0}"
SKIP_MONITORING_CHECKS="${VIGILCARE_SKIP_MONITORING_CHECKS:-0}"
METRICS_COLLECTOR_WAIT="${VIGILCARE_METRICS_COLLECTOR_WAIT:-35}"
RECORDED_AT="${VIGILCARE_RECORDED_AT:-2026-06-27T10:00:00Z}"
RECORDED_AT_BP="${VIGILCARE_RECORDED_AT_BP:-2026-06-27T10:05:00Z}"
PASS_COUNT=0
FAIL_COUNT=0
FAILED_TESTS=()
ALL_BATCH_STATUSES=(
UPLOADED
IN_ENTRY
PENDING_VERIFICATION
REJECTED
VERIFIED
AWAITING_CLINICAL_APPROVAL
APPROVED
PROMOTED
)
log() {
printf '%s\n' "$*"
}
section() {
log ""
log "== $1 =="
}
pass() {
PASS_COUNT=$((PASS_COUNT + 1))
log " PASS: $1"
}
fail() {
FAIL_COUNT=$((FAIL_COUNT + 1))
FAILED_TESTS+=("$1")
log " FAIL: $1"
}
require_cmd() {
local cmd="$1"
if ! command -v "$cmd" >/dev/null 2>&1; then
log "ERROR: required command not found: $cmd"
exit 1
fi
}
ensure_fixture_pdf() {
if [[ -f "$FIXTURE_PDF" ]]; then
return 0
fi
mkdir -p "$(dirname "$FIXTURE_PDF")"
cat >"$FIXTURE_PDF" <<'EOF'
%PDF-1.0
1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj 2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj 3 0 obj<</Type/Page/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<<>>>>endobj
xref
0 4
0000000000 65535 f
0000000009 00000 n
0000000058 00000 n
0000000115 00000 n
trailer<</Size 4/Root 1 0 R>>
startxref
206
%%EOF
EOF
}
new_uuid() {
if command -v uuidgen >/dev/null 2>&1; then
uuidgen
else
cat /proc/sys/kernel/random/uuid
fi
}
new_idempotency_key() {
printf 'phase8-%s' "$(new_uuid)"
}
compose_service_running() {
local service="$1"
"${COMPOSE[@]}" ps --status running --services 2>/dev/null | grep -qx "$service"
}
http_code() {
curl -sS -o /dev/null -w '%{http_code}' "$@"
}
json_post() {
local url="$1"
local body="$2"
local token="${3:-}"
if [[ -n "$token" ]]; then
curl -sS -X POST "$url" \
-H "Authorization: Bearer $token" \
-H 'Content-Type: application/json' \
-d "$body"
else
curl -sS -X POST "$url" \
-H 'Content-Type: application/json' \
-d "$body"
fi
}
json_put() {
local url="$1"
local body="$2"
local token="$3"
curl -sS -X PUT "$url" \
-H "Authorization: Bearer $token" \
-H 'Content-Type: application/json' \
-d "$body"
}
login() {
local username="$1"
local password="${2:-password}"
json_post "$API_URL/api/v1/auth/login" \
"{\"username\":\"$username\",\"password\":\"$password\"}"
}
extract_data_field() {
local json="$1"
local field="$2"
jq -er ".data.$field // empty" <<<"$json"
}
extract_error_code() {
local json="$1"
jq -er '.error.code // empty' <<<"$json" 2>/dev/null || true
}
assert_api_reachable() {
local code
code="$(http_code "$API_URL/swagger/index.html" || true)"
if [[ "$code" != "200" ]]; then
log "ERROR: API not reachable at $API_URL (HTTP $code)."
log "Start infrastructure with: docker compose up -d"
log "Apply migrations with: dotnet ef database update --project VigilCareRecordsAPI"
log "Start API with: dotnet run --project VigilCareRecordsAPI"
exit 1
fi
}
fetch_metrics() {
curl -sS "$API_URL/metrics"
}
metric_gauge_status() {
local status="$1"
local line value
line="$(fetch_metrics | grep -E "digitization_batches_by_status\\{status=\"${status}\"\\}" | head -1 || true)"
value="$(awk '{print $2}' <<<"$line")"
if [[ -z "$value" ]]; then
printf '0'
else
printf '%s' "$value"
fi
}
metric_counter_rejection() {
local category="$1"
local line value
line="$(fetch_metrics | grep -E "digitization_rejection_total\\{reason_category=\"${category}\"\\}" | head -1 || true)"
value="$(awk '{print $2}' <<<"$line")"
if [[ -z "$value" ]]; then
printf '0'
else
printf '%s' "$value"
fi
}
metric_histogram_count() {
local line value
line="$(fetch_metrics | grep -E '^digitization_promotion_duration_seconds_count ' | head -1 || true)"
value="$(awk '{print $2}' <<<"$line")"
if [[ -z "$value" ]]; then
printf '0'
else
printf '%s' "$value"
fi
}
psql_available() {
[[ "$SKIP_DB_CHECKS" == "1" ]] && return 1
compose_service_running postgres && return 0
command -v psql >/dev/null 2>&1 && return 0
return 1
}
psql_query() {
if [[ "$SKIP_DB_CHECKS" == "1" ]]; then
return 1
fi
if compose_service_running postgres; then
"${COMPOSE[@]}" exec -T postgres \
psql -U "$PG_USER" -d "$PG_DB" -Atqc "$1"
elif command -v psql >/dev/null 2>&1; then
PGPASSWORD="$PG_PASSWORD" psql -h "$PG_HOST" -p "$PG_PORT" -U "$PG_USER" -d "$PG_DB" -Atqc "$1"
else
return 1
fi
}
upload_batch() {
local token="$1"
local batch_type="${2:-VITALS_SHEET}"
local track="${3:-BACKFILL}"
curl -sS -X POST "$API_URL/api/v1/digitization-batches" \
-H "Authorization: Bearer $token" \
-F "file=@${FIXTURE_PDF};type=application/pdf" \
-F "batchType=$batch_type" \
-F "track=$track"
}
assign_batch() {
local token="$1"
local batch_id="$2"
local entry_clerk_id="$3"
curl -sS -X PATCH "$API_URL/api/v1/digitization-batches/$batch_id/assign" \
-H "Authorization: Bearer $token" \
-H 'Content-Type: application/json' \
-d "{\"entryClerkUserId\":\"$entry_clerk_id\"}"
}
verify_batch() {
local token="$1"
local batch_id="$2"
local body="$3"
json_post "$API_URL/api/v1/digitization-batches/$batch_id/verify" "$body" "$token"
}
reject_batch() {
local token="$1"
local batch_id="$2"
local reason="$3"
json_post "$API_URL/api/v1/digitization-batches/$batch_id/reject" \
"{\"reason\":\"$reason\"}" "$token"
}
approve_batch() {
local token="$1"
local batch_id="$2"
local idempotency_key="$3"
local body='{"enableRetroactiveAlerts":false}'
if [[ -n "${4:-}" ]]; then
body="$4"
fi
curl -sS -X POST "$API_URL/api/v1/digitization-batches/$batch_id/approve" \
-H "Authorization: Bearer $token" \
-H 'Content-Type: application/json' \
-H "Idempotency-Key: $idempotency_key" \
-d "$body"
}
create_assigned_batch() {
local intake_token="$1"
local upload_json batch_id entry_id
upload_json="$(upload_batch "$intake_token")"
if [[ "$(jq -er '.success' <<<"$upload_json")" != "true" ]]; then
return 1
fi
batch_id="$(extract_data_field "$upload_json" id)"
entry_id="$(extract_data_field "$(login entry1)" userId)"
assign_batch "$intake_token" "$batch_id" "$entry_id" >/dev/null
printf '%s' "$batch_id"
}
create_pending_vitals_batch() {
local intake_token="$1"
local entry_token batch_id submit_code
batch_id="$(create_assigned_batch "$intake_token")" || return 1
entry_token="$(extract_data_field "$(login entry1)" token)"
json_put \
"$API_URL/api/v1/digitization-batches/$batch_id/draft/patient" \
'{"fullName":"Phase8 Verify Patient","dateOfBirth":"1985-03-15","sex":"M","noKnownAllergies":true,"noActiveMedications":true}' \
"$entry_token" >/dev/null
json_put \
"$API_URL/api/v1/digitization-batches/$batch_id/draft/encounter" \
'{"admissionDate":"2025-01-01T08:00:00Z","department":"ICU","admissionReason":"Phase 8 verification"}' \
"$entry_token" >/dev/null
json_post \
"$API_URL/api/v1/digitization-batches/$batch_id/draft/observations" \
"{\"observationCode\":\"HEART_RATE\",\"value\":88,\"unit\":\"bpm\",\"recordedAt\":\"$RECORDED_AT\"}" \
"$entry_token" >/dev/null
submit_code="$(http_code -X POST \
"$API_URL/api/v1/digitization-batches/$batch_id/submit-for-verification" \
-H "Authorization: Bearer $entry_token")"
if [[ "$submit_code" != "200" ]]; then
return 1
fi
printf '%s' "$batch_id"
}
create_batch_ready_for_approval() {
local intake_token="$1"
local entry_token verifier_token batch_id entry_id verify_json verify_status submit_code
batch_id="$(create_assigned_batch "$intake_token")" || return 1
entry_id="$(extract_data_field "$(login entry1)" userId)"
entry_token="$(extract_data_field "$(login entry1)" token)"
json_put \
"$API_URL/api/v1/digitization-batches/$batch_id/draft/patient" \
'{"fullName":"Phase8 Promote Patient","dateOfBirth":"1990-05-15","sex":"M","bloodType":"A+","noKnownAllergies":true}' \
"$entry_token" >/dev/null
json_put \
"$API_URL/api/v1/digitization-batches/$batch_id/draft/encounter" \
'{"admissionDate":"2024-01-15T08:00:00Z","department":"General Medicine","roomBed":"301-A","admissionReason":"Routine checkup","status":"active"}' \
"$entry_token" >/dev/null
json_post \
"$API_URL/api/v1/digitization-batches/$batch_id/draft/observations" \
"{\"observationCode\":\"HR\",\"value\":88,\"unit\":\"bpm\",\"recordedAt\":\"$RECORDED_AT\"}" \
"$entry_token" >/dev/null
json_post \
"$API_URL/api/v1/digitization-batches/$batch_id/draft/observations" \
"{\"observationCode\":\"TEMP\",\"value\":37.2,\"unit\":\"C\",\"recordedAt\":\"$RECORDED_AT\"}" \
"$entry_token" >/dev/null
json_post \
"$API_URL/api/v1/digitization-batches/$batch_id/draft/observations" \
"{\"observationCode\":\"BP_SYS\",\"value\":120,\"unit\":\"mmHg\",\"recordedAt\":\"$RECORDED_AT_BP\"}" \
"$entry_token" >/dev/null
submit_code="$(http_code -X POST \
"$API_URL/api/v1/digitization-batches/$batch_id/submit-for-verification" \
-H "Authorization: Bearer $entry_token")"
if [[ "$submit_code" != "200" ]]; then
return 1
fi
verifier_token="$(extract_data_field "$(login verifier1)" token)"
verify_json="$(verify_batch "$verifier_token" "$batch_id" \
'{"fieldChecks":[{"fieldName":"patient.fullName","status":"ok","note":null},{"fieldName":"observation.heartRate","status":"ok","note":null}],"passed":true}')"
verify_status="$(extract_data_field "$verify_json" status)"
if [[ "$(jq -er '.success' <<<"$verify_json")" != "true" || "$verify_status" != "AWAITING_CLINICAL_APPROVAL" ]]; then
return 1
fi
printf '%s' "$batch_id"
}
test_metrics_endpoint() {
section "1. Prometheus metrics endpoint"
local metrics
metrics="$(fetch_metrics)"
if grep -q '^# HELP ' <<<"$metrics" && grep -q '^# TYPE ' <<<"$metrics"; then
pass "/metrics serves Prometheus exposition format"
else
fail "/metrics serves Prometheus exposition format"
fi
local metric
for metric in \
digitization_batches_by_status \
digitization_promotion_duration_seconds \
digitization_rejection_total \
digitization_queue_age_seconds \
http_request_duration_seconds; do
if grep -q "$metric" <<<"$metrics"; then
pass "metric registered: $metric"
else
fail "metric registered: $metric"
fi
done
local missing=0 status
for status in "${ALL_BATCH_STATUSES[@]}"; do
if ! grep -q "digitization_batches_by_status{status=\"${status}\"}" <<<"$metrics"; then
missing=$((missing + 1))
fi
done
if [[ "$missing" -eq 0 ]]; then
pass "digitization_batches_by_status exposes all 8 status labels"
else
fail "digitization_batches_by_status exposes all 8 status labels ($missing missing)"
fi
if grep -q 'digitization_rejection_total{reason_category="verification_failed"}' <<<"$metrics" \
&& grep -q 'digitization_rejection_total{reason_category="clinical_rejected"}' <<<"$metrics"; then
pass "digitization_rejection_total exposes both reason_category labels"
else
fail "digitization_rejection_total exposes both reason_category labels"
fi
}
test_gauge_updates_after_upload() {
section "2. Gauge metrics update after creating a batch"
local intake_token before after upload_json
intake_token="$(extract_data_field "$(login intake1)" token)"
before="$(metric_gauge_status UPLOADED)"
upload_json="$(upload_batch "$intake_token")"
if [[ "$(jq -er '.success' <<<"$upload_json")" != "true" ]]; then
fail "upload batch for gauge test"
return
fi
pass "uploaded batch for gauge test"
log " waiting ${METRICS_COLLECTOR_WAIT}s for MetricsCollectorService..."
sleep "$METRICS_COLLECTOR_WAIT"
after="$(metric_gauge_status UPLOADED)"
if awk -v before="$before" -v after="$after" 'BEGIN { exit (after > before) ? 0 : 1 }'; then
pass "digitization_batches_by_status UPLOADED increased ($before -> $after)"
else
fail "digitization_batches_by_status UPLOADED increased ($before -> $after)"
fi
}
test_rejection_counter() {
section "3. Rejection counter increments"
local intake_token verifier_token batch_id before after verify_json
intake_token="$(extract_data_field "$(login intake1)" token)"
batch_id="$(create_pending_vitals_batch "$intake_token")" || {
fail "setup pending batch for rejection counter"
return
}
before="$(metric_counter_rejection verification_failed)"
verifier_token="$(extract_data_field "$(login verifier1)" token)"
verify_json="$(verify_batch "$verifier_token" "$batch_id" \
'{"fieldChecks":[{"fieldName":"patient.fullName","status":"error","note":"Name mismatch"}],"passed":false}')"
if [[ "$(jq -er '.success' <<<"$verify_json")" != "true" ]]; then
fail "verify with passed=false returns success envelope"
return
fi
pass "verification failed via VerifyAsync (Passed=false)"
after="$(metric_counter_rejection verification_failed)"
if awk -v before="$before" -v after="$after" 'BEGIN { exit (after > before) ? 0 : 1 }'; then
pass "digitization_rejection_total verification_failed incremented ($before -> $after)"
else
fail "digitization_rejection_total verification_failed incremented ($before -> $after)"
fi
}
test_promotion_histogram() {
section "4. Promotion duration histogram"
local intake_token approver_token batch_id before after approve_json
before="$(metric_histogram_count)"
intake_token="$(extract_data_field "$(login intake1)" token)"
batch_id="$(create_batch_ready_for_approval "$intake_token")" || {
fail "setup batch ready for approval"
return
}
approver_token="$(extract_data_field "$(login approver1)" token)"
approve_json="$(approve_batch "$approver_token" "$batch_id" "$(new_idempotency_key)" \
'{"enableRetroactiveAlerts":false}')"
if [[ "$(jq -er '.success' <<<"$approve_json")" != "true" ]]; then
fail "approve batch for promotion histogram"
return
fi
pass "batch promoted via ApproveAndPromoteAsync"
after="$(metric_histogram_count)"
if awk -v before="$before" -v after="$after" 'BEGIN { exit (after > before) ? 0 : 1 }'; then
pass "digitization_promotion_duration_seconds_count increased ($before -> $after)"
else
fail "digitization_promotion_duration_seconds_count increased ($before -> $after)"
fi
if grep -q '^digitization_promotion_duration_seconds_sum ' <<<"$(fetch_metrics)"; then
pass "digitization_promotion_duration_seconds_sum present after promotion"
else
fail "digitization_promotion_duration_seconds_sum present after promotion"
fi
}
test_work_queue_overview() {
section "5. Work queue overview endpoint"
local admin_token entry_token overview_json code missing=0 status
admin_token="$(extract_data_field "$(login admin1)" token)"
overview_json="$(curl -sS "$API_URL/api/v1/work-queue/overview" \
-H "Authorization: Bearer $admin_token")"
if [[ "$(jq -er '.success' <<<"$overview_json")" == "true" ]]; then
pass "GET /work-queue/overview returns success for administrator"
else
fail "GET /work-queue/overview returns success for administrator"
return
fi
for status in "${ALL_BATCH_STATUSES[@]}"; do
if [[ "$(jq -er --arg s "$status" '.data.statusCounts[$s] | type' <<<"$overview_json")" != "number" ]]; then
missing=$((missing + 1))
fi
done
if [[ "$missing" -eq 0 ]]; then
pass "overview statusCounts includes all 8 statuses"
else
fail "overview statusCounts includes all 8 statuses ($missing missing)"
fi
if jq -er '.data.rejectRate | type' <<<"$overview_json" | grep -qx number; then
pass "overview rejectRate is numeric"
else
fail "overview rejectRate is numeric"
fi
if jq -er '.data.averageTimeInQueueMinutes | type' <<<"$overview_json" | grep -qx number \
&& jq -er '.data.oldestPendingVerificationMinutes | type' <<<"$overview_json" | grep -qx number; then
pass "overview queue age fields are numeric"
else
fail "overview queue age fields are numeric"
fi
entry_token="$(extract_data_field "$(login entry1)" token)"
code="$(http_code "$API_URL/api/v1/work-queue/overview" \
-H "Authorization: Bearer $entry_token")"
if [[ "$code" == "403" ]]; then
pass "GET /work-queue/overview returns 403 for non-administrator"
else
fail "GET /work-queue/overview returns 403 for non-administrator (HTTP $code)"
fi
}
test_audit_trail_events() {
section "6. Cursor-paginated audit trail"
local admin_token intake_token batch_id page1 page2 code cursor encoded_cursor
admin_token="$(extract_data_field "$(login admin1)" token)"
intake_token="$(extract_data_field "$(login intake1)" token)"
batch_id="$(create_batch_ready_for_approval "$intake_token")" || {
fail "setup batch with multiple audit events"
return
}
page1="$(curl -sS "$API_URL/api/v1/digitization-batches/$batch_id/events?pageSize=2" \
-H "Authorization: Bearer $admin_token")"
if [[ "$(jq -er '.success' <<<"$page1")" != "true" ]]; then
fail "GET /events returns success envelope"
return
fi
pass "GET /events returns success envelope"
if [[ "$(jq -er '.data.items | length' <<<"$page1")" -ge 1 ]]; then
pass "events page contains at least one item"
else
fail "events page contains at least one item"
fi
if jq -er '.data.items[0] | has("actorUsername") and has("actorFullName") and has("eventType")' \
<<<"$page1" | grep -qx true; then
pass "event items include actorUsername and actorFullName"
else
fail "event items include actorUsername and actorFullName"
fi
if [[ "$(jq -er '.data.hasMore' <<<"$page1")" == "true" ]]; then
cursor="$(extract_data_field "$page1" nextCursor)"
encoded_cursor="$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "$cursor")"
page2="$(curl -sS "$API_URL/api/v1/digitization-batches/$batch_id/events?pageSize=2&after=$encoded_cursor" \
-H "Authorization: Bearer $admin_token")"
if [[ "$(jq -er '.success' <<<"$page2")" == "true" \
&& "$(jq -er '.data.items | length' <<<"$page2")" -ge 1 ]]; then
pass "cursor pagination returns a second page"
else
fail "cursor pagination returns a second page"
fi
else
log " SKIP: cursor second-page check (batch has <=2 events)"
fi
code="$(http_code "$API_URL/api/v1/digitization-batches/$batch_id/events?after=not-a-date" \
-H "Authorization: Bearer $admin_token")"
if [[ "$code" == "400" ]]; then
pass "invalid cursor returns 400"
else
fail "invalid cursor returns 400 (HTTP $code)"
fi
code="$(http_code "$API_URL/api/v1/digitization-batches/00000000-0000-0000-0000-000000000000/events" \
-H "Authorization: Bearer $admin_token")"
if [[ "$code" == "404" ]]; then
pass "missing batch returns 404"
else
fail "missing batch returns 404 (HTTP $code)"
fi
}
test_promotion_retry_schema() {
section "7. Promotion retry infrastructure"
if ! psql_available; then
log " SKIP: promotion_attempts table check (postgres not reachable)"
return
fi
local table_exists
table_exists="$(psql_query "
SELECT EXISTS (
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = 'promotion_attempts'
);
")"
if [[ "$table_exists" == "t" ]]; then
pass "promotion_attempts table exists"
else
fail "promotion_attempts table exists"
fi
local index_count
index_count="$(psql_query "
SELECT count(*)
FROM pg_indexes
WHERE tablename = 'promotion_attempts'
AND indexname IN (
'ix_promotion_attempts_batch_attempt',
'ix_promotion_attempts_pending_retry'
);
")"
if [[ "$index_count" == "2" ]]; then
pass "promotion_attempts has batch/attempt and pending-retry indexes"
else
fail "promotion_attempts has batch/attempt and pending-retry indexes (got $index_count)"
fi
log " NOTE: full PromotionRetryService retry flow requires simulating a deferred promotion failure (manual test in phase-8 plan section 7)."
}
test_monitoring_stack() {
section "8. Prometheus and Grafana stack"
if [[ "$SKIP_MONITORING_CHECKS" == "1" ]]; then
log " SKIP: VIGILCARE_SKIP_MONITORING_CHECKS=1"
return
fi
if ! compose_service_running prometheus; then
log " SKIP: prometheus container not running (docker compose up -d prometheus grafana)"
return
fi
local prom_health grafana_health target_health query_status
prom_health="$(http_code "$PROMETHEUS_URL/-/healthy" || true)"
if [[ "$prom_health" == "200" ]]; then
pass "Prometheus healthy at $PROMETHEUS_URL"
else
fail "Prometheus healthy at $PROMETHEUS_URL (HTTP $prom_health)"
fi
if compose_service_running grafana; then
grafana_health="$(http_code "$GRAFANA_URL/api/health" || true)"
if [[ "$grafana_health" == "200" ]]; then
pass "Grafana healthy at $GRAFANA_URL"
else
fail "Grafana healthy at $GRAFANA_URL (HTTP $grafana_health)"
fi
else
log " SKIP: grafana container not running"
fi
target_health="$(curl -sS "$PROMETHEUS_URL/api/v1/targets" | jq -er '.data.activeTargets[0].health' 2>/dev/null || true)"
if [[ "$target_health" == "up" ]]; then
pass "Prometheus scrape target is up"
elif [[ "$target_health" == "down" ]]; then
fail "Prometheus scrape target is up (currently down — is the API running on port 5217?)"
else
fail "Prometheus scrape target health could not be determined"
fi
query_status="$(curl -sS "$PROMETHEUS_URL/api/v1/query?query=digitization_batches_by_status" | jq -er '.status' 2>/dev/null || true)"
if [[ "$query_status" == "success" ]]; then
pass "Prometheus query API returns digitization_batches_by_status"
else
fail "Prometheus query API returns digitization_batches_by_status"
fi
if [[ -f "$REPO_ROOT/prometheus.yml" ]]; then
if grep -q 'host.docker.internal:5217' "$REPO_ROOT/prometheus.yml" \
&& grep -q 'vigilcare-records-api' "$REPO_ROOT/prometheus.yml"; then
pass "prometheus.yml targets host.docker.internal:5217"
else
fail "prometheus.yml targets host.docker.internal:5217"
fi
else
fail "prometheus.yml exists in repo root"
fi
}
main() {
require_cmd curl
require_cmd jq
require_cmd docker
require_cmd python3
ensure_fixture_pdf
log "VigilCare Records — Phase 8 verification"
log "API: $API_URL"
log "Prometheus: $PROMETHEUS_URL"
log "Grafana: $GRAFANA_URL"
assert_api_reachable
test_metrics_endpoint
test_gauge_updates_after_upload
test_rejection_counter
test_promotion_histogram
test_work_queue_overview
test_audit_trail_events
test_promotion_retry_schema
test_monitoring_stack
log ""
log "Results: $PASS_COUNT passed, $FAIL_COUNT failed"
if (( FAIL_COUNT > 0 )); then
log "Failed checks:"
for item in "${FAILED_TESTS[@]}"; do
log " - $item"
done
exit 1
fi
log "All Phase 8 verification checks passed."
}
main "$@"