vigilcare-records/scripts/run-vigilcare-records-phase-8-verification.sh

#!/usr/bin/env bash
# Runs Phase 8 verification checks from docs/plans/phase-8-plan.md.
#
# Covers Prometheus metrics, MetricsCollectorService gauges, rejection counter,
# promotion duration histogram, work queue overview, cursor-paginated audit trail,
# promotion retry schema, and the Prometheus/Grafana Docker stack.
#
# Prerequisites:
#   docker compose up -d          (PostgreSQL, Redis, MinIO, Prometheus, Grafana)
#   dotnet ef database update --project VigilCareRecordsAPI
#   dotnet run --project VigilCareRecordsAPI
#   Phase 1–7 seed data (intake1, entry1, verifier1, approver1, admin1)
#
# Environment overrides:
#   VIGILCARE_API_URL                  default: http://localhost:5217
#   VIGILCARE_PROMETHEUS_URL           default: http://localhost:9095
#   VIGILCARE_GRAFANA_URL              default: http://localhost:3013
#   VIGILCARE_COMPOSE_FILE             default: <repo>/docker-compose.yml
#   VIGILCARE_PG_HOST                  default: localhost
#   VIGILCARE_PG_PORT                  default: 5437
#   VIGILCARE_PG_DB                    default: vigilcare_records
#   VIGILCARE_PG_USER                  default: postgres
#   VIGILCARE_PG_PASSWORD              default: password
#   VIGILCARE_SKIP_DB_CHECKS           set to 1 to skip PostgreSQL assertions
#   VIGILCARE_SKIP_MONITORING_CHECKS   set to 1 to skip Prometheus/Grafana checks
#   VIGILCARE_METRICS_COLLECTOR_WAIT   default: 35 (seconds; MetricsCollectorService interval is 30s)
#   VIGILCARE_RECORDED_AT              default: 2026-06-27T10:00:00Z

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
FIXTURE_PDF="$SCRIPT_DIR/fixtures/test-scan.pdf"

API_URL="${VIGILCARE_API_URL:-http://localhost:5217}"
PROMETHEUS_URL="${VIGILCARE_PROMETHEUS_URL:-http://localhost:9095}"
GRAFANA_URL="${VIGILCARE_GRAFANA_URL:-http://localhost:3013}"
COMPOSE_FILE="${VIGILCARE_COMPOSE_FILE:-$REPO_ROOT/docker-compose.yml}"
COMPOSE=(docker compose -f "$COMPOSE_FILE")
PG_HOST="${VIGILCARE_PG_HOST:-localhost}"
PG_PORT="${VIGILCARE_PG_PORT:-5437}"
PG_DB="${VIGILCARE_PG_DB:-vigilcare_records}"
PG_USER="${VIGILCARE_PG_USER:-postgres}"
PG_PASSWORD="${VIGILCARE_PG_PASSWORD:-password}"
SKIP_DB_CHECKS="${VIGILCARE_SKIP_DB_CHECKS:-0}"
SKIP_MONITORING_CHECKS="${VIGILCARE_SKIP_MONITORING_CHECKS:-0}"
METRICS_COLLECTOR_WAIT="${VIGILCARE_METRICS_COLLECTOR_WAIT:-35}"
RECORDED_AT="${VIGILCARE_RECORDED_AT:-2026-06-27T10:00:00Z}"
RECORDED_AT_BP="${VIGILCARE_RECORDED_AT_BP:-2026-06-27T10:05:00Z}"

PASS_COUNT=0
FAIL_COUNT=0
FAILED_TESTS=()

ALL_BATCH_STATUSES=(
  UPLOADED
  IN_ENTRY
  PENDING_VERIFICATION
  REJECTED
  VERIFIED
  AWAITING_CLINICAL_APPROVAL
  APPROVED
  PROMOTED
)

log() {
  printf '%s\n' "$*"
}

section() {
  log ""
  log "== $1 =="
}

pass() {
  PASS_COUNT=$((PASS_COUNT + 1))
  log "  PASS: $1"
}

fail() {
  FAIL_COUNT=$((FAIL_COUNT + 1))
  FAILED_TESTS+=("$1")
  log "  FAIL: $1"
}

require_cmd() {
  local cmd="$1"
  if ! command -v "$cmd" >/dev/null 2>&1; then
    log "ERROR: required command not found: $cmd"
    exit 1
  fi
}

ensure_fixture_pdf() {
  if [[ -f "$FIXTURE_PDF" ]]; then
    return 0
  fi
  mkdir -p "$(dirname "$FIXTURE_PDF")"
  cat >"$FIXTURE_PDF" <<'EOF'
%PDF-1.0
1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj 2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj 3 0 obj<</Type/Page/MediaBox[0 0 612 792]/Parent 2 0 R/Resources<<>>>>endobj
xref
0 4
0000000000 65535 f 
0000000009 00000 n 
0000000058 00000 n 
0000000115 00000 n 
trailer<</Size 4/Root 1 0 R>>
startxref
206
%%EOF
EOF
}

new_uuid() {
  if command -v uuidgen >/dev/null 2>&1; then
    uuidgen
  else
    cat /proc/sys/kernel/random/uuid
  fi
}

new_idempotency_key() {
  printf 'phase8-%s' "$(new_uuid)"
}

compose_service_running() {
  local service="$1"
  "${COMPOSE[@]}" ps --status running --services 2>/dev/null | grep -qx "$service"
}

http_code() {
  curl -sS -o /dev/null -w '%{http_code}' "$@"
}

json_post() {
  local url="$1"
  local body="$2"
  local token="${3:-}"
  if [[ -n "$token" ]]; then
    curl -sS -X POST "$url" \
      -H "Authorization: Bearer $token" \
      -H 'Content-Type: application/json' \
      -d "$body"
  else
    curl -sS -X POST "$url" \
      -H 'Content-Type: application/json' \
      -d "$body"
  fi
}

json_put() {
  local url="$1"
  local body="$2"
  local token="$3"
  curl -sS -X PUT "$url" \
    -H "Authorization: Bearer $token" \
    -H 'Content-Type: application/json' \
    -d "$body"
}

login() {
  local username="$1"
  local password="${2:-password}"
  json_post "$API_URL/api/v1/auth/login" \
    "{\"username\":\"$username\",\"password\":\"$password\"}"
}

extract_data_field() {
  local json="$1"
  local field="$2"
  jq -er ".data.$field // empty" <<<"$json"
}

extract_error_code() {
  local json="$1"
  jq -er '.error.code // empty' <<<"$json" 2>/dev/null || true
}

assert_api_reachable() {
  local code
  code="$(http_code "$API_URL/swagger/index.html" || true)"
  if [[ "$code" != "200" ]]; then
    log "ERROR: API not reachable at $API_URL (HTTP $code)."
    log "Start infrastructure with: docker compose up -d"
    log "Apply migrations with: dotnet ef database update --project VigilCareRecordsAPI"
    log "Start API with: dotnet run --project VigilCareRecordsAPI"
    exit 1
  fi
}

fetch_metrics() {
  curl -sS "$API_URL/metrics"
}

metric_gauge_status() {
  local status="$1"
  local line value
  line="$(fetch_metrics | grep -E "digitization_batches_by_status\\{status=\"${status}\"\\}" | head -1 || true)"
  value="$(awk '{print $2}' <<<"$line")"
  if [[ -z "$value" ]]; then
    printf '0'
  else
    printf '%s' "$value"
  fi
}

metric_counter_rejection() {
  local category="$1"
  local line value
  line="$(fetch_metrics | grep -E "digitization_rejection_total\\{reason_category=\"${category}\"\\}" | head -1 || true)"
  value="$(awk '{print $2}' <<<"$line")"
  if [[ -z "$value" ]]; then
    printf '0'
  else
    printf '%s' "$value"
  fi
}

metric_histogram_count() {
  local line value
  line="$(fetch_metrics | grep -E '^digitization_promotion_duration_seconds_count ' | head -1 || true)"
  value="$(awk '{print $2}' <<<"$line")"
  if [[ -z "$value" ]]; then
    printf '0'
  else
    printf '%s' "$value"
  fi
}

psql_available() {
  [[ "$SKIP_DB_CHECKS" == "1" ]] && return 1
  compose_service_running postgres && return 0
  command -v psql >/dev/null 2>&1 && return 0
  return 1
}

psql_query() {
  if [[ "$SKIP_DB_CHECKS" == "1" ]]; then
    return 1
  fi
  if compose_service_running postgres; then
    "${COMPOSE[@]}" exec -T postgres \
      psql -U "$PG_USER" -d "$PG_DB" -Atqc "$1"
  elif command -v psql >/dev/null 2>&1; then
    PGPASSWORD="$PG_PASSWORD" psql -h "$PG_HOST" -p "$PG_PORT" -U "$PG_USER" -d "$PG_DB" -Atqc "$1"
  else
    return 1
  fi
}

upload_batch() {
  local token="$1"
  local batch_type="${2:-VITALS_SHEET}"
  local track="${3:-BACKFILL}"

  curl -sS -X POST "$API_URL/api/v1/digitization-batches" \
    -H "Authorization: Bearer $token" \
    -F "file=@${FIXTURE_PDF};type=application/pdf" \
    -F "batchType=$batch_type" \
    -F "track=$track"
}

assign_batch() {
  local token="$1"
  local batch_id="$2"
  local entry_clerk_id="$3"

  curl -sS -X PATCH "$API_URL/api/v1/digitization-batches/$batch_id/assign" \
    -H "Authorization: Bearer $token" \
    -H 'Content-Type: application/json' \
    -d "{\"entryClerkUserId\":\"$entry_clerk_id\"}"
}

verify_batch() {
  local token="$1"
  local batch_id="$2"
  local body="$3"
  json_post "$API_URL/api/v1/digitization-batches/$batch_id/verify" "$body" "$token"
}

reject_batch() {
  local token="$1"
  local batch_id="$2"
  local reason="$3"
  json_post "$API_URL/api/v1/digitization-batches/$batch_id/reject" \
    "{\"reason\":\"$reason\"}" "$token"
}

approve_batch() {
  local token="$1"
  local batch_id="$2"
  local idempotency_key="$3"
  local body='{"enableRetroactiveAlerts":false}'
  if [[ -n "${4:-}" ]]; then
    body="$4"
  fi

  curl -sS -X POST "$API_URL/api/v1/digitization-batches/$batch_id/approve" \
    -H "Authorization: Bearer $token" \
    -H 'Content-Type: application/json' \
    -H "Idempotency-Key: $idempotency_key" \
    -d "$body"
}

create_assigned_batch() {
  local intake_token="$1"
  local upload_json batch_id entry_id

  upload_json="$(upload_batch "$intake_token")"
  if [[ "$(jq -er '.success' <<<"$upload_json")" != "true" ]]; then
    return 1
  fi

  batch_id="$(extract_data_field "$upload_json" id)"
  entry_id="$(extract_data_field "$(login entry1)" userId)"
  assign_batch "$intake_token" "$batch_id" "$entry_id" >/dev/null
  printf '%s' "$batch_id"
}

create_pending_vitals_batch() {
  local intake_token="$1"
  local entry_token batch_id submit_code

  batch_id="$(create_assigned_batch "$intake_token")" || return 1
  entry_token="$(extract_data_field "$(login entry1)" token)"

  json_put \
    "$API_URL/api/v1/digitization-batches/$batch_id/draft/patient" \
    '{"fullName":"Phase8 Verify Patient","dateOfBirth":"1985-03-15","sex":"M","noKnownAllergies":true,"noActiveMedications":true}' \
    "$entry_token" >/dev/null

  json_put \
    "$API_URL/api/v1/digitization-batches/$batch_id/draft/encounter" \
    '{"admissionDate":"2025-01-01T08:00:00Z","department":"ICU","admissionReason":"Phase 8 verification"}' \
    "$entry_token" >/dev/null

  json_post \
    "$API_URL/api/v1/digitization-batches/$batch_id/draft/observations" \
    "{\"observationCode\":\"HEART_RATE\",\"value\":88,\"unit\":\"bpm\",\"recordedAt\":\"$RECORDED_AT\"}" \
    "$entry_token" >/dev/null

  submit_code="$(http_code -X POST \
    "$API_URL/api/v1/digitization-batches/$batch_id/submit-for-verification" \
    -H "Authorization: Bearer $entry_token")"
  if [[ "$submit_code" != "200" ]]; then
    return 1
  fi

  printf '%s' "$batch_id"
}

create_batch_ready_for_approval() {
  local intake_token="$1"
  local entry_token verifier_token batch_id entry_id verify_json verify_status submit_code

  batch_id="$(create_assigned_batch "$intake_token")" || return 1
  entry_id="$(extract_data_field "$(login entry1)" userId)"
  entry_token="$(extract_data_field "$(login entry1)" token)"

  json_put \
    "$API_URL/api/v1/digitization-batches/$batch_id/draft/patient" \
    '{"fullName":"Phase8 Promote Patient","dateOfBirth":"1990-05-15","sex":"M","bloodType":"A+","noKnownAllergies":true}' \
    "$entry_token" >/dev/null

  json_put \
    "$API_URL/api/v1/digitization-batches/$batch_id/draft/encounter" \
    '{"admissionDate":"2024-01-15T08:00:00Z","department":"General Medicine","roomBed":"301-A","admissionReason":"Routine checkup","status":"active"}' \
    "$entry_token" >/dev/null

  json_post \
    "$API_URL/api/v1/digitization-batches/$batch_id/draft/observations" \
    "{\"observationCode\":\"HR\",\"value\":88,\"unit\":\"bpm\",\"recordedAt\":\"$RECORDED_AT\"}" \
    "$entry_token" >/dev/null

  json_post \
    "$API_URL/api/v1/digitization-batches/$batch_id/draft/observations" \
    "{\"observationCode\":\"TEMP\",\"value\":37.2,\"unit\":\"C\",\"recordedAt\":\"$RECORDED_AT\"}" \
    "$entry_token" >/dev/null

  json_post \
    "$API_URL/api/v1/digitization-batches/$batch_id/draft/observations" \
    "{\"observationCode\":\"BP_SYS\",\"value\":120,\"unit\":\"mmHg\",\"recordedAt\":\"$RECORDED_AT_BP\"}" \
    "$entry_token" >/dev/null

  submit_code="$(http_code -X POST \
    "$API_URL/api/v1/digitization-batches/$batch_id/submit-for-verification" \
    -H "Authorization: Bearer $entry_token")"
  if [[ "$submit_code" != "200" ]]; then
    return 1
  fi

  verifier_token="$(extract_data_field "$(login verifier1)" token)"
  verify_json="$(verify_batch "$verifier_token" "$batch_id" \
    '{"fieldChecks":[{"fieldName":"patient.fullName","status":"ok","note":null},{"fieldName":"observation.heartRate","status":"ok","note":null}],"passed":true}')"
  verify_status="$(extract_data_field "$verify_json" status)"

  if [[ "$(jq -er '.success' <<<"$verify_json")" != "true" || "$verify_status" != "AWAITING_CLINICAL_APPROVAL" ]]; then
    return 1
  fi

  printf '%s' "$batch_id"
}

test_metrics_endpoint() {
  section "1. Prometheus metrics endpoint"

  local metrics
  metrics="$(fetch_metrics)"

  if grep -q '^# HELP ' <<<"$metrics" && grep -q '^# TYPE ' <<<"$metrics"; then
    pass "/metrics serves Prometheus exposition format"
  else
    fail "/metrics serves Prometheus exposition format"
  fi

  local metric
  for metric in \
    digitization_batches_by_status \
    digitization_promotion_duration_seconds \
    digitization_rejection_total \
    digitization_queue_age_seconds \
    http_request_duration_seconds; do
    if grep -q "$metric" <<<"$metrics"; then
      pass "metric registered: $metric"
    else
      fail "metric registered: $metric"
    fi
  done

  local missing=0 status
  for status in "${ALL_BATCH_STATUSES[@]}"; do
    if ! grep -q "digitization_batches_by_status{status=\"${status}\"}" <<<"$metrics"; then
      missing=$((missing + 1))
    fi
  done
  if [[ "$missing" -eq 0 ]]; then
    pass "digitization_batches_by_status exposes all 8 status labels"
  else
    fail "digitization_batches_by_status exposes all 8 status labels ($missing missing)"
  fi

  if grep -q 'digitization_rejection_total{reason_category="verification_failed"}' <<<"$metrics" \
    && grep -q 'digitization_rejection_total{reason_category="clinical_rejected"}' <<<"$metrics"; then
    pass "digitization_rejection_total exposes both reason_category labels"
  else
    fail "digitization_rejection_total exposes both reason_category labels"
  fi
}

test_gauge_updates_after_upload() {
  section "2. Gauge metrics update after creating a batch"

  local intake_token before after upload_json

  intake_token="$(extract_data_field "$(login intake1)" token)"
  before="$(metric_gauge_status UPLOADED)"

  upload_json="$(upload_batch "$intake_token")"
  if [[ "$(jq -er '.success' <<<"$upload_json")" != "true" ]]; then
    fail "upload batch for gauge test"
    return
  fi
  pass "uploaded batch for gauge test"

  log "  waiting ${METRICS_COLLECTOR_WAIT}s for MetricsCollectorService..."
  sleep "$METRICS_COLLECTOR_WAIT"

  after="$(metric_gauge_status UPLOADED)"
  if awk -v before="$before" -v after="$after" 'BEGIN { exit (after > before) ? 0 : 1 }'; then
    pass "digitization_batches_by_status UPLOADED increased ($before -> $after)"
  else
    fail "digitization_batches_by_status UPLOADED increased ($before -> $after)"
  fi
}

test_rejection_counter() {
  section "3. Rejection counter increments"

  local intake_token verifier_token batch_id before after verify_json

  intake_token="$(extract_data_field "$(login intake1)" token)"
  batch_id="$(create_pending_vitals_batch "$intake_token")" || {
    fail "setup pending batch for rejection counter"
    return
  }

  before="$(metric_counter_rejection verification_failed)"
  verifier_token="$(extract_data_field "$(login verifier1)" token)"

  verify_json="$(verify_batch "$verifier_token" "$batch_id" \
    '{"fieldChecks":[{"fieldName":"patient.fullName","status":"error","note":"Name mismatch"}],"passed":false}')"
  if [[ "$(jq -er '.success' <<<"$verify_json")" != "true" ]]; then
    fail "verify with passed=false returns success envelope"
    return
  fi
  pass "verification failed via VerifyAsync (Passed=false)"

  after="$(metric_counter_rejection verification_failed)"
  if awk -v before="$before" -v after="$after" 'BEGIN { exit (after > before) ? 0 : 1 }'; then
    pass "digitization_rejection_total verification_failed incremented ($before -> $after)"
  else
    fail "digitization_rejection_total verification_failed incremented ($before -> $after)"
  fi
}

test_promotion_histogram() {
  section "4. Promotion duration histogram"

  local intake_token approver_token batch_id before after approve_json

  before="$(metric_histogram_count)"
  intake_token="$(extract_data_field "$(login intake1)" token)"
  batch_id="$(create_batch_ready_for_approval "$intake_token")" || {
    fail "setup batch ready for approval"
    return
  }

  approver_token="$(extract_data_field "$(login approver1)" token)"
  approve_json="$(approve_batch "$approver_token" "$batch_id" "$(new_idempotency_key)" \
    '{"enableRetroactiveAlerts":false}')"

  if [[ "$(jq -er '.success' <<<"$approve_json")" != "true" ]]; then
    fail "approve batch for promotion histogram"
    return
  fi
  pass "batch promoted via ApproveAndPromoteAsync"

  after="$(metric_histogram_count)"
  if awk -v before="$before" -v after="$after" 'BEGIN { exit (after > before) ? 0 : 1 }'; then
    pass "digitization_promotion_duration_seconds_count increased ($before -> $after)"
  else
    fail "digitization_promotion_duration_seconds_count increased ($before -> $after)"
  fi

  if grep -q '^digitization_promotion_duration_seconds_sum ' <<<"$(fetch_metrics)"; then
    pass "digitization_promotion_duration_seconds_sum present after promotion"
  else
    fail "digitization_promotion_duration_seconds_sum present after promotion"
  fi
}

test_work_queue_overview() {
  section "5. Work queue overview endpoint"

  local admin_token entry_token overview_json code missing=0 status

  admin_token="$(extract_data_field "$(login admin1)" token)"
  overview_json="$(curl -sS "$API_URL/api/v1/work-queue/overview" \
    -H "Authorization: Bearer $admin_token")"

  if [[ "$(jq -er '.success' <<<"$overview_json")" == "true" ]]; then
    pass "GET /work-queue/overview returns success for administrator"
  else
    fail "GET /work-queue/overview returns success for administrator"
    return
  fi

  for status in "${ALL_BATCH_STATUSES[@]}"; do
    if [[ "$(jq -er --arg s "$status" '.data.statusCounts[$s] | type' <<<"$overview_json")" != "number" ]]; then
      missing=$((missing + 1))
    fi
  done
  if [[ "$missing" -eq 0 ]]; then
    pass "overview statusCounts includes all 8 statuses"
  else
    fail "overview statusCounts includes all 8 statuses ($missing missing)"
  fi

  if jq -er '.data.rejectRate | type' <<<"$overview_json" | grep -qx number; then
    pass "overview rejectRate is numeric"
  else
    fail "overview rejectRate is numeric"
  fi

  if jq -er '.data.averageTimeInQueueMinutes | type' <<<"$overview_json" | grep -qx number \
    && jq -er '.data.oldestPendingVerificationMinutes | type' <<<"$overview_json" | grep -qx number; then
    pass "overview queue age fields are numeric"
  else
    fail "overview queue age fields are numeric"
  fi

  entry_token="$(extract_data_field "$(login entry1)" token)"
  code="$(http_code "$API_URL/api/v1/work-queue/overview" \
    -H "Authorization: Bearer $entry_token")"
  if [[ "$code" == "403" ]]; then
    pass "GET /work-queue/overview returns 403 for non-administrator"
  else
    fail "GET /work-queue/overview returns 403 for non-administrator (HTTP $code)"
  fi
}

test_audit_trail_events() {
  section "6. Cursor-paginated audit trail"

  local admin_token intake_token batch_id page1 page2 code cursor encoded_cursor

  admin_token="$(extract_data_field "$(login admin1)" token)"
  intake_token="$(extract_data_field "$(login intake1)" token)"
  batch_id="$(create_batch_ready_for_approval "$intake_token")" || {
    fail "setup batch with multiple audit events"
    return
  }

  page1="$(curl -sS "$API_URL/api/v1/digitization-batches/$batch_id/events?pageSize=2" \
    -H "Authorization: Bearer $admin_token")"

  if [[ "$(jq -er '.success' <<<"$page1")" != "true" ]]; then
    fail "GET /events returns success envelope"
    return
  fi
  pass "GET /events returns success envelope"

  if [[ "$(jq -er '.data.items | length' <<<"$page1")" -ge 1 ]]; then
    pass "events page contains at least one item"
  else
    fail "events page contains at least one item"
  fi

  if jq -er '.data.items[0] | has("actorUsername") and has("actorFullName") and has("eventType")' \
    <<<"$page1" | grep -qx true; then
    pass "event items include actorUsername and actorFullName"
  else
    fail "event items include actorUsername and actorFullName"
  fi

  if [[ "$(jq -er '.data.hasMore' <<<"$page1")" == "true" ]]; then
    cursor="$(extract_data_field "$page1" nextCursor)"
    encoded_cursor="$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "$cursor")"
    page2="$(curl -sS "$API_URL/api/v1/digitization-batches/$batch_id/events?pageSize=2&after=$encoded_cursor" \
      -H "Authorization: Bearer $admin_token")"
    if [[ "$(jq -er '.success' <<<"$page2")" == "true" \
      && "$(jq -er '.data.items | length' <<<"$page2")" -ge 1 ]]; then
      pass "cursor pagination returns a second page"
    else
      fail "cursor pagination returns a second page"
    fi
  else
    log "  SKIP: cursor second-page check (batch has <=2 events)"
  fi

  code="$(http_code "$API_URL/api/v1/digitization-batches/$batch_id/events?after=not-a-date" \
    -H "Authorization: Bearer $admin_token")"
  if [[ "$code" == "400" ]]; then
    pass "invalid cursor returns 400"
  else
    fail "invalid cursor returns 400 (HTTP $code)"
  fi

  code="$(http_code "$API_URL/api/v1/digitization-batches/00000000-0000-0000-0000-000000000000/events" \
    -H "Authorization: Bearer $admin_token")"
  if [[ "$code" == "404" ]]; then
    pass "missing batch returns 404"
  else
    fail "missing batch returns 404 (HTTP $code)"
  fi
}

test_promotion_retry_schema() {
  section "7. Promotion retry infrastructure"

  if ! psql_available; then
    log "  SKIP: promotion_attempts table check (postgres not reachable)"
    return
  fi

  local table_exists
  table_exists="$(psql_query "
    SELECT EXISTS (
      SELECT 1
      FROM information_schema.tables
      WHERE table_schema = 'public'
        AND table_name = 'promotion_attempts'
    );
  ")"

  if [[ "$table_exists" == "t" ]]; then
    pass "promotion_attempts table exists"
  else
    fail "promotion_attempts table exists"
  fi

  local index_count
  index_count="$(psql_query "
    SELECT count(*)
    FROM pg_indexes
    WHERE tablename = 'promotion_attempts'
      AND indexname IN (
        'ix_promotion_attempts_batch_attempt',
        'ix_promotion_attempts_pending_retry'
      );
  ")"

  if [[ "$index_count" == "2" ]]; then
    pass "promotion_attempts has batch/attempt and pending-retry indexes"
  else
    fail "promotion_attempts has batch/attempt and pending-retry indexes (got $index_count)"
  fi

  log "  NOTE: full PromotionRetryService retry flow requires simulating a deferred promotion failure (manual test in phase-8 plan section 7)."
}

test_monitoring_stack() {
  section "8. Prometheus and Grafana stack"

  if [[ "$SKIP_MONITORING_CHECKS" == "1" ]]; then
    log "  SKIP: VIGILCARE_SKIP_MONITORING_CHECKS=1"
    return
  fi

  if ! compose_service_running prometheus; then
    log "  SKIP: prometheus container not running (docker compose up -d prometheus grafana)"
    return
  fi

  local prom_health grafana_health target_health query_status

  prom_health="$(http_code "$PROMETHEUS_URL/-/healthy" || true)"
  if [[ "$prom_health" == "200" ]]; then
    pass "Prometheus healthy at $PROMETHEUS_URL"
  else
    fail "Prometheus healthy at $PROMETHEUS_URL (HTTP $prom_health)"
  fi

  if compose_service_running grafana; then
    grafana_health="$(http_code "$GRAFANA_URL/api/health" || true)"
    if [[ "$grafana_health" == "200" ]]; then
      pass "Grafana healthy at $GRAFANA_URL"
    else
      fail "Grafana healthy at $GRAFANA_URL (HTTP $grafana_health)"
    fi
  else
    log "  SKIP: grafana container not running"
  fi

  target_health="$(curl -sS "$PROMETHEUS_URL/api/v1/targets" | jq -er '.data.activeTargets[0].health' 2>/dev/null || true)"
  if [[ "$target_health" == "up" ]]; then
    pass "Prometheus scrape target is up"
  elif [[ "$target_health" == "down" ]]; then
    fail "Prometheus scrape target is up (currently down — is the API running on port 5217?)"
  else
    fail "Prometheus scrape target health could not be determined"
  fi

  query_status="$(curl -sS "$PROMETHEUS_URL/api/v1/query?query=digitization_batches_by_status" | jq -er '.status' 2>/dev/null || true)"
  if [[ "$query_status" == "success" ]]; then
    pass "Prometheus query API returns digitization_batches_by_status"
  else
    fail "Prometheus query API returns digitization_batches_by_status"
  fi

  if [[ -f "$REPO_ROOT/prometheus.yml" ]]; then
    if grep -q 'host.docker.internal:5217' "$REPO_ROOT/prometheus.yml" \
      && grep -q 'vigilcare-records-api' "$REPO_ROOT/prometheus.yml"; then
      pass "prometheus.yml targets host.docker.internal:5217"
    else
      fail "prometheus.yml targets host.docker.internal:5217"
    fi
  else
    fail "prometheus.yml exists in repo root"
  fi
}

main() {
  require_cmd curl
  require_cmd jq
  require_cmd docker
  require_cmd python3
  ensure_fixture_pdf

  log "VigilCare Records — Phase 8 verification"
  log "API: $API_URL"
  log "Prometheus: $PROMETHEUS_URL"
  log "Grafana: $GRAFANA_URL"

  assert_api_reachable

  test_metrics_endpoint
  test_gauge_updates_after_upload
  test_rejection_counter
  test_promotion_histogram
  test_work_queue_overview
  test_audit_trail_events
  test_promotion_retry_schema
  test_monitoring_stack

  log ""
  log "Results: $PASS_COUNT passed, $FAIL_COUNT failed"
  if (( FAIL_COUNT > 0 )); then
    log "Failed checks:"
    for item in "${FAILED_TESTS[@]}"; do
      log "  - $item"
    done
    exit 1
  fi

  log "All Phase 8 verification checks passed."
}

main "$@"