From ddde7fee31b0ef32a222568b4586819c9dd22fb5 Mon Sep 17 00:00:00 2001 From: voltsrage Date: Thu, 18 Jun 2026 16:16:57 +0800 Subject: [PATCH] test: run verification test --- README.md | 69 +++++- scripts/run-phase11-verification.sh | 334 ++++++++++++++++++++++++++++ 2 files changed, 395 insertions(+), 8 deletions(-) create mode 100755 scripts/run-phase11-verification.sh diff --git a/README.md b/README.md index e54934f..faaf4e5 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,9 @@ An `OutboxEvent` is written in the same transaction as any observation or alert, - **Patient Registration** — register patients with MRN generation; optional blood type, allergies, and emergency contact; paginated list with name (`ILIKE`) and MRN (exact) search; patient detail with active encounter summary - **Encounter Management** — open encounters against a patient with optional room/bed and admission reason; encounter status state machine (`scheduled → active → discharged / cancelled`) with 409 on illegal transitions; optional discharge diagnosis on discharge; encounter timeline as a merged chronological view across status changes, observation summaries, and alerts - **Alert Threshold Management** — configure per-observation-code numeric bounds (`criticalLow`, `warningLow`, `warningHigh`, `criticalHigh`) for 12 observation codes; thresholds pre-loaded into Redis on startup; write-through cache invalidation on update -- **Observation Ingest** — `POST /encounters/:id/observations` accepts single or small batch (up to 10); idempotency via `Idempotency-Key` header (partial unique index); plausibility validation per observation code; synchronous critical alert creation within the ingest transaction; warning breach deferred to Kafka consumer; outbox event written in the same commit; cursor-paginated history on `(encounter_id, observation_code, recorded_at DESC)` +- **Observation Ingest** — `POST /encounters/:id/observations` accepts single or small batch (up to 10); idempotency via `Idempotency-Key` header (partial unique index); plausibility validation per observation code; synchronous critical alert creation within the ingest transaction; warning-range breaches evaluated asynchronously by `WarningAlertService` (Kafka consumer group `warning-evaluator`); outbox event written in the same commit; cursor-paginated history on `(encounter_id, observation_code, recorded_at DESC)` +- **Warning Threshold Alerts** — `WarningEvaluator` reads thresholds from Redis; creates `WARNING`-severity alerts for values above `warningHigh` or below `warningLow` that are not also critical breaches; idempotent `INSERT WHERE NOT EXISTS` per encounter and alert type while status is `OPEN` or `ACKNOWLEDGED`; warning alerts are indexed in Elasticsearch but not published to the RabbitMQ paging queue +- **Clinical Order Management** — `POST /encounters/:id/orders` create; `GET /encounters/:id/orders` list with optional status filter; `GET /orders/:id` detail; `PATCH /orders/:id/status` status transitions; `PATCH /orders/:id/result` record result and transition to `Resulted`; status machine enforces `Pending → InProgress → Resulted` and terminal `Cancelled` - **Clinical Alert Lifecycle** — paginated alert list per encounter and globally; acknowledge with clinician ID and optional note; resolve (must be acknowledged first); global list filterable by status, severity, and department - **Outbox Relay** — `IHostedService` polling every 500ms; reads unprocessed outbox rows, publishes to Kafka, marks processed; partitioned by `encounterId` for per-encounter ordering - **Kafka Pipeline** — three topics (`observation.recorded`, `alert.generated`, `encounter.status.changed`) with six partitions each; KRaft mode, no Zookeeper; `KAFKA_AUTO_CREATE_TOPICS_ENABLE=false` — topics are provisioned explicitly by `KafkaTopicProvisioner` @@ -59,7 +61,8 @@ An `OutboxEvent` is written in the same transaction as any observation or alert, - **RabbitMQ Notification Workers** — `NotificationPublisherService` reads `alert.generated` from Kafka and publishes paging jobs to `alerts.paging.queue`; `PagingWorkerService` sends the page and waits for acknowledgment; if no ack arrives before timeout it NACKs to `alerts.paging.dlq` with `x-message-ttl = 300000ms`; if the host is stopping, in-flight paging messages are NACKed with `requeue=true` so they are retried after restart and do not false-escalate; `EscalationWorkerService` pages the on-call backup and sets alert status to `escalated`; `DischargeSummaryWorkerService` reads `encounter.status.changed`, generates a discharge summary, and stores it in MinIO under `/discharge-summaries/{encounterId}/summary.pdf` - **Data Lake Writer** — `DataLakeWriterService` (consumer group `data-lake-writer`) buffers `observation.recorded`, `alert.generated`, and `encounter.status.changed` events, flushes date-partitioned Parquet files to MinIO (`/observations/`, `/alerts/`, `/encounters/`), and commits Kafka offsets only after successful uploads; `kafka_partition` and `kafka_offset` columns provide audit lineage - **Reconciliation Jobs** — three scheduled checks: (1) unacknowledged CRITICAL alerts older than 30 minutes, (2) pending orders without results after 4 hours, (3) active inpatients with no observation in 2 hours; each finding creates a `reconciliation_alerts` row and publishes to RabbitMQ -- **Standard Envelope** — all responses use a consistent `{ success, statusCode, data, error }` wrapper; validation errors use the same shape; `ApiBehaviorOptions` overridden so model validation also produces the standard envelope +- **Standard Envelope** — all responses use a consistent `{ success, statusCode, data, error }` wrapper; validation errors use the same shape; `ApiBehaviorOptions` overridden so model validation also produces the standard envelope with field-level `details` +- **Input Validation** — FluentValidation validators on all request DTOs (patient registration, encounter open, observation ingest, alert acknowledge, alert thresholds, orders); invalid requests return 400 before reaching the service layer - **Observability** — Serilog structured logging enriched with `correlationId`, `encounterId`, `patientId` on alert paths; Seq sink (`http://localhost:5345`); Prometheus (`http://localhost:9101`) scrapes `GET /metrics`; eight application metric families via `ClinicalMetrics` and three background collectors (`AlertsUnacknowledgedCollector`, `OutboxPendingCollector`, `KafkaConsumerLagCollector`); Grafana clinical dashboard (`http://localhost:3101`, admin/admin) with `alerts_unacknowledged_gauge` as the primary safety panel; per-request correlation IDs in request logs and `X-Correlation-Id` response headers - **Swagger UI** — OpenAPI spec via Swashbuckle (Development only) @@ -85,6 +88,7 @@ IHostedServices (background): OutboxRelayService → PostgreSQL outbox → Kafka (every 500ms) EsIndexerService → Kafka → Elasticsearch (consumer group: es-indexer) SepsisEngineService → Kafka → Redis SIRS state → PostgreSQL alert (consumer group: sepsis-engine) + WarningAlertService → Kafka → WarningEvaluator → PostgreSQL WARNING alert (consumer group: warning-evaluator) NotificationPublisherService → Kafka → RabbitMQ paging.queue (consumer group: notification-publisher) PagingWorkerService → RabbitMQ paging.queue → log page → NACK on timeout (or requeue on shutdown) EscalationWorkerService → RabbitMQ escalation.queue → update alert status @@ -117,6 +121,7 @@ IHostedServices (background): | Dashboards | Prometheus 2.52 + Grafana 10.4 | | Data lake format | Parquet.Net 4.x | | Docs | Swagger / OpenAPI (Swashbuckle) | +| Validation | FluentValidation.AspNetCore | | Testing | xUnit + Testcontainers + WebApplicationFactory | --- @@ -133,6 +138,7 @@ VigilCareClinicalAPI/ │ ├── ObservationsController.cs # Ingest POST, cursor-paginated GET │ ├── AlertThresholdsController.cs # Threshold CRUD + cache invalidation │ ├── AlertsController.cs # Alert list (global + per-encounter), acknowledge, resolve +│ ├── OrdersController.cs # Order create, list, get, status transition, record result │ └── AnalyticsController.cs # Elasticsearch-backed patient search, trend, alert summary, population ├── Domains/ │ ├── Entities/ @@ -149,7 +155,7 @@ VigilCareClinicalAPI/ │ ├── EncounterType.cs # Inpatient, Outpatient, Emergency │ ├── AlertSeverity.cs # Warning, Critical │ ├── AlertStatus.cs # Open, Acknowledged, Resolved, Escalated -│ ├── AlertType.cs # Threshold breach, sepsis, systolic BP, AVPU, glucose, … +│ ├── AlertType.cs # Threshold breach, sepsis, warning*, systolic BP, AVPU, glucose, … │ ├── BloodType.cs # A+, O-, AB-, … with ToDbString/FromDbString │ ├── ObservationSource.cs # Device, Manual, Lab │ └── OrderType.cs / ReconciliationCheckType.cs / Department.cs / OrderStatus.cs @@ -165,8 +171,11 @@ VigilCareClinicalAPI/ │ ├── ObservationService.cs # Ingest transaction: idempotency → plausibility → threshold → alert → outbox; emits Prometheus counters │ ├── ObservationQueryService.cs # Cursor-paginated history │ ├── AlertService.cs # Acknowledge, resolve, list +│ ├── OrderService.cs # Order lifecycle; status machine; ConflictException on illegal transitions +│ ├── WarningEvaluator.cs # Warning-range threshold evaluation; idempotent alert INSERT │ ├── AnalyticsService.cs # Elasticsearch query wrappers │ └── PlausibilityValidator.cs # Per-code numeric range guard +├── Validators/ # FluentValidation — RegisterPatient, OpenEncounter, IngestObservation, … ├── Observability/ │ └── Metrics/ │ └── ClinicalMetrics.cs # Eight Prometheus metric families (counters, histogram, gauges) @@ -182,6 +191,7 @@ VigilCareClinicalAPI/ │ │ ├── ElasticIndexProvisioner.cs # Creates patient_encounters, observations, clinical_alerts indices │ │ └── EsIndexerService.cs # consumer group: es-indexer; upserts Elasticsearch documents │ ├── SepsisEngineService.cs # consumer group: sepsis-engine; SIRS eval via Redis TTL keys +│ ├── WarningAlertService.cs # consumer group: warning-evaluator; observation.recorded → WARNING alerts │ ├── Notifications/ │ │ ├── NotificationPublisherService.cs # consumer group: notification-publisher; alert.generated → RabbitMQ paging.queue │ │ ├── PagingWorkerService.cs # RabbitMQ consumer; logs page; NACK on ack timeout → DLQ, requeue on graceful shutdown @@ -250,7 +260,10 @@ tests/ ├── ReconciliationTests.cs # Three reconciliation checks, deduplication, RabbitMQ publish ├── ObservabilityPhase8Tests.cs # /metrics families and correlation header behavior ├── DataLakePhase9Tests.cs # Kafka → MinIO Parquet flow and schema checks - └── ClinicalDemographicsAndObservationTests.cs # Patient/encounter enrichment, expanded observation alerts + ├── ClinicalDemographicsAndObservationTests.cs # Patient/encounter enrichment, expanded observation alerts + ├── WarningAlertTests.cs # WarningEvaluator — warning created, normal/critical skipped, idempotent + ├── OrderLifecycleTests.cs # Orders API — create, list, record result, illegal transition 409 + └── ValidationTests.cs # FluentValidation — empty fields, threshold ordering, order description scripts/ ├── run-api-redis-tests.sh # Phase 1 — patient/encounter/threshold + Redis cache @@ -261,10 +274,11 @@ scripts/ ├── run-reconciliation-tests.sh # Phase 7 — reconciliation scheduler checks ├── run-phase8-verification.sh # Phase 8 — Prometheus metrics, alerts_unacknowledged_gauge, correlation headers ├── run-phase9-verification.sh # Phase 9 — data lake tests, Kafka offsets, MinIO Parquet, DuckDB schema -└── run-phase10-verification.sh # Phase 10 — 12 Redis thresholds, clinical enrichment, ES pipeline, integration tests +├── run-phase10-verification.sh # Phase 10 — 12 Redis thresholds, clinical enrichment, ES pipeline, integration tests +└── run-phase11-verification.sh # Phase 11 — warning alerts, orders API, validation, integration tests docs/ -├── plans/ # Phase 1–10 implementation and verification guides +├── plans/ # Phase 1–11 implementation and verification guides ├── decisions/ │ ├── data-lake-design.md # Parquet vs JSON, partitioning, replay rationale │ └── sepsis-engine-design.md # SIRS sliding window and idempotent alert design @@ -405,6 +419,9 @@ Integration tests use `WebApplicationFactory` with a `Testing` environment and T | `ObservabilityPhase8Tests` | 8 | All eight `/metrics` families, correlation headers, ingest counter increment | | `DataLakePhase9Tests` | 9 | Kafka → MinIO Parquet flow and schema checks | | `ClinicalDemographicsAndObservationTests` | 10 | Patient clinical fields, encounter enrichment, expanded observation codes, critical glucose alert | +| `WarningAlertTests` | 11 | WarningEvaluator — warning HR alert, normal/critical skipped, duplicate idempotent | +| `OrderLifecycleTests` | 11 | Orders API create, list, record result, cancel-resulted 409 | +| `ValidationTests` | 11 | FluentValidation 400 on empty first name, invalid threshold order, empty order description | ### Verification Scripts @@ -414,6 +431,7 @@ With the API running (`dotnet run`) and Docker Compose up: ./scripts/run-phase8-verification.sh # Prometheus target UP, eight metrics, alerts_unacknowledged_gauge live update ./scripts/run-phase9-verification.sh # DataLakePhase9Tests, Kafka consumer group, MinIO Parquet, DuckDB schema ./scripts/run-phase10-verification.sh # 12 Redis thresholds, clinical enrichment, ES pipeline, Phase 10 integration tests +./scripts/run-phase11-verification.sh # Warning alert pipeline, orders API, FluentValidation, Phase 11 integration tests ``` Per-phase test runners (subset of `dotnet test`): @@ -656,6 +674,39 @@ open → acknowledged → resolved | `clinicianId` | string | yes | Clinician identifier | | `note` | string | no | Optional acknowledgment note | +### Orders + +| Method | Path | Description | +|---|---|---| +| POST | `/encounters/{id}/orders` | Create a clinical order for an active encounter | +| GET | `/encounters/{id}/orders` | List orders for an encounter; optional `status`, `page`, `pageSize` | +| GET | `/orders/{id}` | Order detail with encounter | +| PATCH | `/orders/{id}/status` | Transition order status | +| PATCH | `/orders/{id}/result` | Record a result; transitions to `Resulted` | + +**Order status machine:** + +``` +pending → in_progress → resulted + → cancelled +``` + +`PATCH /orders/{id}/status` and `PATCH /orders/{id}/result` return **409** on illegal transitions (e.g. cancelling a resulted order). + +**POST body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `orderType` | string | yes | `Lab`, `Imaging`, `Medication`, `Procedure` | +| `description` | string | yes | Order description | +| `orderedBy` | string | yes | Ordering clinician | + +**PATCH `/orders/{id}/result` body:** + +| Field | Type | Required | Description | +|---|---|---|---| +| `resultSummary` | string | no | Free-text result summary | + ### Analytics (Elasticsearch) | Method | Path | Description | @@ -775,6 +826,7 @@ orderedBy string required (max 200) status string pending | in_progress | resulted | cancelled (default: pending) orderedAt DateTimeOffset resultedAt DateTimeOffset? +resultSummary string? Free-text result summary (set on record result) ``` Indexes: `(encounter_id, ordered_at DESC)`, partial `(status, ordered_at) WHERE status IN ('pending', 'in_progress')` @@ -879,7 +931,7 @@ Exchange: `clinical.notifications.exchange` (direct) | Topic | Partition key | Consumer groups | |---|---|---| -| `observation.recorded` | `encounterId` | `es-indexer`, `sepsis-engine`, `data-lake-writer` | +| `observation.recorded` | `encounterId` | `es-indexer`, `sepsis-engine`, `warning-evaluator`, `data-lake-writer` | | `alert.generated` | `encounterId` | `es-indexer`, `notification-publisher`, `data-lake-writer` | | `encounter.status.changed` | `encounterId` | `es-indexer`, `data-lake-writer` | @@ -979,7 +1031,7 @@ Observation history uses cursor pagination on `(recorded_at DESC, id DESC)`. Off ## Implemented Phases -All ten phases from the project roadmap are implemented and covered by integration tests and/or verification scripts. +Eleven phases from the project roadmap are implemented. Phases 1–10 and Step 4 of Phase 11 are covered by integration tests (`dotnet test` — 64 passing). Phase 11 Step 5 manual verification (docker compose end-to-end) is documented in `docs/plans/phase-11-plan.md`. | Phase | Feature | Status | |---|---|---| @@ -993,5 +1045,6 @@ All ten phases from the project roadmap are implemented and covered by integrati | 8 | Prometheus metrics (`GET /metrics`); eight metric families and three collectors; Grafana clinical dashboard; `ObservabilityPhase8Tests`; `run-phase8-verification.sh` | Done | | 9 | Data lake writer — `data-lake-writer` consumer group; date-partitioned Parquet flush to MinIO; `DataLakePhase9Tests`; `run-phase9-verification.sh`; design doc in `docs/decisions/data-lake-design.md` | Done | | 10 | Clinical data model expansion — `BloodType`, patient allergies/emergency contact, encounter room/bed/admission/discharge fields; five new observation codes (`SYSTOLIC_BP`, `DIASTOLIC_BP`, `LACTATE_MMOL_L`, `AVPU`, `SUPPLEMENTAL_O2`); `GLUCOSE_MG_DL` threshold fix; 12 seeded thresholds; `ClinicalDemographicsAndObservationTests`; `run-phase10-verification.sh` | Done | +| 11 | Warning alert consumer (`WarningAlertService` / `warning-evaluator`); 10 `Warning*` alert types; Orders API (`OrdersController`, `OrderService`); FluentValidation on all request DTOs; `WarningAlertTests`, `OrderLifecycleTests`, `ValidationTests`; `run-phase11-verification.sh` | Done (Step 5 E2E verification via script) | **Optional follow-up:** execute and document the Kafka replay demonstration for the data lake (reset `data-lake-writer` offsets, clear MinIO prefixes, restart API, confirm Parquet rebuild). See `docs/plans/phase-9-plan.md` § Replay demonstration. diff --git a/scripts/run-phase11-verification.sh b/scripts/run-phase11-verification.sh new file mode 100755 index 0000000..5130108 --- /dev/null +++ b/scripts/run-phase11-verification.sh @@ -0,0 +1,334 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" + +BASE_URL="${BASE_URL:-http://localhost:5270}" +ES_URL="${ES_URL:-http://localhost:9200}" +RABBITMQ_MGMT_URL="${RABBITMQ_MGMT_URL:-http://localhost:15674}" +RABBITMQ_USER="${RABBITMQ_USER:-guest}" +RABBITMQ_PASS="${RABBITMQ_PASS:-guest}" +REDIS_PORT="${REDIS_PORT:-6382}" + +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5436}" +PGDATABASE="${PGDATABASE:-vigilcare}" +PGUSER="${PGUSER:-postgres}" +PGPASSWORD="${PGPASSWORD:-password}" + +TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}" +TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~WarningAlertTests|FullyQualifiedName~OrderLifecycleTests|FullyQualifiedName~ValidationTests}" +FULL_TEST="${FULL_TEST:-0}" + +WARNING_WAIT_SECS="${WARNING_WAIT_SECS:-45}" +INDEX_WAIT_SECS="${INDEX_WAIT_SECS:-60}" +WARNING_CONSUMER_GROUP="${WARNING_CONSUMER_GROUP:-warning-evaluator}" +ES_CONSUMER_GROUP="${ES_CONSUMER_GROUP:-es-indexer}" + +SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")" +RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + +TMP_FILES=() +cleanup() { + local f + for f in "${TMP_FILES[@]}"; do + rm -f "${f}" "${f}.status" 2>/dev/null || true + done +} +trap cleanup EXIT + +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing dependency: $1" + exit 1 + } +} + +need curl +need jq +need dotnet + +if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then + echo "Missing dependency: docker compose (${COMPOSE_FILE})" + exit 1 +fi + +compose() { + docker compose -f "${COMPOSE_FILE}" "$@" +} + +kafka_exec() { + compose exec -T kafka "$@" +} + +redis_cmd() { + if command -v redis-cli >/dev/null 2>&1; then + redis-cli -p "${REDIS_PORT}" "$@" + else + compose exec -T redis redis-cli "$@" + fi +} + +psql_cmd() { + local sql="$1" + if command -v psql >/dev/null 2>&1; then + PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" + else + compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" + fi +} + +request() { + local method="$1" + local url="$2" + local body="${3:-}" + local tmp + tmp="$(mktemp)" + TMP_FILES+=("${tmp}") + local status + + if [[ -n "${body}" ]]; then + status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \ + -H "Content-Type: application/json" -d "${body}")" + else + status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")" + fi + + echo "${status}" > "${tmp}.status" + echo "${tmp}" +} + +assert_status() { + local expected="$1" + local body_file="$2" + local status + status="$(<"${body_file}.status")" + if [[ "${status}" != "${expected}" ]]; then + echo "Expected HTTP ${expected}, got ${status}" + cat "${body_file}" + echo + exit 1 + fi +} + +rabbit_api() { + curl -sS -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" "${RABBITMQ_MGMT_URL}/api/${1}" +} + +queue_stat() { + local queue="$1" + local field="$2" + rabbit_api "queues/%2F/${queue}" | jq -r "${field} // 0" +} + +consumer_group_lag() { + local group="$1" + kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \ + --bootstrap-server localhost:9092 \ + --describe \ + --group "${group}" 2>/dev/null | \ + awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }' +} + +wait_for_consumer_lag_zero() { + local group="$1" + local max_secs="$2" + local elapsed=0 + local lag="unknown" + while (( elapsed < max_secs )); do + lag="$(consumer_group_lag "${group}")" + if [[ "${lag}" == "0" ]]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})" + exit 1 +} + +es_alert_hits() { + local alert_type="$1" + local encounter_id="$2" + local payload + payload="$(jq -nc \ + --arg type "${alert_type}" \ + --arg enc "${encounter_id}" \ + '{query:{bool:{must:[{term:{alertType:$type}},{term:{encounterId:$enc}}]}},size:0,track_total_hits:true}')" + local body + body="$(curl -sS "${ES_URL}/clinical_alerts/_search" \ + -H "Content-Type: application/json" \ + -d "${payload}")" + jq -r '.hits.total.value // .hits.total // 0' <<< "${body}" +} + +echo "Phase 11 verification starting..." +echo "Repo root: ${ROOT_DIR}" +echo "API: ${BASE_URL}" + +echo "[1/10] Preflight API, Elasticsearch, PostgreSQL, Redis, RabbitMQ, Kafka" +api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" +es_status="$(curl -sS -o /dev/null -w "%{http_code}" "${ES_URL}/_cluster/health" || true)" +rabbit_status="$(curl -sS -o /dev/null -w "%{http_code}" -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" "${RABBITMQ_MGMT_URL}/api/overview" || true)" +[[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; } +[[ "${es_status}" == "200" ]] || { echo "Elasticsearch not ready (${es_status})"; exit 1; } +[[ "${rabbit_status}" == "200" ]] || { echo "RabbitMQ management API not ready (${rabbit_status})"; exit 1; } +redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; } +psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; } +kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || { + echo "Kafka not ready" + exit 1 +} +redis_cmd EXISTS "threshold:HEART_RATE" | grep -q '^1$' || { + echo "Missing Redis threshold:HEART_RATE — restart API to run ThresholdCacheLoader" + exit 1 +} +echo "OK: infrastructure preflight passed" + +echo "[2/10] Create patient and active encounter" +patient_payload="$(jq -nc \ + --arg fn "Phase11" \ + --arg ln "Verify${SCRIPT_RUN_ID}" \ + '{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')" +patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")" +assert_status "201" "${patient_resp}" +patient_id="$(jq -r '.data.id' "${patient_resp}")" + +enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase11"}' +enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")" +assert_status "201" "${enc_resp}" +encounter_id="$(jq -r '.data.id' "${enc_resp}")" +echo "OK: patient=${patient_id} encounter=${encounter_id}" + +echo "[3/10] Warning-range heart rate — synchronous ingest must not create critical alert" +paging_publish_before="$(queue_stat "alerts.paging.queue" '.message_stats.publish')" + +hr_payload="$(jq -nc \ + --arg recordedAt "${RECORDED_AT}" \ + --arg key "phase11-warning-hr-${encounter_id}" \ + '{observations:[{observationCode:"HEART_RATE",value:105,unit:"bpm",source:"DEVICE",recordedAt:$recordedAt,idempotencyKey:$key}]}')" +hr_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr_payload}")" +assert_status "201" "${hr_resp}" +[[ "$(jq -r '.data.alertGenerated' "${hr_resp}")" == "false" ]] || { + echo "Expected alertGenerated=false for warning-range heart rate 105" + cat "${hr_resp}" + exit 1 +} +echo "OK: ingest returned alertGenerated=false" + +echo "[4/10] Wait for warning-evaluator and verify WARNING_HEART_RATE in PostgreSQL" +wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}" + +warning_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND severity = 'WARNING' AND alert_type = 'WARNING_HEART_RATE' AND status = 'OPEN'")" +[[ "${warning_count}" == "1" ]] || { + echo "Expected 1 WARNING_HEART_RATE alert in PostgreSQL, found ${warning_count}" + psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true + exit 1 +} +echo "OK: WARNING_HEART_RATE alert exists in PostgreSQL" + +echo "[5/10] Verify warning alert indexed in Elasticsearch" +wait_for_consumer_lag_zero "${ES_CONSUMER_GROUP}" "${INDEX_WAIT_SECS}" +es_hits="$(es_alert_hits "WARNING_HEART_RATE" "${encounter_id}")" +[[ "${es_hits}" -ge 1 ]] || { + echo "Expected Elasticsearch hit for WARNING_HEART_RATE on encounter ${encounter_id}, found ${es_hits}" + exit 1 +} +echo "OK: clinical_alerts index contains WARNING_HEART_RATE" + +echo "[6/10] Verify warning alert was not published to alerts.paging.queue" +paging_publish_after="$(queue_stat "alerts.paging.queue" '.message_stats.publish')" +[[ "${paging_publish_after}" == "${paging_publish_before}" ]] || { + echo "Expected no new paging publishes for WARNING alert (before=${paging_publish_before}, after=${paging_publish_after})" + exit 1 +} +echo "OK: alerts.paging.queue publish count unchanged" + +echo "[7/10] Orders API — create, list, record result, illegal cancel" +order_payload='{"orderType":"Lab","description":"CBC with differential","orderedBy":"Dr. Phase11"}' +order_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/orders" "${order_payload}")" +assert_status "201" "${order_resp}" +order_id="$(jq -r '.data.id' "${order_resp}")" +[[ "$(jq -r '.data.status' "${order_resp}")" == "Pending" ]] || { + echo "Expected order status Pending" + cat "${order_resp}" + exit 1 +} + +list_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/orders")" +assert_status "200" "${list_resp}" +[[ "$(jq -r '.data.totalCount' "${list_resp}")" -ge 1 ]] || { + echo "Expected totalCount >= 1 on order list" + cat "${list_resp}" + exit 1 +} + +result_payload='{"resultSummary":"WBC 8.5, Hgb 14.2, Plt 210 — all within normal limits"}' +result_resp="$(request PATCH "${BASE_URL}/api/v1/orders/${order_id}/result" "${result_payload}")" +assert_status "200" "${result_resp}" +[[ "$(jq -r '.data.status' "${result_resp}")" == "Resulted" ]] || { + echo "Expected order status Resulted after recording result" + cat "${result_resp}" + exit 1 +} +[[ "$(jq -r '.data.resultSummary' "${result_resp}")" == *"8.5"* ]] || { + echo "Expected resultSummary to contain lab values" + cat "${result_resp}" + exit 1 +} + +cancel_payload='{"status":"Cancelled"}' +cancel_resp="$(request PATCH "${BASE_URL}/api/v1/orders/${order_id}/status" "${cancel_payload}")" +assert_status "409" "${cancel_resp}" +[[ "$(jq -r '.error.code' "${cancel_resp}")" == "ILLEGAL_ORDER_STATUS_TRANSITION" ]] || { + echo "Expected ILLEGAL_ORDER_STATUS_TRANSITION on cancel of resulted order" + cat "${cancel_resp}" + exit 1 +} +echo "OK: order lifecycle and illegal transition guard verified" + +echo "[8/10] Input validation — FluentValidation 400 responses" +invalid_patient="$(jq -nc '{firstName:"",lastName:"Test",dateOfBirth:"1990-01-01",gender:"M"}')" +val_patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${invalid_patient}")" +assert_status "400" "${val_patient_resp}" +[[ "$(jq -r '.error.code' "${val_patient_resp}")" == "VALIDATION_ERROR" ]] || { + echo "Expected VALIDATION_ERROR for empty firstName" + cat "${val_patient_resp}" + exit 1 +} + +invalid_threshold='{"observationCode":"TEST_CODE","displayName":"Test","unit":"units","criticalLow":50,"warningLow":30}' +val_threshold_resp="$(request POST "${BASE_URL}/api/v1/alert-thresholds" "${invalid_threshold}")" +assert_status "400" "${val_threshold_resp}" +threshold_details="$(jq -c '.error.details // []' "${val_threshold_resp}")" +echo "${threshold_details}" | grep -qi "CriticalLow must be less than WarningLow" || { + echo "Expected threshold ordering validation message" + cat "${val_threshold_resp}" + exit 1 +} + +invalid_order='{"orderType":"Lab","description":"","orderedBy":"Dr. Test"}' +val_order_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/orders" "${invalid_order}")" +assert_status "400" "${val_order_resp}" +[[ "$(jq -r '.error.code' "${val_order_resp}")" == "VALIDATION_ERROR" ]] || { + echo "Expected VALIDATION_ERROR for empty order description" + cat "${val_order_resp}" + exit 1 +} +echo "OK: validation rejects empty firstName, invalid threshold order, empty order description" + +echo "[9/10] Run Phase 11 integration tests" +dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}" + +if [[ "${FULL_TEST}" == "1" ]]; then + echo "[10/10] Run full test suite (FULL_TEST=1)" + dotnet test "${ROOT_DIR}/VigilCareClinicalAPI.sln" 2>/dev/null || dotnet test "${ROOT_DIR}" +else + echo "[10/10] Skipping full suite (set FULL_TEST=1 to run all 64 tests)" +fi + +echo +echo "Phase 11 verification checks passed."