#!/usr/bin/env bash # Phase 24 — ward-outage-reconnect-01 end-to-end verification. # # Runs Phase A (central baseline), Phase B (gateway replay while central down), # and Phase C (central restart + sync verification). # # Optional env: # SKIP_DOCKER=1 — assume docker stack already up # SKIP_PHASE_A=1 — skip central replay; use newest ICU encounter on gateway # CENTRAL_URL — default http://localhost:5270 # GATEWAY_URL — default http://localhost:5081 # SCENARIO_SPEED — simulator speed (default 0 = instant) set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" cd "$ROOT" CENTRAL="${CENTRAL_URL:-http://localhost:5270}" GW="${GATEWAY_URL:-http://localhost:5081}" GW_ID="${GATEWAY_ID:-22222222-2222-2222-2222-222222222222}" SCENARIO="${ROOT}/VigilCare.Simulator/Scenarios/List/ward-outage-reconnect-01.json" SPEED="${SCENARIO_SPEED:-0}" LOG_DIR="${TMPDIR:-/tmp}/vigilcare-phase24-$$" CENTRAL_PID="" WE_STARTED_CENTRAL=0 STOPPED_CENTRAL=0 mkdir -p "$LOG_DIR" log() { echo "==> $*"; } die() { echo "ERROR: $*" >&2; exit 1; } wait_for_url() { local url="$1" max="${2:-60}" i for i in $(seq 1 "$max"); do if curl -sf "$url" >/dev/null 2>&1; then return 0 fi sleep 2 done return 1 } central_login() { local user="$1" pass="$2" curl -sf -X POST "${CENTRAL}/api/v1/auth/login" \ -H "Content-Type: application/json" \ -d "{\"username\":\"${user}\",\"password\":\"${pass}\"}" \ | jq -r '.data.accessToken' } ensure_central_running() { if curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1; then log "Central API already running at $CENTRAL" return fi log "Starting central API (log: $LOG_DIR/central.log)" dotnet run --project VigilCareClinicalAPI --launch-profile http \ >"$LOG_DIR/central.log" 2>&1 & CENTRAL_PID=$! WE_STARTED_CENTRAL=1 wait_for_url "${CENTRAL}/health/ready" 90 \ || die "Central API did not become ready — see $LOG_DIR/central.log" log "Central API ready" } stop_central() { if [[ "$STOPPED_CENTRAL" -eq 1 ]]; then return fi log "Stopping central API to simulate outage" if [[ -n "$CENTRAL_PID" ]] && kill -0 "$CENTRAL_PID" 2>/dev/null; then kill "$CENTRAL_PID" 2>/dev/null || true wait "$CENTRAL_PID" 2>/dev/null || true CENTRAL_PID="" else pkill -f "VigilCareClinicalAPI" 2>/dev/null || true fi for _ in $(seq 1 20); do curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1 || break sleep 1 done STOPPED_CENTRAL=1 } restart_central() { if curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1; then log "Central API already running" STOPPED_CENTRAL=0 return fi log "Restarting central API" dotnet run --project VigilCareClinicalAPI --launch-profile http \ >>"$LOG_DIR/central.log" 2>&1 & CENTRAL_PID=$! WE_STARTED_CENTRAL=1 wait_for_url "${CENTRAL}/health/ready" 90 \ || die "Central API failed to restart — see $LOG_DIR/central.log" STOPPED_CENTRAL=0 log "Central API back online" } configure_gateway_sync() { local sync_jwt="$1" log "Configuring gateway encounter sync (GATEWAY_SYNC_JWT)" GATEWAY_SYNC_JWT="$sync_jwt" docker compose --profile ward-gateway up -d --force-recreate ward-gateway-api wait_for_url "${GW}/health/ready" 60 \ || die "Gateway not ready after restart" } wait_for_encounter_on_gateway() { local encounter_id="$1" jwt="$2" max_attempts="${3:-60}" local i for i in $(seq 1 "$max_attempts"); do if curl -sf "${GW}/api/v1/encounters/${encounter_id}" \ -H "Authorization: Bearer ${jwt}" >/dev/null 2>&1; then log "Encounter on gateway after ${i} attempt(s)" return 0 fi sleep 3 done return 1 } wait_for_gateway_encounter() { local jwt="$1" encounter_id="" i for i in $(seq 1 45); do encounter_id=$(curl -sf "${GW}/api/v1/encounters?status=ACTIVE&department=ICU" \ -H "Authorization: Bearer ${jwt}" \ | jq -r '.data.items[0].encounterId // empty') if [[ -n "$encounter_id" ]]; then echo "$encounter_id" return 0 fi sleep 4 done return 1 } parse_encounter_from_replay() { local log_file="$1" grep -oE 'Encounter opened: [0-9a-f-]{36}' "$log_file" \ | head -1 \ | sed 's/Encounter opened: //' } cleanup() { if [[ "${KEEP_CENTRAL_DOWN:-0}" == "1" ]]; then return fi if [[ "$STOPPED_CENTRAL" -eq 1 ]]; then restart_central || true fi } trap cleanup EXIT command -v jq >/dev/null 2>&1 || die "jq is required" command -v python3 >/dev/null 2>&1 || die "python3 is required (for mint-gateway-jwt.sh)" log "Phase 24 verification — ward-outage-reconnect-01" if [[ "${SKIP_DOCKER:-0}" != "1" ]]; then log "Starting docker infrastructure" docker compose up -d log "Building ward gateway API image" docker compose --profile ward-gateway build ward-gateway-api log "Starting ward gateway profile" docker compose --profile ward-gateway up -d else log "SKIP_DOCKER=1 — assuming stack is already up" log "Rebuilding ward gateway API image (picks up local code changes)" docker compose --profile ward-gateway build ward-gateway-api fi wait_for_url "${GW}/health/ready" 60 \ || die "Gateway not ready at $GW — run: docker compose --profile ward-gateway up -d" ensure_central_running PHYSICIAN_JWT=$(central_login "physician.demo" "DemoPhysician1!") ADMIN_JWT=$(central_login "admin.demo" "DemoAdmin1!") GATEWAY_JWT=$("${ROOT}/scripts/mint-gateway-jwt.sh" nurse.demo NURSE) export GATEWAY_JWT configure_gateway_sync "$PHYSICIAN_JWT" ENCOUNTER_ID="" if [[ "${SKIP_PHASE_A:-0}" != "1" ]]; then log "Phase A — baseline replay on central" PHASE_A_LOG="$LOG_DIR/phase-a.log" dotnet run --project VigilCare.Simulator -- replay "$SCENARIO" \ --speed "$SPEED" --base-url "$CENTRAL" \ 2>&1 | tee "$PHASE_A_LOG" ENCOUNTER_ID=$(parse_encounter_from_replay "$PHASE_A_LOG") [[ -n "$ENCOUNTER_ID" ]] || die "Phase A did not report an encounter id" log "Phase A encounter: $ENCOUNTER_ID" configure_gateway_sync "$PHYSICIAN_JWT" wait_for_encounter_on_gateway "$ENCOUNTER_ID" "$GATEWAY_JWT" 60 \ || die "Encounter $ENCOUNTER_ID not replicated to gateway after Phase A" fi if [[ -z "$ENCOUNTER_ID" ]]; then log "Waiting for ICU encounter on gateway replica" ENCOUNTER_ID=$(wait_for_gateway_encounter "$GATEWAY_JWT") \ || die "No ACTIVE ICU encounter on gateway — run Phase A or set GATEWAY_SYNC_JWT" fi log "Using encounter $ENCOUNTER_ID" wait_for_encounter_on_gateway "$ENCOUNTER_ID" "$GATEWAY_JWT" 10 \ || die "Encounter $ENCOUNTER_ID not available on gateway" stop_central log "Waiting for gateway to detect central unreachable (~35s)" sleep 35 log "Phase B — replay against gateway while central is down" dotnet run --project VigilCare.Simulator -- replay "$SCENARIO" \ --gateway --encounter-id "$ENCOUNTER_ID" --gateway-token "$GATEWAY_JWT" \ --speed "$SPEED" log "Verify critical potassium alert acknowledged locally by RN-Wu" GW_ALERTS=$(curl -sf "${GW}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \ -H "Authorization: Bearer ${GATEWAY_JWT}") echo "$GW_ALERTS" | jq -e \ '.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | (.status == "ACKNOWLEDGED" or .status == "Acknowledged")' \ >/dev/null \ || die "Expected ACKNOWLEDGED critical potassium alert on gateway" echo "$GW_ALERTS" | jq -e \ '.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | .acknowledgedBy == "RN-Wu"' \ >/dev/null \ || die "Expected acknowledgedBy RN-Wu on gateway" BUFFER_DEPTH=$(docker exec "$(docker ps -qf name=ward-gateway-db)" \ psql -U postgres -d vigilcare_ward -tAc \ "SELECT COUNT(*) FROM buffered_sync_items WHERE NOT synced;") log "Gateway buffer depth (unsynced items): $BUFFER_DEPTH" [[ "$BUFFER_DEPTH" -gt 0 ]] || die "Expected buffered sync items on gateway after Phase B" log "Phase C — restart central and wait for sync" restart_central log "Waiting up to 180s for gateway-synced alert on central" SYNCED_ALERT=0 for _ in $(seq 1 36); do CENTRAL_ALERTS=$(curl -sf "${CENTRAL}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \ -H "Authorization: Bearer ${PHYSICIAN_JWT}" || echo "") if echo "$CENTRAL_ALERTS" | jq -e \ '.data.items[]? | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true) | .acknowledgedBy == "RN-Wu"' \ >/dev/null 2>&1; then SYNCED_ALERT=1 break fi sleep 5 done [[ "$SYNCED_ALERT" -eq 1 ]] || die "Timed out waiting for gateway-synced RN-Wu ack on central" BUFFER=$(curl -sf "${CENTRAL}/api/v1/operations/gateways" \ -H "Authorization: Bearer ${ADMIN_JWT}" \ | jq -r ".data[] | select(.id == \"${GW_ID}\") | .reportedBufferDepth // empty" \ | head -1) log "Gateway reported buffer depth: ${BUFFER:-unknown}" log "Verify central received alert acknowledgement" CENTRAL_ALERTS=$(curl -sf "${CENTRAL}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \ -H "Authorization: Bearer ${PHYSICIAN_JWT}") echo "$CENTRAL_ALERTS" | jq -e \ '.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true)' \ >/dev/null \ || die "Critical potassium alert not synced from gateway to central" ACK_BY=$(echo "$CENTRAL_ALERTS" | jq -r \ '.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true) | .acknowledgedBy // empty' \ | head -1) if [[ "$ACK_BY" == "RN-Wu" ]]; then log "Central acknowledgedBy = RN-Wu" else log "Checking central DB for acknowledged_by (API returned: ${ACK_BY:-})" docker exec "$(docker ps -qf name=postgres)" psql -U postgres -d vigilcare -tAc \ "SELECT acknowledged_by FROM clinical_alerts WHERE encounter_id = '${ENCOUNTER_ID}' AND alert_type = 'CRITICAL_POTASSIUM_MEQ_L' LIMIT 1;" \ | grep -q "RN-Wu" \ || die "Central acknowledged_by is not RN-Wu after sync" log "Central DB acknowledged_by = RN-Wu" fi log "Phase 24 verification passed." log "Logs: $LOG_DIR"