vigilcare-clinical/scripts/run-phase24-verification.sh

#!/usr/bin/env bash
# Phase 24 — ward-outage-reconnect-01 end-to-end verification.
#
# Runs Phase A (central baseline), Phase B (gateway replay while central down),
# and Phase C (central restart + sync verification).
#
# Optional env:
#   SKIP_DOCKER=1       — assume docker stack already up
#   SKIP_PHASE_A=1      — skip central replay; use newest ICU encounter on gateway
#   CENTRAL_URL         — default http://localhost:5270
#   GATEWAY_URL         — default http://localhost:5081
#   SCENARIO_SPEED      — simulator speed (default 0 = instant)

set -euo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT"

CENTRAL="${CENTRAL_URL:-http://localhost:5270}"
GW="${GATEWAY_URL:-http://localhost:5081}"
GW_ID="${GATEWAY_ID:-22222222-2222-2222-2222-222222222222}"
SCENARIO="${ROOT}/VigilCare.Simulator/Scenarios/List/ward-outage-reconnect-01.json"
SPEED="${SCENARIO_SPEED:-0}"
LOG_DIR="${TMPDIR:-/tmp}/vigilcare-phase24-$$"
CENTRAL_PID=""
WE_STARTED_CENTRAL=0
STOPPED_CENTRAL=0

mkdir -p "$LOG_DIR"

log() { echo "==> $*"; }
die() { echo "ERROR: $*" >&2; exit 1; }

wait_for_url() {
  local url="$1" max="${2:-60}" i
  for i in $(seq 1 "$max"); do
    if curl -sf "$url" >/dev/null 2>&1; then
      return 0
    fi
    sleep 2
  done
  return 1
}

central_login() {
  local user="$1" pass="$2"
  curl -sf -X POST "${CENTRAL}/api/v1/auth/login" \
    -H "Content-Type: application/json" \
    -d "{\"username\":\"${user}\",\"password\":\"${pass}\"}" \
    | jq -r '.data.accessToken'
}

ensure_central_running() {
  if curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1; then
    log "Central API already running at $CENTRAL"
    return
  fi

  log "Starting central API (log: $LOG_DIR/central.log)"
  dotnet run --project VigilCareClinicalAPI --launch-profile http \
    >"$LOG_DIR/central.log" 2>&1 &
  CENTRAL_PID=$!
  WE_STARTED_CENTRAL=1

  wait_for_url "${CENTRAL}/health/ready" 90 \
    || die "Central API did not become ready — see $LOG_DIR/central.log"
  log "Central API ready"
}

stop_central() {
  if [[ "$STOPPED_CENTRAL" -eq 1 ]]; then
    return
  fi

  log "Stopping central API to simulate outage"
  if [[ -n "$CENTRAL_PID" ]] && kill -0 "$CENTRAL_PID" 2>/dev/null; then
    kill "$CENTRAL_PID" 2>/dev/null || true
    wait "$CENTRAL_PID" 2>/dev/null || true
    CENTRAL_PID=""
  else
    pkill -f "VigilCareClinicalAPI" 2>/dev/null || true
  fi

  for _ in $(seq 1 20); do
    curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1 || break
    sleep 1
  done

  STOPPED_CENTRAL=1
}

restart_central() {
  if curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1; then
    log "Central API already running"
    STOPPED_CENTRAL=0
    return
  fi

  log "Restarting central API"
  dotnet run --project VigilCareClinicalAPI --launch-profile http \
    >>"$LOG_DIR/central.log" 2>&1 &
  CENTRAL_PID=$!
  WE_STARTED_CENTRAL=1
  wait_for_url "${CENTRAL}/health/ready" 90 \
    || die "Central API failed to restart — see $LOG_DIR/central.log"
  STOPPED_CENTRAL=0
  log "Central API back online"
}

configure_gateway_sync() {
  local sync_jwt="$1"
  log "Configuring gateway encounter sync (GATEWAY_SYNC_JWT)"
  GATEWAY_SYNC_JWT="$sync_jwt" docker compose --profile ward-gateway up -d --force-recreate ward-gateway-api
  wait_for_url "${GW}/health/ready" 60 \
    || die "Gateway not ready after restart"
}

wait_for_encounter_on_gateway() {
  local encounter_id="$1" jwt="$2" max_attempts="${3:-60}"
  local i
  for i in $(seq 1 "$max_attempts"); do
    if curl -sf "${GW}/api/v1/encounters/${encounter_id}" \
      -H "Authorization: Bearer ${jwt}" >/dev/null 2>&1; then
      log "Encounter on gateway after ${i} attempt(s)"
      return 0
    fi
    sleep 3
  done
  return 1
}

wait_for_gateway_encounter() {
  local jwt="$1" encounter_id="" i
  for i in $(seq 1 45); do
    encounter_id=$(curl -sf "${GW}/api/v1/encounters?status=ACTIVE&department=ICU" \
      -H "Authorization: Bearer ${jwt}" \
      | jq -r '.data.items[0].encounterId // empty')
    if [[ -n "$encounter_id" ]]; then
      echo "$encounter_id"
      return 0
    fi
    sleep 4
  done
  return 1
}

parse_encounter_from_replay() {
  local log_file="$1"
  grep -oE 'Encounter opened: [0-9a-f-]{36}' "$log_file" \
    | head -1 \
    | sed 's/Encounter opened: //'
}

cleanup() {
  if [[ "${KEEP_CENTRAL_DOWN:-0}" == "1" ]]; then
    return
  fi
  if [[ "$STOPPED_CENTRAL" -eq 1 ]]; then
    restart_central || true
  fi
}
trap cleanup EXIT

command -v jq >/dev/null 2>&1 || die "jq is required"
command -v python3 >/dev/null 2>&1 || die "python3 is required (for mint-gateway-jwt.sh)"

log "Phase 24 verification — ward-outage-reconnect-01"

if [[ "${SKIP_DOCKER:-0}" != "1" ]]; then
  log "Starting docker infrastructure"
  docker compose up -d
  log "Building ward gateway API image"
  docker compose --profile ward-gateway build ward-gateway-api
  log "Starting ward gateway profile"
  docker compose --profile ward-gateway up -d
else
  log "SKIP_DOCKER=1 — assuming stack is already up"
  log "Rebuilding ward gateway API image (picks up local code changes)"
  docker compose --profile ward-gateway build ward-gateway-api
fi

wait_for_url "${GW}/health/ready" 60 \
  || die "Gateway not ready at $GW — run: docker compose --profile ward-gateway up -d"

ensure_central_running

PHYSICIAN_JWT=$(central_login "physician.demo" "DemoPhysician1!")
ADMIN_JWT=$(central_login "admin.demo" "DemoAdmin1!")
GATEWAY_JWT=$("${ROOT}/scripts/mint-gateway-jwt.sh" nurse.demo NURSE)
export GATEWAY_JWT

configure_gateway_sync "$PHYSICIAN_JWT"

ENCOUNTER_ID=""

if [[ "${SKIP_PHASE_A:-0}" != "1" ]]; then
  log "Phase A — baseline replay on central"
  PHASE_A_LOG="$LOG_DIR/phase-a.log"
  dotnet run --project VigilCare.Simulator -- replay "$SCENARIO" \
    --speed "$SPEED" --base-url "$CENTRAL" \
    2>&1 | tee "$PHASE_A_LOG"

  ENCOUNTER_ID=$(parse_encounter_from_replay "$PHASE_A_LOG")
  [[ -n "$ENCOUNTER_ID" ]] || die "Phase A did not report an encounter id"
  log "Phase A encounter: $ENCOUNTER_ID"

  configure_gateway_sync "$PHYSICIAN_JWT"
  wait_for_encounter_on_gateway "$ENCOUNTER_ID" "$GATEWAY_JWT" 60 \
    || die "Encounter $ENCOUNTER_ID not replicated to gateway after Phase A"
fi

if [[ -z "$ENCOUNTER_ID" ]]; then
  log "Waiting for ICU encounter on gateway replica"
  ENCOUNTER_ID=$(wait_for_gateway_encounter "$GATEWAY_JWT") \
    || die "No ACTIVE ICU encounter on gateway — run Phase A or set GATEWAY_SYNC_JWT"
fi

log "Using encounter $ENCOUNTER_ID"

wait_for_encounter_on_gateway "$ENCOUNTER_ID" "$GATEWAY_JWT" 10 \
  || die "Encounter $ENCOUNTER_ID not available on gateway"

stop_central
log "Waiting for gateway to detect central unreachable (~35s)"
sleep 35

log "Phase B — replay against gateway while central is down"
dotnet run --project VigilCare.Simulator -- replay "$SCENARIO" \
  --gateway --encounter-id "$ENCOUNTER_ID" --gateway-token "$GATEWAY_JWT" \
  --speed "$SPEED"

log "Verify critical potassium alert acknowledged locally by RN-Wu"
GW_ALERTS=$(curl -sf "${GW}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \
  -H "Authorization: Bearer ${GATEWAY_JWT}")
echo "$GW_ALERTS" | jq -e \
  '.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | (.status == "ACKNOWLEDGED" or .status == "Acknowledged")' \
  >/dev/null \
  || die "Expected ACKNOWLEDGED critical potassium alert on gateway"

echo "$GW_ALERTS" | jq -e \
  '.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | .acknowledgedBy == "RN-Wu"' \
  >/dev/null \
  || die "Expected acknowledgedBy RN-Wu on gateway"

BUFFER_DEPTH=$(docker exec "$(docker ps -qf name=ward-gateway-db)" \
  psql -U postgres -d vigilcare_ward -tAc \
  "SELECT COUNT(*) FROM buffered_sync_items WHERE NOT synced;")
log "Gateway buffer depth (unsynced items): $BUFFER_DEPTH"
[[ "$BUFFER_DEPTH" -gt 0 ]] || die "Expected buffered sync items on gateway after Phase B"

log "Phase C — restart central and wait for sync"
restart_central
log "Waiting up to 180s for gateway-synced alert on central"
SYNCED_ALERT=0
for _ in $(seq 1 36); do
  CENTRAL_ALERTS=$(curl -sf "${CENTRAL}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \
    -H "Authorization: Bearer ${PHYSICIAN_JWT}" || echo "")
  if echo "$CENTRAL_ALERTS" | jq -e \
    '.data.items[]? | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true) | .acknowledgedBy == "RN-Wu"' \
    >/dev/null 2>&1; then
    SYNCED_ALERT=1
    break
  fi
  sleep 5
done
[[ "$SYNCED_ALERT" -eq 1 ]] || die "Timed out waiting for gateway-synced RN-Wu ack on central"

BUFFER=$(curl -sf "${CENTRAL}/api/v1/operations/gateways" \
  -H "Authorization: Bearer ${ADMIN_JWT}" \
  | jq -r ".data[] | select(.id == \"${GW_ID}\") | .reportedBufferDepth // empty" \
  | head -1)
log "Gateway reported buffer depth: ${BUFFER:-unknown}"

log "Verify central received alert acknowledgement"
CENTRAL_ALERTS=$(curl -sf "${CENTRAL}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \
  -H "Authorization: Bearer ${PHYSICIAN_JWT}")
echo "$CENTRAL_ALERTS" | jq -e \
  '.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true)' \
  >/dev/null \
  || die "Critical potassium alert not synced from gateway to central"

ACK_BY=$(echo "$CENTRAL_ALERTS" | jq -r \
  '.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true) | .acknowledgedBy // empty' \
  | head -1)
if [[ "$ACK_BY" == "RN-Wu" ]]; then
  log "Central acknowledgedBy = RN-Wu"
else
  log "Checking central DB for acknowledged_by (API returned: ${ACK_BY:-<empty>})"
  docker exec "$(docker ps -qf name=postgres)" psql -U postgres -d vigilcare -tAc \
    "SELECT acknowledged_by FROM clinical_alerts WHERE encounter_id = '${ENCOUNTER_ID}' AND alert_type = 'CRITICAL_POTASSIUM_MEQ_L' LIMIT 1;" \
    | grep -q "RN-Wu" \
    || die "Central acknowledged_by is not RN-Wu after sync"
  log "Central DB acknowledged_by = RN-Wu"
fi

log "Phase 24 verification passed."
log "Logs: $LOG_DIR"