Run initial test for Climate Resilience Verification Suite

Add first part of Alert Quality Analytics
This commit is contained in:
voltsrage
2026-06-24 03:01:00 +08:00
parent 032cd1d240
commit 185dc93fa1
50 changed files with 4116 additions and 55 deletions
+11 -1
View File
@@ -87,7 +87,7 @@ An `OutboxEvent` is written in the same transaction as any observation or alert,
- **MRN Sequence Generation** — MRN numbers are generated via a PostgreSQL sequence (`mrn_seq`) instead of MAX+1 queries; eliminates race conditions under concurrent patient registration; configurable prefix and digit count via `PatientOptions` - **MRN Sequence Generation** — MRN numbers are generated via a PostgreSQL sequence (`mrn_seq`) instead of MAX+1 queries; eliminates race conditions under concurrent patient registration; configurable prefix and digit count via `PatientOptions`
- **FHIR Bundle Transaction Rollback** — `FhirBundleProcessor` wraps all bundle entry processing in a database transaction; on any entry failure, the transaction is rolled back and the response includes the `OperationOutcome` for the failed entry; prevents partial state from orphaned Patient/Encounter records - **FHIR Bundle Transaction Rollback** — `FhirBundleProcessor` wraps all bundle entry processing in a database transaction; on any entry failure, the transaction is rolled back and the response includes the `OperationOutcome` for the failed entry; prevents partial state from orphaned Patient/Encounter records
- **Clinician Feedback Mode** — six quick ratings per alert (useful, too early, too late, false positive, missing context, would act); optional notes; Feedback Summary with aggregate stats and JSON/CSV export; client-side persistence for product research - **Clinician Feedback Mode** — six quick ratings per alert (useful, too early, too late, false positive, missing context, would act); optional notes; Feedback Summary with aggregate stats and JSON/CSV export; client-side persistence for product research
- **Console Replay Simulator** — standalone `VigilCare.Simulator` .NET console app replays JSON scenario files against the live API with configurable speed (`--speed 0` instant, `60` = 60× faster); commands: `replay`, `replay-all`, `validate`, `dry-run`; optional `--poll` shows alerts, NEWS2, GCS, SOFA, and sepsis bundle state during replay; eleven sample scenarios in `VigilCare.Simulator/Scenarios/List/` (including GCS neurological decline, SOFA sepsis progression, and SpO₂/FiO₂ fallback); user guide in `docs/simulator-guide.md` - **Console Replay Simulator** — standalone `VigilCare.Simulator` .NET console app replays JSON scenario files against the live API with configurable speed (`--speed 0` instant, `60` = 60× faster); commands: `replay`, `replay-all`, `validate`, `dry-run`; optional `--poll` shows alerts, NEWS2, GCS, SOFA, and sepsis bundle state during replay; `--gateway` targets the ward gateway (`http://localhost:5081`) with `--encounter-id`, `--skip-setup`, and `--gateway-token`; `alert_ack` events poll for open alerts on central before acknowledging (handles async alert pipeline at `--speed 0`); twelve sample scenarios in `VigilCare.Simulator/Scenarios/List/` (including ward outage reconnect, GCS neurological decline, SOFA sepsis progression, and SpO₂/FiO₂ fallback); user guide in `docs/simulator-guide.md`
- **RabbitMQ Notification Workers** — `NotificationPublisherService` reads `alert.generated` from Kafka and publishes paging jobs to `alerts.paging.queue`; `PagingWorkerService` sends the page and waits for acknowledgment; if no ack arrives before timeout it NACKs to `alerts.paging.dlq` with `x-message-ttl = 300000ms`; if the host is stopping, in-flight paging messages are NACKed with `requeue=true` so they are retried after restart and do not false-escalate; `EscalationWorkerService` pages the on-call backup and sets alert status to `escalated`; `DischargeSummaryWorkerService` reads `encounter.status.changed`, generates a discharge summary, and stores it in MinIO under `/discharge-summaries/{encounterId}/summary.pdf` - **RabbitMQ Notification Workers** — `NotificationPublisherService` reads `alert.generated` from Kafka and publishes paging jobs to `alerts.paging.queue`; `PagingWorkerService` sends the page and waits for acknowledgment; if no ack arrives before timeout it NACKs to `alerts.paging.dlq` with `x-message-ttl = 300000ms`; if the host is stopping, in-flight paging messages are NACKed with `requeue=true` so they are retried after restart and do not false-escalate; `EscalationWorkerService` pages the on-call backup and sets alert status to `escalated`; `DischargeSummaryWorkerService` reads `encounter.status.changed`, generates a discharge summary, and stores it in MinIO under `/discharge-summaries/{encounterId}/summary.pdf`
- **Data Lake Writer** — `DataLakeWriterService` (consumer group `data-lake-writer`) buffers `observation.recorded`, `alert.generated`, and `encounter.status.changed` events, flushes date-partitioned Parquet files to MinIO (`/observations/`, `/alerts/`, `/encounters/`), and commits Kafka offsets only for topic-partitions where all uploads succeeded; failed partition buffers are retained in memory and retried on the next flush cycle (prevents data loss from partial upload failures); shutdown flush uses an uncanceled token so MinIO writes complete on Ctrl+C; `kafka_partition` and `kafka_offset` columns provide audit lineage - **Data Lake Writer** — `DataLakeWriterService` (consumer group `data-lake-writer`) buffers `observation.recorded`, `alert.generated`, and `encounter.status.changed` events, flushes date-partitioned Parquet files to MinIO (`/observations/`, `/alerts/`, `/encounters/`), and commits Kafka offsets only for topic-partitions where all uploads succeeded; failed partition buffers are retained in memory and retried on the next flush cycle (prevents data loss from partial upload failures); shutdown flush uses an uncanceled token so MinIO writes complete on Ctrl+C; `kafka_partition` and `kafka_offset` columns provide audit lineage
- **Reconciliation Jobs** — three scheduled checks: (1) unacknowledged CRITICAL alerts older than 30 minutes, (2) pending orders without results after 4 hours, (3) active inpatients with no observation in 2 hours; each finding creates a `reconciliation_alerts` row and publishes to RabbitMQ - **Reconciliation Jobs** — three scheduled checks: (1) unacknowledged CRITICAL alerts older than 30 minutes, (2) pending orders without results after 4 hours, (3) active inpatients with no observation in 2 hours; each finding creates a `reconciliation_alerts` row and publishes to RabbitMQ
@@ -593,6 +593,7 @@ scripts/
├── run-phase20-verification.sh # Phase 20 — Gateway registry tests + site/gateway/heartbeat curl checks ├── run-phase20-verification.sh # Phase 20 — Gateway registry tests + site/gateway/heartbeat curl checks
├── run-phase21-verification.sh # Phase 21 — Ward gateway local-first path, partition tests, sync upload ├── run-phase21-verification.sh # Phase 21 — Ward gateway local-first path, partition tests, sync upload
├── run-phase22-verification.sh # Phase 22 — Dashboard gap analysis fixes (SOFA/GCS/qSOFA history, patient banner, timeline, medication markers) ├── run-phase22-verification.sh # Phase 22 — Dashboard gap analysis fixes (SOFA/GCS/qSOFA history, patient banner, timeline, medication markers)
├── run-phase24-verification.sh # Phase 24 — Ward outage reconnect scenario (central + gateway replay, ack sync)
├── run-phase29-verification.sh # Phase 29 — Simulator scenario expansion + clinical validation ├── run-phase29-verification.sh # Phase 29 — Simulator scenario expansion + clinical validation
├── run-phase30-verification.sh # Phase 30 — FHIR R4 ingest integration tests + manual bundle/metadata checks ├── run-phase30-verification.sh # Phase 30 — FHIR R4 ingest integration tests + manual bundle/metadata checks
└── run-phase31-verification.sh # Phase 31 — RBAC integration tests + JWT login + audit log query └── run-phase31-verification.sh # Phase 31 — RBAC integration tests + JWT login + audit log query
@@ -791,6 +792,14 @@ dotnet run --project VigilCare.Simulator -- replay \
Other commands: `validate <file>`, `dry-run <file>`, `replay-all <directory>`. See `docs/simulator-guide.md` for the full user guide. Other commands: `validate <file>`, `dry-run <file>`, `replay-all <directory>`. See `docs/simulator-guide.md` for the full user guide.
**Ward outage reconnect (Phase 24):** scenario `ward-outage-reconnect-01.json` exercises critical hyperkalemia alerting and nurse acknowledgment during a central outage, then sync back to central when connectivity returns. Manual procedure in `docs/simulator-guide.md` §10; automated end-to-end check:
```bash
./scripts/run-phase24-verification.sh
```
Requires Docker Compose (central API, ward gateway stack) and a running central API on `http://localhost:5270`. Use `SKIP_DOCKER=1` when the stack is already up. Notes from verification runs: `docs/resilience/phase-24-verification-notes.md`.
### Run the Dashboard ### Run the Dashboard
With the API running, start the Vue frontend: With the API running, start the Vue frontend:
@@ -860,6 +869,7 @@ With the API running (`dotnet run`) and Docker Compose up:
```bash ```bash
./scripts/run-phase22-verification.sh # Dashboard gap analysis fixes — SOFA/GCS/qSOFA history, patient banner, timeline ./scripts/run-phase22-verification.sh # Dashboard gap analysis fixes — SOFA/GCS/qSOFA history, patient banner, timeline
./scripts/run-phase21-verification.sh # Ward gateway local-first path, partition tests, sync upload verification ./scripts/run-phase21-verification.sh # Ward gateway local-first path, partition tests, sync upload verification
./scripts/run-phase24-verification.sh # Ward outage reconnect — gateway replay, local ack, central sync after restart
./scripts/run-phase20-verification.sh # Gateway registry tests, site/gateway/heartbeat API verification ./scripts/run-phase20-verification.sh # Gateway registry tests, site/gateway/heartbeat API verification
./scripts/run-phase8-verification.sh # Prometheus target UP, ten metrics, alerts_unacknowledged_gauge live update ./scripts/run-phase8-verification.sh # Prometheus target UP, ten metrics, alerts_unacknowledged_gauge live update
./scripts/run-phase9-verification.sh # DataLakePhase9Tests, Kafka consumer group, MinIO Parquet, DuckDB schema ./scripts/run-phase9-verification.sh # DataLakePhase9Tests, Kafka consumer group, MinIO Parquet, DuckDB schema
@@ -27,6 +27,11 @@ public class VigilCareApiClient
_http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
} }
public void SetBearerToken(string token)
{
_http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
}
public async Task<PatientResponse> RegisterPatientAsync(RegisterPatientRequest req) public async Task<PatientResponse> RegisterPatientAsync(RegisterPatientRequest req)
{ {
var response = await _http.PostAsJsonAsync("/api/v1/patients", req); var response = await _http.PostAsJsonAsync("/api/v1/patients", req);
@@ -44,8 +49,34 @@ public class VigilCareApiClient
} }
public async Task SendObservationBatchAsync( public async Task SendObservationBatchAsync(
Guid encounterId, List<IngestObservationRequest> observations) Guid encounterId, List<IngestObservationRequest> observations,
ReplayTarget target = ReplayTarget.Central)
{ {
if (target == ReplayTarget.Gateway)
{
foreach (var obs in observations)
{
var response = await _http.PostAsJsonAsync(
$"/api/v1/encounters/{encounterId}/observations",
new
{
observationCode = obs.ObservationCode,
value = obs.Value,
unit = obs.Unit,
source = obs.Source,
recordedAt = obs.RecordedAt,
idempotencyKey = obs.IdempotencyKey ?? Guid.NewGuid().ToString()
});
if (!response.IsSuccessStatusCode)
{
var body = await response.Content.ReadAsStringAsync();
throw new HttpRequestException(
$"Observation ingest failed ({(int)response.StatusCode} {response.StatusCode}): {body}");
}
}
return;
}
foreach (var chunk in observations.Chunk(10)) foreach (var chunk in observations.Chunk(10))
{ {
var batch = new BatchIngestRequest(chunk.ToList()); var batch = new BatchIngestRequest(chunk.ToList());
@@ -161,4 +192,50 @@ public class VigilCareApiClient
var envelope = await response.Content.ReadFromJsonAsync<ApiResponse<QsofaResponse>>(); var envelope = await response.Content.ReadFromJsonAsync<ApiResponse<QsofaResponse>>();
return envelope?.Data; return envelope?.Data;
} }
public async Task<bool> TryAcknowledgeAlertAsync(
Guid encounterId, string alertType, string clinicianId, string? note,
TimeSpan? waitForAlert = null, CancellationToken ct = default)
{
const int pollIntervalMs = 500;
var deadline = waitForAlert.HasValue
? DateTimeOffset.UtcNow.Add(waitForAlert.Value)
: (DateTimeOffset?)null;
while (true)
{
ct.ThrowIfCancellationRequested();
var alerts = await GetAlertsAsync(encounterId);
var alert = alerts.FirstOrDefault(a =>
IsMatchingAlertType(a.AlertType, alertType)
&& IsOpenAlertStatus(a.Status));
if (alert is not null)
{
var response = await _http.PostAsJsonAsync(
$"/api/v1/alerts/{alert.Id}/acknowledge",
new { note, clinicianId }, ct);
return response.IsSuccessStatusCode;
}
if (!deadline.HasValue || DateTimeOffset.UtcNow >= deadline.Value)
return false;
await Task.Delay(pollIntervalMs, ct);
}
}
private static bool IsOpenAlertStatus(string status) =>
string.Equals(status, "OPEN", StringComparison.OrdinalIgnoreCase);
private static bool IsMatchingAlertType(string actual, string expected)
{
if (string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase))
return true;
static string Normalize(string s) =>
s.Replace("_", "", StringComparison.Ordinal).ToUpperInvariant();
return Normalize(actual) == Normalize(expected);
}
} }
+61 -9
View File
@@ -11,15 +11,35 @@ public static class ReplayCommand
var pollIntervalOpt = new Option<int>("--poll-interval", () => 5, "Seconds between polls"); var pollIntervalOpt = new Option<int>("--poll-interval", () => 5, "Seconds between polls");
var usernameOpt = new Option<string>("--username", () => "physician.demo", "API login username"); var usernameOpt = new Option<string>("--username", () => "physician.demo", "API login username");
var passwordOpt = new Option<string>("--password", () => "DemoPhysician1!", "API login password"); var passwordOpt = new Option<string>("--password", () => "DemoPhysician1!", "API login password");
var gatewayOpt = new Option<bool>("--gateway", () => false,
"Target ward gateway API (default base URL http://localhost:5081)");
var encounterIdOpt = new Option<Guid?>("--encounter-id",
"Use existing encounter (required for --gateway when replica already synced)");
var skipSetupOpt = new Option<bool>("--skip-setup", () => false,
"Skip patient/encounter registration — use --encounter-id");
var gatewayTokenOpt = new Option<string?>("--gateway-token",
"Bearer token for ward gateway (default: GATEWAY_JWT env var when --gateway)");
var command = new Command("replay", "Replay a scenario against the API") var command = new Command("replay", "Replay a scenario against the API")
{ {
fileArg, speedOpt, baseUrlOpt, pollOpt, pollIntervalOpt, usernameOpt, passwordOpt fileArg, speedOpt, baseUrlOpt, pollOpt, pollIntervalOpt, usernameOpt, passwordOpt,
gatewayOpt, encounterIdOpt, skipSetupOpt, gatewayTokenOpt
}; };
command.SetHandler(async (FileInfo file, double speed, string baseUrl, bool poll, int pollInterval, command.SetHandler(async context =>
string username, string password) =>
{ {
var file = context.ParseResult.GetValueForArgument(fileArg);
var speed = context.ParseResult.GetValueForOption(speedOpt);
var baseUrl = context.ParseResult.GetValueForOption(baseUrlOpt)!;
var poll = context.ParseResult.GetValueForOption(pollOpt);
var pollInterval = context.ParseResult.GetValueForOption(pollIntervalOpt);
var username = context.ParseResult.GetValueForOption(usernameOpt)!;
var password = context.ParseResult.GetValueForOption(passwordOpt)!;
var gateway = context.ParseResult.GetValueForOption(gatewayOpt);
var encounterId = context.ParseResult.GetValueForOption(encounterIdOpt);
var skipSetup = context.ParseResult.GetValueForOption(skipSetupOpt);
var gatewayToken = context.ParseResult.GetValueForOption(gatewayTokenOpt);
var scenario = ScenarioLoader.Load(file.FullName); var scenario = ScenarioLoader.Load(file.FullName);
var errors = ScenarioValidator.Validate(scenario); var errors = ScenarioValidator.Validate(scenario);
if (errors.Count > 0) if (errors.Count > 0)
@@ -29,19 +49,51 @@ public static class ReplayCommand
return; return;
} }
using var http = new HttpClient { BaseAddress = new Uri(baseUrl) }; if (skipSetup && !encounterId.HasValue)
{
SimulatorConsole.Error("--skip-setup requires --encounter-id");
return;
}
if (gateway && !encounterId.HasValue)
{
SimulatorConsole.Error("--gateway requires --encounter-id");
return;
}
var effectiveBaseUrl = gateway ? "http://localhost:5081" : baseUrl;
using var http = new HttpClient { BaseAddress = new Uri(effectiveBaseUrl) };
var client = new VigilCareApiClient(http); var client = new VigilCareApiClient(http);
SimulatorConsole.Info($"Authenticating as {username}..."); if (gateway)
await client.LoginAsync(username, password); {
SimulatorConsole.Info("Authenticated."); var token = gatewayToken ?? Environment.GetEnvironmentVariable("GATEWAY_JWT");
if (string.IsNullOrWhiteSpace(token))
{
SimulatorConsole.Error(
"Gateway mode requires --gateway-token or GATEWAY_JWT (ward gateway has no /auth/login endpoint)");
return;
}
client.SetBearerToken(token);
SimulatorConsole.Info("Using gateway bearer token.");
}
else
{
SimulatorConsole.Info($"Authenticating as {username}...");
await client.LoginAsync(username, password);
SimulatorConsole.Info("Authenticated.");
}
var poller = poll ? new ApiPoller(client) : null; var poller = poll ? new ApiPoller(client) : null;
var engine = new ReplayEngine(client, poller); var engine = new ReplayEngine(client, poller);
var options = new ReplayOptions(speed, poll, pollInterval); var options = new ReplayOptions(
speed, poll, pollInterval, DryRun: false,
Target: gateway ? ReplayTarget.Gateway : ReplayTarget.Central,
ExistingEncounterId: encounterId);
await engine.RunAsync(scenario, options); await engine.RunAsync(scenario, options);
}, fileArg, speedOpt, baseUrlOpt, pollOpt, pollIntervalOpt, usernameOpt, passwordOpt); });
return command; return command;
} }
+60 -11
View File
@@ -20,25 +20,43 @@ public class ReplayEngine
// --- Phase 1: Setup --- // --- Phase 1: Setup ---
SimulatorConsole.Header(scenario.Scenario.Name, scenario.Scenario.Description); SimulatorConsole.Header(scenario.Scenario.Name, scenario.Scenario.Description);
PatientResponse patient;
EncounterResponse encounter;
if (options.DryRun) if (options.DryRun)
{ {
SimulatorConsole.DryRun("Would register patient: " + if (options.ExistingEncounterId.HasValue)
$"{scenario.Patient.FirstName} {scenario.Patient.LastName}"); {
SimulatorConsole.DryRun("Would open encounter: " + result.EncounterId = options.ExistingEncounterId.Value;
$"{scenario.Encounter.Department} / {scenario.Encounter.EncounterType}"); if (options.Target == ReplayTarget.Gateway)
SimulatorConsole.DryRun($"Gateway mode — would use existing encounter {result.EncounterId}");
else
SimulatorConsole.DryRun($"Would use existing encounter {result.EncounterId}");
}
else
{
SimulatorConsole.DryRun("Would register patient: " +
$"{scenario.Patient.FirstName} {scenario.Patient.LastName}");
SimulatorConsole.DryRun("Would open encounter: " +
$"{scenario.Encounter.Department} / {scenario.Encounter.EncounterType}");
}
}
else if (options.Target == ReplayTarget.Gateway && options.ExistingEncounterId.HasValue)
{
result.EncounterId = options.ExistingEncounterId.Value;
SimulatorConsole.Info($"Gateway mode — using existing encounter {result.EncounterId}");
}
else if (options.ExistingEncounterId.HasValue)
{
result.EncounterId = options.ExistingEncounterId.Value;
SimulatorConsole.Info($"Using existing encounter {result.EncounterId}");
} }
else else
{ {
patient = await _client.RegisterPatientAsync(new RegisterPatientRequest( var patient = await _client.RegisterPatientAsync(new RegisterPatientRequest(
scenario.Patient.FirstName, scenario.Patient.FirstName,
scenario.Patient.LastName, scenario.Patient.LastName,
DateOnly.Parse(scenario.Patient.DateOfBirth), DateOnly.Parse(scenario.Patient.DateOfBirth),
scenario.Patient.Gender)); scenario.Patient.Gender));
encounter = await _client.OpenEncounterAsync(patient.Id, new OpenEncounterRequest( var encounter = await _client.OpenEncounterAsync(patient.Id, new OpenEncounterRequest(
scenario.Encounter.EncounterType, scenario.Encounter.EncounterType,
DepartmentMapper.ToApiDepartment(scenario.Encounter.Department), DepartmentMapper.ToApiDepartment(scenario.Encounter.Department),
scenario.Encounter.AttendingPhysician, scenario.Encounter.AttendingPhysician,
@@ -91,6 +109,9 @@ public class ReplayEngine
case "order_result": case "order_result":
await ReplayOrderResult(evt, simTimestamp, options, result); await ReplayOrderResult(evt, simTimestamp, options, result);
break; break;
case "alert_ack":
await ReplayAlertAck(evt, simTimestamp, options, result, ct);
break;
} }
} }
@@ -115,7 +136,9 @@ public class ReplayEngine
{ {
var observations = new List<IngestObservationRequest>(); var observations = new List<IngestObservationRequest>();
var offsetMinutes = cluster[0].OffsetMinutes; var offsetMinutes = cluster[0].OffsetMinutes;
var recordedAt = _scenarioStartTime.AddMinutes(offsetMinutes); var recordedAt = options.Target == ReplayTarget.Gateway
? DateTimeOffset.UtcNow
: _scenarioStartTime.AddMinutes(offsetMinutes);
foreach (var evt in cluster) foreach (var evt in cluster)
{ {
@@ -135,7 +158,7 @@ public class ReplayEngine
} }
if (!options.DryRun && observations.Count > 0) if (!options.DryRun && observations.Count > 0)
await _client.SendObservationBatchAsync(result.EncounterId, observations); await _client.SendObservationBatchAsync(result.EncounterId, observations, options.Target);
} }
private async Task ReplayMedication( private async Task ReplayMedication(
@@ -238,4 +261,30 @@ public class ReplayEngine
"Lab" or "LAB" or "lab" => "LAB", "Lab" or "LAB" or "lab" => "LAB",
_ => throw new InvalidOperationException($"Unknown observation source: '{source}'") _ => throw new InvalidOperationException($"Unknown observation source: '{source}'")
}; };
private async Task ReplayAlertAck(
ScenarioEvent evt, string simTime, ReplayOptions options, ReplayResult result,
CancellationToken ct)
{
var alertType = evt.Data.GetProperty("alertType").GetString()!;
var clinicianId = evt.Data.GetProperty("clinicianId").GetString()!;
var note = evt.Data.TryGetProperty("note", out var n) ? n.GetString() : null;
if (options.DryRun)
{
SimulatorConsole.DryRun($"[{simTime}] ACK {alertType} by {clinicianId}");
return;
}
var waitForAlert = options.Target == ReplayTarget.Central
? TimeSpan.FromSeconds(30)
: (TimeSpan?)null;
var ok = await _client.TryAcknowledgeAlertAsync(
result.EncounterId, alertType, clinicianId, note, waitForAlert, ct);
if (ok)
SimulatorConsole.Event(simTime, $"ACK {alertType} by {clinicianId}");
else
SimulatorConsole.Warn($"[{simTime}] ACK failed — no open {alertType} alert found");
}
} }
+5 -1
View File
@@ -1,5 +1,9 @@
public enum ReplayTarget { Central, Gateway }
public record ReplayOptions( public record ReplayOptions(
double Speed = 60, double Speed = 60,
bool Poll = false, bool Poll = false,
int PollIntervalSeconds = 5, int PollIntervalSeconds = 5,
bool DryRun = false); bool DryRun = false,
ReplayTarget Target = ReplayTarget.Central,
Guid? ExistingEncounterId = null);
@@ -0,0 +1,92 @@
{
"scenario": {
"id": "ward-outage-reconnect-01",
"name": "ICU Ward Isolation — Critical Potassium During Uplink Loss",
"description": "Post-surgical ICU patient with stable vitals, then critical hyperkalemia during simulated central link loss. Nurse acknowledges on ward gateway. Reconnect syncs to central without duplicate pages.",
"durationMinutes": 90,
"tags": ["climate-resilience", "gateway", "critical-value", "sync", "potassium"]
},
"patient": {
"firstName": "James",
"lastName": "Wu",
"dateOfBirth": "1968-11-02",
"gender": "Male"
},
"encounter": {
"department": "Icu",
"encounterType": "Inpatient",
"attendingPhysician": "Dr. Elena Park",
"roomBed": "ICU-3B-12",
"admissionReason": "Post-op monitoring — abdominal surgery"
},
"events": [
{
"offsetMinutes": 0,
"type": "observation",
"data": { "code": "HEART_RATE", "value": 82, "unit": "bpm", "source": "Device" },
"note": "Baseline vitals — stable post-op ICU admission"
},
{
"offsetMinutes": 0,
"type": "observation",
"data": { "code": "TEMP_C", "value": 36.8, "unit": "°C", "source": "Manual" }
},
{
"offsetMinutes": 0,
"type": "observation",
"data": { "code": "SPO2", "value": 98, "unit": "%", "source": "Device" }
},
{
"offsetMinutes": 0,
"type": "observation",
"data": { "code": "POTASSIUM_MEQ_L", "value": 4.2, "unit": "mEq/L", "source": "Lab" }
},
{
"offsetMinutes": 45,
"type": "observation",
"data": { "code": "POTASSIUM_MEQ_L", "value": 6.8, "unit": "mEq/L", "source": "Lab" },
"note": "Critical hyperkalemia — must alert locally even if central is down"
},
{
"offsetMinutes": 50,
"type": "alert_ack",
"data": {
"alertType": "CRITICAL_POTASSIUM_MEQ_L",
"clinicianId": "RN-Wu",
"note": "Calcium gluconate ordered, ECG at bedside"
}
},
{
"offsetMinutes": 60,
"type": "observation",
"data": { "code": "HEART_RATE", "value": 78, "unit": "bpm", "source": "Device" }
},
{
"offsetMinutes": 75,
"type": "observation",
"data": { "code": "POTASSIUM_MEQ_L", "value": 5.4, "unit": "mEq/L", "source": "Lab" },
"note": "Repeat lab — improving after treatment"
}
],
"expectedOutcomes": [
{
"afterOffsetMinutes": 45,
"type": "alert",
"alertType": "CRITICAL_POTASSIUM_MEQ_L",
"description": "Critical potassium alert fires at K+ 6.8 mEq/L — locally on gateway during outage"
},
{
"afterOffsetMinutes": 50,
"type": "alert",
"alertType": "CRITICAL_POTASSIUM_MEQ_L",
"description": "Alert acknowledged by RN-Wu — buffered for sync to central"
},
{
"afterOffsetMinutes": 75,
"type": "score",
"scoreType": "NEWS2",
"expectedMinimum": 0,
"description": "NEWS2 unavailable on gateway during outage — replays on central after sync (Tier 3 deferred)"
}
]
}
@@ -29,7 +29,7 @@ public static class ScenarioValidator
private static readonly HashSet<string> ValidEventTypes = new() private static readonly HashSet<string> ValidEventTypes = new()
{ {
"observation", "order", "medication", "order_result" "observation", "order", "medication", "order_result", "alert_ack"
}; };
private static readonly HashSet<string> ValidOrderTypes = new() private static readonly HashSet<string> ValidOrderTypes = new()
@@ -131,6 +131,19 @@ public static class ScenarioValidator
"(sepsis bundle orders are auto-created when SOFA_SEPSIS or QSOFA_SCREEN fires)"); "(sepsis bundle orders are auto-created when SOFA_SEPSIS or QSOFA_SCREEN fires)");
} }
} }
if (evt.Type == "alert_ack")
{
var alertType = evt.Data.TryGetProperty("alertType", out var alertTypeProp)
? alertTypeProp.GetString() : null;
if (string.IsNullOrWhiteSpace(alertType))
errors.Add($"events[{i}]: alert_ack.alertType is required");
var clinicianId = evt.Data.TryGetProperty("clinicianId", out var clinicianProp)
? clinicianProp.GetString() : null;
if (string.IsNullOrWhiteSpace(clinicianId))
errors.Add($"events[{i}]: alert_ack.clinicianId is required");
}
} }
return errors; return errors;
+1 -1
View File
@@ -43,7 +43,7 @@
"required": ["offsetMinutes", "type", "data"], "required": ["offsetMinutes", "type", "data"],
"properties": { "properties": {
"offsetMinutes": { "type": "number", "minimum": 0 }, "offsetMinutes": { "type": "number", "minimum": 0 },
"type": { "type": "string", "enum": ["observation", "order", "medication", "order_result"] }, "type": { "type": "string", "enum": ["observation", "order", "order_result", "medication", "alert_ack"] },
"data": { "type": "object" }, "data": { "type": "object" },
"note": { "type": "string" } "note": { "type": "string" }
} }
@@ -101,6 +101,41 @@ public class WardGatewayLocalPathTests : IAsyncLifetime
b.ItemType == BufferedSyncItemType.Ack && !b.Synced)).Should().Be(1); b.ItemType == BufferedSyncItemType.Ack && !b.Synced)).Should().Be(1);
} }
[Fact]
public async Task Acknowledge_WithClinicianId_UsesScenarioAttribution()
{
await _client.PostAsJsonAsync(
$"/api/v1/encounters/{_encounterId}/observations",
new
{
observationCode = "HEART_RATE",
value = 160,
unit = "bpm",
source = "DEVICE",
recordedAt = DateTimeOffset.UtcNow,
idempotencyKey = Guid.NewGuid().ToString()
});
var alertsResp = await _client.GetAsync($"/api/v1/encounters/{_encounterId}/alerts");
var alertsDoc = await alertsResp.Content.ReadFromJsonAsync<JsonDocument>();
var alertId = alertsDoc!.RootElement
.GetProperty("data").GetProperty("items")[0].GetProperty("id").GetGuid();
var ackResp = await _client.PostAsJsonAsync(
$"/api/v1/alerts/{alertId}/acknowledge",
new AcknowledgeAlertRequest("Calcium gluconate ordered.", "RN-Wu"));
ackResp.StatusCode.Should().Be(HttpStatusCode.OK);
using var scope = _fixture.Services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<GatewayDbContext>();
var alert = await db.ClinicalAlerts.SingleAsync(a => a.Id == alertId);
alert.AcknowledgedBy.Should().Be("RN-Wu");
var bufferedAck = await db.BufferedSyncItems.SingleAsync(b =>
b.ItemType == BufferedSyncItemType.Ack && !b.Synced);
bufferedAck.Payload.Should().Contain("RN-Wu");
}
[Fact] [Fact]
public async Task CentralDown_ReachabilityReportsDegraded() public async Task CentralDown_ReachabilityReportsDegraded()
{ {
@@ -53,7 +53,7 @@ public class AlertsController : ControllerBase
[HttpPost("api/v1/alerts/{id:guid}/acknowledge")] [HttpPost("api/v1/alerts/{id:guid}/acknowledge")]
public async Task<IActionResult> Acknowledge(Guid id, [FromBody] AcknowledgeAlertRequest req) public async Task<IActionResult> Acknowledge(Guid id, [FromBody] AcknowledgeAlertRequest req)
{ {
var clinicianId = User.Identity?.Name ?? "unknown"; var clinicianId = req.ClinicianId ?? User.Identity?.Name ?? "unknown";
var alert = await _alerts.AcknowledgeAsync(id, req, clinicianId); var alert = await _alerts.AcknowledgeAsync(id, req, clinicianId);
return Ok(ApiResponse<LocalClinicalAlert>.Ok(alert)); return Ok(ApiResponse<LocalClinicalAlert>.Ok(alert));
} }
@@ -1 +1,5 @@
public record AcknowledgeAlertRequest(string? Note); using System.Text.Json.Serialization;
public record AcknowledgeAlertRequest(
[property: JsonPropertyName("note")] string? Note,
[property: JsonPropertyName("clinicianId")] string? ClinicianId = null);
@@ -5,5 +5,6 @@ public class AcknowledgeAlertRequestValidator : AbstractValidator<AcknowledgeAle
public AcknowledgeAlertRequestValidator() public AcknowledgeAlertRequestValidator()
{ {
RuleFor(x => x.Note).MaximumLength(1000).When(x => x.Note is not null); RuleFor(x => x.Note).MaximumLength(1000).When(x => x.Note is not null);
RuleFor(x => x.ClinicianId).MaximumLength(200).When(x => x.ClinicianId is not null);
} }
} }
@@ -0,0 +1,174 @@
using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using FluentAssertions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
[Collection("Integration")]
public class AlertQualityAnalyticsTests : IAsyncLifetime
{
private readonly ApiFixture _fixture;
private readonly HttpClient _client;
private Guid _alertId;
public AlertQualityAnalyticsTests(ApiFixture fixture)
{
_fixture = fixture;
_client = fixture.CreateClient();
_client.AsNurse();
}
public async Task InitializeAsync()
{
using var scope = _fixture.Services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
await DbResetHelper.ResetAsync(db);
var patient = new Patient
{
Id = Guid.NewGuid(), Mrn = "MRN-QA-001", FirstName = "Quality", LastName = "Analytics",
DateOfBirth = new DateOnly(1985, 3, 10), Gender = "F", CreatedAt = DateTimeOffset.UtcNow
};
var encounter = new Encounter
{
Id = Guid.NewGuid(), PatientId = patient.Id, EncounterType = EncounterType.Inpatient,
Status = EncounterStatus.Active, Department = Department.Icu,
AttendingPhysician = "Dr. QA", AdmittedAt = DateTimeOffset.UtcNow,
CreatedAt = DateTimeOffset.UtcNow
};
var alert = new ClinicalAlert
{
Id = Guid.NewGuid(), EncounterId = encounter.Id, PatientId = patient.Id,
AlertType = AlertType.News2Warning, Severity = AlertSeverity.Warning,
Details = "NEWS2 score 6 (MEDIUM).", Status = AlertStatus.Open,
TriggeredAt = DateTimeOffset.UtcNow.AddMinutes(-30)
};
db.Patients.Add(patient);
db.Encounters.Add(encounter);
db.ClinicalAlerts.Add(alert);
await db.SaveChangesAsync();
_alertId = alert.Id;
await _client.PostAsJsonAsync(
$"/api/v1/alerts/{_alertId}/acknowledge",
new AcknowledgeAlertRequest("Reviewed for QA test."));
}
public Task DisposeAsync() => Task.CompletedTask;
[Theory]
[InlineData("Useful")]
[InlineData("TooEarly")]
[InlineData("TooLate")]
[InlineData("FalsePositive")]
[InlineData("MissingContext")]
[InlineData("WouldAct")]
public async Task SubmitFeedback_AllTypes_Returns201(string feedbackType)
{
await ResetAcknowledgedAlertAsync();
var resp = await _client.PostAsJsonAsync(
$"/api/v1/alerts/{_alertId}/feedback",
new { feedbackType, comment = $"QA {feedbackType}" });
resp.StatusCode.Should().Be(HttpStatusCode.Created);
var body = await resp.Content.ReadFromJsonAsync<JsonDocument>();
body!.RootElement.GetProperty("data").GetProperty("id").GetGuid()
.Should().NotBe(Guid.Empty);
}
[Fact]
public async Task SubmitFeedback_DuplicateUser_Returns409()
{
await ResetAcknowledgedAlertAsync();
await _client.PostAsJsonAsync(
$"/api/v1/alerts/{_alertId}/feedback",
new { feedbackType = "Useful" });
var resp = await _client.PostAsJsonAsync(
$"/api/v1/alerts/{_alertId}/feedback",
new { feedbackType = "FalsePositive" });
resp.StatusCode.Should().Be(HttpStatusCode.Conflict);
var body = await resp.Content.ReadFromJsonAsync<JsonDocument>();
body!.RootElement.GetProperty("error").GetProperty("code").GetString()
.Should().Be("FEEDBACK_ALREADY_SUBMITTED");
}
[Fact]
public async Task SubmitFeedback_OpenAlert_Returns400()
{
using var scope = _fixture.Services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var openAlert = new ClinicalAlert
{
Id = Guid.NewGuid(),
EncounterId = (await db.Encounters.FirstAsync()).Id,
PatientId = (await db.Patients.FirstAsync()).Id,
AlertType = AlertType.GcsWarning,
Severity = AlertSeverity.Warning,
Details = "Open alert.",
Status = AlertStatus.Open,
TriggeredAt = DateTimeOffset.UtcNow
};
db.ClinicalAlerts.Add(openAlert);
await db.SaveChangesAsync();
var resp = await _client.PostAsJsonAsync(
$"/api/v1/alerts/{openAlert.Id}/feedback",
new { feedbackType = "Useful" });
resp.StatusCode.Should().Be(HttpStatusCode.BadRequest);
}
[Fact]
public async Task SubmitFeedback_WritesAuditLog()
{
await ResetAcknowledgedAlertAsync();
await _client.PostAsJsonAsync(
$"/api/v1/alerts/{_alertId}/feedback",
new { feedbackType = "Useful", comment = "audit check" });
using var scope = _fixture.Services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var logged = await db.ClinicalAuditLogs
.AnyAsync(l => l.EntityId == _alertId
&& l.Action == AuditAction.AlertFeedbackSubmitted);
logged.Should().BeTrue();
}
[Fact]
public async Task QualityMetricsSummary_ReturnsAggregate()
{
var resp = await _client.GetAsync("/api/v1/alerts/quality-metrics/summary");
resp.StatusCode.Should().Be(HttpStatusCode.OK);
var body = await resp.Content.ReadFromJsonAsync<JsonDocument>();
body!.RootElement.GetProperty("data").GetProperty("totalAlerts").GetInt32()
.Should().BeGreaterThanOrEqualTo(0);
}
[Fact]
public async Task QualityMetrics_FilterByType()
{
var resp = await _client.GetAsync(
"/api/v1/alerts/quality-metrics?alertType=NEWS2_WARNING");
resp.StatusCode.Should().Be(HttpStatusCode.OK);
var body = await resp.Content.ReadFromJsonAsync<JsonDocument>();
body!.RootElement.GetProperty("data").GetProperty("items").GetArrayLength()
.Should().BeGreaterThanOrEqualTo(0);
}
private async Task ResetAcknowledgedAlertAsync()
{
using var scope = _fixture.Services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var existing = await db.AlertFeedbacks
.Where(f => f.AlertId == _alertId)
.ToListAsync();
db.AlertFeedbacks.RemoveRange(existing);
await db.SaveChangesAsync();
}
}
@@ -22,6 +22,8 @@ public static class DbResetHelper
DELETE FROM reconciliation_alerts; DELETE FROM reconciliation_alerts;
DELETE FROM outbox_events; DELETE FROM outbox_events;
DELETE FROM orders; DELETE FROM orders;
DELETE FROM alert_feedbacks;
DELETE FROM alert_quality_metrics;
DELETE FROM clinical_alerts; DELETE FROM clinical_alerts;
DELETE FROM gcs_scores; DELETE FROM gcs_scores;
DELETE FROM sofa_scores; DELETE FROM sofa_scores;
@@ -17,4 +17,5 @@ public static class ClinicalPermissions
public const string FhirRead = "fhir:read"; public const string FhirRead = "fhir:read";
public const string AuditRead = "audit:read"; public const string AuditRead = "audit:read";
public const string UsersAdmin = "users:admin"; public const string UsersAdmin = "users:admin";
public const string AlertsFeedback = "alerts:feedback";
} }
@@ -16,6 +16,7 @@ public static class ClinicalRolePermissionMap
ClinicalPermissions.AnalyticsRead, ClinicalPermissions.AnalyticsRead,
ClinicalPermissions.OrdersWrite, ClinicalPermissions.OrdersWrite,
ClinicalPermissions.MedicationsWrite, ClinicalPermissions.MedicationsWrite,
ClinicalPermissions.AlertsFeedback,
}, },
[ClinicalRole.Physician] = new(StringComparer.Ordinal) [ClinicalRole.Physician] = new(StringComparer.Ordinal)
{ {
@@ -31,6 +32,7 @@ public static class ClinicalRolePermissionMap
ClinicalPermissions.AnalyticsRead, ClinicalPermissions.AnalyticsRead,
ClinicalPermissions.OrdersWrite, ClinicalPermissions.OrdersWrite,
ClinicalPermissions.MedicationsWrite, ClinicalPermissions.MedicationsWrite,
ClinicalPermissions.AlertsFeedback,
}, },
[ClinicalRole.Admin] = new(StringComparer.Ordinal) [ClinicalRole.Admin] = new(StringComparer.Ordinal)
{ {
@@ -51,6 +53,7 @@ public static class ClinicalRolePermissionMap
ClinicalPermissions.FhirRead, ClinicalPermissions.FhirRead,
ClinicalPermissions.AuditRead, ClinicalPermissions.AuditRead,
ClinicalPermissions.UsersAdmin, ClinicalPermissions.UsersAdmin,
ClinicalPermissions.AlertsFeedback,
}, },
[ClinicalRole.Integration] = new(StringComparer.Ordinal) [ClinicalRole.Integration] = new(StringComparer.Ordinal)
{ {
@@ -0,0 +1,176 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
public sealed class AlertQualityAggregatorService : BackgroundService
{
private readonly IServiceScopeFactory _scopes;
private readonly IOptions<AlertQualityOptions> _options;
private readonly ClinicalMetrics _metrics;
private readonly ILogger<AlertQualityAggregatorService> _logger;
public AlertQualityAggregatorService(
IServiceScopeFactory scopes,
IOptions<AlertQualityOptions> options,
ClinicalMetrics metrics,
ILogger<AlertQualityAggregatorService> logger)
{
_scopes = scopes;
_options = options;
_metrics = metrics;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var interval = TimeSpan.FromMinutes(Math.Max(1, _options.Value.IntervalMinutes));
using var timer = new PeriodicTimer(interval);
// Run once at startup, then on interval
await AggregateAsync(stoppingToken);
while (await timer.WaitForNextTickAsync(stoppingToken))
await AggregateAsync(stoppingToken);
}
private async Task AggregateAsync(CancellationToken ct)
{
try
{
await using var scope = _scopes.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var windowHours = Math.Max(1, _options.Value.WindowHours);
var windowEnd = AlignToHour(DateTimeOffset.UtcNow);
var windowStart = windowEnd.AddHours(-windowHours);
foreach (AlertType alertType in Enum.GetValues<AlertType>())
{
var alerts = await db.ClinicalAlerts
.AsNoTracking()
.Where(a => a.AlertType == alertType
&& a.TriggeredAt >= windowStart
&& a.TriggeredAt < windowEnd)
.Select(a => new
{
a.Status,
a.TriggeredAt,
a.AcknowledgedAt,
a.ResolvedAt
})
.ToListAsync(ct);
if (alerts.Count == 0)
continue;
var alertIds = await db.ClinicalAlerts
.AsNoTracking()
.Where(a => a.AlertType == alertType
&& a.TriggeredAt >= windowStart
&& a.TriggeredAt < windowEnd)
.Select(a => a.Id)
.ToListAsync(ct);
var feedbacks = await db.AlertFeedbacks
.AsNoTracking()
.Where(f => alertIds.Contains(f.AlertId))
.Select(f => f.FeedbackType)
.ToListAsync(ct);
var total = alerts.Count;
var acknowledged = alerts.Count(a =>
a.Status is AlertStatus.Acknowledged or AlertStatus.Resolved);
var resolved = alerts.Count(a => a.Status == AlertStatus.Resolved);
var escalated = alerts.Count(a => a.Status == AlertStatus.Escalated);
var useful = feedbacks.Count(f => f == AlertFeedbackType.Useful);
var falsePositive = feedbacks.Count(f => f == AlertFeedbackType.FalsePositive);
var wouldAct = feedbacks.Count(f => f == AlertFeedbackType.WouldAct);
var feedbackCount = feedbacks.Count;
var ackDurations = alerts
.Where(a => a.AcknowledgedAt.HasValue)
.Select(a => (a.AcknowledgedAt!.Value - a.TriggeredAt).TotalSeconds)
.ToList();
var resolveDurations = alerts
.Where(a => a.ResolvedAt.HasValue && a.AcknowledgedAt.HasValue)
.Select(a => (a.ResolvedAt!.Value - a.AcknowledgedAt!.Value).TotalSeconds)
.ToList();
var metric = new AlertQualityMetric
{
Id = Guid.NewGuid(),
AlertType = alertType,
WindowStart = windowStart,
WindowEnd = windowEnd,
TotalAlerts = total,
AcknowledgedCount = acknowledged,
ResolvedCount = resolved,
EscalatedCount = escalated,
FeedbackUsefulCount = useful,
FeedbackFalsePositiveCount = falsePositive,
FeedbackWouldActCount = wouldAct,
FeedbackCount = feedbackCount,
AcknowledgementRate = SafeRate(acknowledged, total),
FalsePositiveRate = SafeRate(falsePositive, feedbackCount),
UsefulRate = SafeRate(useful, feedbackCount),
WouldActRate = SafeRate(wouldAct, feedbackCount),
AvgSecondsToAcknowledge = ackDurations.Count > 0 ? ackDurations.Average() : 0,
AvgSecondsToResolution = resolveDurations.Count > 0 ? resolveDurations.Average() : 0,
ComputedAt = DateTimeOffset.UtcNow
};
var existing = await db.AlertQualityMetrics
.FirstOrDefaultAsync(m =>
m.AlertType == alertType
&& m.WindowStart == windowStart
&& m.WindowEnd == windowEnd, ct);
if (existing is null)
db.AlertQualityMetrics.Add(metric);
else
{
existing.TotalAlerts = metric.TotalAlerts;
existing.AcknowledgedCount = metric.AcknowledgedCount;
existing.ResolvedCount = metric.ResolvedCount;
existing.EscalatedCount = metric.EscalatedCount;
existing.FeedbackUsefulCount = metric.FeedbackUsefulCount;
existing.FeedbackFalsePositiveCount = metric.FeedbackFalsePositiveCount;
existing.FeedbackWouldActCount = metric.FeedbackWouldActCount;
existing.FeedbackCount = metric.FeedbackCount;
existing.AcknowledgementRate = metric.AcknowledgementRate;
existing.FalsePositiveRate = metric.FalsePositiveRate;
existing.UsefulRate = metric.UsefulRate;
existing.WouldActRate = metric.WouldActRate;
existing.AvgSecondsToAcknowledge = metric.AvgSecondsToAcknowledge;
existing.AvgSecondsToResolution = metric.AvgSecondsToResolution;
existing.ComputedAt = metric.ComputedAt;
metric = existing;
}
await db.SaveChangesAsync(ct);
var typeLabel = alertType.ToDbString();
_metrics.AlertAcknowledgementRate.WithLabels(typeLabel).Set(metric.AcknowledgementRate);
_metrics.AlertFalsePositiveRate.WithLabels(typeLabel).Set(metric.FalsePositiveRate);
_metrics.AlertUsefulRate.WithLabels(typeLabel).Set(metric.UsefulRate);
_metrics.AlertAvgAckSeconds.WithLabels(typeLabel).Set(metric.AvgSecondsToAcknowledge);
}
_logger.LogDebug(
"Alert quality aggregation complete for window {Start} {End}",
windowStart, windowEnd);
}
catch (Exception ex)
{
_logger.LogError(ex, "AlertQualityAggregatorService failed");
}
}
private static DateTimeOffset AlignToHour(DateTimeOffset value) =>
new(value.Year, value.Month, value.Day, value.Hour, 0, 0, value.Offset);
private static double SafeRate(int numerator, int denominator) =>
denominator == 0 ? 0.0 : (double)numerator / denominator;
}
@@ -0,0 +1,10 @@
public class AlertQualityOptions
{
public const string Section = "AlertQuality";
/// <summary>Aggregation interval in minutes. Default: 60.</summary>
public int IntervalMinutes { get; set; } = 60;
/// <summary>Snapshot window size in hours. Default: 1.</summary>
public int WindowHours { get; set; } = 1;
}
@@ -0,0 +1,73 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
[ApiController]
[Produces("application/json")]
[Authorize]
public class AlertQualityMetricsController : ControllerBase
{
private readonly IAlertQualityMetricsService _metrics;
public AlertQualityMetricsController(IAlertQualityMetricsService metrics) =>
_metrics = metrics;
/// <summary>
/// Returns alert quality metric snapshots for a time range, optionally filtered by alert type.
/// </summary>
[HttpGet("api/v1/alerts/quality-metrics")]
[AuthorizePermission(ClinicalPermissions.AnalyticsRead)]
[ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status200OK)]
[ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status400BadRequest)]
public async Task<IActionResult> List(
[FromQuery] string? alertType,
[FromQuery] DateTimeOffset? from,
[FromQuery] DateTimeOffset? to)
{
AlertType? parsedType = null;
if (!string.IsNullOrEmpty(alertType))
{
try
{
parsedType = AlertTypeExtensions.FromDbString(alertType);
}
catch (ArgumentOutOfRangeException)
{
return BadRequest(ApiResponse<object>.Fail(
400, "Invalid alert type filter.", "INVALID_ALERT_TYPE"));
}
}
var periodEnd = to ?? DateTimeOffset.UtcNow;
var periodStart = from ?? periodEnd.AddDays(-7);
if (periodStart >= periodEnd)
{
return BadRequest(ApiResponse<object>.Fail(
400, "'from' must be before 'to'.", "INVALID_DATE_RANGE"));
}
var items = await _metrics.ListAsync(parsedType, periodStart, periodEnd);
return Ok(ApiResponse<object>.Ok(new
{
periodStart,
periodEnd,
items
}));
}
/// <summary>
/// Returns aggregate alert quality rates across all alert types for a time range.
/// </summary>
[HttpGet("api/v1/alerts/quality-metrics/summary")]
[AuthorizePermission(ClinicalPermissions.AnalyticsRead)]
[ProducesResponseType(typeof(ApiResponse<AlertQualitySummaryResponse>), StatusCodes.Status200OK)]
public async Task<IActionResult> Summary(
[FromQuery] DateTimeOffset? from,
[FromQuery] DateTimeOffset? to)
{
var periodEnd = to ?? DateTimeOffset.UtcNow;
var periodStart = from ?? periodEnd.AddDays(-7);
var summary = await _metrics.GetSummaryAsync(periodStart, periodEnd);
return Ok(ApiResponse<AlertQualitySummaryResponse>.Ok(summary));
}
}
@@ -173,4 +173,30 @@ public class AlertsController : ControllerBase
var alert = await _alerts.ResolveAsync(id); var alert = await _alerts.ResolveAsync(id);
return Ok(ApiResponse<ClinicalAlert>.Ok(alert)); return Ok(ApiResponse<ClinicalAlert>.Ok(alert));
} }
/// <summary>
/// Submits clinician feedback for an acknowledged or resolved alert.
/// One submission per user per alert.
/// </summary>
[HttpPost("api/v1/alerts/{id:guid}/feedback")]
[AuthorizePermission(ClinicalPermissions.AlertsFeedback)]
[ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status201Created)]
[ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status400BadRequest)]
[ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status404NotFound)]
[ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status409Conflict)]
public async Task<IActionResult> SubmitFeedback(
Guid id, [FromBody] SubmitAlertFeedbackRequest req)
{
var feedback = await _alerts.SubmitFeedbackAsync(id, req.FeedbackType, req.Comment);
return Created(
$"/api/v1/alerts/{id}/feedback/{feedback.Id}",
ApiResponse<object>.Ok(new
{
id = feedback.Id,
alertId = feedback.AlertId,
feedbackType = feedback.FeedbackType.ToString(),
comment = feedback.Comment,
createdAt = feedback.CreatedAt
}));
}
} }
@@ -34,6 +34,8 @@ public class AppDbContext : DbContext
public DbSet<WardGateway> WardGateways => Set<WardGateway>(); public DbSet<WardGateway> WardGateways => Set<WardGateway>();
public DbSet<ClinicalSyncBatch> ClinicalSyncBatches => Set<ClinicalSyncBatch>(); public DbSet<ClinicalSyncBatch> ClinicalSyncBatches => Set<ClinicalSyncBatch>();
public DbSet<ClinicalSyncConflict> ClinicalSyncConflicts => Set<ClinicalSyncConflict>(); public DbSet<ClinicalSyncConflict> ClinicalSyncConflicts => Set<ClinicalSyncConflict>();
public DbSet<AlertFeedback> AlertFeedbacks => Set<AlertFeedback>();
public DbSet<AlertQualityMetric> AlertQualityMetrics => Set<AlertQualityMetric>();
protected override void OnModelCreating(ModelBuilder modelBuilder) protected override void OnModelCreating(ModelBuilder modelBuilder)
{ {
@@ -0,0 +1,27 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
public class AlertFeedbackConfiguration : IEntityTypeConfiguration<AlertFeedback>
{
public void Configure(EntityTypeBuilder<AlertFeedback> builder)
{
builder.ToTable("alert_feedbacks");
builder.HasKey(f => f.Id);
builder.Property(f => f.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
builder.Property(f => f.AlertId).HasColumnName("alert_id").IsRequired();
builder.Property(f => f.UserId).HasColumnName("user_id").IsRequired();
builder.Property(f => f.FeedbackType).HasColumnName("feedback_type").HasMaxLength(30).IsRequired()
.HasConversion(v => v.ToDbString(), v => AlertFeedbackTypeExtensions.FromDbString(v));
builder.Property(f => f.Comment).HasColumnName("comment").HasMaxLength(1000);
builder.Property(f => f.CreatedAt).HasColumnName("created_at").HasDefaultValueSql("NOW()");
builder.HasOne(f => f.Alert)
.WithMany(a => a.Feedbacks)
.HasForeignKey(f => f.AlertId)
.OnDelete(DeleteBehavior.Cascade);
builder.HasIndex(f => f.AlertId);
builder.HasIndex(f => f.FeedbackType);
builder.HasIndex(f => new { f.AlertId, f.UserId }).IsUnique();
}
}
@@ -0,0 +1,34 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
public class AlertQualityMetricConfiguration : IEntityTypeConfiguration<AlertQualityMetric>
{
public void Configure(EntityTypeBuilder<AlertQualityMetric> builder)
{
builder.ToTable("alert_quality_metrics");
builder.HasKey(m => m.Id);
builder.Property(m => m.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
builder.Property(m => m.AlertType).HasColumnName("alert_type").HasMaxLength(50).IsRequired()
.HasConversion(v => v.ToDbString(), v => AlertTypeExtensions.FromDbString(v));
builder.Property(m => m.WindowStart).HasColumnName("window_start").IsRequired();
builder.Property(m => m.WindowEnd).HasColumnName("window_end").IsRequired();
builder.Property(m => m.TotalAlerts).HasColumnName("total_alerts").IsRequired();
builder.Property(m => m.AcknowledgedCount).HasColumnName("acknowledged_count").IsRequired();
builder.Property(m => m.ResolvedCount).HasColumnName("resolved_count").IsRequired();
builder.Property(m => m.EscalatedCount).HasColumnName("escalated_count").IsRequired();
builder.Property(m => m.FeedbackUsefulCount).HasColumnName("feedback_useful_count").IsRequired();
builder.Property(m => m.FeedbackFalsePositiveCount).HasColumnName("feedback_false_positive_count").IsRequired();
builder.Property(m => m.FeedbackWouldActCount).HasColumnName("feedback_would_act_count").IsRequired();
builder.Property(m => m.FeedbackCount).HasColumnName("feedback_count").IsRequired();
builder.Property(m => m.AcknowledgementRate).HasColumnName("acknowledgement_rate").IsRequired();
builder.Property(m => m.FalsePositiveRate).HasColumnName("false_positive_rate").IsRequired();
builder.Property(m => m.UsefulRate).HasColumnName("useful_rate").IsRequired();
builder.Property(m => m.WouldActRate).HasColumnName("would_act_rate").IsRequired();
builder.Property(m => m.AvgSecondsToAcknowledge).HasColumnName("avg_seconds_to_acknowledge").IsRequired();
builder.Property(m => m.AvgSecondsToResolution).HasColumnName("avg_seconds_to_resolution").IsRequired();
builder.Property(m => m.ComputedAt).HasColumnName("computed_at").HasDefaultValueSql("NOW()");
builder.HasIndex(m => new { m.AlertType, m.WindowStart, m.WindowEnd }).IsUnique();
builder.HasIndex(m => m.WindowStart);
}
}
@@ -66,6 +66,8 @@ public class ClinicalAlertConfiguration : IEntityTypeConfiguration<ClinicalAlert
builder.Property(a => a.ClientAlertId).HasColumnName("client_alert_id"); builder.Property(a => a.ClientAlertId).HasColumnName("client_alert_id");
builder.Property(a => a.SyncedFromGateway).HasColumnName("synced_from_gateway").HasDefaultValue(false); builder.Property(a => a.SyncedFromGateway).HasColumnName("synced_from_gateway").HasDefaultValue(false);
builder.Property(a => a.TriggeredAt).HasColumnName("triggered_at").HasDefaultValueSql("NOW()"); builder.Property(a => a.TriggeredAt).HasColumnName("triggered_at").HasDefaultValueSql("NOW()");
builder.Property(a => a.FeedbackReceived).HasColumnName("feedback_received")
.HasDefaultValue(false);
builder.HasOne(a => a.Encounter) builder.HasOne(a => a.Encounter)
.WithMany(e => e.Alerts) .WithMany(e => e.Alerts)
@@ -0,0 +1,10 @@
public class AlertFeedback
{
public Guid Id { get; set; }
public Guid AlertId { get; set; }
public ClinicalAlert Alert { get; set; } = null!;
public Guid UserId { get; set; }
public AlertFeedbackType FeedbackType { get; set; }
public string? Comment { get; set; }
public DateTimeOffset CreatedAt { get; set; }
}
@@ -0,0 +1,22 @@
public class AlertQualityMetric
{
public Guid Id { get; set; }
public AlertType AlertType { get; set; }
public DateTimeOffset WindowStart { get; set; }
public DateTimeOffset WindowEnd { get; set; }
public int TotalAlerts { get; set; }
public int AcknowledgedCount { get; set; }
public int ResolvedCount { get; set; }
public int EscalatedCount { get; set; }
public int FeedbackUsefulCount { get; set; }
public int FeedbackFalsePositiveCount { get; set; }
public int FeedbackWouldActCount { get; set; }
public int FeedbackCount { get; set; }
public double AcknowledgementRate { get; set; }
public double FalsePositiveRate { get; set; }
public double UsefulRate { get; set; }
public double WouldActRate { get; set; }
public double AvgSecondsToAcknowledge { get; set; }
public double AvgSecondsToResolution { get; set; }
public DateTimeOffset ComputedAt { get; set; }
}
@@ -15,6 +15,8 @@ public class ClinicalAlert
public DateTimeOffset TriggeredAt { get; set; } public DateTimeOffset TriggeredAt { get; set; }
public Guid? ClientAlertId { get; set; } public Guid? ClientAlertId { get; set; }
public bool SyncedFromGateway { get; set; } public bool SyncedFromGateway { get; set; }
public bool FeedbackReceived { get; set; }
public Encounter Encounter { get; set; } = null!; public Encounter Encounter { get; set; } = null!;
public List<AlertFeedback> Feedbacks { get; set; } = new();
} }
@@ -0,0 +1,34 @@
public enum AlertFeedbackType
{
Useful,
TooEarly,
TooLate,
FalsePositive,
MissingContext,
WouldAct
}
public static class AlertFeedbackTypeExtensions
{
public static string ToDbString(this AlertFeedbackType t) => t switch
{
AlertFeedbackType.Useful => "USEFUL",
AlertFeedbackType.TooEarly => "TOO_EARLY",
AlertFeedbackType.TooLate => "TOO_LATE",
AlertFeedbackType.FalsePositive => "FALSE_POSITIVE",
AlertFeedbackType.MissingContext => "MISSING_CONTEXT",
AlertFeedbackType.WouldAct => "WOULD_ACT",
_ => throw new ArgumentOutOfRangeException(nameof(t))
};
public static AlertFeedbackType FromDbString(string v) => v switch
{
"USEFUL" => AlertFeedbackType.Useful,
"TOO_EARLY" => AlertFeedbackType.TooEarly,
"TOO_LATE" => AlertFeedbackType.TooLate,
"FALSE_POSITIVE" => AlertFeedbackType.FalsePositive,
"MISSING_CONTEXT" => AlertFeedbackType.MissingContext,
"WOULD_ACT" => AlertFeedbackType.WouldAct,
_ => throw new ArgumentOutOfRangeException(nameof(v))
};
}
@@ -10,7 +10,8 @@ public enum AuditAction
PatientUpdated, PatientUpdated,
SuppressionWindowSet, SuppressionWindowSet,
UserLogin, UserLogin,
AuthorizationDenied AuthorizationDenied,
AlertFeedbackSubmitted,
} }
public static class AuditActionExtensions public static class AuditActionExtensions
@@ -28,6 +29,7 @@ public static class AuditActionExtensions
AuditAction.SuppressionWindowSet => "SUPPRESSION_WINDOW_SET", AuditAction.SuppressionWindowSet => "SUPPRESSION_WINDOW_SET",
AuditAction.UserLogin => "USER_LOGIN", AuditAction.UserLogin => "USER_LOGIN",
AuditAction.AuthorizationDenied => "AUTHORIZATION_DENIED", AuditAction.AuthorizationDenied => "AUTHORIZATION_DENIED",
AuditAction.AlertFeedbackSubmitted => "ALERT_FEEDBACK_SUBMITTED",
_ => throw new ArgumentOutOfRangeException(nameof(a)) _ => throw new ArgumentOutOfRangeException(nameof(a))
}; };
@@ -44,6 +46,7 @@ public static class AuditActionExtensions
"SUPPRESSION_WINDOW_SET" => AuditAction.SuppressionWindowSet, "SUPPRESSION_WINDOW_SET" => AuditAction.SuppressionWindowSet,
"USER_LOGIN" => AuditAction.UserLogin, "USER_LOGIN" => AuditAction.UserLogin,
"AUTHORIZATION_DENIED" => AuditAction.AuthorizationDenied, "AUTHORIZATION_DENIED" => AuditAction.AuthorizationDenied,
"ALERT_FEEDBACK_SUBMITTED" => AuditAction.AlertFeedbackSubmitted,
_ => throw new ArgumentOutOfRangeException(nameof(v)) _ => throw new ArgumentOutOfRangeException(nameof(v))
}; };
} }
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,114 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace VigilCareClinicalAPI.Migrations
{
/// <inheritdoc />
public partial class AddAlertQualityAnalytics : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "feedback_received",
table: "clinical_alerts",
type: "boolean",
nullable: false,
defaultValue: false);
migrationBuilder.CreateTable(
name: "alert_feedbacks",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false, defaultValueSql: "gen_random_uuid()"),
alert_id = table.Column<Guid>(type: "uuid", nullable: false),
user_id = table.Column<Guid>(type: "uuid", nullable: false),
feedback_type = table.Column<string>(type: "character varying(30)", maxLength: 30, nullable: false),
comment = table.Column<string>(type: "character varying(1000)", maxLength: 1000, nullable: true),
created_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false, defaultValueSql: "NOW()")
},
constraints: table =>
{
table.PrimaryKey("PK_alert_feedbacks", x => x.id);
table.ForeignKey(
name: "FK_alert_feedbacks_clinical_alerts_alert_id",
column: x => x.alert_id,
principalTable: "clinical_alerts",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "alert_quality_metrics",
columns: table => new
{
id = table.Column<Guid>(type: "uuid", nullable: false, defaultValueSql: "gen_random_uuid()"),
alert_type = table.Column<string>(type: "character varying(50)", maxLength: 50, nullable: false),
window_start = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
window_end = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
total_alerts = table.Column<int>(type: "integer", nullable: false),
acknowledged_count = table.Column<int>(type: "integer", nullable: false),
resolved_count = table.Column<int>(type: "integer", nullable: false),
escalated_count = table.Column<int>(type: "integer", nullable: false),
feedback_useful_count = table.Column<int>(type: "integer", nullable: false),
feedback_false_positive_count = table.Column<int>(type: "integer", nullable: false),
feedback_would_act_count = table.Column<int>(type: "integer", nullable: false),
feedback_count = table.Column<int>(type: "integer", nullable: false),
acknowledgement_rate = table.Column<double>(type: "double precision", nullable: false),
false_positive_rate = table.Column<double>(type: "double precision", nullable: false),
useful_rate = table.Column<double>(type: "double precision", nullable: false),
would_act_rate = table.Column<double>(type: "double precision", nullable: false),
avg_seconds_to_acknowledge = table.Column<double>(type: "double precision", nullable: false),
avg_seconds_to_resolution = table.Column<double>(type: "double precision", nullable: false),
computed_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false, defaultValueSql: "NOW()")
},
constraints: table =>
{
table.PrimaryKey("PK_alert_quality_metrics", x => x.id);
});
migrationBuilder.CreateIndex(
name: "IX_alert_feedbacks_alert_id",
table: "alert_feedbacks",
column: "alert_id");
migrationBuilder.CreateIndex(
name: "IX_alert_feedbacks_alert_id_user_id",
table: "alert_feedbacks",
columns: new[] { "alert_id", "user_id" },
unique: true);
migrationBuilder.CreateIndex(
name: "IX_alert_feedbacks_feedback_type",
table: "alert_feedbacks",
column: "feedback_type");
migrationBuilder.CreateIndex(
name: "IX_alert_quality_metrics_alert_type_window_start_window_end",
table: "alert_quality_metrics",
columns: new[] { "alert_type", "window_start", "window_end" },
unique: true);
migrationBuilder.CreateIndex(
name: "IX_alert_quality_metrics_window_start",
table: "alert_quality_metrics",
column: "window_start");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "alert_feedbacks");
migrationBuilder.DropTable(
name: "alert_quality_metrics");
migrationBuilder.DropColumn(
name: "feedback_received",
table: "clinical_alerts");
}
}
}
@@ -21,6 +21,145 @@ namespace VigilCareClinicalAPI.Migrations
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("AlertFeedback", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id")
.HasDefaultValueSql("gen_random_uuid()");
b.Property<Guid>("AlertId")
.HasColumnType("uuid")
.HasColumnName("alert_id");
b.Property<string>("Comment")
.HasMaxLength(1000)
.HasColumnType("character varying(1000)")
.HasColumnName("comment");
b.Property<DateTimeOffset>("CreatedAt")
.ValueGeneratedOnAdd()
.HasColumnType("timestamp with time zone")
.HasColumnName("created_at")
.HasDefaultValueSql("NOW()");
b.Property<string>("FeedbackType")
.IsRequired()
.HasMaxLength(30)
.HasColumnType("character varying(30)")
.HasColumnName("feedback_type");
b.Property<Guid>("UserId")
.HasColumnType("uuid")
.HasColumnName("user_id");
b.HasKey("Id");
b.HasIndex("AlertId");
b.HasIndex("FeedbackType");
b.HasIndex("AlertId", "UserId")
.IsUnique();
b.ToTable("alert_feedbacks", (string)null);
});
modelBuilder.Entity("AlertQualityMetric", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid")
.HasColumnName("id")
.HasDefaultValueSql("gen_random_uuid()");
b.Property<int>("AcknowledgedCount")
.HasColumnType("integer")
.HasColumnName("acknowledged_count");
b.Property<double>("AcknowledgementRate")
.HasColumnType("double precision")
.HasColumnName("acknowledgement_rate");
b.Property<string>("AlertType")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("character varying(50)")
.HasColumnName("alert_type");
b.Property<double>("AvgSecondsToAcknowledge")
.HasColumnType("double precision")
.HasColumnName("avg_seconds_to_acknowledge");
b.Property<double>("AvgSecondsToResolution")
.HasColumnType("double precision")
.HasColumnName("avg_seconds_to_resolution");
b.Property<DateTimeOffset>("ComputedAt")
.ValueGeneratedOnAdd()
.HasColumnType("timestamp with time zone")
.HasColumnName("computed_at")
.HasDefaultValueSql("NOW()");
b.Property<int>("EscalatedCount")
.HasColumnType("integer")
.HasColumnName("escalated_count");
b.Property<double>("FalsePositiveRate")
.HasColumnType("double precision")
.HasColumnName("false_positive_rate");
b.Property<int>("FeedbackCount")
.HasColumnType("integer")
.HasColumnName("feedback_count");
b.Property<int>("FeedbackFalsePositiveCount")
.HasColumnType("integer")
.HasColumnName("feedback_false_positive_count");
b.Property<int>("FeedbackUsefulCount")
.HasColumnType("integer")
.HasColumnName("feedback_useful_count");
b.Property<int>("FeedbackWouldActCount")
.HasColumnType("integer")
.HasColumnName("feedback_would_act_count");
b.Property<int>("ResolvedCount")
.HasColumnType("integer")
.HasColumnName("resolved_count");
b.Property<int>("TotalAlerts")
.HasColumnType("integer")
.HasColumnName("total_alerts");
b.Property<double>("UsefulRate")
.HasColumnType("double precision")
.HasColumnName("useful_rate");
b.Property<DateTimeOffset>("WindowEnd")
.HasColumnType("timestamp with time zone")
.HasColumnName("window_end");
b.Property<DateTimeOffset>("WindowStart")
.HasColumnType("timestamp with time zone")
.HasColumnName("window_start");
b.Property<double>("WouldActRate")
.HasColumnType("double precision")
.HasColumnName("would_act_rate");
b.HasKey("Id");
b.HasIndex("WindowStart");
b.HasIndex("AlertType", "WindowStart", "WindowEnd")
.IsUnique();
b.ToTable("alert_quality_metrics", (string)null);
});
modelBuilder.Entity("AlertThreshold", b => modelBuilder.Entity("AlertThreshold", b =>
{ {
b.Property<Guid>("Id") b.Property<Guid>("Id")
@@ -117,6 +256,12 @@ namespace VigilCareClinicalAPI.Migrations
.HasColumnType("uuid") .HasColumnType("uuid")
.HasColumnName("encounter_id"); .HasColumnName("encounter_id");
b.Property<bool>("FeedbackReceived")
.ValueGeneratedOnAdd()
.HasColumnType("boolean")
.HasDefaultValue(false)
.HasColumnName("feedback_received");
b.Property<string>("ObservationCode") b.Property<string>("ObservationCode")
.HasMaxLength(50) .HasMaxLength(50)
.HasColumnType("character varying(50)") .HasColumnType("character varying(50)")
@@ -1476,6 +1621,17 @@ namespace VigilCareClinicalAPI.Migrations
}); });
}); });
modelBuilder.Entity("AlertFeedback", b =>
{
b.HasOne("ClinicalAlert", "Alert")
.WithMany("Feedbacks")
.HasForeignKey("AlertId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Alert");
});
modelBuilder.Entity("ClinicalAlert", b => modelBuilder.Entity("ClinicalAlert", b =>
{ {
b.HasOne("Encounter", "Encounter") b.HasOne("Encounter", "Encounter")
@@ -1668,6 +1824,11 @@ namespace VigilCareClinicalAPI.Migrations
b.Navigation("Site"); b.Navigation("Site");
}); });
modelBuilder.Entity("ClinicalAlert", b =>
{
b.Navigation("Feedbacks");
});
modelBuilder.Entity("ClinicalSite", b => modelBuilder.Entity("ClinicalSite", b =>
{ {
b.Navigation("Gateways"); b.Navigation("Gateways");
@@ -0,0 +1,20 @@
public record AlertQualityMetricResponse(
Guid Id,
string AlertType,
DateTimeOffset WindowStart,
DateTimeOffset WindowEnd,
int TotalAlerts,
int AcknowledgedCount,
int ResolvedCount,
int EscalatedCount,
int FeedbackUsefulCount,
int FeedbackFalsePositiveCount,
int FeedbackWouldActCount,
int FeedbackCount,
double AcknowledgementRate,
double FalsePositiveRate,
double UsefulRate,
double WouldActRate,
double AvgSecondsToAcknowledge,
double AvgSecondsToResolution,
DateTimeOffset ComputedAt);
@@ -0,0 +1,11 @@
public record AlertQualitySummaryResponse(
DateTimeOffset PeriodStart,
DateTimeOffset PeriodEnd,
int TotalAlerts,
int TotalFeedback,
double AcknowledgementRate,
double FalsePositiveRate,
double UsefulRate,
double WouldActRate,
double AvgSecondsToAcknowledge,
double AvgSecondsToResolution);
@@ -0,0 +1 @@
public record SubmitAlertFeedbackRequest(AlertFeedbackType FeedbackType, string? Comment);
@@ -132,35 +132,55 @@ public sealed class ClinicalMetrics
Buckets = new[] { 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0 } Buckets = new[] { 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0 }
}); });
// --- Gauges (set by background collectors, not incremented inline) --- // --- Gauges (set by background collectors, not incremented inline) ---
// The most clinically significant panel. A non-zero value means a patient's // The most clinically significant panel. A non-zero value means a patient's
// critical alert has gone unacknowledged for more than 5 minutes. // critical alert has gone unacknowledged for more than 5 minutes.
// In a real deployment this panel drives an on-call pager alert at the nurse station. // In a real deployment this panel drives an on-call pager alert at the nurse station.
public readonly Gauge AlertsUnacknowledgedGauge = Metrics.CreateGauge( public readonly Gauge AlertsUnacknowledgedGauge = Metrics.CreateGauge(
"alerts_unacknowledged_gauge", "alerts_unacknowledged_gauge",
"Count of open CRITICAL alerts older than 5 minutes with no acknowledgment."); "Count of open CRITICAL alerts older than 5 minutes with no acknowledgment.");
// Per consumer group so the dashboard can show whether es-indexer, sepsis-engine, // Per consumer group so the dashboard can show whether es-indexer, sepsis-engine,
// or data-lake-writer is falling behind the observation stream. // or data-lake-writer is falling behind the observation stream.
public readonly Gauge KafkaConsumerLag = Metrics.CreateGauge( public readonly Gauge KafkaConsumerLag = Metrics.CreateGauge(
"kafka_consumer_lag", "kafka_consumer_lag",
"Approximate consumer group lag in messages, labeled by consumer group.", "Approximate consumer group lag in messages, labeled by consumer group.",
labelNames: new[] { "consumer_group" }); labelNames: new[] { "consumer_group" });
// An outbox that is growing means the relay is not keeping up or Kafka is unavailable. // An outbox that is growing means the relay is not keeping up or Kafka is unavailable.
// In a patient safety system, a growing outbox delays alert delivery to all consumers. // In a patient safety system, a growing outbox delays alert delivery to all consumers.
public readonly Gauge OutboxPendingEvents = Metrics.CreateGauge( public readonly Gauge OutboxPendingEvents = Metrics.CreateGauge(
"outbox_pending_events", "outbox_pending_events",
"Count of outbox events not yet relayed to Kafka."); "Count of outbox events not yet relayed to Kafka.");
public readonly Gauge WardGatewaysOffline = Metrics.CreateGauge( public readonly Gauge WardGatewaysOffline = Metrics.CreateGauge(
"ward_gateways_offline_gauge", "ward_gateways_offline_gauge",
"Ward gateways with status OFFLINE or DEGRADED", "Ward gateways with status OFFLINE or DEGRADED",
labelNames: new[] { "site_code" }); labelNames: new[] { "site_code" });
public readonly Gauge WardGatewayBufferDepth = Metrics.CreateGauge( public readonly Gauge WardGatewayBufferDepth = Metrics.CreateGauge(
"ward_gateway_buffer_depth", "ward_gateway_buffer_depth",
"Reported unsynced event count per gateway", "Reported unsynced event count per gateway",
labelNames: new[] { "gateway_code", "department" }); labelNames: new[] { "gateway_code", "department" });
public readonly Gauge AlertAcknowledgementRate = Metrics.CreateGauge(
"vigilcare_alert_acknowledgement_rate",
"Alert acknowledgement rate by type.",
labelNames: new[] { "alert_type" });
public readonly Gauge AlertFalsePositiveRate = Metrics.CreateGauge(
"vigilcare_alert_false_positive_rate",
"Clinician-reported false positive rate by type.",
labelNames: new[] { "alert_type" });
public readonly Gauge AlertUsefulRate = Metrics.CreateGauge(
"vigilcare_alert_useful_rate",
"Clinician-reported useful rate by type.",
labelNames: new[] { "alert_type" });
public readonly Gauge AlertAvgAckSeconds = Metrics.CreateGauge(
"vigilcare_alert_avg_ack_seconds",
"Average seconds from trigger to acknowledgement by type.",
labelNames: new[] { "alert_type" });
} }
+5
View File
@@ -151,6 +151,9 @@ try
builder.Services.Configure<GatewayMonitoringOptions>( builder.Services.Configure<GatewayMonitoringOptions>(
builder.Configuration.GetSection(GatewayMonitoringOptions.Section)); builder.Configuration.GetSection(GatewayMonitoringOptions.Section));
builder.Services.Configure<AlertQualityOptions>(
builder.Configuration.GetSection(AlertQualityOptions.Section));
builder.Services.AddCors(options => builder.Services.AddCors(options =>
{ {
options.AddPolicy("Dashboard", policy => options.AddPolicy("Dashboard", policy =>
@@ -212,6 +215,7 @@ try
builder.Services.AddScoped<IGatewayRegistryService, GatewayRegistryService>(); builder.Services.AddScoped<IGatewayRegistryService, GatewayRegistryService>();
builder.Services.AddScoped<IOperationsService, OperationsService>(); builder.Services.AddScoped<IOperationsService, OperationsService>();
builder.Services.AddHostedService<GatewayStaleDetectorService>(); builder.Services.AddHostedService<GatewayStaleDetectorService>();
builder.Services.AddScoped<IAlertQualityMetricsService, AlertQualityMetricsService>();
builder.Services.AddHostedService<ThresholdCacheLoader>(); builder.Services.AddHostedService<ThresholdCacheLoader>();
builder.Services.AddHostedService<KafkaTopicProvisioner>(); builder.Services.AddHostedService<KafkaTopicProvisioner>();
@@ -238,6 +242,7 @@ try
builder.Services.AddHostedService<SofaScoringService>(); builder.Services.AddHostedService<SofaScoringService>();
builder.Services.AddHostedService<PatientPhiMigrationService>(); builder.Services.AddHostedService<PatientPhiMigrationService>();
builder.Services.AddHostedService<WardGatewayMetricsCollector>(); builder.Services.AddHostedService<WardGatewayMetricsCollector>();
builder.Services.AddHostedService<AlertQualityAggregatorService>();
builder.Services.AddHealthChecks() builder.Services.AddHealthChecks()
.AddDbContextCheck<AppDbContext>("postgresql", tags: new[] { "ready" }) .AddDbContextCheck<AppDbContext>("postgresql", tags: new[] { "ready" })
@@ -0,0 +1,87 @@
using Microsoft.EntityFrameworkCore;
public class AlertQualityMetricsService : IAlertQualityMetricsService
{
private readonly AppDbContext _db;
public AlertQualityMetricsService(AppDbContext db) => _db = db;
public async Task<IReadOnlyList<AlertQualityMetricResponse>> ListAsync(
AlertType? alertType, DateTimeOffset from, DateTimeOffset to)
{
var query = _db.AlertQualityMetrics
.AsNoTracking()
.Where(m => m.WindowStart >= from && m.WindowEnd <= to);
if (alertType.HasValue)
query = query.Where(m => m.AlertType == alertType.Value);
var rows = await query
.OrderByDescending(m => m.WindowStart)
.ThenBy(m => m.AlertType)
.ToListAsync();
return rows.Select(Map).ToList();
}
public async Task<AlertQualitySummaryResponse> GetSummaryAsync(
DateTimeOffset from, DateTimeOffset to)
{
var snapshots = await _db.AlertQualityMetrics
.AsNoTracking()
.Where(m => m.WindowStart >= from && m.WindowEnd <= to)
.ToListAsync();
if (snapshots.Count == 0)
{
return new AlertQualitySummaryResponse(
from, to, 0, 0, 0, 0, 0, 0, 0, 0);
}
var totalAlerts = snapshots.Sum(s => s.TotalAlerts);
var totalFeedback = snapshots.Sum(s => s.FeedbackCount);
var totalAcknowledged = snapshots.Sum(s => s.AcknowledgedCount);
var totalUseful = snapshots.Sum(s => s.FeedbackUsefulCount);
var totalFalsePositive = snapshots.Sum(s => s.FeedbackFalsePositiveCount);
var totalWouldAct = snapshots.Sum(s => s.FeedbackWouldActCount);
var weightedAckSeconds = snapshots.Sum(s => s.AvgSecondsToAcknowledge * s.TotalAlerts);
var weightedResolveSeconds = snapshots.Sum(s => s.AvgSecondsToResolution * s.ResolvedCount);
return new AlertQualitySummaryResponse(
PeriodStart: from,
PeriodEnd: to,
TotalAlerts: totalAlerts,
TotalFeedback: totalFeedback,
AcknowledgementRate: totalAlerts == 0 ? 0 : (double)totalAcknowledged / totalAlerts,
FalsePositiveRate: totalFeedback == 0 ? 0 : (double)totalFalsePositive / totalFeedback,
UsefulRate: totalFeedback == 0 ? 0 : (double)totalUseful / totalFeedback,
WouldActRate: totalFeedback == 0 ? 0 : (double)totalWouldAct / totalFeedback,
AvgSecondsToAcknowledge: totalAlerts == 0 ? 0 : weightedAckSeconds / totalAlerts,
AvgSecondsToResolution: snapshots.Sum(s => s.ResolvedCount) == 0
? 0
: weightedResolveSeconds / snapshots.Sum(s => s.ResolvedCount));
}
private static AlertQualityMetricResponse Map(AlertQualityMetric m) =>
new(
m.Id,
m.AlertType.ToString(),
m.WindowStart,
m.WindowEnd,
m.TotalAlerts,
m.AcknowledgedCount,
m.ResolvedCount,
m.EscalatedCount,
m.FeedbackUsefulCount,
m.FeedbackFalsePositiveCount,
m.FeedbackWouldActCount,
m.FeedbackCount,
m.AcknowledgementRate,
m.FalsePositiveRate,
m.UsefulRate,
m.WouldActRate,
m.AvgSecondsToAcknowledge,
m.AvgSecondsToResolution,
m.ComputedAt);
}
@@ -163,6 +163,54 @@ public class AlertService : IAlertService
return alert; return alert;
} }
public async Task<AlertFeedback> SubmitFeedbackAsync(
Guid alertId, AlertFeedbackType type, string? comment)
{
if (!_currentUser.IsAuthenticated || _currentUser.UserId is null)
throw new ValidationException("Authentication required.", "AUTH_REQUIRED");
var userId = _currentUser.UserId.Value;
var alert = await _db.ClinicalAlerts.FindAsync(alertId);
if (alert is null)
throw new NotFoundException("Alert not found.", "ALERT_NOT_FOUND");
if (alert.Status == AlertStatus.Open)
throw new ValidationException(
"Feedback can only be submitted on acknowledged or resolved alerts.",
"ALERT_NOT_REVIEWABLE");
var alreadySubmitted = await _db.AlertFeedbacks
.AnyAsync(f => f.AlertId == alertId && f.UserId == userId);
if (alreadySubmitted)
throw new ConflictException(
"You have already submitted feedback for this alert.",
"FEEDBACK_ALREADY_SUBMITTED");
var feedback = new AlertFeedback
{
Id = Guid.NewGuid(),
AlertId = alertId,
UserId = userId,
FeedbackType = type,
Comment = string.IsNullOrWhiteSpace(comment) ? null : comment.Trim(),
CreatedAt = DateTimeOffset.UtcNow
};
_db.AlertFeedbacks.Add(feedback);
alert.FeedbackReceived = true;
await _db.SaveChangesAsync();
await _audit.WriteAsync(
AuditAction.AlertFeedbackSubmitted,
"ClinicalAlert",
alert.Id,
newValue: new { feedbackType = type.ToDbString(), feedback.UserId },
reason: comment);
return feedback;
}
private async Task<int> ResolveSuppressionWindowMinutesAsync( private async Task<int> ResolveSuppressionWindowMinutesAsync(
AlertType alertType, int defaultWindowMinutes) AlertType alertType, int defaultWindowMinutes)
{ {
@@ -0,0 +1,8 @@
public interface IAlertQualityMetricsService
{
Task<IReadOnlyList<AlertQualityMetricResponse>> ListAsync(
AlertType? alertType, DateTimeOffset from, DateTimeOffset to);
Task<AlertQualitySummaryResponse> GetSummaryAsync(
DateTimeOffset from, DateTimeOffset to);
}
@@ -13,4 +13,5 @@ public interface IAlertService
Task<ClinicalAlert> ResolveAsync(Guid id); Task<ClinicalAlert> ResolveAsync(Guid id);
Task ApplySyncedAcknowledgmentAsync(Guid alertId, SyncedAlertAcknowledgment ack, CancellationToken ct); Task ApplySyncedAcknowledgmentAsync(Guid alertId, SyncedAlertAcknowledgment ack, CancellationToken ct);
Task ApplySyncedResolutionAsync(Guid alertId, SyncedAlertResolution resolve, CancellationToken ct); Task ApplySyncedResolutionAsync(Guid alertId, SyncedAlertResolution resolve, CancellationToken ct);
Task<AlertFeedback> SubmitFeedbackAsync(Guid alertId, AlertFeedbackType type, string? comment);
} }
@@ -0,0 +1,10 @@
using FluentValidation;
public class SubmitAlertFeedbackRequestValidator : AbstractValidator<SubmitAlertFeedbackRequest>
{
public SubmitAlertFeedbackRequestValidator()
{
RuleFor(r => r.FeedbackType).IsInEnum();
RuleFor(r => r.Comment).MaximumLength(1000);
}
}
+4
View File
@@ -214,5 +214,9 @@
"GatewayMonitoring": { "GatewayMonitoring": {
"StaleThresholdMinutes": 10, "StaleThresholdMinutes": 10,
"PollIntervalMinutes": 5 "PollIntervalMinutes": 5
},
"AlertQuality": {
"IntervalMinutes": 60,
"WindowHours": 1
} }
} }
+1 -1
View File
@@ -204,7 +204,7 @@ Documentation complete when:
- [ ] [resilience/README.md](resilience/README.md) experiment index populated - [ ] [resilience/README.md](resilience/README.md) experiment index populated
- [ ] Six chaos experiments documented with baseline/broken/fixed artifacts - [ ] Six chaos experiments documented with baseline/broken/fixed artifacts
- [ ] Interview questions #2932 in [interview-questions.md](interview-questions.md) - [ ] Interview questions #2932 in [interview-questions.md](interview-questions.md)
- [ ] `scripts/verify-phase-24.ps1` passes - [ ] `scripts/run-phase24-verification.sh` passes
Implementation complete when all phase exit gates in plans 2024 pass and integration tests green. Implementation complete when all phase exit gates in plans 2024 pass and integration tests green.
@@ -0,0 +1,145 @@
# Phase 24 verification notes
Notes from getting `scripts/run-phase24-verification.sh` to pass against local Docker + dev central API (`http://localhost:5270`).
## Automated verification
```bash
./scripts/run-phase24-verification.sh
```
Useful env overrides:
| Variable | Purpose |
|----------|---------|
| `SKIP_DOCKER=1` | Stack already up; skip compose |
| `SKIP_PHASE_A=1` | Skip central baseline replay |
| `SCENARIO_SPEED=0` | Instant replay (default) |
| `KEEP_CENTRAL_DOWN=1` | Leave central stopped after Phase B (debug) |
Companion script: `scripts/mint-gateway-jwt.sh` — dev JWT for gateway (`issuer: vigilcare-gateway`).
---
## Fixes applied during verification
### 1. Gateway encounter missing after Phase A
**Symptom:** Phase B could not find encounter on gateway.
**Cause:** `docker compose up -d` without recreate did not re-run `SyncOnceAsync` after central had new encounters.
**Fix:** Script calls `docker compose up -d --force-recreate ward-gateway-api` in `configure_gateway_sync` so sync runs on a fresh gateway container.
---
### 2. Phase B observation POST returned 400
**Symptom:** Gateway rejected observation batches from simulator.
**Cause:** Central API accepts batch ingest; gateway expects a **single** observation per POST with `observationCode` (not centrals batch shape).
**Fix:** `VigilCareApiClient.SendObservationBatchAsync` — gateway path posts one observation at a time using gateway field names.
---
### 3. Phase B `recordedAt` rejected (future timestamp)
**Symptom:** 400 — timestamp more than ~5 minutes in the future.
**Cause:** `--speed 0` uses scenario `offsetMinutes` as simulated future times; gateway validates `recordedAt` against wall clock.
**Fix:** `ReplayEngine` uses `DateTimeOffset.UtcNow` for observations when `Target == Gateway`.
**Trade-off:** Gateway replay does not preserve simulated timeline spacing at speed 0; acceptable for outage demo (alerts still fire on values).
---
### 4. No critical potassium alert at K+ 6.1
**Symptom:** Only warning-tier alert; Phase B ack step had nothing to match.
**Cause:** Seeded threshold `CriticalHigh` for potassium is **6.5 mEq/L**; 6.1 is above warning (5.5) but below critical.
**Fix:** Scenario `ward-outage-reconnect-01.json` — critical observation value **6.1 → 6.8**.
---
### 5. Alert ack / jq verification failed (alert type mismatch)
**Symptom:** `TryAcknowledgeAlertAsync` returned false; jq filters found no alert.
**Cause:** API stores/returns enum-style names (e.g. `CriticalPotassiumMeqL`); scenario uses `CRITICAL_POTASSIUM_MEQ_L`.
**Fix:** `VigilCareApiClient.IsMatchingAlertType` — flexible match (case, underscores, suffix).
**Fix:** Verification script uses jq filters that accept both naming styles.
---
### 6. `acknowledgedBy` was `nurse.demo` instead of `RN-Wu`
**Symptom:** After gateway ack, synced alert on central showed JWT subject, not scenario nurse.
**Cause:** Gateway `AcknowledgeAlertRequest` had no body field for clinician; controller used `User.Identity.Name` only. Stale Docker image also hid fixes until rebuild.
**Fix:** Optional `clinicianId` on gateway ack request; controller uses `req.ClinicianId ?? User.Identity?.Name`. Script rebuilds gateway image before run.
---
### 7. Phase C verify timeout
**Symptom:** Script gave up before central showed synced ack with `RN-Wu`.
**Cause:** Central restart + gateway sync can exceed 120s on a loaded machine.
**Fix:** Poll up to **180s** for alert with `syncedFromGateway` and `acknowledgedBy = RN-Wu`.
---
### 8. Phase A central ack fails at `--speed 0` (non-fatal in script)
**Symptom:** Phase A `alert_ack` against central sometimes failed before the poll fix.
**Cause:** Alert pipeline is async; at speed 0 the ack event could run before the critical alert row existed.
**Fix:** Central replay now polls for up to 30s before posting `alert_ack` (`TryAcknowledgeAlertAsync` with `waitForAlert`). The verification script still treats Phase A ack as non-fatal; Phase B (gateway outage path) remains the authoritative ack/sync test.
---
## Code changes summary (already merged)
| Area | Files |
|------|--------|
| Scenario + schema | `ward-outage-reconnect-01.json`, `schema.json`, `ScenarioValidator.cs` |
| Gateway replay | `ReplayOptions.cs`, `ReplayCommand.cs`, `ReplayEngine.cs`, `VigilCareApiClient.cs` |
| Gateway ack attribution | `AcknowledgeAlertRequest.cs`, `AcknowledgeAlertRequestValidator.cs`, `AlertsController.cs` |
| Tests | `WardGatewayLocalPathTests.cs` |
| Automation | `run-phase24-verification.sh`, `mint-gateway-jwt.sh` |
| Docs | `simulator-guide.md` §10, phase-24 plan footnote on gateway vs central ack |
---
## Remaining doc / optional code follow-ups
| Item | Priority | Notes |
|------|----------|--------|
| `phase-24-plan.md` embedded scenario snippet still showed K+ 6.1 | Doc | Updated to 6.8 to match scenario + thresholds |
| `ext-climate-resilience-roadmap.md` referenced `verify-phase-24.ps1` | Doc | Updated to `run-phase24-verification.sh` |
| Central async ack at speed 0 | Done | `ReplayEngine` polls up to 30s before central `alert_ack` |
| Gateway has no `/auth/login` | By design | Use `--gateway-token` / `mint-gateway-jwt.sh` |
| `CentralApiOptions` default port 5080 vs dev 5270 | OK | Docker compose overrides via `host.docker.internal:5270` |
| README mention of Phase 24 script | Done | Ward outage section + verification script list |
| Gateway mode unit tests | Optional | No simulator tests for `--gateway` path yet |
---
## Expected pass criteria (Phase C)
After central restarts:
1. Buffered alert appears on central with `syncedFromGateway` (or equivalent sync marker).
2. `acknowledgedBy` = **RN-Wu** (not gateway JWT subject).
3. Gateway buffer depth returns to **0** (or heartbeat shows no pending acks).
Last successful run: `./scripts/run-phase24-verification.sh` with `SKIP_DOCKER=1` — all phases passed.
+91
View File
@@ -17,6 +17,7 @@ Think of it as a flight simulator, but for clinical decision support.
7. [Reading the Output](#7-reading-the-output) 7. [Reading the Output](#7-reading-the-output)
8. [Creating Your Own Scenarios](#8-creating-your-own-scenarios) 8. [Creating Your Own Scenarios](#8-creating-your-own-scenarios)
9. [Troubleshooting](#9-troubleshooting) 9. [Troubleshooting](#9-troubleshooting)
10. [Ward Outage Scenario (Climate Resilience)](#10-ward-outage-scenario-climate-resilience)
--- ---
@@ -76,6 +77,12 @@ dotnet run --project VigilCare.Simulator -- replay <scenario-file> [options]
| `--base-url <url>` | `http://localhost:5270` | API address (change if your API runs elsewhere) | | `--base-url <url>` | `http://localhost:5270` | API address (change if your API runs elsewhere) |
| `--poll` | off | Show alerts and scores after each set of vitals | | `--poll` | off | Show alerts and scores after each set of vitals |
| `--poll-interval <seconds>` | 5 | How often to check for alerts when polling | | `--poll-interval <seconds>` | 5 | How often to check for alerts when polling |
| `--username <name>` | `physician.demo` | API login username |
| `--password <secret>` | (demo default) | API login password |
| `--gateway` | off | Target the ward gateway API at `http://localhost:5081` |
| `--encounter-id <guid>` | — | Use an existing encounter (required with `--gateway`; also used with `--skip-setup`) |
| `--skip-setup` | off | Skip patient/encounter registration — requires `--encounter-id` |
| `--gateway-token <jwt>` | `$GATEWAY_JWT` | Bearer token for gateway replay (required with `--gateway`; use `./scripts/mint-gateway-jwt.sh`) |
**Example -- run the stable baseline scenario in real-time with polling:** **Example -- run the stable baseline scenario in real-time with polling:**
@@ -200,6 +207,9 @@ The key concept is **offsetMinutes** -- each event happens at a certain number o
| `medication` | A drug being administered | Ceftriaxone 1g IV | | `medication` | A drug being administered | Ceftriaxone 1g IV |
| `order` | A clinical order being placed | "Blood cultures", "Chest X-ray" | | `order` | A clinical order being placed | "Blood cultures", "Chest X-ray" |
| `order_result` | Result of a prior order | "Positive for E. coli" | | `order_result` | Result of a prior order | "Positive for E. coli" |
| `alert_ack` | Acknowledgement of an open alert | RN acknowledges critical potassium alert |
`alert_ack` events require `alertType` and `clinicianId` in `data`; optional `note`. On **gateway** replay (`--gateway`), the simulator sends `clinicianId` in the acknowledge request so the ward records the bedside nurse label (e.g. `RN-Wu`) and syncs it to central. On **central** replay, attribution comes from the logged-in API user.
### Vital Sign Codes ### Vital Sign Codes
@@ -236,6 +246,7 @@ The simulator ships with 8 scenarios covering different clinical situations:
| **dka-electrolyte-01** | Diabetic ketoacidosis with potassium and glucose derangement. | Varies | | **dka-electrolyte-01** | Diabetic ketoacidosis with potassium and glucose derangement. | Varies |
| **hypothermia-elderly-01** | Elderly patient with severe hypothermia. Slow HR, dropping temperature. | Varies | | **hypothermia-elderly-01** | Elderly patient with severe hypothermia. Slow HR, dropping temperature. | Varies |
| **medication-false-alarm-01** | Beta-blocker causing bradycardia. Tests whether the system correctly handles medication-induced vital changes. | 3 hours | | **medication-false-alarm-01** | Beta-blocker causing bradycardia. Tests whether the system correctly handles medication-induced vital changes. | 3 hours |
| **ward-outage-reconnect-01** | ICU patient with critical hyperkalemia during simulated central outage. Validates gateway-local alerting and alert acknowledgement sync. | 90 minutes |
All scenario files are in: `VigilCare.Simulator/Scenarios/List/` All scenario files are in: `VigilCare.Simulator/Scenarios/List/`
@@ -398,3 +409,83 @@ Use `--base-url`:
dotnet run --project VigilCare.Simulator -- replay scenario.json \ dotnet run --project VigilCare.Simulator -- replay scenario.json \
--base-url http://192.168.1.50:5270 --base-url http://192.168.1.50:5270
``` ```
---
## 10. Ward Outage Scenario (Climate Resilience)
The `ward-outage-reconnect-01` scenario validates Tier 1 safety during a central API outage. Observations and alerts continue on the ward gateway; acknowledgements are recorded locally and synced when the uplink returns.
### Automated verification (recommended)
From the repo root, with Docker running:
```bash
./scripts/run-phase24-verification.sh
```
This script runs all three phases automatically: central baseline replay, gateway replay during simulated central outage, and post-reconnect sync checks. Logs go to `/tmp/vigilcare-phase24-*`.
Useful flags:
| Env var | Effect |
|---------|--------|
| `SKIP_DOCKER=1` | Assume `docker compose` stack is already up |
| `SKIP_PHASE_A=1` | Skip central replay; use an existing ICU encounter on the gateway |
Helper for manual gateway API calls (ward gateway has no login endpoint):
```bash
export GATEWAY_JWT=$(./scripts/mint-gateway-jwt.sh nurse.demo NURSE)
```
### Manual procedure
1. Full stack running with the ward gateway profile:
```bash
docker compose --profile ward-gateway up -d
```
2. Central API running and the gateway encounter replica synced
3. Note an active ICU encounter id from the gateway:
```bash
curl "http://localhost:5081/api/v1/encounters?status=ACTIVE&department=ICU" \
-H "Authorization: Bearer $JWT"
```
### Procedure
**Phase A — Baseline on central (optional):**
```bash
dotnet run --project VigilCare.Simulator -- replay \
VigilCare.Simulator/Scenarios/List/ward-outage-reconnect-01.json \
--speed 0 --base-url http://localhost:5270
```
**Phase B — Stop central, replay against gateway:**
```bash
# Stop central API process/container
dotnet run --project VigilCare.Simulator -- replay \
VigilCare.Simulator/Scenarios/List/ward-outage-reconnect-01.json \
--gateway --encounter-id <ENCOUNTER-GUID> --speed 60 --poll
```
The `--gateway` flag targets `http://localhost:5081` automatically. `--encounter-id` must reference an encounter already replicated on the gateway. The `alert_ack` event at T+50 min sends `clinicianId: "RN-Wu"` from the scenario; the gateway honors this in `acknowledged_by` and in the sync buffer (login user is only used when `clinicianId` is omitted).
**Phase A note:** Central replay attributes acks to the JWT user (`physician.demo` by default). Use `--username nurse.demo` if you want a nurse role on central; gateway Phase B is the authoritative climate-resilience path for scenario attribution.
**Phase C — Restart central, verify sync:**
- Wait for `SyncUploaderService` to drain the buffer
- Poll `GET /api/v1/operations/gateways` — buffer depth should reach 0
- Confirm observations on central with preserved `recorded_at` timestamps
### Success criteria
- Critical potassium alert created on gateway at T+45 min while central is down
- Ack recorded locally at T+50 min with `acknowledged_by` = `RN-Wu`
- After reconnect: central has observations, alert, and ack; no duplicate paging logs
@@ -0,0 +1,154 @@
{
"uid": "vigilcare-alert-quality",
"title": "VigilCare Alert Quality",
"tags": ["vigilcare", "clinical", "alerts", "prometheus"],
"timezone": "browser",
"schemaVersion": 38,
"version": 1,
"refresh": "30s",
"time": {
"from": "now-24h",
"to": "now"
},
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "Acknowledgement Rate by Type",
"description": "Share of alerts acknowledged or resolved within the aggregation window.",
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "vigilcare_alert_acknowledgement_rate",
"legendFormat": "{{alert_type}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.7 },
{ "color": "green", "value": 0.9 }
]
}
},
"overrides": []
},
"options": {
"legend": { "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi" }
}
},
{
"id": 2,
"type": "timeseries",
"title": "False Positive Rate by Type",
"description": "Clinician-reported false positive share among submitted feedback.",
"gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "vigilcare_alert_false_positive_rate",
"legendFormat": "{{alert_type}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 0.2 },
{ "color": "red", "value": 0.4 }
]
}
},
"overrides": []
},
"options": {
"legend": { "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi" }
}
},
{
"id": 3,
"type": "timeseries",
"title": "Useful Rate by Type",
"description": "Clinician-reported useful share among submitted feedback.",
"gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "vigilcare_alert_useful_rate",
"legendFormat": "{{alert_type}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.5 },
{ "color": "green", "value": 0.75 }
]
}
},
"overrides": []
},
"options": {
"legend": { "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi" }
}
},
{
"id": 4,
"type": "timeseries",
"title": "Avg Ack Seconds by Type",
"description": "Average seconds from alert trigger to acknowledgement.",
"gridPos": { "x": 12, "y": 8, "w": 12, "h": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "vigilcare_alert_avg_ack_seconds",
"legendFormat": "{{alert_type}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 120 },
{ "color": "red", "value": 300 }
]
}
},
"overrides": []
},
"options": {
"legend": { "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi" }
}
}
]
}
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
# Mint a dev JWT for the ward gateway API (issuer vigilcare-gateway).
# Usage: mint-gateway-jwt.sh [username] [clinical_role]
# Prints the token to stdout.
set -euo pipefail
USERNAME="${1:-nurse.demo}"
ROLE="${2:-NURSE}"
python3 - "$USERNAME" "$ROLE" <<'PY'
import json, hmac, hashlib, base64, datetime, os, sys
def b64url(data: bytes) -> str:
return base64.urlsafe_b64encode(data).decode().rstrip("=")
username = sys.argv[1]
role = sys.argv[2].upper()
secret = os.environ.get(
"GATEWAY_JWT_SECRET", "dev-signing-key-minimum-32-bytes-long!!").encode()
user_id = os.environ.get(
"GATEWAY_JWT_USER_ID", "11111111-1111-1111-1111-111111111111")
display = os.environ.get("GATEWAY_JWT_DISPLAY_NAME", "Demo Nurse")
header = b64url(json.dumps({"alg": "HS256", "typ": "JWT"}, separators=(",", ":")).encode())
now = int(datetime.datetime.now(datetime.timezone.utc).timestamp())
payload = {
"iss": "vigilcare-gateway",
"aud": "vigilcare-dashboard",
"exp": now + 86400,
"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier": user_id,
"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name": username,
"display_name": display,
"clinical_role": role,
}
payload_b64 = b64url(json.dumps(payload, separators=(",", ":")).encode())
signing_input = f"{header}.{payload_b64}".encode()
sig = b64url(hmac.new(secret, signing_input, hashlib.sha256).digest())
print(f"{header}.{payload_b64}.{sig}", end="")
PY
+297
View File
@@ -0,0 +1,297 @@
#!/usr/bin/env bash
# Phase 24 — ward-outage-reconnect-01 end-to-end verification.
#
# Runs Phase A (central baseline), Phase B (gateway replay while central down),
# and Phase C (central restart + sync verification).
#
# Optional env:
# SKIP_DOCKER=1 — assume docker stack already up
# SKIP_PHASE_A=1 — skip central replay; use newest ICU encounter on gateway
# CENTRAL_URL — default http://localhost:5270
# GATEWAY_URL — default http://localhost:5081
# SCENARIO_SPEED — simulator speed (default 0 = instant)
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT"
CENTRAL="${CENTRAL_URL:-http://localhost:5270}"
GW="${GATEWAY_URL:-http://localhost:5081}"
GW_ID="${GATEWAY_ID:-22222222-2222-2222-2222-222222222222}"
SCENARIO="${ROOT}/VigilCare.Simulator/Scenarios/List/ward-outage-reconnect-01.json"
SPEED="${SCENARIO_SPEED:-0}"
LOG_DIR="${TMPDIR:-/tmp}/vigilcare-phase24-$$"
CENTRAL_PID=""
WE_STARTED_CENTRAL=0
STOPPED_CENTRAL=0
mkdir -p "$LOG_DIR"
log() { echo "==> $*"; }
die() { echo "ERROR: $*" >&2; exit 1; }
wait_for_url() {
local url="$1" max="${2:-60}" i
for i in $(seq 1 "$max"); do
if curl -sf "$url" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
return 1
}
central_login() {
local user="$1" pass="$2"
curl -sf -X POST "${CENTRAL}/api/v1/auth/login" \
-H "Content-Type: application/json" \
-d "{\"username\":\"${user}\",\"password\":\"${pass}\"}" \
| jq -r '.data.accessToken'
}
ensure_central_running() {
if curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1; then
log "Central API already running at $CENTRAL"
return
fi
log "Starting central API (log: $LOG_DIR/central.log)"
dotnet run --project VigilCareClinicalAPI --launch-profile http \
>"$LOG_DIR/central.log" 2>&1 &
CENTRAL_PID=$!
WE_STARTED_CENTRAL=1
wait_for_url "${CENTRAL}/health/ready" 90 \
|| die "Central API did not become ready — see $LOG_DIR/central.log"
log "Central API ready"
}
stop_central() {
if [[ "$STOPPED_CENTRAL" -eq 1 ]]; then
return
fi
log "Stopping central API to simulate outage"
if [[ -n "$CENTRAL_PID" ]] && kill -0 "$CENTRAL_PID" 2>/dev/null; then
kill "$CENTRAL_PID" 2>/dev/null || true
wait "$CENTRAL_PID" 2>/dev/null || true
CENTRAL_PID=""
else
pkill -f "VigilCareClinicalAPI" 2>/dev/null || true
fi
for _ in $(seq 1 20); do
curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1 || break
sleep 1
done
STOPPED_CENTRAL=1
}
restart_central() {
if curl -sf "${CENTRAL}/health/ready" >/dev/null 2>&1; then
log "Central API already running"
STOPPED_CENTRAL=0
return
fi
log "Restarting central API"
dotnet run --project VigilCareClinicalAPI --launch-profile http \
>>"$LOG_DIR/central.log" 2>&1 &
CENTRAL_PID=$!
WE_STARTED_CENTRAL=1
wait_for_url "${CENTRAL}/health/ready" 90 \
|| die "Central API failed to restart — see $LOG_DIR/central.log"
STOPPED_CENTRAL=0
log "Central API back online"
}
configure_gateway_sync() {
local sync_jwt="$1"
log "Configuring gateway encounter sync (GATEWAY_SYNC_JWT)"
GATEWAY_SYNC_JWT="$sync_jwt" docker compose --profile ward-gateway up -d --force-recreate ward-gateway-api
wait_for_url "${GW}/health/ready" 60 \
|| die "Gateway not ready after restart"
}
wait_for_encounter_on_gateway() {
local encounter_id="$1" jwt="$2" max_attempts="${3:-60}"
local i
for i in $(seq 1 "$max_attempts"); do
if curl -sf "${GW}/api/v1/encounters/${encounter_id}" \
-H "Authorization: Bearer ${jwt}" >/dev/null 2>&1; then
log "Encounter on gateway after ${i} attempt(s)"
return 0
fi
sleep 3
done
return 1
}
wait_for_gateway_encounter() {
local jwt="$1" encounter_id="" i
for i in $(seq 1 45); do
encounter_id=$(curl -sf "${GW}/api/v1/encounters?status=ACTIVE&department=ICU" \
-H "Authorization: Bearer ${jwt}" \
| jq -r '.data.items[0].encounterId // empty')
if [[ -n "$encounter_id" ]]; then
echo "$encounter_id"
return 0
fi
sleep 4
done
return 1
}
parse_encounter_from_replay() {
local log_file="$1"
grep -oE 'Encounter opened: [0-9a-f-]{36}' "$log_file" \
| head -1 \
| sed 's/Encounter opened: //'
}
cleanup() {
if [[ "${KEEP_CENTRAL_DOWN:-0}" == "1" ]]; then
return
fi
if [[ "$STOPPED_CENTRAL" -eq 1 ]]; then
restart_central || true
fi
}
trap cleanup EXIT
command -v jq >/dev/null 2>&1 || die "jq is required"
command -v python3 >/dev/null 2>&1 || die "python3 is required (for mint-gateway-jwt.sh)"
log "Phase 24 verification — ward-outage-reconnect-01"
if [[ "${SKIP_DOCKER:-0}" != "1" ]]; then
log "Starting docker infrastructure"
docker compose up -d
log "Building ward gateway API image"
docker compose --profile ward-gateway build ward-gateway-api
log "Starting ward gateway profile"
docker compose --profile ward-gateway up -d
else
log "SKIP_DOCKER=1 — assuming stack is already up"
log "Rebuilding ward gateway API image (picks up local code changes)"
docker compose --profile ward-gateway build ward-gateway-api
fi
wait_for_url "${GW}/health/ready" 60 \
|| die "Gateway not ready at $GW — run: docker compose --profile ward-gateway up -d"
ensure_central_running
PHYSICIAN_JWT=$(central_login "physician.demo" "DemoPhysician1!")
ADMIN_JWT=$(central_login "admin.demo" "DemoAdmin1!")
GATEWAY_JWT=$("${ROOT}/scripts/mint-gateway-jwt.sh" nurse.demo NURSE)
export GATEWAY_JWT
configure_gateway_sync "$PHYSICIAN_JWT"
ENCOUNTER_ID=""
if [[ "${SKIP_PHASE_A:-0}" != "1" ]]; then
log "Phase A — baseline replay on central"
PHASE_A_LOG="$LOG_DIR/phase-a.log"
dotnet run --project VigilCare.Simulator -- replay "$SCENARIO" \
--speed "$SPEED" --base-url "$CENTRAL" \
2>&1 | tee "$PHASE_A_LOG"
ENCOUNTER_ID=$(parse_encounter_from_replay "$PHASE_A_LOG")
[[ -n "$ENCOUNTER_ID" ]] || die "Phase A did not report an encounter id"
log "Phase A encounter: $ENCOUNTER_ID"
configure_gateway_sync "$PHYSICIAN_JWT"
wait_for_encounter_on_gateway "$ENCOUNTER_ID" "$GATEWAY_JWT" 60 \
|| die "Encounter $ENCOUNTER_ID not replicated to gateway after Phase A"
fi
if [[ -z "$ENCOUNTER_ID" ]]; then
log "Waiting for ICU encounter on gateway replica"
ENCOUNTER_ID=$(wait_for_gateway_encounter "$GATEWAY_JWT") \
|| die "No ACTIVE ICU encounter on gateway — run Phase A or set GATEWAY_SYNC_JWT"
fi
log "Using encounter $ENCOUNTER_ID"
wait_for_encounter_on_gateway "$ENCOUNTER_ID" "$GATEWAY_JWT" 10 \
|| die "Encounter $ENCOUNTER_ID not available on gateway"
stop_central
log "Waiting for gateway to detect central unreachable (~35s)"
sleep 35
log "Phase B — replay against gateway while central is down"
dotnet run --project VigilCare.Simulator -- replay "$SCENARIO" \
--gateway --encounter-id "$ENCOUNTER_ID" --gateway-token "$GATEWAY_JWT" \
--speed "$SPEED"
log "Verify critical potassium alert acknowledged locally by RN-Wu"
GW_ALERTS=$(curl -sf "${GW}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \
-H "Authorization: Bearer ${GATEWAY_JWT}")
echo "$GW_ALERTS" | jq -e \
'.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | (.status == "ACKNOWLEDGED" or .status == "Acknowledged")' \
>/dev/null \
|| die "Expected ACKNOWLEDGED critical potassium alert on gateway"
echo "$GW_ALERTS" | jq -e \
'.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | .acknowledgedBy == "RN-Wu"' \
>/dev/null \
|| die "Expected acknowledgedBy RN-Wu on gateway"
BUFFER_DEPTH=$(docker exec "$(docker ps -qf name=ward-gateway-db)" \
psql -U postgres -d vigilcare_ward -tAc \
"SELECT COUNT(*) FROM buffered_sync_items WHERE NOT synced;")
log "Gateway buffer depth (unsynced items): $BUFFER_DEPTH"
[[ "$BUFFER_DEPTH" -gt 0 ]] || die "Expected buffered sync items on gateway after Phase B"
log "Phase C — restart central and wait for sync"
restart_central
log "Waiting up to 180s for gateway-synced alert on central"
SYNCED_ALERT=0
for _ in $(seq 1 36); do
CENTRAL_ALERTS=$(curl -sf "${CENTRAL}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \
-H "Authorization: Bearer ${PHYSICIAN_JWT}" || echo "")
if echo "$CENTRAL_ALERTS" | jq -e \
'.data.items[]? | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true) | .acknowledgedBy == "RN-Wu"' \
>/dev/null 2>&1; then
SYNCED_ALERT=1
break
fi
sleep 5
done
[[ "$SYNCED_ALERT" -eq 1 ]] || die "Timed out waiting for gateway-synced RN-Wu ack on central"
BUFFER=$(curl -sf "${CENTRAL}/api/v1/operations/gateways" \
-H "Authorization: Bearer ${ADMIN_JWT}" \
| jq -r ".data[] | select(.id == \"${GW_ID}\") | .reportedBufferDepth // empty" \
| head -1)
log "Gateway reported buffer depth: ${BUFFER:-unknown}"
log "Verify central received alert acknowledgement"
CENTRAL_ALERTS=$(curl -sf "${CENTRAL}/api/v1/encounters/${ENCOUNTER_ID}/alerts" \
-H "Authorization: Bearer ${PHYSICIAN_JWT}")
echo "$CENTRAL_ALERTS" | jq -e \
'.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true)' \
>/dev/null \
|| die "Critical potassium alert not synced from gateway to central"
ACK_BY=$(echo "$CENTRAL_ALERTS" | jq -r \
'.data.items[] | select(.alertType | ascii_downcase | contains("critical") and contains("potassium")) | select(.syncedFromGateway == true) | .acknowledgedBy // empty' \
| head -1)
if [[ "$ACK_BY" == "RN-Wu" ]]; then
log "Central acknowledgedBy = RN-Wu"
else
log "Checking central DB for acknowledged_by (API returned: ${ACK_BY:-<empty>})"
docker exec "$(docker ps -qf name=postgres)" psql -U postgres -d vigilcare -tAc \
"SELECT acknowledged_by FROM clinical_alerts WHERE encounter_id = '${ENCOUNTER_ID}' AND alert_type = 'CRITICAL_POTASSIUM_MEQ_L' LIMIT 1;" \
| grep -q "RN-Wu" \
|| die "Central acknowledged_by is not RN-Wu after sync"
log "Central DB acknowledged_by = RN-Wu"
fi
log "Phase 24 verification passed."
log "Logs: $LOG_DIR"
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
BASE_URL="${BASE_URL:-http://localhost:5270}"
echo "=== Phase 33 verification ==="
dotnet test "${ROOT_DIR}/VigilCareClinicalAPI.Tests" \
--filter "FullyQualifiedName~AlertQualityAnalytics" --no-restore
TOKEN=$(curl -sf -X POST "${BASE_URL}/api/v1/auth/login" \
-H "Content-Type: application/json" \
-d '{"username":"nurse.demo","password":"DemoNurse1!"}' \
| jq -r '.data.accessToken')
ALERT_ID=$(curl -sf "${BASE_URL}/api/v1/alerts?status=ACKNOWLEDGED&pageSize=1" \
-H "Authorization: Bearer ${TOKEN}" | jq -r '.data.items[0].id')
if [[ "${ALERT_ID}" == "null" || -z "${ALERT_ID}" ]]; then
echo "No acknowledged alert found — acknowledge one first via simulator or API"
exit 1
fi
echo "Submit feedback for alert ${ALERT_ID}"
curl -sf -X POST "${BASE_URL}/api/v1/alerts/${ALERT_ID}/feedback" \
-H "Authorization: Bearer ${TOKEN}" \
-H "Content-Type: application/json" \
-d '{"feedbackType":"Useful","comment":"Phase 33 verification"}' \
| jq -e '.data.id != null'
echo "Verify quality metrics summary"
curl -sf "${BASE_URL}/api/v1/alerts/quality-metrics/summary" \
-H "Authorization: Bearer ${TOKEN}" | jq -e '.data.totalAlerts >= 0'
echo "Verify quality metrics list"
curl -sf "${BASE_URL}/api/v1/alerts/quality-metrics?from=$(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ)" \
-H "Authorization: Bearer ${TOKEN}" | jq -e '.data.items != null'
echo "Phase 33 verification complete."