feature: Observability: Prometheus Metrics and Grafana

2026-06-17 16:25:27 +08:00
parent 101040f9d9
commit df99bf3c91
22 changed files with 1462 additions and 132 deletions
@@ -0,0 +1,54 @@
+using Microsoft.EntityFrameworkCore;
+
+public sealed class AlertsUnacknowledgedCollector : BackgroundService
+{
+    // 5 minutes matches the DLQ TTL — an alert that survived escalation is still open.
+    private static readonly TimeSpan UnacknowledgedThreshold = TimeSpan.FromMinutes(5);
+
+    private readonly IServiceScopeFactory _scopes;
+    private readonly ClinicalMetrics _metrics;
+    private readonly ILogger<AlertsUnacknowledgedCollector> _logger;
+
+    public AlertsUnacknowledgedCollector(
+        IServiceScopeFactory scopes,
+        ClinicalMetrics metrics,
+        ILogger<AlertsUnacknowledgedCollector> logger)
+    {
+        _scopes  = scopes;
+        _metrics = metrics;
+        _logger  = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken ct)
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromSeconds(30));
+        while (await timer.WaitForNextTickAsync(ct))
+            await CollectAsync(ct);
+    }
+
+    private async Task CollectAsync(CancellationToken ct)
+    {
+        try
+        {
+            await using var scope = _scopes.CreateAsyncScope();
+            var db     = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+            var cutoff = DateTimeOffset.UtcNow - UnacknowledgedThreshold;
+
+            var count = await db.ClinicalAlerts
+                .CountAsync(a => a.Severity   == AlertSeverity.Critical
+                              && a.Status      == AlertStatus.Open
+                              && a.TriggeredAt < cutoff, ct);
+
+            _metrics.AlertsUnacknowledgedGauge.Set(count);
+
+            if (count > 0)
+                _logger.LogWarning(
+                    "[PATIENT-SAFETY] alerts_unacknowledged_gauge={Count} " +
+                    "(CRITICAL alerts open > 5 min)", count);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "AlertsUnacknowledgedCollector failed");
+        }
+    }
+}