feature: Observability: Prometheus Metrics and Grafana

VigilCareClinicalAPI/BackgroundServices/Notifications/EscalationWorkerService.cs

@@ -3,21 +3,25 @@ using System.Text.Json;
 using Microsoft.Extensions.Options;
 using RabbitMQ.Client;
 using RabbitMQ.Client.Events;
+using Serilog.Context;
 
 public sealed class EscalationWorkerService : BackgroundService
 {
     private readonly IOptions<RabbitMqOptions> _opts;
     private readonly IServiceScopeFactory _scopes;
+    private readonly ClinicalMetrics _metrics;
     private readonly ILogger<EscalationWorkerService> _logger;
 
     public EscalationWorkerService(
         IOptions<RabbitMqOptions> opts,
         IServiceScopeFactory scopes,
+        ClinicalMetrics metrics,
         ILogger<EscalationWorkerService> logger)
     {
-        _opts   = opts;
-        _scopes = scopes;
-        _logger = logger;
+        _opts    = opts;
+        _scopes  = scopes;
+        _metrics = metrics;
+        _logger  = logger;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -57,15 +61,20 @@ public sealed class EscalationWorkerService : BackgroundService
         var payload     = Encoding.UTF8.GetString(ea.Body.Span);
         var doc         = JsonDocument.Parse(payload);
         var alertId     = Guid.Parse(doc.RootElement.GetProperty("alertId").GetString()!);
-        var encounterId = doc.RootElement.GetProperty("encounterId").GetString();
+        var encounterId = Guid.Parse(doc.RootElement.GetProperty("encounterId").GetString()!);
 
-        _logger.LogCritical(
-            "[ESCALATION] Paging on-call backup — AlertId={AlertId} EncounterId={EncounterId}",
-            alertId, encounterId);
+        using (LogContext.PushProperty("EncounterId", encounterId))
+        using (LogContext.PushProperty("AlertId", alertId))
+        {
+            _logger.LogCritical("[ESCALATION] Paging on-call backup for alert {AlertId}", alertId);
+        }
 
         try
         {
-            await UpdateAlertStatusEscalatedAsync(alertId, ct);
+            var escalated = await UpdateAlertStatusEscalatedAsync(alertId, ct);
+            if (escalated)
+                _metrics.EscalationsTotal.Inc();
+
             channel.BasicAck(ea.DeliveryTag, multiple: false);
 
             _logger.LogWarning(
@@ -78,18 +87,19 @@ public sealed class EscalationWorkerService : BackgroundService
         }
     }
 
-    private async Task UpdateAlertStatusEscalatedAsync(Guid alertId, CancellationToken ct)
+    private async Task<bool> UpdateAlertStatusEscalatedAsync(Guid alertId, CancellationToken ct)
     {
         await using var scope = _scopes.CreateAsyncScope();
         var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
 
         var alert = await db.ClinicalAlerts.FindAsync(new object[] { alertId }, ct);
-        if (alert is null) return;
+        if (alert is null) return false;
 
         // Only escalate if still open — if acknowledged between NACK and TTL expiry, leave it.
-        if (alert.Status != AlertStatus.Open) return;
+        if (alert.Status != AlertStatus.Open) return false;
 
         alert.Status = AlertStatus.Escalated;
         await db.SaveChangesAsync(ct);
+        return true;
     }
 }

VigilCareClinicalAPI/BackgroundServices/Notifications/PagingWorkerService.cs

@@ -51,6 +51,12 @@ public sealed class PagingWorkerService : BackgroundService
             {
                 await HandlePageAsync(channel, ea, stoppingToken);
             }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                // Host is stopping while we were waiting for ack — requeue so restart does not false-escalate.
+                _logger.LogInformation("PagingWorker stopping — requeueing in-flight page message");
+                channel.BasicNack(ea.DeliveryTag, multiple: false, requeue: true);
+            }
             catch (Exception ex)
             {
                 _logger.LogError(ex, "PagingWorker failed — NACKing to DLQ");