feature: Observability: Prometheus Metrics and Grafana

VigilCareClinicalAPI/BackgroundServices/Metrics/AlertsUnacknowledgedCollector.cs

@@ -0,0 +1,54 @@
+using Microsoft.EntityFrameworkCore;
+
+public sealed class AlertsUnacknowledgedCollector : BackgroundService
+{
+    // 5 minutes matches the DLQ TTL — an alert that survived escalation is still open.
+    private static readonly TimeSpan UnacknowledgedThreshold = TimeSpan.FromMinutes(5);
+
+    private readonly IServiceScopeFactory _scopes;
+    private readonly ClinicalMetrics _metrics;
+    private readonly ILogger<AlertsUnacknowledgedCollector> _logger;
+
+    public AlertsUnacknowledgedCollector(
+        IServiceScopeFactory scopes,
+        ClinicalMetrics metrics,
+        ILogger<AlertsUnacknowledgedCollector> logger)
+    {
+        _scopes  = scopes;
+        _metrics = metrics;
+        _logger  = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken ct)
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromSeconds(30));
+        while (await timer.WaitForNextTickAsync(ct))
+            await CollectAsync(ct);
+    }
+
+    private async Task CollectAsync(CancellationToken ct)
+    {
+        try
+        {
+            await using var scope = _scopes.CreateAsyncScope();
+            var db     = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+            var cutoff = DateTimeOffset.UtcNow - UnacknowledgedThreshold;
+
+            var count = await db.ClinicalAlerts
+                .CountAsync(a => a.Severity   == AlertSeverity.Critical
+                              && a.Status      == AlertStatus.Open
+                              && a.TriggeredAt < cutoff, ct);
+
+            _metrics.AlertsUnacknowledgedGauge.Set(count);
+
+            if (count > 0)
+                _logger.LogWarning(
+                    "[PATIENT-SAFETY] alerts_unacknowledged_gauge={Count} " +
+                    "(CRITICAL alerts open > 5 min)", count);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "AlertsUnacknowledgedCollector failed");
+        }
+    }
+}

VigilCareClinicalAPI/BackgroundServices/Metrics/KafkaConsumerLagCollector.cs

@@ -0,0 +1,91 @@
+using Confluent.Kafka;
+using Confluent.Kafka.Admin;
+using Microsoft.Extensions.Options;
+
+public sealed class KafkaConsumerLagCollector : BackgroundService
+{
+    private static readonly string[] Groups =
+    {
+        "es-indexer",
+        "sepsis-engine",
+        "notification-publisher",
+        "data-lake-writer",
+    };
+
+    private readonly KafkaOptions _kafkaOptions;
+    private readonly ClinicalMetrics _metrics;
+    private readonly ILogger<KafkaConsumerLagCollector> _logger;
+
+    public KafkaConsumerLagCollector(
+        IOptions<KafkaOptions> kafkaOptions,
+        ClinicalMetrics metrics,
+        ILogger<KafkaConsumerLagCollector> logger)
+    {
+        _kafkaOptions = kafkaOptions.Value;
+        _metrics      = metrics;
+        _logger       = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken ct)
+    {
+        // Initial delay so Kafka is reachable before the first collection.
+        await Task.Delay(TimeSpan.FromSeconds(20), ct);
+
+        using var timer = new PeriodicTimer(TimeSpan.FromSeconds(30));
+        while (await timer.WaitForNextTickAsync(ct))
+        {
+            foreach (var group in Groups)
+            {
+                try { await CollectGroupLagAsync(group, ct); }
+                catch (Exception ex)
+                {
+                    _logger.LogDebug(ex, "KafkaConsumerLagCollector: failed for group {Group}", group);
+                }
+            }
+        }
+    }
+
+    private async Task CollectGroupLagAsync(string groupId, CancellationToken ct)
+    {
+        var adminConfig = new AdminClientConfig
+            { BootstrapServers = _kafkaOptions.BootstrapServers };
+
+        using var admin = new AdminClientBuilder(adminConfig).Build();
+
+        // List the committed offsets for this consumer group across all its partitions.
+        var result = await admin.ListConsumerGroupOffsetsAsync(
+            new[] { new ConsumerGroupTopicPartitions(groupId, null) },
+            new ListConsumerGroupOffsetsOptions { RequireStableOffsets = false });
+
+        var partitions = result.FirstOrDefault()?.Partitions ?? [];
+        if (!partitions.Any())
+        {
+            _metrics.KafkaConsumerLag.WithLabels(groupId).Set(0);
+            return;
+        }
+
+        // Query high watermarks using a temporary consumer (does not join the group).
+        using var tempConsumer = new ConsumerBuilder<Ignore, Ignore>(new ConsumerConfig
+        {
+            BootstrapServers = _kafkaOptions.BootstrapServers,
+            GroupId          = $"__lag-probe",
+        }).Build();
+
+        long totalLag = 0;
+        foreach (var tpo in partitions)
+        {
+            if (tpo.Error.IsError || tpo.Offset == Offset.Unset) continue;
+
+            var watermarks = tempConsumer.QueryWatermarkOffsets(
+                tpo.TopicPartition, TimeSpan.FromSeconds(5));
+
+            // Committed offset is the next offset the consumer will read.
+            // High watermark is the latest available offset + 1.
+            // Lag = messages the consumer has not yet read.
+            var lag = watermarks.High.Value - tpo.Offset.Value;
+            totalLag += Math.Max(0L, lag);
+        }
+
+        _metrics.KafkaConsumerLag.WithLabels(groupId).Set(totalLag);
+    }
+}

VigilCareClinicalAPI/BackgroundServices/Metrics/OutboxPendingCollector.cs

@@ -0,0 +1,42 @@
+using Microsoft.EntityFrameworkCore;
+
+public sealed class OutboxPendingCollector : BackgroundService
+{
+    private readonly IServiceScopeFactory _scopes;
+    private readonly ClinicalMetrics _metrics;
+    private readonly ILogger<OutboxPendingCollector> _logger;
+
+    public OutboxPendingCollector(
+        IServiceScopeFactory scopes,
+        ClinicalMetrics metrics,
+        ILogger<OutboxPendingCollector> logger)
+    {
+        _scopes  = scopes;
+        _metrics = metrics;
+        _logger  = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken ct)
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromSeconds(30));
+        while (await timer.WaitForNextTickAsync(ct))
+            await CollectAsync(ct);
+    }
+
+    private async Task CollectAsync(CancellationToken ct)
+    {
+        try
+        {
+            await using var scope = _scopes.CreateAsyncScope();
+            var db    = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+            var count = await db.OutboxEvents
+                .CountAsync(e => e.ProcessedAt == null, ct);
+
+            _metrics.OutboxPendingEvents.Set(count);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "OutboxPendingCollector failed");
+        }
+    }
+}