feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job

2026-06-27 13:29:04 +08:00
parent e22d33b654
commit 04bdb7e85c
31 changed files with 4482 additions and 259 deletions
@@ -0,0 +1,131 @@
+using Microsoft.EntityFrameworkCore;
+
+/// <summary>
+/// Background service that periodically queries the database to update
+/// gauge metrics for Prometheus. Runs every 30 seconds.
+///
+/// Metrics updated:
+/// - digitization_batches_by_status: count per status
+/// - digitization_queue_age_seconds: age of oldest PendingVerification batch
+///
+/// Design decisions:
+/// - Uses IServiceScopeFactory (not injected AppDbContext) because
+///   BackgroundService is a singleton and AppDbContext is scoped.
+///   Each collection cycle creates a fresh scope.
+/// - 30-second interval balances freshness against DB load. Prometheus
+///   typically scrapes every 15-60 seconds, so 30 seconds ensures the
+///   gauge is never more than one scrape interval stale.
+/// - Explicit zero-setting for empty statuses prevents stale gauge values
+///   from persisting after all batches of a status are processed.
+/// - Uses UpdatedAt (not CreatedAt) for queue age because UpdatedAt
+///   reflects when the batch entered PendingVerification.
+/// </summary>
+public class MetricsCollectorService : BackgroundService
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly ILogger<MetricsCollectorService> _logger;
+    private static readonly TimeSpan CollectionInterval = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    /// All batch statuses that should be reported as gauge values.
+    /// If a status has zero batches, the gauge is set to 0 (not omitted).
+    /// </summary>
+    private static readonly BatchStatus[] AllStatuses =
+    {
+        BatchStatus.Uploaded,
+        BatchStatus.InEntry,
+        BatchStatus.PendingVerification,
+        BatchStatus.Rejected,
+        BatchStatus.Verified,
+        BatchStatus.AwaitingClinicalApproval,
+        BatchStatus.Approved,
+        BatchStatus.Promoted
+    };
+
+    public MetricsCollectorService(
+        IServiceScopeFactory scopeFactory,
+        ILogger<MetricsCollectorService> logger)
+    {
+        _scopeFactory = scopeFactory;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation(
+            "MetricsCollectorService started. Collection interval: {Interval}s",
+            CollectionInterval.TotalSeconds);
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await CollectMetricsAsync(stoppingToken);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                // Graceful shutdown — do not log as error
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex,
+                    "MetricsCollectorService failed to collect metrics");
+                // Continue running — transient DB errors should not kill the collector
+            }
+
+            await Task.Delay(CollectionInterval, stoppingToken);
+        }
+
+        _logger.LogInformation("MetricsCollectorService stopped");
+    }
+
+    private async Task CollectMetricsAsync(CancellationToken ct)
+    {
+        using var scope = _scopeFactory.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+
+        // --- Batch counts by status ---
+        // Single query: GROUP BY status, returns dictionary
+        var statusCounts = await db.DigitizationBatches
+            .AsNoTracking()
+            .GroupBy(b => b.Status)
+            .Select(g => new { Status = g.Key, Count = g.Count() })
+            .ToDictionaryAsync(x => x.Status, x => x.Count, ct);
+
+        // Set gauge for every status — zero out statuses with no batches
+        foreach (var status in AllStatuses)
+        {
+            var count = statusCounts.GetValueOrDefault(status, 0);
+            DiagnosticsMetrics.BatchesByStatus
+                .WithLabels(status.ToDbString())
+                .Set(count);
+        }
+
+        // --- Queue age: oldest batch in PendingVerification ---
+        var oldestPendingUpdatedAt = await db.DigitizationBatches
+            .AsNoTracking()
+            .Where(b => b.Status == BatchStatus.PendingVerification)
+            .OrderBy(b => b.UpdatedAt)
+            .Select(b => (DateTimeOffset?)b.UpdatedAt)
+            .FirstOrDefaultAsync(ct);
+
+        if (oldestPendingUpdatedAt.HasValue)
+        {
+            var ageSeconds = (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds;
+            DiagnosticsMetrics.QueueAgeSeconds.Set(ageSeconds);
+        }
+        else
+        {
+            // No batches in PendingVerification — queue is empty
+            DiagnosticsMetrics.QueueAgeSeconds.Set(0);
+        }
+
+        _logger.LogDebug(
+            "Metrics collected: {StatusCount} status groups, queue age {QueueAge}s",
+            statusCounts.Count,
+            oldestPendingUpdatedAt.HasValue
+                ? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds
+                : 0);
+    }
+}