using Microsoft.EntityFrameworkCore; /// /// Background service that periodically queries the database to update /// gauge metrics for Prometheus. Runs every 30 seconds. /// /// Metrics updated: /// - digitization_batches_by_status: count per status /// - digitization_queue_age_seconds: age of oldest PendingVerification batch /// /// Design decisions: /// - Uses IServiceScopeFactory (not injected AppDbContext) because /// BackgroundService is a singleton and AppDbContext is scoped. /// Each collection cycle creates a fresh scope. /// - 30-second interval balances freshness against DB load. Prometheus /// typically scrapes every 15-60 seconds, so 30 seconds ensures the /// gauge is never more than one scrape interval stale. /// - Explicit zero-setting for empty statuses prevents stale gauge values /// from persisting after all batches of a status are processed. /// - Uses UpdatedAt (not CreatedAt) for queue age because UpdatedAt /// reflects when the batch entered PendingVerification. /// public class MetricsCollectorService : BackgroundService { private readonly IServiceScopeFactory _scopeFactory; private readonly ILogger _logger; private static readonly TimeSpan CollectionInterval = TimeSpan.FromSeconds(30); /// /// All batch statuses that should be reported as gauge values. /// If a status has zero batches, the gauge is set to 0 (not omitted). /// private static readonly BatchStatus[] AllStatuses = { BatchStatus.Uploaded, BatchStatus.InEntry, BatchStatus.PendingVerification, BatchStatus.Rejected, BatchStatus.Verified, BatchStatus.AwaitingClinicalApproval, BatchStatus.Approved, BatchStatus.Promoted, BatchStatus.Cancelled }; public MetricsCollectorService( IServiceScopeFactory scopeFactory, ILogger logger) { _scopeFactory = scopeFactory; _logger = logger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation( "MetricsCollectorService started. Collection interval: {Interval}s", CollectionInterval.TotalSeconds); while (!stoppingToken.IsCancellationRequested) { try { await CollectMetricsAsync(stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { // Graceful shutdown — do not log as error break; } catch (Exception ex) { _logger.LogError(ex, "MetricsCollectorService failed to collect metrics"); // Continue running — transient DB errors should not kill the collector } await Task.Delay(CollectionInterval, stoppingToken); } _logger.LogInformation("MetricsCollectorService stopped"); } private async Task CollectMetricsAsync(CancellationToken ct) { using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); // --- Batch counts by status --- // Single query: GROUP BY status, returns dictionary var statusCounts = await db.DigitizationBatches .AsNoTracking() .GroupBy(b => b.Status) .Select(g => new { Status = g.Key, Count = g.Count() }) .ToDictionaryAsync(x => x.Status, x => x.Count, ct); // Set gauge for every status — zero out statuses with no batches foreach (var status in AllStatuses) { var count = statusCounts.GetValueOrDefault(status, 0); DiagnosticsMetrics.BatchesByStatus .WithLabels(status.ToDbString()) .Set(count); } // --- Queue age: oldest batch in PendingVerification --- var oldestPendingUpdatedAt = await db.DigitizationBatches .AsNoTracking() .Where(b => b.Status == BatchStatus.PendingVerification) .OrderBy(b => b.UpdatedAt) .Select(b => (DateTimeOffset?)b.UpdatedAt) .FirstOrDefaultAsync(ct); if (oldestPendingUpdatedAt.HasValue) { var ageSeconds = (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds; DiagnosticsMetrics.QueueAgeSeconds.Set(ageSeconds); } else { // No batches in PendingVerification — queue is empty DiagnosticsMetrics.QueueAgeSeconds.Set(0); } // --- Promotion retry gauges --- var pendingRetries = await db.PromotionAttempts .AsNoTracking() .CountAsync(a => !a.Succeeded && a.NextRetryAt != null, ct); DiagnosticsMetrics.PromotionPendingRetries.Set(pendingRetries); var exhaustedCount = await db.DigitizationBatches .AsNoTracking() .Where(b => b.Status == BatchStatus.Approved) .Where(b => db.PromotionAttempts .Any(a => a.BatchId == b.Id && !a.Succeeded && a.NextRetryAt == null)) .CountAsync(ct); DiagnosticsMetrics.PromotionExhaustedTotal.Set(exhaustedCount); // --- Approval queue age: oldest APPROVED batch --- var oldestApprovedUpdatedAt = await db.DigitizationBatches .AsNoTracking() .Where(b => b.Status == BatchStatus.Approved) .OrderBy(b => b.UpdatedAt) .Select(b => (DateTimeOffset?)b.UpdatedAt) .FirstOrDefaultAsync(ct); DiagnosticsMetrics.ApprovalQueueAgeSeconds.Set( oldestApprovedUpdatedAt.HasValue ? (DateTimeOffset.UtcNow - oldestApprovedUpdatedAt.Value).TotalSeconds : 0); _logger.LogDebug( "Metrics collected: {StatusCount} status groups, queue age {QueueAge}s, " + "pending retries {PendingRetries}, exhausted {Exhausted}", statusCounts.Count, oldestPendingUpdatedAt.HasValue ? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds : 0, pendingRetries, exhaustedCount); } }