using Microsoft.EntityFrameworkCore;
///
/// Background service that periodically queries the database to update
/// gauge metrics for Prometheus. Runs every 30 seconds.
///
/// Metrics updated:
/// - digitization_batches_by_status: count per status
/// - digitization_queue_age_seconds: age of oldest PendingVerification batch
///
/// Design decisions:
/// - Uses IServiceScopeFactory (not injected AppDbContext) because
/// BackgroundService is a singleton and AppDbContext is scoped.
/// Each collection cycle creates a fresh scope.
/// - 30-second interval balances freshness against DB load. Prometheus
/// typically scrapes every 15-60 seconds, so 30 seconds ensures the
/// gauge is never more than one scrape interval stale.
/// - Explicit zero-setting for empty statuses prevents stale gauge values
/// from persisting after all batches of a status are processed.
/// - Uses UpdatedAt (not CreatedAt) for queue age because UpdatedAt
/// reflects when the batch entered PendingVerification.
///
public class MetricsCollectorService : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger _logger;
private static readonly TimeSpan CollectionInterval = TimeSpan.FromSeconds(30);
///
/// All batch statuses that should be reported as gauge values.
/// If a status has zero batches, the gauge is set to 0 (not omitted).
///
private static readonly BatchStatus[] AllStatuses =
{
BatchStatus.Uploaded,
BatchStatus.InEntry,
BatchStatus.PendingVerification,
BatchStatus.Rejected,
BatchStatus.Verified,
BatchStatus.AwaitingClinicalApproval,
BatchStatus.Approved,
BatchStatus.Promoted,
BatchStatus.Cancelled
};
public MetricsCollectorService(
IServiceScopeFactory scopeFactory,
ILogger logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation(
"MetricsCollectorService started. Collection interval: {Interval}s",
CollectionInterval.TotalSeconds);
while (!stoppingToken.IsCancellationRequested)
{
try
{
await CollectMetricsAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// Graceful shutdown — do not log as error
break;
}
catch (Exception ex)
{
_logger.LogError(ex,
"MetricsCollectorService failed to collect metrics");
// Continue running — transient DB errors should not kill the collector
}
await Task.Delay(CollectionInterval, stoppingToken);
}
_logger.LogInformation("MetricsCollectorService stopped");
}
private async Task CollectMetricsAsync(CancellationToken ct)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService();
// --- Batch counts by status ---
// Single query: GROUP BY status, returns dictionary
var statusCounts = await db.DigitizationBatches
.AsNoTracking()
.GroupBy(b => b.Status)
.Select(g => new { Status = g.Key, Count = g.Count() })
.ToDictionaryAsync(x => x.Status, x => x.Count, ct);
// Set gauge for every status — zero out statuses with no batches
foreach (var status in AllStatuses)
{
var count = statusCounts.GetValueOrDefault(status, 0);
DiagnosticsMetrics.BatchesByStatus
.WithLabels(status.ToDbString())
.Set(count);
}
// --- Queue age: oldest batch in PendingVerification ---
var oldestPendingUpdatedAt = await db.DigitizationBatches
.AsNoTracking()
.Where(b => b.Status == BatchStatus.PendingVerification)
.OrderBy(b => b.UpdatedAt)
.Select(b => (DateTimeOffset?)b.UpdatedAt)
.FirstOrDefaultAsync(ct);
if (oldestPendingUpdatedAt.HasValue)
{
var ageSeconds = (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds;
DiagnosticsMetrics.QueueAgeSeconds.Set(ageSeconds);
}
else
{
// No batches in PendingVerification — queue is empty
DiagnosticsMetrics.QueueAgeSeconds.Set(0);
}
// --- Promotion retry gauges ---
var pendingRetries = await db.PromotionAttempts
.AsNoTracking()
.CountAsync(a => !a.Succeeded && a.NextRetryAt != null, ct);
DiagnosticsMetrics.PromotionPendingRetries.Set(pendingRetries);
var exhaustedCount = await db.DigitizationBatches
.AsNoTracking()
.Where(b => b.Status == BatchStatus.Approved)
.Where(b => db.PromotionAttempts
.Any(a => a.BatchId == b.Id && !a.Succeeded && a.NextRetryAt == null))
.CountAsync(ct);
DiagnosticsMetrics.PromotionExhaustedTotal.Set(exhaustedCount);
// --- Approval queue age: oldest APPROVED batch ---
var oldestApprovedUpdatedAt = await db.DigitizationBatches
.AsNoTracking()
.Where(b => b.Status == BatchStatus.Approved)
.OrderBy(b => b.UpdatedAt)
.Select(b => (DateTimeOffset?)b.UpdatedAt)
.FirstOrDefaultAsync(ct);
DiagnosticsMetrics.ApprovalQueueAgeSeconds.Set(
oldestApprovedUpdatedAt.HasValue
? (DateTimeOffset.UtcNow - oldestApprovedUpdatedAt.Value).TotalSeconds
: 0);
_logger.LogDebug(
"Metrics collected: {StatusCount} status groups, queue age {QueueAge}s, " +
"pending retries {PendingRetries}, exhausted {Exhausted}",
statusCounts.Count,
oldestPendingUpdatedAt.HasValue
? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds
: 0,
pendingRetries,
exhaustedCount);
}
}