vigilcare-records/VigilCareRecordsAPI/BackgroundServices/MetricsCollectorService.cs

using Microsoft.EntityFrameworkCore;

/// <summary>
/// Background service that periodically queries the database to update
/// gauge metrics for Prometheus. Runs every 30 seconds.
///
/// Metrics updated:
/// - digitization_batches_by_status: count per status
/// - digitization_queue_age_seconds: age of oldest PendingVerification batch
///
/// Design decisions:
/// - Uses IServiceScopeFactory (not injected AppDbContext) because
///   BackgroundService is a singleton and AppDbContext is scoped.
///   Each collection cycle creates a fresh scope.
/// - 30-second interval balances freshness against DB load. Prometheus
///   typically scrapes every 15-60 seconds, so 30 seconds ensures the
///   gauge is never more than one scrape interval stale.
/// - Explicit zero-setting for empty statuses prevents stale gauge values
///   from persisting after all batches of a status are processed.
/// - Uses UpdatedAt (not CreatedAt) for queue age because UpdatedAt
///   reflects when the batch entered PendingVerification.
/// </summary>
public class MetricsCollectorService : BackgroundService
{
    private readonly IServiceScopeFactory _scopeFactory;
    private readonly ILogger<MetricsCollectorService> _logger;
    private static readonly TimeSpan CollectionInterval = TimeSpan.FromSeconds(30);

    /// <summary>
    /// All batch statuses that should be reported as gauge values.
    /// If a status has zero batches, the gauge is set to 0 (not omitted).
    /// </summary>
    private static readonly BatchStatus[] AllStatuses =
    {
        BatchStatus.Uploaded,
        BatchStatus.InEntry,
        BatchStatus.PendingVerification,
        BatchStatus.Rejected,
        BatchStatus.Verified,
        BatchStatus.AwaitingClinicalApproval,
        BatchStatus.Approved,
        BatchStatus.Promoted,
        BatchStatus.Cancelled
    };

    public MetricsCollectorService(
        IServiceScopeFactory scopeFactory,
        ILogger<MetricsCollectorService> logger)
    {
        _scopeFactory = scopeFactory;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation(
            "MetricsCollectorService started. Collection interval: {Interval}s",
            CollectionInterval.TotalSeconds);

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                await CollectMetricsAsync(stoppingToken);
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                // Graceful shutdown — do not log as error
                break;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex,
                    "MetricsCollectorService failed to collect metrics");
                // Continue running — transient DB errors should not kill the collector
            }

            await Task.Delay(CollectionInterval, stoppingToken);
        }

        _logger.LogInformation("MetricsCollectorService stopped");
    }

    private async Task CollectMetricsAsync(CancellationToken ct)
    {
        using var scope = _scopeFactory.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

        // --- Batch counts by status ---
        // Single query: GROUP BY status, returns dictionary
        var statusCounts = await db.DigitizationBatches
            .AsNoTracking()
            .GroupBy(b => b.Status)
            .Select(g => new { Status = g.Key, Count = g.Count() })
            .ToDictionaryAsync(x => x.Status, x => x.Count, ct);

        // Set gauge for every status — zero out statuses with no batches
        foreach (var status in AllStatuses)
        {
            var count = statusCounts.GetValueOrDefault(status, 0);
            DiagnosticsMetrics.BatchesByStatus
                .WithLabels(status.ToDbString())
                .Set(count);
        }

        // --- Queue age: oldest batch in PendingVerification ---
        var oldestPendingUpdatedAt = await db.DigitizationBatches
            .AsNoTracking()
            .Where(b => b.Status == BatchStatus.PendingVerification)
            .OrderBy(b => b.UpdatedAt)
            .Select(b => (DateTimeOffset?)b.UpdatedAt)
            .FirstOrDefaultAsync(ct);

        if (oldestPendingUpdatedAt.HasValue)
        {
            var ageSeconds = (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds;
            DiagnosticsMetrics.QueueAgeSeconds.Set(ageSeconds);
        }
        else
        {
            // No batches in PendingVerification — queue is empty
            DiagnosticsMetrics.QueueAgeSeconds.Set(0);
        }

        // --- Promotion retry gauges ---
        var pendingRetries = await db.PromotionAttempts
            .AsNoTracking()
            .CountAsync(a => !a.Succeeded && a.NextRetryAt != null, ct);
        DiagnosticsMetrics.PromotionPendingRetries.Set(pendingRetries);

        var exhaustedCount = await db.DigitizationBatches
            .AsNoTracking()
            .Where(b => b.Status == BatchStatus.Approved)
            .Where(b => db.PromotionAttempts
                .Any(a => a.BatchId == b.Id && !a.Succeeded && a.NextRetryAt == null))
            .CountAsync(ct);
        DiagnosticsMetrics.PromotionExhaustedTotal.Set(exhaustedCount);

        // --- Approval queue age: oldest APPROVED batch ---
        var oldestApprovedUpdatedAt = await db.DigitizationBatches
            .AsNoTracking()
            .Where(b => b.Status == BatchStatus.Approved)
            .OrderBy(b => b.UpdatedAt)
            .Select(b => (DateTimeOffset?)b.UpdatedAt)
            .FirstOrDefaultAsync(ct);

        DiagnosticsMetrics.ApprovalQueueAgeSeconds.Set(
            oldestApprovedUpdatedAt.HasValue
                ? (DateTimeOffset.UtcNow - oldestApprovedUpdatedAt.Value).TotalSeconds
                : 0);

        _logger.LogDebug(
            "Metrics collected: {StatusCount} status groups, queue age {QueueAge}s, " +
            "pending retries {PendingRetries}, exhausted {Exhausted}",
            statusCounts.Count,
            oldestPendingUpdatedAt.HasValue
                ? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds
                : 0,
            pendingRetries,
            exhaustedCount);
    }
}