vigilcare-records/VigilCareRecordsAPI/BackgroundServices/PromotionRetryService.cs

using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;

/// <summary>
/// Background service that retries failed promotion attempts with exponential backoff.
///
/// When VigilCareClinical is unreachable in a split deployment, promotion fails and
/// the batch remains in Approved status. This service:
/// 1. Finds batches in Approved status that have a failed PromotionAttempt
///    with a NextRetryAt in the past.
/// 2. Retries promotion via IPromotionService.
/// 3. On failure, records a new PromotionAttempt with exponentially increasing NextRetryAt.
/// 4. After MaxRetryAttempts, sets NextRetryAt to null (manual intervention required)
///    and logs a critical warning.
///
/// The batch NEVER reverts to draft — it stays Approved until promotion succeeds
/// or an administrator manually intervenes.
/// </summary>
public class PromotionRetryService : BackgroundService
{
    private readonly IServiceScopeFactory _scopeFactory;
    private readonly PromotionRetryOptions _options;
    private readonly ILogger<PromotionRetryService> _logger;

    public PromotionRetryService(
        IServiceScopeFactory scopeFactory,
        IOptions<PromotionRetryOptions> options,
        ILogger<PromotionRetryService> logger)
    {
        _scopeFactory = scopeFactory;
        _options = options.Value;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation(
            "PromotionRetryService started. Poll interval: {PollInterval}s, " +
            "max retries: {MaxRetries}, initial delay: {InitialDelay}s, " +
            "max delay: {MaxDelay}s, backoff multiplier: {Multiplier}",
            _options.PollIntervalSeconds,
            _options.MaxRetryAttempts,
            _options.InitialDelaySeconds,
            _options.MaxDelaySeconds,
            _options.BackoffMultiplier);

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                await ProcessPendingRetriesAsync(stoppingToken);
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                break;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex,
                    "PromotionRetryService encountered an error during processing");
            }

            await Task.Delay(
                TimeSpan.FromSeconds(_options.PollIntervalSeconds), stoppingToken);
        }

        _logger.LogInformation("PromotionRetryService stopped");
    }

    private async Task ProcessPendingRetriesAsync(CancellationToken ct)
    {
        using var scope = _scopeFactory.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
        var promotionService = scope.ServiceProvider.GetRequiredService<IPromotionService>();
        var now = DateTimeOffset.UtcNow;

        // Find batches that need retry:
        // 1. Batch is in Approved status (not yet promoted)
        // 2. Has a failed PromotionAttempt with NextRetryAt <= now
        var pendingRetries = await db.PromotionAttempts
            .Include(a => a.Batch)
            .Where(a => !a.Succeeded
                     && a.NextRetryAt != null
                     && a.NextRetryAt <= now
                     && a.Batch.Status == BatchStatus.Approved)
            .OrderBy(a => a.NextRetryAt)
            .Take(10) // Process up to 10 retries per poll cycle
            .ToListAsync(ct);

        if (pendingRetries.Count == 0)
            return;

        _logger.LogInformation(
            "PromotionRetryService found {Count} batches pending retry",
            pendingRetries.Count);

        foreach (var attempt in pendingRetries)
        {
            await RetryPromotionAsync(db, promotionService, attempt, ct);
        }
    }

    private async Task RetryPromotionAsync(
        AppDbContext db,
        IPromotionService promotionService,
        PromotionAttempt lastAttempt,
        CancellationToken ct)
    {
        var batchId = lastAttempt.BatchId;
        var nextAttemptNumber = lastAttempt.AttemptNumber + 1;

        _logger.LogInformation(
            "Retrying promotion for batch {BatchId}, attempt {Attempt}/{MaxAttempts}",
            batchId, nextAttemptNumber, _options.MaxRetryAttempts);

        try
        {
            // Verify batch is still in Approved status
            var batch = await db.DigitizationBatches.FindAsync(
                new object[] { batchId }, ct);

            if (batch is null || batch.Status != BatchStatus.Approved)
            {
                _logger.LogWarning(
                    "Batch {BatchId} is no longer in Approved status, skipping retry",
                    batchId);
                return;
            }

            var approverUserId = batch.ApprovedByUserId
                ?? throw new InvalidOperationException(
                    $"Batch {batchId} is in Approved status but has no ApprovedByUserId");

            // Attempt promotion with a unique idempotency key per attempt
            await promotionService.PromoteAsync(batchId, approverUserId);

            // --- Success path ---
            var successAttempt = new PromotionAttempt
            {
                Id = Guid.NewGuid(),
                BatchId = batchId,
                AttemptNumber = nextAttemptNumber,
                Succeeded = true,
                ErrorMessage = null,
                AttemptedAt = DateTimeOffset.UtcNow,
                NextRetryAt = null
            };

            db.PromotionAttempts.Add(successAttempt);

            // Clear the previous attempt's NextRetryAt so it won't be picked up again
            lastAttempt.NextRetryAt = null;

            // Audit event
            db.DigitizationEvents.Add(new DigitizationEvent
            {
                Id = Guid.NewGuid(),
                BatchId = batchId,
                EventType = DigitizationEventType.PromotionRetrySucceeded,
                ActorUserId = approverUserId,
                OccurredAt = DateTimeOffset.UtcNow,
                MetadataJson = JsonSerializer.Serialize(new
                {
                    attemptNumber = nextAttemptNumber,
                    retriedAt = DateTimeOffset.UtcNow
                })
            });

            await db.SaveChangesAsync(ct);

            _logger.LogInformation(
                "Batch {BatchId} successfully promoted on retry attempt {Attempt}",
                batchId, nextAttemptNumber);
        }
        catch (OperationCanceledException) when (ct.IsCancellationRequested)
        {
            throw; // Let the outer loop handle shutdown
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex,
                "Promotion retry failed for batch {BatchId}, attempt {Attempt}",
                batchId, nextAttemptNumber);

            // Calculate next retry delay with exponential backoff
            var delay = CalculateBackoffDelay(nextAttemptNumber);
            DateTimeOffset? nextRetryAt = null;

            if (nextAttemptNumber < _options.MaxRetryAttempts)
            {
                nextRetryAt = DateTimeOffset.UtcNow.Add(delay);

                _logger.LogInformation(
                    "Scheduling retry for batch {BatchId} at {NextRetryAt} " +
                    "(delay: {DelaySec:F0}s, attempt {Attempt}/{Max})",
                    batchId, nextRetryAt, delay.TotalSeconds,
                    nextAttemptNumber, _options.MaxRetryAttempts);
            }
            else
            {
                // Exhausted all retries — manual intervention required
                _logger.LogCritical(
                    "Batch {BatchId} has exhausted all {Max} retry attempts. " +
                    "Manual intervention required. Last error: {Error}",
                    batchId, _options.MaxRetryAttempts, ex.Message);
            }

            // Record failed attempt
            var failedAttempt = new PromotionAttempt
            {
                Id = Guid.NewGuid(),
                BatchId = batchId,
                AttemptNumber = nextAttemptNumber,
                Succeeded = false,
                ErrorMessage = ex.Message.Length > 2000
                    ? ex.Message[..2000]
                    : ex.Message,
                AttemptedAt = DateTimeOffset.UtcNow,
                NextRetryAt = nextRetryAt
            };

            db.PromotionAttempts.Add(failedAttempt);

            // Clear the previous attempt's NextRetryAt
            lastAttempt.NextRetryAt = null;

            // Audit event
            var batch = await db.DigitizationBatches.FindAsync(
                new object[] { batchId }, ct);
            var actorId = batch?.ApprovedByUserId ?? Guid.Empty;

            db.DigitizationEvents.Add(new DigitizationEvent
            {
                Id = Guid.NewGuid(),
                BatchId = batchId,
                EventType = nextRetryAt.HasValue
                    ? DigitizationEventType.PromotionRetryFailed
                    : DigitizationEventType.PromotionRetryExhausted,
                ActorUserId = actorId,
                OccurredAt = DateTimeOffset.UtcNow,
                MetadataJson = JsonSerializer.Serialize(new
                {
                    attemptNumber = nextAttemptNumber,
                    error = ex.Message,
                    nextRetryAt = nextRetryAt?.ToString("o"),
                    exhausted = !nextRetryAt.HasValue
                })
            });

            await db.SaveChangesAsync(ct);
        }
    }

    /// <summary>
    /// Calculates exponential backoff delay for a given attempt number.
    /// Formula: min(initialDelay * multiplier^(attempt-1), maxDelay)
    /// With jitter: adds up to 10% random jitter to prevent thundering herd
    /// when multiple replicas retry simultaneously.
    /// </summary>
    private TimeSpan CalculateBackoffDelay(int attemptNumber)
    {
        var baseDelay = _options.InitialDelaySeconds
            * Math.Pow(_options.BackoffMultiplier, attemptNumber - 1);

        var cappedDelay = Math.Min(baseDelay, _options.MaxDelaySeconds);

        // Add up to 10% jitter
        var jitter = cappedDelay * 0.1 * Random.Shared.NextDouble();
        var finalDelay = cappedDelay + jitter;

        return TimeSpan.FromSeconds(finalDelay);
    }
}