using System.Text.Json; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; /// /// Background service that retries failed promotion attempts with exponential backoff. /// /// When VigilCareClinical is unreachable in a split deployment, promotion fails and /// the batch remains in Approved status. This service: /// 1. Finds batches in Approved status that have a failed PromotionAttempt /// with a NextRetryAt in the past. /// 2. Retries promotion via IPromotionService. /// 3. On failure, records a new PromotionAttempt with exponentially increasing NextRetryAt. /// 4. After MaxRetryAttempts, sets NextRetryAt to null (manual intervention required) /// and logs a critical warning. /// /// The batch NEVER reverts to draft — it stays Approved until promotion succeeds /// or an administrator manually intervenes. /// public class PromotionRetryService : BackgroundService { private readonly IServiceScopeFactory _scopeFactory; private readonly PromotionRetryOptions _options; private readonly ILogger _logger; public PromotionRetryService( IServiceScopeFactory scopeFactory, IOptions options, ILogger logger) { _scopeFactory = scopeFactory; _options = options.Value; _logger = logger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation( "PromotionRetryService started. Poll interval: {PollInterval}s, " + "max retries: {MaxRetries}, initial delay: {InitialDelay}s, " + "max delay: {MaxDelay}s, backoff multiplier: {Multiplier}", _options.PollIntervalSeconds, _options.MaxRetryAttempts, _options.InitialDelaySeconds, _options.MaxDelaySeconds, _options.BackoffMultiplier); while (!stoppingToken.IsCancellationRequested) { try { await ProcessPendingRetriesAsync(stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } catch (Exception ex) { _logger.LogError(ex, "PromotionRetryService encountered an error during processing"); } await Task.Delay( TimeSpan.FromSeconds(_options.PollIntervalSeconds), stoppingToken); } _logger.LogInformation("PromotionRetryService stopped"); } private async Task ProcessPendingRetriesAsync(CancellationToken ct) { using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var promotionService = scope.ServiceProvider.GetRequiredService(); var now = DateTimeOffset.UtcNow; // Find batches that need retry: // 1. Batch is in Approved status (not yet promoted) // 2. Has a failed PromotionAttempt with NextRetryAt <= now var pendingRetries = await db.PromotionAttempts .Include(a => a.Batch) .Where(a => !a.Succeeded && a.NextRetryAt != null && a.NextRetryAt <= now && a.Batch.Status == BatchStatus.Approved) .OrderBy(a => a.NextRetryAt) .Take(10) // Process up to 10 retries per poll cycle .ToListAsync(ct); if (pendingRetries.Count == 0) return; _logger.LogInformation( "PromotionRetryService found {Count} batches pending retry", pendingRetries.Count); foreach (var attempt in pendingRetries) { await RetryPromotionAsync(db, promotionService, attempt, ct); } } private async Task RetryPromotionAsync( AppDbContext db, IPromotionService promotionService, PromotionAttempt lastAttempt, CancellationToken ct) { var batchId = lastAttempt.BatchId; var nextAttemptNumber = lastAttempt.AttemptNumber + 1; _logger.LogInformation( "Retrying promotion for batch {BatchId}, attempt {Attempt}/{MaxAttempts}", batchId, nextAttemptNumber, _options.MaxRetryAttempts); try { // Verify batch is still in Approved status var batch = await db.DigitizationBatches.FindAsync( new object[] { batchId }, ct); if (batch is null || batch.Status != BatchStatus.Approved) { _logger.LogWarning( "Batch {BatchId} is no longer in Approved status, skipping retry", batchId); return; } var approverUserId = batch.ApprovedByUserId ?? throw new InvalidOperationException( $"Batch {batchId} is in Approved status but has no ApprovedByUserId"); // Attempt promotion with a unique idempotency key per attempt await promotionService.PromoteAsync(batchId, approverUserId); // --- Success path --- DiagnosticsMetrics.PromotionRetryTotal.WithLabels("success").Inc(); var successAttempt = new PromotionAttempt { Id = Guid.NewGuid(), BatchId = batchId, AttemptNumber = nextAttemptNumber, Succeeded = true, ErrorMessage = null, AttemptedAt = DateTimeOffset.UtcNow, NextRetryAt = null }; db.PromotionAttempts.Add(successAttempt); // Clear the previous attempt's NextRetryAt so it won't be picked up again lastAttempt.NextRetryAt = null; // Audit event db.DigitizationEvents.Add(new DigitizationEvent { Id = Guid.NewGuid(), BatchId = batchId, EventType = DigitizationEventType.PromotionRetrySucceeded, ActorUserId = approverUserId, OccurredAt = DateTimeOffset.UtcNow, MetadataJson = JsonSerializer.Serialize(new { attemptNumber = nextAttemptNumber, retriedAt = DateTimeOffset.UtcNow }) }); await db.SaveChangesAsync(ct); _logger.LogInformation( "Batch {BatchId} successfully promoted on retry attempt {Attempt}", batchId, nextAttemptNumber); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; // Let the outer loop handle shutdown } catch (Exception ex) { _logger.LogWarning(ex, "Promotion retry failed for batch {BatchId}, attempt {Attempt}", batchId, nextAttemptNumber); // Calculate next retry delay with exponential backoff var delay = CalculateBackoffDelay(nextAttemptNumber); DateTimeOffset? nextRetryAt = null; if (nextAttemptNumber < _options.MaxRetryAttempts) { nextRetryAt = DateTimeOffset.UtcNow.Add(delay); DiagnosticsMetrics.PromotionRetryTotal.WithLabels("failure").Inc(); _logger.LogInformation( "Scheduling retry for batch {BatchId} at {NextRetryAt} " + "(delay: {DelaySec:F0}s, attempt {Attempt}/{Max})", batchId, nextRetryAt, delay.TotalSeconds, nextAttemptNumber, _options.MaxRetryAttempts); } else { DiagnosticsMetrics.PromotionRetryTotal.WithLabels("exhausted").Inc(); _logger.LogCritical( "Batch {BatchId} has exhausted all {Max} retry attempts. " + "Manual intervention required. Last error: {Error}", batchId, _options.MaxRetryAttempts, ex.Message); } // Record failed attempt var failedAttempt = new PromotionAttempt { Id = Guid.NewGuid(), BatchId = batchId, AttemptNumber = nextAttemptNumber, Succeeded = false, ErrorMessage = ex.Message.Length > 2000 ? ex.Message[..2000] : ex.Message, AttemptedAt = DateTimeOffset.UtcNow, NextRetryAt = nextRetryAt }; db.PromotionAttempts.Add(failedAttempt); // Clear the previous attempt's NextRetryAt lastAttempt.NextRetryAt = null; // Audit event var batch = await db.DigitizationBatches.FindAsync( new object[] { batchId }, ct); var actorId = batch?.ApprovedByUserId ?? Guid.Empty; db.DigitizationEvents.Add(new DigitizationEvent { Id = Guid.NewGuid(), BatchId = batchId, EventType = nextRetryAt.HasValue ? DigitizationEventType.PromotionRetryFailed : DigitizationEventType.PromotionRetryExhausted, ActorUserId = actorId, OccurredAt = DateTimeOffset.UtcNow, MetadataJson = JsonSerializer.Serialize(new { attemptNumber = nextAttemptNumber, error = ex.Message, nextRetryAt = nextRetryAt?.ToString("o"), exhausted = !nextRetryAt.HasValue }) }); await db.SaveChangesAsync(ct); } } /// /// Calculates exponential backoff delay for a given attempt number. /// Formula: min(initialDelay * multiplier^(attempt-1), maxDelay) /// With jitter: adds up to 10% random jitter to prevent thundering herd /// when multiple replicas retry simultaneously. /// private TimeSpan CalculateBackoffDelay(int attemptNumber) { var baseDelay = _options.InitialDelaySeconds * Math.Pow(_options.BackoffMultiplier, attemptNumber - 1); var cappedDelay = Math.Min(baseDelay, _options.MaxDelaySeconds); // Add up to 10% jitter var jitter = cappedDelay * 0.1 * Random.Shared.NextDouble(); var finalDelay = cappedDelay + jitter; return TimeSpan.FromSeconds(finalDelay); } }