using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
///
/// Background service that retries failed promotion attempts with exponential backoff.
///
/// When VigilCareClinical is unreachable in a split deployment, promotion fails and
/// the batch remains in Approved status. This service:
/// 1. Finds batches in Approved status that have a failed PromotionAttempt
/// with a NextRetryAt in the past.
/// 2. Retries promotion via IPromotionService.
/// 3. On failure, records a new PromotionAttempt with exponentially increasing NextRetryAt.
/// 4. After MaxRetryAttempts, sets NextRetryAt to null (manual intervention required)
/// and logs a critical warning.
///
/// The batch NEVER reverts to draft — it stays Approved until promotion succeeds
/// or an administrator manually intervenes.
///
public class PromotionRetryService : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly PromotionRetryOptions _options;
private readonly ILogger _logger;
public PromotionRetryService(
IServiceScopeFactory scopeFactory,
IOptions options,
ILogger logger)
{
_scopeFactory = scopeFactory;
_options = options.Value;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation(
"PromotionRetryService started. Poll interval: {PollInterval}s, " +
"max retries: {MaxRetries}, initial delay: {InitialDelay}s, " +
"max delay: {MaxDelay}s, backoff multiplier: {Multiplier}",
_options.PollIntervalSeconds,
_options.MaxRetryAttempts,
_options.InitialDelaySeconds,
_options.MaxDelaySeconds,
_options.BackoffMultiplier);
while (!stoppingToken.IsCancellationRequested)
{
try
{
await ProcessPendingRetriesAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex,
"PromotionRetryService encountered an error during processing");
}
await Task.Delay(
TimeSpan.FromSeconds(_options.PollIntervalSeconds), stoppingToken);
}
_logger.LogInformation("PromotionRetryService stopped");
}
private async Task ProcessPendingRetriesAsync(CancellationToken ct)
{
using var scope = _scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService();
var promotionService = scope.ServiceProvider.GetRequiredService();
var now = DateTimeOffset.UtcNow;
// Find batches that need retry:
// 1. Batch is in Approved status (not yet promoted)
// 2. Has a failed PromotionAttempt with NextRetryAt <= now
var pendingRetries = await db.PromotionAttempts
.Include(a => a.Batch)
.Where(a => !a.Succeeded
&& a.NextRetryAt != null
&& a.NextRetryAt <= now
&& a.Batch.Status == BatchStatus.Approved)
.OrderBy(a => a.NextRetryAt)
.Take(10) // Process up to 10 retries per poll cycle
.ToListAsync(ct);
if (pendingRetries.Count == 0)
return;
_logger.LogInformation(
"PromotionRetryService found {Count} batches pending retry",
pendingRetries.Count);
foreach (var attempt in pendingRetries)
{
await RetryPromotionAsync(db, promotionService, attempt, ct);
}
}
private async Task RetryPromotionAsync(
AppDbContext db,
IPromotionService promotionService,
PromotionAttempt lastAttempt,
CancellationToken ct)
{
var batchId = lastAttempt.BatchId;
var nextAttemptNumber = lastAttempt.AttemptNumber + 1;
_logger.LogInformation(
"Retrying promotion for batch {BatchId}, attempt {Attempt}/{MaxAttempts}",
batchId, nextAttemptNumber, _options.MaxRetryAttempts);
try
{
// Verify batch is still in Approved status
var batch = await db.DigitizationBatches.FindAsync(
new object[] { batchId }, ct);
if (batch is null || batch.Status != BatchStatus.Approved)
{
_logger.LogWarning(
"Batch {BatchId} is no longer in Approved status, skipping retry",
batchId);
return;
}
var approverUserId = batch.ApprovedByUserId
?? throw new InvalidOperationException(
$"Batch {batchId} is in Approved status but has no ApprovedByUserId");
// Attempt promotion with a unique idempotency key per attempt
await promotionService.PromoteAsync(batchId, approverUserId);
// --- Success path ---
var successAttempt = new PromotionAttempt
{
Id = Guid.NewGuid(),
BatchId = batchId,
AttemptNumber = nextAttemptNumber,
Succeeded = true,
ErrorMessage = null,
AttemptedAt = DateTimeOffset.UtcNow,
NextRetryAt = null
};
db.PromotionAttempts.Add(successAttempt);
// Clear the previous attempt's NextRetryAt so it won't be picked up again
lastAttempt.NextRetryAt = null;
// Audit event
db.DigitizationEvents.Add(new DigitizationEvent
{
Id = Guid.NewGuid(),
BatchId = batchId,
EventType = DigitizationEventType.PromotionRetrySucceeded,
ActorUserId = approverUserId,
OccurredAt = DateTimeOffset.UtcNow,
MetadataJson = JsonSerializer.Serialize(new
{
attemptNumber = nextAttemptNumber,
retriedAt = DateTimeOffset.UtcNow
})
});
await db.SaveChangesAsync(ct);
_logger.LogInformation(
"Batch {BatchId} successfully promoted on retry attempt {Attempt}",
batchId, nextAttemptNumber);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
throw; // Let the outer loop handle shutdown
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Promotion retry failed for batch {BatchId}, attempt {Attempt}",
batchId, nextAttemptNumber);
// Calculate next retry delay with exponential backoff
var delay = CalculateBackoffDelay(nextAttemptNumber);
DateTimeOffset? nextRetryAt = null;
if (nextAttemptNumber < _options.MaxRetryAttempts)
{
nextRetryAt = DateTimeOffset.UtcNow.Add(delay);
_logger.LogInformation(
"Scheduling retry for batch {BatchId} at {NextRetryAt} " +
"(delay: {DelaySec:F0}s, attempt {Attempt}/{Max})",
batchId, nextRetryAt, delay.TotalSeconds,
nextAttemptNumber, _options.MaxRetryAttempts);
}
else
{
// Exhausted all retries — manual intervention required
_logger.LogCritical(
"Batch {BatchId} has exhausted all {Max} retry attempts. " +
"Manual intervention required. Last error: {Error}",
batchId, _options.MaxRetryAttempts, ex.Message);
}
// Record failed attempt
var failedAttempt = new PromotionAttempt
{
Id = Guid.NewGuid(),
BatchId = batchId,
AttemptNumber = nextAttemptNumber,
Succeeded = false,
ErrorMessage = ex.Message.Length > 2000
? ex.Message[..2000]
: ex.Message,
AttemptedAt = DateTimeOffset.UtcNow,
NextRetryAt = nextRetryAt
};
db.PromotionAttempts.Add(failedAttempt);
// Clear the previous attempt's NextRetryAt
lastAttempt.NextRetryAt = null;
// Audit event
var batch = await db.DigitizationBatches.FindAsync(
new object[] { batchId }, ct);
var actorId = batch?.ApprovedByUserId ?? Guid.Empty;
db.DigitizationEvents.Add(new DigitizationEvent
{
Id = Guid.NewGuid(),
BatchId = batchId,
EventType = nextRetryAt.HasValue
? DigitizationEventType.PromotionRetryFailed
: DigitizationEventType.PromotionRetryExhausted,
ActorUserId = actorId,
OccurredAt = DateTimeOffset.UtcNow,
MetadataJson = JsonSerializer.Serialize(new
{
attemptNumber = nextAttemptNumber,
error = ex.Message,
nextRetryAt = nextRetryAt?.ToString("o"),
exhausted = !nextRetryAt.HasValue
})
});
await db.SaveChangesAsync(ct);
}
}
///
/// Calculates exponential backoff delay for a given attempt number.
/// Formula: min(initialDelay * multiplier^(attempt-1), maxDelay)
/// With jitter: adds up to 10% random jitter to prevent thundering herd
/// when multiple replicas retry simultaneously.
///
private TimeSpan CalculateBackoffDelay(int attemptNumber)
{
var baseDelay = _options.InitialDelaySeconds
* Math.Pow(_options.BackoffMultiplier, attemptNumber - 1);
var cappedDelay = Math.Min(baseDelay, _options.MaxDelaySeconds);
// Add up to 10% jitter
var jitter = cappedDelay * 0.1 * Random.Shared.NextDouble();
var finalDelay = cappedDelay + jitter;
return TimeSpan.FromSeconds(finalDelay);
}
}