feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job
This commit is contained in:
voltsrage
@@ -0,0 +1,274 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Background service that retries failed promotion attempts with exponential backoff.
|
||||
///
|
||||
/// When VigilCareClinical is unreachable in a split deployment, promotion fails and
|
||||
/// the batch remains in Approved status. This service:
|
||||
/// 1. Finds batches in Approved status that have a failed PromotionAttempt
|
||||
/// with a NextRetryAt in the past.
|
||||
/// 2. Retries promotion via IPromotionService.
|
||||
/// 3. On failure, records a new PromotionAttempt with exponentially increasing NextRetryAt.
|
||||
/// 4. After MaxRetryAttempts, sets NextRetryAt to null (manual intervention required)
|
||||
/// and logs a critical warning.
|
||||
///
|
||||
/// The batch NEVER reverts to draft — it stays Approved until promotion succeeds
|
||||
/// or an administrator manually intervenes.
|
||||
/// </summary>
|
||||
public class PromotionRetryService : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly PromotionRetryOptions _options;
|
||||
private readonly ILogger<PromotionRetryService> _logger;
|
||||
|
||||
public PromotionRetryService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IOptions<PromotionRetryOptions> options,
|
||||
ILogger<PromotionRetryService> logger)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"PromotionRetryService started. Poll interval: {PollInterval}s, " +
|
||||
"max retries: {MaxRetries}, initial delay: {InitialDelay}s, " +
|
||||
"max delay: {MaxDelay}s, backoff multiplier: {Multiplier}",
|
||||
_options.PollIntervalSeconds,
|
||||
_options.MaxRetryAttempts,
|
||||
_options.InitialDelaySeconds,
|
||||
_options.MaxDelaySeconds,
|
||||
_options.BackoffMultiplier);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessPendingRetriesAsync(stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"PromotionRetryService encountered an error during processing");
|
||||
}
|
||||
|
||||
await Task.Delay(
|
||||
TimeSpan.FromSeconds(_options.PollIntervalSeconds), stoppingToken);
|
||||
}
|
||||
|
||||
_logger.LogInformation("PromotionRetryService stopped");
|
||||
}
|
||||
|
||||
private async Task ProcessPendingRetriesAsync(CancellationToken ct)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var promotionService = scope.ServiceProvider.GetRequiredService<IPromotionService>();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Find batches that need retry:
|
||||
// 1. Batch is in Approved status (not yet promoted)
|
||||
// 2. Has a failed PromotionAttempt with NextRetryAt <= now
|
||||
var pendingRetries = await db.PromotionAttempts
|
||||
.Include(a => a.Batch)
|
||||
.Where(a => !a.Succeeded
|
||||
&& a.NextRetryAt != null
|
||||
&& a.NextRetryAt <= now
|
||||
&& a.Batch.Status == BatchStatus.Approved)
|
||||
.OrderBy(a => a.NextRetryAt)
|
||||
.Take(10) // Process up to 10 retries per poll cycle
|
||||
.ToListAsync(ct);
|
||||
|
||||
if (pendingRetries.Count == 0)
|
||||
return;
|
||||
|
||||
_logger.LogInformation(
|
||||
"PromotionRetryService found {Count} batches pending retry",
|
||||
pendingRetries.Count);
|
||||
|
||||
foreach (var attempt in pendingRetries)
|
||||
{
|
||||
await RetryPromotionAsync(db, promotionService, attempt, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RetryPromotionAsync(
|
||||
AppDbContext db,
|
||||
IPromotionService promotionService,
|
||||
PromotionAttempt lastAttempt,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var batchId = lastAttempt.BatchId;
|
||||
var nextAttemptNumber = lastAttempt.AttemptNumber + 1;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Retrying promotion for batch {BatchId}, attempt {Attempt}/{MaxAttempts}",
|
||||
batchId, nextAttemptNumber, _options.MaxRetryAttempts);
|
||||
|
||||
try
|
||||
{
|
||||
// Verify batch is still in Approved status
|
||||
var batch = await db.DigitizationBatches.FindAsync(
|
||||
new object[] { batchId }, ct);
|
||||
|
||||
if (batch is null || batch.Status != BatchStatus.Approved)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Batch {BatchId} is no longer in Approved status, skipping retry",
|
||||
batchId);
|
||||
return;
|
||||
}
|
||||
|
||||
var approverUserId = batch.ApprovedByUserId
|
||||
?? throw new InvalidOperationException(
|
||||
$"Batch {batchId} is in Approved status but has no ApprovedByUserId");
|
||||
|
||||
// Attempt promotion with a unique idempotency key per attempt
|
||||
await promotionService.PromoteAsync(batchId, approverUserId);
|
||||
|
||||
// --- Success path ---
|
||||
var successAttempt = new PromotionAttempt
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
BatchId = batchId,
|
||||
AttemptNumber = nextAttemptNumber,
|
||||
Succeeded = true,
|
||||
ErrorMessage = null,
|
||||
AttemptedAt = DateTimeOffset.UtcNow,
|
||||
NextRetryAt = null
|
||||
};
|
||||
|
||||
db.PromotionAttempts.Add(successAttempt);
|
||||
|
||||
// Clear the previous attempt's NextRetryAt so it won't be picked up again
|
||||
lastAttempt.NextRetryAt = null;
|
||||
|
||||
// Audit event
|
||||
db.DigitizationEvents.Add(new DigitizationEvent
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
BatchId = batchId,
|
||||
EventType = DigitizationEventType.PromotionRetrySucceeded,
|
||||
ActorUserId = approverUserId,
|
||||
OccurredAt = DateTimeOffset.UtcNow,
|
||||
MetadataJson = JsonSerializer.Serialize(new
|
||||
{
|
||||
attemptNumber = nextAttemptNumber,
|
||||
retriedAt = DateTimeOffset.UtcNow
|
||||
})
|
||||
});
|
||||
|
||||
await db.SaveChangesAsync(ct);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Batch {BatchId} successfully promoted on retry attempt {Attempt}",
|
||||
batchId, nextAttemptNumber);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw; // Let the outer loop handle shutdown
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Promotion retry failed for batch {BatchId}, attempt {Attempt}",
|
||||
batchId, nextAttemptNumber);
|
||||
|
||||
// Calculate next retry delay with exponential backoff
|
||||
var delay = CalculateBackoffDelay(nextAttemptNumber);
|
||||
DateTimeOffset? nextRetryAt = null;
|
||||
|
||||
if (nextAttemptNumber < _options.MaxRetryAttempts)
|
||||
{
|
||||
nextRetryAt = DateTimeOffset.UtcNow.Add(delay);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Scheduling retry for batch {BatchId} at {NextRetryAt} " +
|
||||
"(delay: {DelaySec:F0}s, attempt {Attempt}/{Max})",
|
||||
batchId, nextRetryAt, delay.TotalSeconds,
|
||||
nextAttemptNumber, _options.MaxRetryAttempts);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Exhausted all retries — manual intervention required
|
||||
_logger.LogCritical(
|
||||
"Batch {BatchId} has exhausted all {Max} retry attempts. " +
|
||||
"Manual intervention required. Last error: {Error}",
|
||||
batchId, _options.MaxRetryAttempts, ex.Message);
|
||||
}
|
||||
|
||||
// Record failed attempt
|
||||
var failedAttempt = new PromotionAttempt
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
BatchId = batchId,
|
||||
AttemptNumber = nextAttemptNumber,
|
||||
Succeeded = false,
|
||||
ErrorMessage = ex.Message.Length > 2000
|
||||
? ex.Message[..2000]
|
||||
: ex.Message,
|
||||
AttemptedAt = DateTimeOffset.UtcNow,
|
||||
NextRetryAt = nextRetryAt
|
||||
};
|
||||
|
||||
db.PromotionAttempts.Add(failedAttempt);
|
||||
|
||||
// Clear the previous attempt's NextRetryAt
|
||||
lastAttempt.NextRetryAt = null;
|
||||
|
||||
// Audit event
|
||||
var batch = await db.DigitizationBatches.FindAsync(
|
||||
new object[] { batchId }, ct);
|
||||
var actorId = batch?.ApprovedByUserId ?? Guid.Empty;
|
||||
|
||||
db.DigitizationEvents.Add(new DigitizationEvent
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
BatchId = batchId,
|
||||
EventType = nextRetryAt.HasValue
|
||||
? DigitizationEventType.PromotionRetryFailed
|
||||
: DigitizationEventType.PromotionRetryExhausted,
|
||||
ActorUserId = actorId,
|
||||
OccurredAt = DateTimeOffset.UtcNow,
|
||||
MetadataJson = JsonSerializer.Serialize(new
|
||||
{
|
||||
attemptNumber = nextAttemptNumber,
|
||||
error = ex.Message,
|
||||
nextRetryAt = nextRetryAt?.ToString("o"),
|
||||
exhausted = !nextRetryAt.HasValue
|
||||
})
|
||||
});
|
||||
|
||||
await db.SaveChangesAsync(ct);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates exponential backoff delay for a given attempt number.
|
||||
/// Formula: min(initialDelay * multiplier^(attempt-1), maxDelay)
|
||||
/// With jitter: adds up to 10% random jitter to prevent thundering herd
|
||||
/// when multiple replicas retry simultaneously.
|
||||
/// </summary>
|
||||
private TimeSpan CalculateBackoffDelay(int attemptNumber)
|
||||
{
|
||||
var baseDelay = _options.InitialDelaySeconds
|
||||
* Math.Pow(_options.BackoffMultiplier, attemptNumber - 1);
|
||||
|
||||
var cappedDelay = Math.Min(baseDelay, _options.MaxDelaySeconds);
|
||||
|
||||
// Add up to 10% jitter
|
||||
var jitter = cappedDelay * 0.1 * Random.Shared.NextDouble();
|
||||
var finalDelay = cappedDelay + jitter;
|
||||
|
||||
return TimeSpan.FromSeconds(finalDelay);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user