feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job

2026-06-27 13:29:04 +08:00
parent e22d33b654
commit 04bdb7e85c
31 changed files with 4482 additions and 259 deletions
@@ -0,0 +1,131 @@
+using Microsoft.EntityFrameworkCore;
+
+/// <summary>
+/// Background service that periodically queries the database to update
+/// gauge metrics for Prometheus. Runs every 30 seconds.
+///
+/// Metrics updated:
+/// - digitization_batches_by_status: count per status
+/// - digitization_queue_age_seconds: age of oldest PendingVerification batch
+///
+/// Design decisions:
+/// - Uses IServiceScopeFactory (not injected AppDbContext) because
+///   BackgroundService is a singleton and AppDbContext is scoped.
+///   Each collection cycle creates a fresh scope.
+/// - 30-second interval balances freshness against DB load. Prometheus
+///   typically scrapes every 15-60 seconds, so 30 seconds ensures the
+///   gauge is never more than one scrape interval stale.
+/// - Explicit zero-setting for empty statuses prevents stale gauge values
+///   from persisting after all batches of a status are processed.
+/// - Uses UpdatedAt (not CreatedAt) for queue age because UpdatedAt
+///   reflects when the batch entered PendingVerification.
+/// </summary>
+public class MetricsCollectorService : BackgroundService
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly ILogger<MetricsCollectorService> _logger;
+    private static readonly TimeSpan CollectionInterval = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    /// All batch statuses that should be reported as gauge values.
+    /// If a status has zero batches, the gauge is set to 0 (not omitted).
+    /// </summary>
+    private static readonly BatchStatus[] AllStatuses =
+    {
+        BatchStatus.Uploaded,
+        BatchStatus.InEntry,
+        BatchStatus.PendingVerification,
+        BatchStatus.Rejected,
+        BatchStatus.Verified,
+        BatchStatus.AwaitingClinicalApproval,
+        BatchStatus.Approved,
+        BatchStatus.Promoted
+    };
+
+    public MetricsCollectorService(
+        IServiceScopeFactory scopeFactory,
+        ILogger<MetricsCollectorService> logger)
+    {
+        _scopeFactory = scopeFactory;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation(
+            "MetricsCollectorService started. Collection interval: {Interval}s",
+            CollectionInterval.TotalSeconds);
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await CollectMetricsAsync(stoppingToken);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                // Graceful shutdown — do not log as error
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex,
+                    "MetricsCollectorService failed to collect metrics");
+                // Continue running — transient DB errors should not kill the collector
+            }
+
+            await Task.Delay(CollectionInterval, stoppingToken);
+        }
+
+        _logger.LogInformation("MetricsCollectorService stopped");
+    }
+
+    private async Task CollectMetricsAsync(CancellationToken ct)
+    {
+        using var scope = _scopeFactory.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+
+        // --- Batch counts by status ---
+        // Single query: GROUP BY status, returns dictionary
+        var statusCounts = await db.DigitizationBatches
+            .AsNoTracking()
+            .GroupBy(b => b.Status)
+            .Select(g => new { Status = g.Key, Count = g.Count() })
+            .ToDictionaryAsync(x => x.Status, x => x.Count, ct);
+
+        // Set gauge for every status — zero out statuses with no batches
+        foreach (var status in AllStatuses)
+        {
+            var count = statusCounts.GetValueOrDefault(status, 0);
+            DiagnosticsMetrics.BatchesByStatus
+                .WithLabels(status.ToDbString())
+                .Set(count);
+        }
+
+        // --- Queue age: oldest batch in PendingVerification ---
+        var oldestPendingUpdatedAt = await db.DigitizationBatches
+            .AsNoTracking()
+            .Where(b => b.Status == BatchStatus.PendingVerification)
+            .OrderBy(b => b.UpdatedAt)
+            .Select(b => (DateTimeOffset?)b.UpdatedAt)
+            .FirstOrDefaultAsync(ct);
+
+        if (oldestPendingUpdatedAt.HasValue)
+        {
+            var ageSeconds = (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds;
+            DiagnosticsMetrics.QueueAgeSeconds.Set(ageSeconds);
+        }
+        else
+        {
+            // No batches in PendingVerification — queue is empty
+            DiagnosticsMetrics.QueueAgeSeconds.Set(0);
+        }
+
+        _logger.LogDebug(
+            "Metrics collected: {StatusCount} status groups, queue age {QueueAge}s",
+            statusCounts.Count,
+            oldestPendingUpdatedAt.HasValue
+                ? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds
+                : 0);
+    }
+}
@@ -0,0 +1,274 @@
+using System.Text.Json;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+
+/// <summary>
+/// Background service that retries failed promotion attempts with exponential backoff.
+///
+/// When VigilCareClinical is unreachable in a split deployment, promotion fails and
+/// the batch remains in Approved status. This service:
+/// 1. Finds batches in Approved status that have a failed PromotionAttempt
+///    with a NextRetryAt in the past.
+/// 2. Retries promotion via IPromotionService.
+/// 3. On failure, records a new PromotionAttempt with exponentially increasing NextRetryAt.
+/// 4. After MaxRetryAttempts, sets NextRetryAt to null (manual intervention required)
+///    and logs a critical warning.
+///
+/// The batch NEVER reverts to draft — it stays Approved until promotion succeeds
+/// or an administrator manually intervenes.
+/// </summary>
+public class PromotionRetryService : BackgroundService
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly PromotionRetryOptions _options;
+    private readonly ILogger<PromotionRetryService> _logger;
+
+    public PromotionRetryService(
+        IServiceScopeFactory scopeFactory,
+        IOptions<PromotionRetryOptions> options,
+        ILogger<PromotionRetryService> logger)
+    {
+        _scopeFactory = scopeFactory;
+        _options = options.Value;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation(
+            "PromotionRetryService started. Poll interval: {PollInterval}s, " +
+            "max retries: {MaxRetries}, initial delay: {InitialDelay}s, " +
+            "max delay: {MaxDelay}s, backoff multiplier: {Multiplier}",
+            _options.PollIntervalSeconds,
+            _options.MaxRetryAttempts,
+            _options.InitialDelaySeconds,
+            _options.MaxDelaySeconds,
+            _options.BackoffMultiplier);
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await ProcessPendingRetriesAsync(stoppingToken);
+            }
+            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex,
+                    "PromotionRetryService encountered an error during processing");
+            }
+
+            await Task.Delay(
+                TimeSpan.FromSeconds(_options.PollIntervalSeconds), stoppingToken);
+        }
+
+        _logger.LogInformation("PromotionRetryService stopped");
+    }
+
+    private async Task ProcessPendingRetriesAsync(CancellationToken ct)
+    {
+        using var scope = _scopeFactory.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+        var promotionService = scope.ServiceProvider.GetRequiredService<IPromotionService>();
+        var now = DateTimeOffset.UtcNow;
+
+        // Find batches that need retry:
+        // 1. Batch is in Approved status (not yet promoted)
+        // 2. Has a failed PromotionAttempt with NextRetryAt <= now
+        var pendingRetries = await db.PromotionAttempts
+            .Include(a => a.Batch)
+            .Where(a => !a.Succeeded
+                     && a.NextRetryAt != null
+                     && a.NextRetryAt <= now
+                     && a.Batch.Status == BatchStatus.Approved)
+            .OrderBy(a => a.NextRetryAt)
+            .Take(10) // Process up to 10 retries per poll cycle
+            .ToListAsync(ct);
+
+        if (pendingRetries.Count == 0)
+            return;
+
+        _logger.LogInformation(
+            "PromotionRetryService found {Count} batches pending retry",
+            pendingRetries.Count);
+
+        foreach (var attempt in pendingRetries)
+        {
+            await RetryPromotionAsync(db, promotionService, attempt, ct);
+        }
+    }
+
+    private async Task RetryPromotionAsync(
+        AppDbContext db,
+        IPromotionService promotionService,
+        PromotionAttempt lastAttempt,
+        CancellationToken ct)
+    {
+        var batchId = lastAttempt.BatchId;
+        var nextAttemptNumber = lastAttempt.AttemptNumber + 1;
+
+        _logger.LogInformation(
+            "Retrying promotion for batch {BatchId}, attempt {Attempt}/{MaxAttempts}",
+            batchId, nextAttemptNumber, _options.MaxRetryAttempts);
+
+        try
+        {
+            // Verify batch is still in Approved status
+            var batch = await db.DigitizationBatches.FindAsync(
+                new object[] { batchId }, ct);
+
+            if (batch is null || batch.Status != BatchStatus.Approved)
+            {
+                _logger.LogWarning(
+                    "Batch {BatchId} is no longer in Approved status, skipping retry",
+                    batchId);
+                return;
+            }
+
+            var approverUserId = batch.ApprovedByUserId
+                ?? throw new InvalidOperationException(
+                    $"Batch {batchId} is in Approved status but has no ApprovedByUserId");
+
+            // Attempt promotion with a unique idempotency key per attempt
+            await promotionService.PromoteAsync(batchId, approverUserId);
+
+            // --- Success path ---
+            var successAttempt = new PromotionAttempt
+            {
+                Id = Guid.NewGuid(),
+                BatchId = batchId,
+                AttemptNumber = nextAttemptNumber,
+                Succeeded = true,
+                ErrorMessage = null,
+                AttemptedAt = DateTimeOffset.UtcNow,
+                NextRetryAt = null
+            };
+
+            db.PromotionAttempts.Add(successAttempt);
+
+            // Clear the previous attempt's NextRetryAt so it won't be picked up again
+            lastAttempt.NextRetryAt = null;
+
+            // Audit event
+            db.DigitizationEvents.Add(new DigitizationEvent
+            {
+                Id = Guid.NewGuid(),
+                BatchId = batchId,
+                EventType = DigitizationEventType.PromotionRetrySucceeded,
+                ActorUserId = approverUserId,
+                OccurredAt = DateTimeOffset.UtcNow,
+                MetadataJson = JsonSerializer.Serialize(new
+                {
+                    attemptNumber = nextAttemptNumber,
+                    retriedAt = DateTimeOffset.UtcNow
+                })
+            });
+
+            await db.SaveChangesAsync(ct);
+
+            _logger.LogInformation(
+                "Batch {BatchId} successfully promoted on retry attempt {Attempt}",
+                batchId, nextAttemptNumber);
+        }
+        catch (OperationCanceledException) when (ct.IsCancellationRequested)
+        {
+            throw; // Let the outer loop handle shutdown
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex,
+                "Promotion retry failed for batch {BatchId}, attempt {Attempt}",
+                batchId, nextAttemptNumber);
+
+            // Calculate next retry delay with exponential backoff
+            var delay = CalculateBackoffDelay(nextAttemptNumber);
+            DateTimeOffset? nextRetryAt = null;
+
+            if (nextAttemptNumber < _options.MaxRetryAttempts)
+            {
+                nextRetryAt = DateTimeOffset.UtcNow.Add(delay);
+
+                _logger.LogInformation(
+                    "Scheduling retry for batch {BatchId} at {NextRetryAt} " +
+                    "(delay: {DelaySec:F0}s, attempt {Attempt}/{Max})",
+                    batchId, nextRetryAt, delay.TotalSeconds,
+                    nextAttemptNumber, _options.MaxRetryAttempts);
+            }
+            else
+            {
+                // Exhausted all retries — manual intervention required
+                _logger.LogCritical(
+                    "Batch {BatchId} has exhausted all {Max} retry attempts. " +
+                    "Manual intervention required. Last error: {Error}",
+                    batchId, _options.MaxRetryAttempts, ex.Message);
+            }
+
+            // Record failed attempt
+            var failedAttempt = new PromotionAttempt
+            {
+                Id = Guid.NewGuid(),
+                BatchId = batchId,
+                AttemptNumber = nextAttemptNumber,
+                Succeeded = false,
+                ErrorMessage = ex.Message.Length > 2000
+                    ? ex.Message[..2000]
+                    : ex.Message,
+                AttemptedAt = DateTimeOffset.UtcNow,
+                NextRetryAt = nextRetryAt
+            };
+
+            db.PromotionAttempts.Add(failedAttempt);
+
+            // Clear the previous attempt's NextRetryAt
+            lastAttempt.NextRetryAt = null;
+
+            // Audit event
+            var batch = await db.DigitizationBatches.FindAsync(
+                new object[] { batchId }, ct);
+            var actorId = batch?.ApprovedByUserId ?? Guid.Empty;
+
+            db.DigitizationEvents.Add(new DigitizationEvent
+            {
+                Id = Guid.NewGuid(),
+                BatchId = batchId,
+                EventType = nextRetryAt.HasValue
+                    ? DigitizationEventType.PromotionRetryFailed
+                    : DigitizationEventType.PromotionRetryExhausted,
+                ActorUserId = actorId,
+                OccurredAt = DateTimeOffset.UtcNow,
+                MetadataJson = JsonSerializer.Serialize(new
+                {
+                    attemptNumber = nextAttemptNumber,
+                    error = ex.Message,
+                    nextRetryAt = nextRetryAt?.ToString("o"),
+                    exhausted = !nextRetryAt.HasValue
+                })
+            });
+
+            await db.SaveChangesAsync(ct);
+        }
+    }
+
+    /// <summary>
+    /// Calculates exponential backoff delay for a given attempt number.
+    /// Formula: min(initialDelay * multiplier^(attempt-1), maxDelay)
+    /// With jitter: adds up to 10% random jitter to prevent thundering herd
+    /// when multiple replicas retry simultaneously.
+    /// </summary>
+    private TimeSpan CalculateBackoffDelay(int attemptNumber)
+    {
+        var baseDelay = _options.InitialDelaySeconds
+            * Math.Pow(_options.BackoffMultiplier, attemptNumber - 1);
+
+        var cappedDelay = Math.Min(baseDelay, _options.MaxDelaySeconds);
+
+        // Add up to 10% jitter
+        var jitter = cappedDelay * 0.1 * Random.Shared.NextDouble();
+        var finalDelay = cappedDelay + jitter;
+
+        return TimeSpan.FromSeconds(finalDelay);
+    }
+}
@@ -0,0 +1,38 @@
+/// <summary>
+/// Configuration for the promotion retry background job.
+/// All values configurable via appsettings.json under "PromotionRetry".
+/// </summary>
+public class PromotionRetryOptions
+{
+    public const string Section = "PromotionRetry";
+
+    /// <summary>
+    /// How often the retry service checks for stuck batches (in seconds).
+    /// Default: 60 seconds.
+    /// </summary>
+    public int PollIntervalSeconds { get; set; } = 60;
+
+    /// <summary>
+    /// Initial delay before the first retry attempt (in seconds).
+    /// Default: 30 seconds.
+    /// </summary>
+    public int InitialDelaySeconds { get; set; } = 30;
+
+    /// <summary>
+    /// Maximum delay between retries (in seconds). Exponential backoff
+    /// caps at this value. Default: 900 seconds (15 minutes).
+    /// </summary>
+    public int MaxDelaySeconds { get; set; } = 900;
+
+    /// <summary>
+    /// Maximum number of retry attempts before the batch is flagged
+    /// for manual intervention. Default: 10.
+    /// </summary>
+    public int MaxRetryAttempts { get; set; } = 10;
+
+    /// <summary>
+    /// Backoff multiplier. Each retry delay is multiplied by this value.
+    /// Default: 2.0 (doubles each time).
+    /// </summary>
+    public double BackoffMultiplier { get; set; } = 2.0;
+}
@@ -1,7 +1,9 @@

 using System.Security.Claims;
+using System.Text.Json;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;

 /// <summary>
 /// Approval and promotion of verified digitization batches to VigilCareClinical live tables.
@@ -13,11 +15,19 @@ using Microsoft.AspNetCore.Mvc;
 public class ApprovalController : ControllerBase
 {
    private readonly IPromotionService _promotion;
+    private readonly AppDbContext _db;
+    private readonly PromotionRetryOptions _retryOptions;
    private readonly ILogger<ApprovalController> _logger;

-    public ApprovalController(IPromotionService promotion, ILogger<ApprovalController> logger)
+    public ApprovalController(
+        IPromotionService promotion,
+        AppDbContext db,
+        IOptions<PromotionRetryOptions> retryOptions,
+        ILogger<ApprovalController> logger)
    {
        _promotion = promotion;
+        _db = db;
+        _retryOptions = retryOptions.Value;
        _logger = logger;
    }

@@ -28,6 +38,9 @@ public class ApprovalController : ControllerBase
    /// Requires the Idempotency-Key header for safe retries. If the same key is resubmitted,
    /// the original response is returned without re-executing the promotion.
    ///
+    /// If promotion fails due to infrastructure issues, the batch transitions to Approved status
+    /// and automatic retry is scheduled. Returns 202 Accepted with PROMOTION_DEFERRED.
+    ///
    /// Separation of duties: the approver cannot be the entry clerk or verifier of the same batch.
    /// </summary>
    /// <param name="id">The batch ID to approve.</param>
@@ -36,6 +49,7 @@ public class ApprovalController : ControllerBase
    [HttpPost("{id:guid}/approve")]
    [Authorize(Roles = "CLINICAL_APPROVER,ADMINISTRATOR")]
    [ProducesResponseType(typeof(ApiResponse<PromotionResultResponse>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status202Accepted)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status400BadRequest)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status404NotFound)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status409Conflict)]
@@ -61,10 +75,22 @@ public class ApprovalController : ControllerBase
            "Approve request for batch {BatchId} by user {UserId} with idempotency key {Key}",
            id, approverUserId, idempotencyKey);

-        var result = await _promotion.ApproveAndPromoteAsync(
-            id, approverUserId, enableRetroactiveAlerts, idempotencyKey);
+        try
+        {
+            var result = await _promotion.ApproveAndPromoteAsync(
+                id, approverUserId, enableRetroactiveAlerts, idempotencyKey);

-        return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
+            return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
+        }
+        catch (Exception ex) when (ex is not NotFoundException
+                                && ex is not ConflictException
+                                && ex is not ValidationException)
+        {
+            _logger.LogWarning(ex,
+                "Promotion failed for batch {BatchId}, scheduling for retry", id);
+
+            return await DeferPromotionAsync(id, approverUserId, enableRetroactiveAlerts, ex);
+        }
    }

    /// <summary>
@@ -83,4 +109,81 @@ public class ApprovalController : ControllerBase
        var result = await _promotion.GetPromotionResultAsync(id);
        return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
    }
-}
+
+    private async Task<IActionResult> DeferPromotionAsync(
+        Guid batchId,
+        Guid approverUserId,
+        bool enableRetroactiveAlerts,
+        Exception ex)
+    {
+        var batch = await _db.DigitizationBatches.FindAsync(batchId);
+        if (batch is null)
+            throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
+
+        if (batch.Status is not (BatchStatus.Verified or BatchStatus.AwaitingClinicalApproval))
+        {
+            throw new ConflictException(
+                $"Cannot defer promotion for batch in '{batch.Status.ToDbString()}' status.",
+                "ILLEGAL_STATUS_TRANSITION");
+        }
+
+        var now = DateTimeOffset.UtcNow;
+        batch.Status = BatchStatus.Approved;
+        batch.ApprovedByUserId = approverUserId;
+        batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
+        batch.UpdatedAt = now;
+
+        _db.DigitizationEvents.Add(new DigitizationEvent
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            EventType = DigitizationEventType.Approved,
+            ActorUserId = approverUserId,
+            OccurredAt = now,
+            MetadataJson = JsonSerializer.Serialize(new
+            {
+                enableRetroactiveAlerts,
+                promotionDeferred = true
+            })
+        });
+
+        var nextRetryAt = now.Add(TimeSpan.FromSeconds(_retryOptions.InitialDelaySeconds));
+        var attempt = new PromotionAttempt
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            AttemptNumber = 1,
+            Succeeded = false,
+            ErrorMessage = ex.Message.Length > 2000
+                ? ex.Message[..2000]
+                : ex.Message,
+            AttemptedAt = now,
+            NextRetryAt = nextRetryAt
+        };
+
+        _db.PromotionAttempts.Add(attempt);
+
+        _db.DigitizationEvents.Add(new DigitizationEvent
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            EventType = DigitizationEventType.PromotionFailed,
+            ActorUserId = approverUserId,
+            OccurredAt = now,
+            MetadataJson = JsonSerializer.Serialize(new
+            {
+                error = ex.Message,
+                nextRetryAt = nextRetryAt.ToString("o"),
+                scheduledForRetry = true
+            })
+        });
+
+        await _db.SaveChangesAsync();
+
+        return StatusCode(202, ApiResponse<object>.Fail(
+            202,
+            "Batch approved but promotion deferred due to infrastructure issue. " +
+            "Automatic retry has been scheduled.",
+            "PROMOTION_DEFERRED"));
+    }
+}
@@ -15,6 +15,7 @@ public class DigitizationBatchesController : ControllerBase
    private readonly IBatchService _batches;
    private readonly IDocumentStorageService _storage;
    private readonly IPromotionService _promotion;
+    private readonly IBatchEventService _batchEventService;

    private static readonly HashSet<string> _allowedMimeTypes = new()
    {
@@ -22,13 +23,15 @@ public class DigitizationBatchesController : ControllerBase
    };

    public DigitizationBatchesController(
-        IBatchService batches, 
-        IDocumentStorageService storage, 
-        IPromotionService promotion)
+        IBatchService batches,
+        IDocumentStorageService storage,
+        IPromotionService promotion,
+        IBatchEventService batchEventService)
    {
        _batches = batches;
        _storage = storage;
        _promotion = promotion;
+        _batchEventService = batchEventService;
    }

    /// <summary>
@@ -142,4 +145,38 @@ public class DigitizationBatchesController : ControllerBase
        var result = await _promotion.PromoteAsync(id, actorUserId);
        return Ok(ApiResponse<PromotionResult>.Ok(result));
    }
+
+    /// <summary>
+    /// Returns cursor-paginated audit trail events for a batch.
+    /// Events are ordered chronologically (oldest first).
+    /// Pass the "after" parameter with the cursor from the previous page to paginate.
+    /// </summary>
+    /// <param name="id">Batch ID.</param>
+    /// <param name="after">Cursor: ISO-8601 timestamp from the previous page's nextCursor field.</param>
+    /// <param name="pageSize">Number of events per page. Default 50, max 200.</param>
+    [HttpGet("{id:guid}/events")]
+    [Authorize(Roles = "ADMINISTRATOR,VERIFIER,CLINICAL_APPROVER")]
+    [ProducesResponseType(typeof(ApiResponse<CursorPagedResult<BatchEventResponse>>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status400BadRequest)]
+    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status404NotFound)]
+    public async Task<IActionResult> GetEvents(
+        Guid id,
+        [FromQuery] string? after = null,
+        [FromQuery] int pageSize = 50)
+    {
+        DateTimeOffset? afterCursor = null;
+        if (!string.IsNullOrWhiteSpace(after))
+        {
+            if (!DateTimeOffset.TryParse(after, out var parsed))
+                return BadRequest(ApiResponse<object>.Fail(
+                    400,
+                    "Invalid cursor format. Expected ISO-8601 timestamp.",
+                    "INVALID_CURSOR"));
+
+            afterCursor = parsed;
+        }
+
+        var result = await _batchEventService.GetEventsAsync(id, afterCursor, pageSize);
+        return Ok(ApiResponse<CursorPagedResult<BatchEventResponse>>.Ok(result));
+    }
 }
@@ -22,6 +22,7 @@ public class AppDbContext : DbContext
    public DbSet<ClinicalAlert> ClinicalAlerts => Set<ClinicalAlert>();
    public DbSet<LiveEncounter> LiveEncounters => Set<LiveEncounter>();
    public DbSet<LiveObservation> LiveObservations => Set<LiveObservation>();
+    public DbSet<PromotionAttempt> PromotionAttempts => Set<PromotionAttempt>();

    protected override void OnModelCreating(ModelBuilder modelBuilder)
    {
@@ -0,0 +1,31 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+public class PromotionAttemptConfiguration : IEntityTypeConfiguration<PromotionAttempt>
+{
+    public void Configure(EntityTypeBuilder<PromotionAttempt> builder)
+    {
+        builder.ToTable("promotion_attempts");
+        builder.HasKey(a => a.Id);
+        builder.Property(a => a.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
+        builder.Property(a => a.BatchId).HasColumnName("batch_id").IsRequired();
+        builder.Property(a => a.AttemptNumber).HasColumnName("attempt_number").IsRequired();
+        builder.Property(a => a.Succeeded).HasColumnName("succeeded").HasDefaultValue(false);
+        builder.Property(a => a.ErrorMessage).HasColumnName("error_message").HasMaxLength(2000);
+        builder.Property(a => a.AttemptedAt).HasColumnName("attempted_at").HasDefaultValueSql("NOW()");
+        builder.Property(a => a.NextRetryAt).HasColumnName("next_retry_at");
+
+        builder.HasOne(a => a.Batch)
+            .WithMany()
+            .HasForeignKey(a => a.BatchId)
+            .OnDelete(DeleteBehavior.Restrict);
+
+        builder.HasIndex(a => new { a.BatchId, a.AttemptNumber })
+            .IsUnique()
+            .HasDatabaseName("ix_promotion_attempts_batch_attempt");
+
+        builder.HasIndex(a => a.NextRetryAt)
+            .HasFilter("succeeded = false AND next_retry_at IS NOT NULL")
+            .HasDatabaseName("ix_promotion_attempts_pending_retry");
+    }
+}
@@ -0,0 +1,57 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace VigilCareRecordsAPI.Data.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddPromotionAttempts : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "promotion_attempts",
+                columns: table => new
+                {
+                    id = table.Column<Guid>(type: "uuid", nullable: false, defaultValueSql: "gen_random_uuid()"),
+                    batch_id = table.Column<Guid>(type: "uuid", nullable: false),
+                    attempt_number = table.Column<int>(type: "integer", nullable: false),
+                    succeeded = table.Column<bool>(type: "boolean", nullable: false, defaultValue: false),
+                    error_message = table.Column<string>(type: "character varying(2000)", maxLength: 2000, nullable: true),
+                    attempted_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false, defaultValueSql: "NOW()"),
+                    next_retry_at = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_promotion_attempts", x => x.id);
+                    table.ForeignKey(
+                        name: "FK_promotion_attempts_digitization_batches_batch_id",
+                        column: x => x.batch_id,
+                        principalTable: "digitization_batches",
+                        principalColumn: "id",
+                        onDelete: ReferentialAction.Restrict);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "ix_promotion_attempts_batch_attempt",
+                table: "promotion_attempts",
+                columns: new[] { "batch_id", "attempt_number" },
+                unique: true);
+
+            migrationBuilder.CreateIndex(
+                name: "ix_promotion_attempts_pending_retry",
+                table: "promotion_attempts",
+                column: "next_retry_at",
+                filter: "succeeded = false AND next_retry_at IS NOT NULL");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "promotion_attempts");
+        }
+    }
+}
@@ -1055,6 +1055,56 @@ namespace VigilCareRecordsAPI.Data.Migrations
                        });
                });

+            modelBuilder.Entity("PromotionAttempt", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uuid")
+                        .HasColumnName("id")
+                        .HasDefaultValueSql("gen_random_uuid()");
+
+                    b.Property<int>("AttemptNumber")
+                        .HasColumnType("integer")
+                        .HasColumnName("attempt_number");
+
+                    b.Property<DateTimeOffset>("AttemptedAt")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("attempted_at")
+                        .HasDefaultValueSql("NOW()");
+
+                    b.Property<Guid>("BatchId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("batch_id");
+
+                    b.Property<string>("ErrorMessage")
+                        .HasMaxLength(2000)
+                        .HasColumnType("character varying(2000)")
+                        .HasColumnName("error_message");
+
+                    b.Property<DateTimeOffset?>("NextRetryAt")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("next_retry_at");
+
+                    b.Property<bool>("Succeeded")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("boolean")
+                        .HasDefaultValue(false)
+                        .HasColumnName("succeeded");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("NextRetryAt")
+                        .HasDatabaseName("ix_promotion_attempts_pending_retry")
+                        .HasFilter("succeeded = false AND next_retry_at IS NOT NULL");
+
+                    b.HasIndex("BatchId", "AttemptNumber")
+                        .IsUnique()
+                        .HasDatabaseName("ix_promotion_attempts_batch_attempt");
+
+                    b.ToTable("promotion_attempts", (string)null);
+                });
+
            modelBuilder.Entity("RefreshToken", b =>
                {
                    b.Property<Guid>("Id")
@@ -1345,6 +1395,17 @@ namespace VigilCareRecordsAPI.Data.Migrations
                    b.Navigation("Patient");
                });

+            modelBuilder.Entity("PromotionAttempt", b =>
+                {
+                    b.HasOne("DigitizationBatch", "Batch")
+                        .WithMany()
+                        .HasForeignKey("BatchId")
+                        .OnDelete(DeleteBehavior.Restrict)
+                        .IsRequired();
+
+                    b.Navigation("Batch");
+                });
+
            modelBuilder.Entity("RefreshToken", b =>
                {
                    b.HasOne("RefreshToken", "ReplacedByToken")
@@ -0,0 +1,89 @@
+using Prometheus;
+
+/// <summary>
+/// Application-level Prometheus metrics for the digitization pipeline.
+/// All metrics are static singletons — safe for concurrent use across
+/// all services and background workers.
+///
+/// prometheus-net throws InvalidOperationException if a metric with the
+/// same name but different label configuration is registered twice.
+/// Static fields guarantee each metric is created exactly once.
+/// </summary>
+public static class DiagnosticsMetrics
+{
+    /// <summary>
+    /// Gauge: count of digitization batches per status.
+    /// Updated periodically by MetricsCollectorService.
+    /// Labels: status (UPLOADED, IN_ENTRY, PENDING_VERIFICATION, etc.)
+    ///
+    /// This is a gauge (not a counter) because statuses change — a batch
+    /// moves from UPLOADED to IN_ENTRY, decrementing one label and
+    /// incrementing another. The gauge is set to the current count
+    /// each collection cycle.
+    /// </summary>
+    public static readonly Gauge BatchesByStatus = Metrics.CreateGauge(
+        "digitization_batches_by_status",
+        "Number of digitization batches grouped by current status.",
+        new GaugeConfiguration
+        {
+            LabelNames = new[] { "status" }
+        });
+
+    /// <summary>
+    /// Histogram: how long a promotion operation takes in seconds.
+    /// Recorded in PromotionService when a batch transitions to Promoted.
+    /// Buckets tuned for typical promotion durations (50ms to 30s).
+    ///
+    /// The p50/p95/p99 can be derived from the bucket boundaries in
+    /// Grafana using histogram_quantile().
+    /// </summary>
+    public static readonly Histogram PromotionDuration = Metrics.CreateHistogram(
+        "digitization_promotion_duration_seconds",
+        "Duration of batch promotion operations in seconds.",
+        new HistogramConfiguration
+        {
+            Buckets = new[] { 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0 }
+        });
+
+    /// <summary>
+    /// Counter: total number of batch rejections.
+    /// Incremented in VerificationService on every rejection.
+    /// Labels: reason_category (verification_failed, clinical_rejected)
+    ///
+    /// Supervisors need to distinguish between verification-stage rejections
+    /// (data entry errors) and clinical-stage rejections (clinical judgment
+    /// issues). The label enables separate alerting thresholds.
+    /// </summary>
+    public static readonly Counter RejectionTotal = Metrics.CreateCounter(
+        "digitization_rejection_total",
+        "Total number of digitization batch rejections.",
+        new CounterConfiguration
+        {
+            LabelNames = new[] { "reason_category" }
+        });
+
+    private static readonly string[] RejectionReasonCategories =
+    {
+        "verification_failed",
+        "clinical_rejected"
+    };
+
+    static DiagnosticsMetrics()
+    {
+        // Expose all reason_category label combinations at 0 before any rejections occur.
+        foreach (var category in RejectionReasonCategories)
+            RejectionTotal.WithLabels(category).Inc(0);
+    }
+
+    /// <summary>
+    /// Gauge: age in seconds of the oldest batch in PendingVerification status.
+    /// Updated periodically by MetricsCollectorService.
+    /// A high value indicates the verification queue is backed up.
+    ///
+    /// This is a gauge because it reflects a point-in-time measurement —
+    /// the age of the oldest pending batch right now.
+    /// </summary>
+    public static readonly Gauge QueueAgeSeconds = Metrics.CreateGauge(
+        "digitization_queue_age_seconds",
+        "Age in seconds of the oldest batch in pending_verification status.");
+}
@@ -0,0 +1,17 @@
+/// <summary>
+/// Tracks individual promotion attempts for a batch. Used by the
+/// PromotionRetryService to determine retry timing and attempt count.
+/// Each row represents one attempt (successful or failed).
+/// </summary>
+public class PromotionAttempt
+{
+    public Guid Id { get; set; }
+    public Guid BatchId { get; set; }
+    public int AttemptNumber { get; set; }
+    public bool Succeeded { get; set; }
+    public string? ErrorMessage { get; set; }
+    public DateTimeOffset AttemptedAt { get; set; }
+    public DateTimeOffset? NextRetryAt { get; set; }
+
+    public DigitizationBatch Batch { get; set; } = null!;
+}
@@ -0,0 +1,15 @@
+/// <summary>
+/// A single audit trail event for a digitization batch.
+/// Includes the actor's username and full name for display
+/// without requiring a separate user lookup.
+/// </summary>
+public record BatchEventResponse(
+    Guid Id,
+    Guid BatchId,
+    string EventType,
+    Guid ActorUserId,
+    string ActorUsername,
+    string ActorFullName,
+    DateTimeOffset OccurredAt,
+    string? MetadataJson
+);
@@ -0,0 +1,15 @@
+/// <summary>
+/// Cursor-paginated result set. The cursor is the OccurredAt timestamp
+/// of the last item in this page. Pass it as the "after" query parameter
+/// to get the next page.
+///
+/// This uses the N+1 fetch pattern: fetch pageSize+1 rows, return pageSize,
+/// and use the existence of the extra row to determine HasMore without
+/// a separate COUNT query.
+/// </summary>
+public record CursorPagedResult<T>(
+    IReadOnlyList<T> Items,
+    int PageSize,
+    string? NextCursor,
+    bool HasMore
+);
@@ -3,8 +3,28 @@
 /// Returned by GET /api/v1/work-queue/overview.
 /// </summary>
 public record WorkQueueOverviewResponse(
+    /// <summary>
+    /// Count of batches per status. Key is the DB status string
+    /// (e.g. "UPLOADED", "IN_ENTRY", "PENDING_VERIFICATION").
+    /// All 8 statuses are always present, even if count is 0.
+    /// </summary>
    Dictionary<string, int> StatusCounts,
+
+    /// <summary>
+    /// Average time in minutes that batches currently in PendingVerification
+    /// have been waiting. Zero if no batches are pending.
+    /// </summary>
    double AverageTimeInQueueMinutes,
+
+    /// <summary>
+    /// Rejection rate as a decimal (0.0 to 1.0). Calculated as
+    /// rejections / (rejections + verifications) over the last 24 hours.
+    /// </summary>
    double RejectRate,
+
+    /// <summary>
+    /// Age in minutes of the oldest batch in PendingVerification status.
+    /// Zero if no batches are pending.
+    /// </summary>
    double OldestPendingVerificationMinutes
-);
+);
@@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.IdentityModel.Tokens;
 using Minio;
+using Prometheus;
 using Serilog;
 using StackExchange.Redis;

@@ -37,6 +38,9 @@ try
    builder.Services.Configure<SiteConfigOptions>(
    builder.Configuration.GetSection(SiteConfigOptions.Section));

+    builder.Services.Configure<PromotionRetryOptions>(
+    builder.Configuration.GetSection(PromotionRetryOptions.Section));
+
    // JWT Authentication
    var jwtOptions = builder.Configuration.GetSection(JwtOptions.Section).Get<JwtOptions>()!;
    builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
@@ -70,6 +74,10 @@ try
    builder.Services.AddScoped<IDigitizationHistoryService, DigitizationHistoryService>();
    builder.Services.AddScoped<IAttestationService, AttestationService>();
    builder.Services.AddScoped<ILiveCaptureService, LiveCaptureService>();
+    builder.Services.AddScoped<IBatchEventService, BatchEventService>();
+
+    builder.Services.AddHostedService<MetricsCollectorService>();
+    builder.Services.AddHostedService<PromotionRetryService>();

    builder.Services.AddControllers();
    builder.Services.AddEndpointsApiExplorer();
@@ -90,9 +98,11 @@ try
        });
    }

+    app.UseHttpMetrics(); 
    app.UseAuthentication();
    app.UseAuthorization();
    app.MapControllers();
+    app.MapMetrics(); 

    if (!app.Environment.IsEnvironment("Testing"))
    {
@@ -14,7 +14,7 @@
      "dotnetRunMessages": true,
      "launchBrowser": true,
      "launchUrl": "swagger",
-      "applicationUrl": "http://localhost:5217",
+      "applicationUrl": "http://0.0.0.0:5217",
      "environmentVariables": {
        "ASPNETCORE_ENVIRONMENT": "Development"
      }
@@ -24,7 +24,7 @@
      "dotnetRunMessages": true,
      "launchBrowser": true,
      "launchUrl": "swagger",
-      "applicationUrl": "https://localhost:7223;http://localhost:5217",
+      "applicationUrl": "https://localhost:7223;http://0.0.0.0:5217",
      "environmentVariables": {
        "ASPNETCORE_ENVIRONMENT": "Development"
      }
@@ -0,0 +1,78 @@
+using Microsoft.EntityFrameworkCore;
+
+/// <summary>
+/// Provides cursor-paginated access to the audit trail of digitization events
+/// for a given batch. Events are ordered by occurred_at ascending with id
+/// as a tie-breaker for events at the same timestamp.
+/// </summary>
+public class BatchEventService : IBatchEventService
+{
+    private readonly AppDbContext _db;
+    private const int MaxPageSize = 200;
+    private const int DefaultPageSize = 50;
+
+    public BatchEventService(AppDbContext db)
+    {
+        _db = db;
+    }
+
+    public async Task<CursorPagedResult<BatchEventResponse>> GetEventsAsync(
+        Guid batchId, DateTimeOffset? after, int pageSize)
+    {
+        // Validate batch exists
+        var batchExists = await _db.DigitizationBatches
+            .AnyAsync(b => b.Id == batchId);
+
+        if (!batchExists)
+            throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
+
+        // Clamp page size
+        pageSize = Math.Clamp(pageSize, 1, MaxPageSize);
+
+        // Build query
+        var query = _db.DigitizationEvents
+            .AsNoTracking()
+            .Include(e => e.Actor)
+            .Where(e => e.BatchId == batchId);
+
+        // Apply cursor filter — only events strictly after the cursor timestamp
+        if (after.HasValue)
+        {
+            query = query.Where(e => e.OccurredAt > after.Value);
+        }
+
+        // Fetch pageSize+1 rows to determine HasMore without a COUNT query
+        var events = await query
+            .OrderBy(e => e.OccurredAt)
+            .ThenBy(e => e.Id) // tie-breaker for events at the same timestamp
+            .Take(pageSize + 1)
+            .Select(e => new BatchEventResponse(
+                e.Id,
+                e.BatchId,
+                e.EventType.ToDbString(),
+                e.ActorUserId,
+                e.Actor.Username,
+                e.Actor.FullName,
+                e.OccurredAt,
+                e.MetadataJson))
+            .ToListAsync();
+
+        var hasMore = events.Count > pageSize;
+        var page = hasMore ? events.Take(pageSize).ToList() : events;
+
+        // Build next cursor from the last item's OccurredAt
+        string? nextCursor = null;
+        if (hasMore && page.Count > 0)
+        {
+            var lastEvent = page[^1];
+            // ISO-8601 round-trip format preserves full precision
+            nextCursor = lastEvent.OccurredAt.ToString("o");
+        }
+
+        return new CursorPagedResult<BatchEventResponse>(
+            Items: page,
+            PageSize: pageSize,
+            NextCursor: nextCursor,
+            HasMore: hasMore);
+    }
+}
@@ -0,0 +1,12 @@
+public interface IBatchEventService
+{
+    /// <summary>
+    /// Returns cursor-paginated audit trail events for a batch.
+    /// Events are ordered by OccurredAt ascending (oldest first).
+    /// </summary>
+    /// <param name="batchId">The batch to query events for.</param>
+    /// <param name="after">Cursor: ISO-8601 timestamp. Only events after this timestamp are returned.</param>
+    /// <param name="pageSize">Number of events per page. Default 50, max 200.</param>
+    Task<CursorPagedResult<BatchEventResponse>> GetEventsAsync(
+        Guid batchId, DateTimeOffset? after, int pageSize);
+}
@@ -1,3 +1,4 @@
+using System.Diagnostics;
 using System.Text.Json;
 using Microsoft.EntityFrameworkCore;

@@ -39,167 +40,181 @@ public class PromotionService : IPromotionService
            }
        }

-        // --- Load batch with all draft data ---
-        var batch = await _db.DigitizationBatches
-            .Include(b => b.DraftPatient)
-            .Include(b => b.DraftEncounter)
-            .Include(b => b.DraftObservations)
-            .FirstOrDefaultAsync(b => b.Id == batchId);
-
-        if (batch is null)
-            throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
-
-        // --- Status validation ---
-        if (batch.Status != BatchStatus.Verified && batch.Status != BatchStatus.AwaitingClinicalApproval)
-            throw new ConflictException(
-                $"Batch must be in 'verified' or 'awaiting_clinical_approval' status to approve. Current status: '{batch.Status.ToDbString()}'.",
-                "ILLEGAL_STATUS_TRANSITION");
-
-        // --- Separation of duties: approver cannot be the entry clerk ---
-        if (batch.EnteredByUserId == approverUserId)
-            throw new ConflictException(
-                "Separation of duties: the entry clerk cannot approve their own batch.",
-                "SEPARATION_OF_DUTIES_VIOLATION");
-
-        // --- Also cannot be the verifier ---
-        if (batch.VerifiedByUserId == approverUserId)
-            throw new ConflictException(
-                "Separation of duties: the verifier cannot also approve the same batch.",
-                "SEPARATION_OF_DUTIES_VIOLATION");
-
-        // --- Validate draft data completeness (corrections reuse the linked live patient) ---
-        if (!batch.SupersedesBatchId.HasValue)
-        {
-            if (batch.DraftPatient is null)
-                throw new ValidationException("Batch has no draft patient data.", "MISSING_DRAFT_PATIENT");
-
-            if (string.IsNullOrWhiteSpace(batch.DraftPatient.FullName))
-                throw new ValidationException("Draft patient must have a full name.", "INVALID_DRAFT_PATIENT");
-        }
-        else if (!batch.PatientId.HasValue)
-        {
-            throw new ValidationException(
-                "Correction batch has no linked patient.",
-                "MISSING_PATIENT");
-        }
-
-        // --- Begin atomic transaction ---
-        await using var transaction = await _db.Database.BeginTransactionAsync();
+        // --- Start timing the promotion ---
+        var stopwatch = Stopwatch.StartNew();

        try
        {
-            var now = DateTimeOffset.UtcNow;
+            // --- Load batch with all draft data ---
+            var batch = await _db.DigitizationBatches
+                .Include(b => b.DraftPatient)
+                .Include(b => b.DraftEncounter)
+                .Include(b => b.DraftObservations)
+                .FirstOrDefaultAsync(b => b.Id == batchId);

-            // === Step 1: Create or update Patient ===
-            Patient patient;
-            if (batch.SupersedesBatchId.HasValue)
+            if (batch is null)
+                throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
+
+            // --- Status validation ---
+            if (batch.Status != BatchStatus.Verified && batch.Status != BatchStatus.AwaitingClinicalApproval)
+                throw new ConflictException(
+                    $"Batch must be in 'verified' or 'awaiting_clinical_approval' status to approve. Current status: '{batch.Status.ToDbString()}'.",
+                    "ILLEGAL_STATUS_TRANSITION");
+
+            // --- Separation of duties: approver cannot be the entry clerk ---
+            if (batch.EnteredByUserId == approverUserId)
+                throw new ConflictException(
+                    "Separation of duties: the entry clerk cannot approve their own batch.",
+                    "SEPARATION_OF_DUTIES_VIOLATION");
+
+            // --- Also cannot be the verifier ---
+            if (batch.VerifiedByUserId == approverUserId)
+                throw new ConflictException(
+                    "Separation of duties: the verifier cannot also approve the same batch.",
+                    "SEPARATION_OF_DUTIES_VIOLATION");
+
+            // --- Validate draft data completeness (corrections reuse the linked live patient) ---
+            if (!batch.SupersedesBatchId.HasValue)
            {
-                patient = await _db.Patients.FirstOrDefaultAsync(p => p.Id == batch.PatientId!.Value)
-                    ?? throw new NotFoundException(
-                        $"Patient {batch.PatientId.Value} not found.",
-                        "PATIENT_NOT_FOUND");
+                if (batch.DraftPatient is null)
+                    throw new ValidationException("Batch has no draft patient data.", "MISSING_DRAFT_PATIENT");
+
+                if (string.IsNullOrWhiteSpace(batch.DraftPatient.FullName))
+                    throw new ValidationException("Draft patient must have a full name.", "INVALID_DRAFT_PATIENT");
            }
-            else
+            else if (!batch.PatientId.HasValue)
            {
-                patient = await CreateOrUpdatePatientAsync(batch.DraftPatient!, now);
+                throw new ValidationException(
+                    "Correction batch has no linked patient.",
+                    "MISSING_PATIENT");
            }

-            // === Step 2: Create or match Encounter (reuse original encounter for corrections) ===
-            var encounter = await ResolveClinicalEncounterAsync(batch, patient.Id, now);
-            await EnsureLiveEncounterAsync(encounter, now);
+            // --- Begin atomic transaction ---
+            await using var transaction = await _db.Database.BeginTransactionAsync();

-            // === Step 3: Insert each DraftObservation as live Observation ===
-            var (observationIds, outboxCount) = await PromoteObservationsAsync(
-                batch, patient.Id, encounter.Id, enableRetroactiveAlerts, now);
-
-            // === Step 3b: Mirror to live_observations for supersession / digitization history ===
-            PromoteLiveObservationsAsync(batch, patient.Id, encounter.Id, now);
-
-            SupersessionResult? supersessionResult = null;
-            if (batch.SupersedesBatchId.HasValue)
+            try
            {
-                supersessionResult = await SupersedeOriginalBatchAsync(
-                    batch.SupersedesBatchId.Value, batchId, approverUserId, now);
-            }
+                var now = DateTimeOffset.UtcNow;

-            // === Step 4: Update batch status to Promoted ===
-            batch.Status = BatchStatus.Promoted;
-            batch.ApprovedByUserId = approverUserId;
-            batch.PatientId = patient.Id;
-            batch.PromotedAt = now;
-            batch.PromotionEncounterId = encounter.Id;
-            batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
-            batch.UpdatedAt = now;
-
-            // === Step 5: Write DigitizationEvent ===
-            var promotionMetadata = new Dictionary<string, object>
-            {
-                ["patientId"] = patient.Id,
-                ["mrn"] = patient.Mrn,
-                ["encounterId"] = encounter.Id,
-                ["observationCount"] = observationIds.Length,
-                ["outboxEventsWritten"] = outboxCount,
-                ["enableRetroactiveAlerts"] = enableRetroactiveAlerts,
-                ["track"] = batch.Track.ToDbString(),
-                ["isCorrection"] = batch.SupersedesBatchId.HasValue
-            };
-
-            if (supersessionResult is not null)
-            {
-                promotionMetadata["supersession"] = new
+                // === Step 1: Create or update Patient ===
+                Patient patient;
+                if (batch.SupersedesBatchId.HasValue)
                {
-                    originalBatchId = supersessionResult.OriginalBatchId,
-                    observationsSuperseded = supersessionResult.ObservationsSuperseded,
-                    observationsReplaced = supersessionResult.ObservationsReplaced
+                    patient = await _db.Patients.FirstOrDefaultAsync(p => p.Id == batch.PatientId!.Value)
+                        ?? throw new NotFoundException(
+                            $"Patient {batch.PatientId.Value} not found.",
+                            "PATIENT_NOT_FOUND");
+                }
+                else
+                {
+                    patient = await CreateOrUpdatePatientAsync(batch.DraftPatient!, now);
+                }
+
+                // === Step 2: Create or match Encounter (reuse original encounter for corrections) ===
+                var encounter = await ResolveClinicalEncounterAsync(batch, patient.Id, now);
+                await EnsureLiveEncounterAsync(encounter, now);
+
+                // === Step 3: Insert each DraftObservation as live Observation ===
+                var (observationIds, outboxCount) = await PromoteObservationsAsync(
+                    batch, patient.Id, encounter.Id, enableRetroactiveAlerts, now);
+
+                // === Step 3b: Mirror to live_observations for supersession / digitization history ===
+                PromoteLiveObservationsAsync(batch, patient.Id, encounter.Id, now);
+
+                SupersessionResult? supersessionResult = null;
+                if (batch.SupersedesBatchId.HasValue)
+                {
+                    supersessionResult = await SupersedeOriginalBatchAsync(
+                        batch.SupersedesBatchId.Value, batchId, approverUserId, now);
+                }
+
+                // === Step 4: Update batch status to Promoted ===
+                batch.Status = BatchStatus.Promoted;
+                batch.ApprovedByUserId = approverUserId;
+                batch.PatientId = patient.Id;
+                batch.PromotedAt = now;
+                batch.PromotionEncounterId = encounter.Id;
+                batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
+                batch.UpdatedAt = now;
+
+                // === Step 5: Write DigitizationEvent ===
+                var promotionMetadata = new Dictionary<string, object>
+                {
+                    ["patientId"] = patient.Id,
+                    ["mrn"] = patient.Mrn,
+                    ["encounterId"] = encounter.Id,
+                    ["observationCount"] = observationIds.Length,
+                    ["outboxEventsWritten"] = outboxCount,
+                    ["enableRetroactiveAlerts"] = enableRetroactiveAlerts,
+                    ["track"] = batch.Track.ToDbString(),
+                    ["isCorrection"] = batch.SupersedesBatchId.HasValue
                };
+
+                if (supersessionResult is not null)
+                {
+                    promotionMetadata["supersession"] = new
+                    {
+                        originalBatchId = supersessionResult.OriginalBatchId,
+                        observationsSuperseded = supersessionResult.ObservationsSuperseded,
+                        observationsReplaced = supersessionResult.ObservationsReplaced
+                    };
+                }
+
+                _db.DigitizationEvents.Add(new DigitizationEvent
+                {
+                    Id = Guid.NewGuid(),
+                    BatchId = batchId,
+                    EventType = batch.SupersedesBatchId.HasValue
+                        ? DigitizationEventType.CorrectionPromoted
+                        : DigitizationEventType.Promoted,
+                    ActorUserId = approverUserId,
+                    OccurredAt = now,
+                    MetadataJson = JsonSerializer.Serialize(promotionMetadata)
+                });
+
+                // === Step 6: Store idempotency record (within same transaction) ===
+                var result = new PromotionResultResponse(
+                    BatchId: batchId,
+                    Status: BatchStatus.Promoted.ToDbString(),
+                    PatientId: patient.Id,
+                    Mrn: patient.Mrn,
+                    EncounterId: encounter.Id,
+                    ObservationIds: observationIds,
+                    PromotedAt: now,
+                    OutboxEventsWritten: outboxCount
+                );
+
+                if (!string.IsNullOrWhiteSpace(idempotencyKey))
+                {
+                    await _idempotency.SaveAsync(
+                        idempotencyKey, "batch_promote", batchId,
+                        200, result, TimeSpan.FromHours(24));
+                }
+
+                // === Step 7: Commit ===
+                await _db.SaveChangesAsync();
+                await transaction.CommitAsync();
+
+                stopwatch.Stop();
+                DiagnosticsMetrics.PromotionDuration.Observe(stopwatch.Elapsed.TotalSeconds);
+
+                _logger.LogInformation(
+                    "Batch {BatchId} promoted in {ElapsedMs}ms: Patient {PatientId} (MRN {Mrn}), " +
+                    "Encounter {EncounterId}, {ObsCount} observations, {OutboxCount} outbox events",
+                    batchId, stopwatch.ElapsedMilliseconds, patient.Id, patient.Mrn, encounter.Id,
+                    observationIds.Length, outboxCount);
+
+                return result;
            }
-
-            _db.DigitizationEvents.Add(new DigitizationEvent
+            catch
            {
-                Id = Guid.NewGuid(),
-                BatchId = batchId,
-                EventType = batch.SupersedesBatchId.HasValue
-                    ? DigitizationEventType.CorrectionPromoted
-                    : DigitizationEventType.Promoted,
-                ActorUserId = approverUserId,
-                OccurredAt = now,
-                MetadataJson = JsonSerializer.Serialize(promotionMetadata)
-            });
-
-            // === Step 6: Store idempotency record (within same transaction) ===
-            var result = new PromotionResultResponse(
-                BatchId: batchId,
-                Status: BatchStatus.Promoted.ToDbString(),
-                PatientId: patient.Id,
-                Mrn: patient.Mrn,
-                EncounterId: encounter.Id,
-                ObservationIds: observationIds,
-                PromotedAt: now,
-                OutboxEventsWritten: outboxCount
-            );
-
-            if (!string.IsNullOrWhiteSpace(idempotencyKey))
-            {
-                await _idempotency.SaveAsync(
-                    idempotencyKey, "batch_promote", batchId,
-                    200, result, TimeSpan.FromHours(24));
+                await transaction.RollbackAsync();
+                throw;
            }
-
-            // === Step 7: Commit ===
-            await _db.SaveChangesAsync();
-            await transaction.CommitAsync();
-
-            _logger.LogInformation(
-                "Batch {BatchId} promoted: Patient {PatientId} (MRN {Mrn}), " +
-                "Encounter {EncounterId}, {ObsCount} observations, {OutboxCount} outbox events",
-                batchId, patient.Id, patient.Mrn, encounter.Id,
-                observationIds.Length, outboxCount);
-
-            return result;
        }
        catch
        {
-            await transaction.RollbackAsync();
+            stopwatch.Stop();
            throw;
        }
    }
@@ -513,117 +528,131 @@ public class PromotionService : IPromotionService

    public async Task<PromotionResult> PromoteAsync(Guid batchId, Guid actorUserId)
    {
-        await using var transaction = await _db.Database.BeginTransactionAsync();
+        var stopwatch = Stopwatch.StartNew();

        try
        {
-            // Load the batch with all draft data
-            var batch = await _db.DigitizationBatches
-                .Include(b => b.DraftPatient)
-                .Include(b => b.DraftEncounter)
-                .Include(b => b.DraftObservations)
-                .FirstOrDefaultAsync(b => b.Id == batchId);
+            await using var transaction = await _db.Database.BeginTransactionAsync();

-            if (batch is null)
-                throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
-
-            if (batch.Status != BatchStatus.Approved)
-                throw new ConflictException(
-                    $"Only approved batches can be promoted. Batch is in '{batch.Status.ToDbString()}' status.",
-                    "ILLEGAL_STATUS_TRANSITION");
-
-            // Resolve or create the live encounter
-            var encounterId = await ResolveEncounterAsync(batch);
-
-            // Promote draft observations to live observations
-            var now = DateTimeOffset.UtcNow;
-            var liveObservations = batch.DraftObservations.Select(draft => new LiveObservation
+            try
            {
-                Id = Guid.NewGuid(),
-                EncounterId = encounterId,
-                PatientId = batch.PatientId,
-                SourceBatchId = batch.Id,
-                ObservationCode = draft.ObservationCode,
-                Value = draft.Value,
-                Unit = draft.Unit,
-                RecordedAt = draft.RecordedAt,
-                Note = draft.Note,
-                CreatedAt = now,
-                IsSuperseded = false,
-                SupersededByBatchId = null,
-                SupersededAt = null
-            }).ToList();
+                // Load the batch with all draft data
+                var batch = await _db.DigitizationBatches
+                    .Include(b => b.DraftPatient)
+                    .Include(b => b.DraftEncounter)
+                    .Include(b => b.DraftObservations)
+                    .FirstOrDefaultAsync(b => b.Id == batchId);

-            _db.LiveObservations.AddRange(liveObservations);
+                if (batch is null)
+                    throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");

-            // Handle supersession if this is a correction batch
-            SupersessionResult? supersessionResult = null;
-            if (batch.SupersedesBatchId.HasValue)
-            {
-                supersessionResult = await SupersedeOriginalBatchAsync(
-                    batch.SupersedesBatchId.Value,
-                    batch.Id,
-                    actorUserId,
-                    now);
-            }
+                if (batch.Status != BatchStatus.Approved)
+                    throw new ConflictException(
+                        $"Only approved batches can be promoted. Batch is in '{batch.Status.ToDbString()}' status.",
+                        "ILLEGAL_STATUS_TRANSITION");

-            // Update batch status to Promoted
-            batch.Status = BatchStatus.Promoted;
-            batch.PromotedAt = now;
-            batch.PromotionEncounterId = encounterId;
-            batch.UpdatedAt = now;
+                // Resolve or create the live encounter
+                var encounterId = await ResolveEncounterAsync(batch);

-            // Record promotion event on the correction batch
-            var promotionMetadata = new Dictionary<string, object>
-            {
-                ["encounterId"] = encounterId,
-                ["observationsPromoted"] = liveObservations.Count,
-                ["isCorrection"] = batch.SupersedesBatchId.HasValue
-            };
-
-            if (supersessionResult is not null)
-            {
-                promotionMetadata["supersession"] = new
+                // Promote draft observations to live observations
+                var now = DateTimeOffset.UtcNow;
+                var liveObservations = batch.DraftObservations.Select(draft => new LiveObservation
                {
-                    originalBatchId = supersessionResult.OriginalBatchId,
-                    observationsSuperseded = supersessionResult.ObservationsSuperseded,
-                    observationsReplaced = supersessionResult.ObservationsReplaced
+                    Id = Guid.NewGuid(),
+                    EncounterId = encounterId,
+                    PatientId = batch.PatientId,
+                    SourceBatchId = batch.Id,
+                    ObservationCode = draft.ObservationCode,
+                    Value = draft.Value,
+                    Unit = draft.Unit,
+                    RecordedAt = draft.RecordedAt,
+                    Note = draft.Note,
+                    CreatedAt = now,
+                    IsSuperseded = false,
+                    SupersededByBatchId = null,
+                    SupersededAt = null
+                }).ToList();
+
+                _db.LiveObservations.AddRange(liveObservations);
+
+                // Handle supersession if this is a correction batch
+                SupersessionResult? supersessionResult = null;
+                if (batch.SupersedesBatchId.HasValue)
+                {
+                    supersessionResult = await SupersedeOriginalBatchAsync(
+                        batch.SupersedesBatchId.Value,
+                        batch.Id,
+                        actorUserId,
+                        now);
+                }
+
+                // Update batch status to Promoted
+                batch.Status = BatchStatus.Promoted;
+                batch.PromotedAt = now;
+                batch.PromotionEncounterId = encounterId;
+                batch.UpdatedAt = now;
+
+                // Record promotion event on the correction batch
+                var promotionMetadata = new Dictionary<string, object>
+                {
+                    ["encounterId"] = encounterId,
+                    ["observationsPromoted"] = liveObservations.Count,
+                    ["isCorrection"] = batch.SupersedesBatchId.HasValue
                };
+
+                if (supersessionResult is not null)
+                {
+                    promotionMetadata["supersession"] = new
+                    {
+                        originalBatchId = supersessionResult.OriginalBatchId,
+                        observationsSuperseded = supersessionResult.ObservationsSuperseded,
+                        observationsReplaced = supersessionResult.ObservationsReplaced
+                    };
+                }
+
+                _db.DigitizationEvents.Add(new DigitizationEvent
+                {
+                    Id = Guid.NewGuid(),
+                    BatchId = batch.Id,
+                    EventType = batch.SupersedesBatchId.HasValue
+                        ? DigitizationEventType.CorrectionPromoted
+                        : DigitizationEventType.Promoted,
+                    ActorUserId = actorUserId,
+                    OccurredAt = now,
+                    MetadataJson = JsonSerializer.Serialize(promotionMetadata)
+                });
+
+                await _db.SaveChangesAsync();
+                await transaction.CommitAsync();
+
+                stopwatch.Stop();
+                DiagnosticsMetrics.PromotionDuration.Observe(stopwatch.Elapsed.TotalSeconds);
+
+                _logger.LogInformation(
+                    "Batch {BatchId} promoted in {ElapsedMs}ms (correction={IsCorrection}, " +
+                    "observations={ObservationCount}, superseded={SupersededCount})",
+                    batchId,
+                    stopwatch.ElapsedMilliseconds,
+                    batch.SupersedesBatchId.HasValue,
+                    liveObservations.Count,
+                    supersessionResult?.ObservationsSuperseded ?? 0);
+
+                return new PromotionResult(
+                    batch.Id,
+                    encounterId,
+                    liveObservations.Count,
+                    batch.SupersedesBatchId.HasValue,
+                    supersessionResult);
            }
-
-            _db.DigitizationEvents.Add(new DigitizationEvent
+            catch
            {
-                Id = Guid.NewGuid(),
-                BatchId = batch.Id,
-                EventType = batch.SupersedesBatchId.HasValue
-                    ? DigitizationEventType.CorrectionPromoted
-                    : DigitizationEventType.Promoted,
-                ActorUserId = actorUserId,
-                OccurredAt = now,
-                MetadataJson = JsonSerializer.Serialize(promotionMetadata)
-            });
-
-            await _db.SaveChangesAsync();
-            await transaction.CommitAsync();
-
-            _logger.LogInformation(
-                "Batch {BatchId} promoted (correction={IsCorrection}, " +
-                "observations={ObservationCount}, superseded={SupersededCount})",
-                batchId,
-                batch.SupersedesBatchId.HasValue,
-                liveObservations.Count,
-                supersessionResult?.ObservationsSuperseded ?? 0);
-
-            return new PromotionResult(
-                batch.Id,
-                encounterId,
-                liveObservations.Count,
-                batch.SupersedesBatchId.HasValue,
-                supersessionResult);
+                await transaction.RollbackAsync();
+                throw;
+            }
        }
        catch
        {
-            await transaction.RollbackAsync();
+            stopwatch.Stop();
            throw;
        }
    }
@@ -138,6 +138,10 @@ public class VerificationService : IVerificationService

            await _db.SaveChangesAsync();

+            DiagnosticsMetrics.RejectionTotal
+                .WithLabels("verification_failed")
+                .Inc();
+
            _logger.LogInformation(
                "Batch {BatchId} verification failed by {VerifierUserId}, rejected",
                batchId, verifierUserId);
@@ -206,6 +210,13 @@ public class VerificationService : IVerificationService

        await _db.SaveChangesAsync();

+        var category = previousStatus == BatchStatus.AwaitingClinicalApproval
+            ? "clinical_rejected"
+            : "verification_failed";
+        DiagnosticsMetrics.RejectionTotal
+            .WithLabels(category)
+            .Inc();
+
        _logger.LogInformation(
            "Batch {BatchId} rejected by {ActorUserId} from {PreviousStatus}: {Reason}",
            batchId, actorUserId, previousStatus.ToDbString(), request.Reason);
@@ -125,6 +125,7 @@ public class WorkQueueService : IWorkQueueService
            .AsNoTracking()
            .Where(e => e.OccurredAt >= cutoff)
            .Where(e => e.EventType == DigitizationEventType.Rejected
+                     || e.EventType == DigitizationEventType.VerificationFailed
                     || e.EventType == DigitizationEventType.Verified
                     || e.EventType == DigitizationEventType.VerifiedPendingClinical)
            .GroupBy(e => e.EventType)
@@ -132,7 +133,8 @@ public class WorkQueueService : IWorkQueueService
            .ToListAsync();

        var rejections = recentEvents
-            .Where(e => e.EventType == DigitizationEventType.Rejected)
+            .Where(e => e.EventType == DigitizationEventType.Rejected
+                     || e.EventType == DigitizationEventType.VerificationFailed)
            .Sum(e => e.Count);

        var verifications = recentEvents
@@ -18,6 +18,8 @@
    </PackageReference>
    <PackageReference Include="Minio" Version="6.0.3" />
    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
+    <PackageReference Include="prometheus-net" Version="8.2.1" />
+    <PackageReference Include="prometheus-net.AspNetCore" Version="8.2.1" />
    <PackageReference Include="Serilog.AspNetCore" Version="8.0.2" />
    <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
    <PackageReference Include="Serilog.Sinks.Seq" Version="9.1.0" />
@@ -60,5 +60,12 @@
      "ALLERGY_UPDATE": false,
      "MIXED": true
    }
+  },
+  "PromotionRetry": {
+    "PollIntervalSeconds": 60,
+    "InitialDelaySeconds": 30,
+    "MaxDelaySeconds": 900,
+    "MaxRetryAttempts": 10,
+    "BackoffMultiplier": 2.0
  }
 }