feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job

2026-06-27 13:29:04 +08:00
parent e22d33b654
commit 04bdb7e85c
31 changed files with 4482 additions and 259 deletions
@@ -1,7 +1,9 @@

 using System.Security.Claims;
+using System.Text.Json;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;

 /// <summary>
 /// Approval and promotion of verified digitization batches to VigilCareClinical live tables.
@@ -13,11 +15,19 @@ using Microsoft.AspNetCore.Mvc;
 public class ApprovalController : ControllerBase
 {
    private readonly IPromotionService _promotion;
+    private readonly AppDbContext _db;
+    private readonly PromotionRetryOptions _retryOptions;
    private readonly ILogger<ApprovalController> _logger;

-    public ApprovalController(IPromotionService promotion, ILogger<ApprovalController> logger)
+    public ApprovalController(
+        IPromotionService promotion,
+        AppDbContext db,
+        IOptions<PromotionRetryOptions> retryOptions,
+        ILogger<ApprovalController> logger)
    {
        _promotion = promotion;
+        _db = db;
+        _retryOptions = retryOptions.Value;
        _logger = logger;
    }

@@ -28,6 +38,9 @@ public class ApprovalController : ControllerBase
    /// Requires the Idempotency-Key header for safe retries. If the same key is resubmitted,
    /// the original response is returned without re-executing the promotion.
    ///
+    /// If promotion fails due to infrastructure issues, the batch transitions to Approved status
+    /// and automatic retry is scheduled. Returns 202 Accepted with PROMOTION_DEFERRED.
+    ///
    /// Separation of duties: the approver cannot be the entry clerk or verifier of the same batch.
    /// </summary>
    /// <param name="id">The batch ID to approve.</param>
@@ -36,6 +49,7 @@ public class ApprovalController : ControllerBase
    [HttpPost("{id:guid}/approve")]
    [Authorize(Roles = "CLINICAL_APPROVER,ADMINISTRATOR")]
    [ProducesResponseType(typeof(ApiResponse<PromotionResultResponse>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status202Accepted)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status400BadRequest)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status404NotFound)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status409Conflict)]
@@ -61,10 +75,22 @@ public class ApprovalController : ControllerBase
            "Approve request for batch {BatchId} by user {UserId} with idempotency key {Key}",
            id, approverUserId, idempotencyKey);

-        var result = await _promotion.ApproveAndPromoteAsync(
-            id, approverUserId, enableRetroactiveAlerts, idempotencyKey);
+        try
+        {
+            var result = await _promotion.ApproveAndPromoteAsync(
+                id, approverUserId, enableRetroactiveAlerts, idempotencyKey);

-        return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
+            return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
+        }
+        catch (Exception ex) when (ex is not NotFoundException
+                                && ex is not ConflictException
+                                && ex is not ValidationException)
+        {
+            _logger.LogWarning(ex,
+                "Promotion failed for batch {BatchId}, scheduling for retry", id);
+
+            return await DeferPromotionAsync(id, approverUserId, enableRetroactiveAlerts, ex);
+        }
    }

    /// <summary>
@@ -83,4 +109,81 @@ public class ApprovalController : ControllerBase
        var result = await _promotion.GetPromotionResultAsync(id);
        return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
    }
-}
+
+    private async Task<IActionResult> DeferPromotionAsync(
+        Guid batchId,
+        Guid approverUserId,
+        bool enableRetroactiveAlerts,
+        Exception ex)
+    {
+        var batch = await _db.DigitizationBatches.FindAsync(batchId);
+        if (batch is null)
+            throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
+
+        if (batch.Status is not (BatchStatus.Verified or BatchStatus.AwaitingClinicalApproval))
+        {
+            throw new ConflictException(
+                $"Cannot defer promotion for batch in '{batch.Status.ToDbString()}' status.",
+                "ILLEGAL_STATUS_TRANSITION");
+        }
+
+        var now = DateTimeOffset.UtcNow;
+        batch.Status = BatchStatus.Approved;
+        batch.ApprovedByUserId = approverUserId;
+        batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
+        batch.UpdatedAt = now;
+
+        _db.DigitizationEvents.Add(new DigitizationEvent
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            EventType = DigitizationEventType.Approved,
+            ActorUserId = approverUserId,
+            OccurredAt = now,
+            MetadataJson = JsonSerializer.Serialize(new
+            {
+                enableRetroactiveAlerts,
+                promotionDeferred = true
+            })
+        });
+
+        var nextRetryAt = now.Add(TimeSpan.FromSeconds(_retryOptions.InitialDelaySeconds));
+        var attempt = new PromotionAttempt
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            AttemptNumber = 1,
+            Succeeded = false,
+            ErrorMessage = ex.Message.Length > 2000
+                ? ex.Message[..2000]
+                : ex.Message,
+            AttemptedAt = now,
+            NextRetryAt = nextRetryAt
+        };
+
+        _db.PromotionAttempts.Add(attempt);
+
+        _db.DigitizationEvents.Add(new DigitizationEvent
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            EventType = DigitizationEventType.PromotionFailed,
+            ActorUserId = approverUserId,
+            OccurredAt = now,
+            MetadataJson = JsonSerializer.Serialize(new
+            {
+                error = ex.Message,
+                nextRetryAt = nextRetryAt.ToString("o"),
+                scheduledForRetry = true
+            })
+        });
+
+        await _db.SaveChangesAsync();
+
+        return StatusCode(202, ApiResponse<object>.Fail(
+            202,
+            "Batch approved but promotion deferred due to infrastructure issue. " +
+            "Automatic retry has been scheduled.",
+            "PROMOTION_DEFERRED"));
+    }
+}