feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job

2026-06-27 13:29:04 +08:00
parent e22d33b654
commit 04bdb7e85c
31 changed files with 4482 additions and 259 deletions
@@ -1,7 +1,9 @@

 using System.Security.Claims;
+using System.Text.Json;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;

 /// <summary>
 /// Approval and promotion of verified digitization batches to VigilCareClinical live tables.
@@ -13,11 +15,19 @@ using Microsoft.AspNetCore.Mvc;
 public class ApprovalController : ControllerBase
 {
    private readonly IPromotionService _promotion;
+    private readonly AppDbContext _db;
+    private readonly PromotionRetryOptions _retryOptions;
    private readonly ILogger<ApprovalController> _logger;

-    public ApprovalController(IPromotionService promotion, ILogger<ApprovalController> logger)
+    public ApprovalController(
+        IPromotionService promotion,
+        AppDbContext db,
+        IOptions<PromotionRetryOptions> retryOptions,
+        ILogger<ApprovalController> logger)
    {
        _promotion = promotion;
+        _db = db;
+        _retryOptions = retryOptions.Value;
        _logger = logger;
    }

@@ -28,6 +38,9 @@ public class ApprovalController : ControllerBase
    /// Requires the Idempotency-Key header for safe retries. If the same key is resubmitted,
    /// the original response is returned without re-executing the promotion.
    ///
+    /// If promotion fails due to infrastructure issues, the batch transitions to Approved status
+    /// and automatic retry is scheduled. Returns 202 Accepted with PROMOTION_DEFERRED.
+    ///
    /// Separation of duties: the approver cannot be the entry clerk or verifier of the same batch.
    /// </summary>
    /// <param name="id">The batch ID to approve.</param>
@@ -36,6 +49,7 @@ public class ApprovalController : ControllerBase
    [HttpPost("{id:guid}/approve")]
    [Authorize(Roles = "CLINICAL_APPROVER,ADMINISTRATOR")]
    [ProducesResponseType(typeof(ApiResponse<PromotionResultResponse>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status202Accepted)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status400BadRequest)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status404NotFound)]
    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status409Conflict)]
@@ -61,10 +75,22 @@ public class ApprovalController : ControllerBase
            "Approve request for batch {BatchId} by user {UserId} with idempotency key {Key}",
            id, approverUserId, idempotencyKey);

-        var result = await _promotion.ApproveAndPromoteAsync(
-            id, approverUserId, enableRetroactiveAlerts, idempotencyKey);
+        try
+        {
+            var result = await _promotion.ApproveAndPromoteAsync(
+                id, approverUserId, enableRetroactiveAlerts, idempotencyKey);

-        return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
+            return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
+        }
+        catch (Exception ex) when (ex is not NotFoundException
+                                && ex is not ConflictException
+                                && ex is not ValidationException)
+        {
+            _logger.LogWarning(ex,
+                "Promotion failed for batch {BatchId}, scheduling for retry", id);
+
+            return await DeferPromotionAsync(id, approverUserId, enableRetroactiveAlerts, ex);
+        }
    }

    /// <summary>
@@ -83,4 +109,81 @@ public class ApprovalController : ControllerBase
        var result = await _promotion.GetPromotionResultAsync(id);
        return Ok(ApiResponse<PromotionResultResponse>.Ok(result));
    }
-}
+
+    private async Task<IActionResult> DeferPromotionAsync(
+        Guid batchId,
+        Guid approverUserId,
+        bool enableRetroactiveAlerts,
+        Exception ex)
+    {
+        var batch = await _db.DigitizationBatches.FindAsync(batchId);
+        if (batch is null)
+            throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
+
+        if (batch.Status is not (BatchStatus.Verified or BatchStatus.AwaitingClinicalApproval))
+        {
+            throw new ConflictException(
+                $"Cannot defer promotion for batch in '{batch.Status.ToDbString()}' status.",
+                "ILLEGAL_STATUS_TRANSITION");
+        }
+
+        var now = DateTimeOffset.UtcNow;
+        batch.Status = BatchStatus.Approved;
+        batch.ApprovedByUserId = approverUserId;
+        batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
+        batch.UpdatedAt = now;
+
+        _db.DigitizationEvents.Add(new DigitizationEvent
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            EventType = DigitizationEventType.Approved,
+            ActorUserId = approverUserId,
+            OccurredAt = now,
+            MetadataJson = JsonSerializer.Serialize(new
+            {
+                enableRetroactiveAlerts,
+                promotionDeferred = true
+            })
+        });
+
+        var nextRetryAt = now.Add(TimeSpan.FromSeconds(_retryOptions.InitialDelaySeconds));
+        var attempt = new PromotionAttempt
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            AttemptNumber = 1,
+            Succeeded = false,
+            ErrorMessage = ex.Message.Length > 2000
+                ? ex.Message[..2000]
+                : ex.Message,
+            AttemptedAt = now,
+            NextRetryAt = nextRetryAt
+        };
+
+        _db.PromotionAttempts.Add(attempt);
+
+        _db.DigitizationEvents.Add(new DigitizationEvent
+        {
+            Id = Guid.NewGuid(),
+            BatchId = batchId,
+            EventType = DigitizationEventType.PromotionFailed,
+            ActorUserId = approverUserId,
+            OccurredAt = now,
+            MetadataJson = JsonSerializer.Serialize(new
+            {
+                error = ex.Message,
+                nextRetryAt = nextRetryAt.ToString("o"),
+                scheduledForRetry = true
+            })
+        });
+
+        await _db.SaveChangesAsync();
+
+        return StatusCode(202, ApiResponse<object>.Fail(
+            202,
+            "Batch approved but promotion deferred due to infrastructure issue. " +
+            "Automatic retry has been scheduled.",
+            "PROMOTION_DEFERRED"));
+    }
+}
@@ -15,6 +15,7 @@ public class DigitizationBatchesController : ControllerBase
    private readonly IBatchService _batches;
    private readonly IDocumentStorageService _storage;
    private readonly IPromotionService _promotion;
+    private readonly IBatchEventService _batchEventService;

    private static readonly HashSet<string> _allowedMimeTypes = new()
    {
@@ -22,13 +23,15 @@ public class DigitizationBatchesController : ControllerBase
    };

    public DigitizationBatchesController(
-        IBatchService batches, 
-        IDocumentStorageService storage, 
-        IPromotionService promotion)
+        IBatchService batches,
+        IDocumentStorageService storage,
+        IPromotionService promotion,
+        IBatchEventService batchEventService)
    {
        _batches = batches;
        _storage = storage;
        _promotion = promotion;
+        _batchEventService = batchEventService;
    }

    /// <summary>
@@ -142,4 +145,38 @@ public class DigitizationBatchesController : ControllerBase
        var result = await _promotion.PromoteAsync(id, actorUserId);
        return Ok(ApiResponse<PromotionResult>.Ok(result));
    }
+
+    /// <summary>
+    /// Returns cursor-paginated audit trail events for a batch.
+    /// Events are ordered chronologically (oldest first).
+    /// Pass the "after" parameter with the cursor from the previous page to paginate.
+    /// </summary>
+    /// <param name="id">Batch ID.</param>
+    /// <param name="after">Cursor: ISO-8601 timestamp from the previous page's nextCursor field.</param>
+    /// <param name="pageSize">Number of events per page. Default 50, max 200.</param>
+    [HttpGet("{id:guid}/events")]
+    [Authorize(Roles = "ADMINISTRATOR,VERIFIER,CLINICAL_APPROVER")]
+    [ProducesResponseType(typeof(ApiResponse<CursorPagedResult<BatchEventResponse>>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status400BadRequest)]
+    [ProducesResponseType(typeof(ApiResponse<object>), StatusCodes.Status404NotFound)]
+    public async Task<IActionResult> GetEvents(
+        Guid id,
+        [FromQuery] string? after = null,
+        [FromQuery] int pageSize = 50)
+    {
+        DateTimeOffset? afterCursor = null;
+        if (!string.IsNullOrWhiteSpace(after))
+        {
+            if (!DateTimeOffset.TryParse(after, out var parsed))
+                return BadRequest(ApiResponse<object>.Fail(
+                    400,
+                    "Invalid cursor format. Expected ISO-8601 timestamp.",
+                    "INVALID_CURSOR"));
+
+            afterCursor = parsed;
+        }
+
+        var result = await _batchEventService.GetEventsAsync(id, afterCursor, pageSize);
+        return Ok(ApiResponse<CursorPagedResult<BatchEventResponse>>.Ok(result));
+    }
 }