From 5db60f46ebd46e0ce8de754777576281fc8b2fdf Mon Sep 17 00:00:00 2001
From: voltsrage <electric5388@hotmail.com>
Date: Sat, 27 Jun 2026 20:58:57 +0800
Subject: [PATCH] fix: MetricsCollectorService does not track APPROVED or
 retry-pending batches

---
 .../MetricsCollectorService.cs                | 37 +++++++++++++++++--
 .../Diagnostics/DiagnosticsMetrics.cs         | 12 ++++++
 docs/vigilcare-records-gap-analysis.md        |  2 +-
 3 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/VigilCareRecordsAPI/BackgroundServices/MetricsCollectorService.cs b/VigilCareRecordsAPI/BackgroundServices/MetricsCollectorService.cs
index deee226..21c772d 100644
--- a/VigilCareRecordsAPI/BackgroundServices/MetricsCollectorService.cs
+++ b/VigilCareRecordsAPI/BackgroundServices/MetricsCollectorService.cs
@@ -39,7 +39,8 @@ public class MetricsCollectorService : BackgroundService
         BatchStatus.Verified,
         BatchStatus.AwaitingClinicalApproval,
         BatchStatus.Approved,
-        BatchStatus.Promoted
+        BatchStatus.Promoted,
+        BatchStatus.Cancelled
     };
 
     public MetricsCollectorService(
@@ -121,11 +122,41 @@ public class MetricsCollectorService : BackgroundService
             DiagnosticsMetrics.QueueAgeSeconds.Set(0);
         }
 
+        // --- Promotion retry gauges ---
+        var pendingRetries = await db.PromotionAttempts
+            .AsNoTracking()
+            .CountAsync(a => !a.Succeeded && a.NextRetryAt != null, ct);
+        DiagnosticsMetrics.PromotionPendingRetries.Set(pendingRetries);
+
+        var exhaustedCount = await db.DigitizationBatches
+            .AsNoTracking()
+            .Where(b => b.Status == BatchStatus.Approved)
+            .Where(b => db.PromotionAttempts
+                .Any(a => a.BatchId == b.Id && !a.Succeeded && a.NextRetryAt == null))
+            .CountAsync(ct);
+        DiagnosticsMetrics.PromotionExhaustedTotal.Set(exhaustedCount);
+
+        // --- Approval queue age: oldest APPROVED batch ---
+        var oldestApprovedUpdatedAt = await db.DigitizationBatches
+            .AsNoTracking()
+            .Where(b => b.Status == BatchStatus.Approved)
+            .OrderBy(b => b.UpdatedAt)
+            .Select(b => (DateTimeOffset?)b.UpdatedAt)
+            .FirstOrDefaultAsync(ct);
+
+        DiagnosticsMetrics.ApprovalQueueAgeSeconds.Set(
+            oldestApprovedUpdatedAt.HasValue
+                ? (DateTimeOffset.UtcNow - oldestApprovedUpdatedAt.Value).TotalSeconds
+                : 0);
+
         _logger.LogDebug(
-            "Metrics collected: {StatusCount} status groups, queue age {QueueAge}s",
+            "Metrics collected: {StatusCount} status groups, queue age {QueueAge}s, " +
+            "pending retries {PendingRetries}, exhausted {Exhausted}",
             statusCounts.Count,
             oldestPendingUpdatedAt.HasValue
                 ? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds
-                : 0);
+                : 0,
+            pendingRetries,
+            exhaustedCount);
     }
 }
\ No newline at end of file
diff --git a/VigilCareRecordsAPI/Diagnostics/DiagnosticsMetrics.cs b/VigilCareRecordsAPI/Diagnostics/DiagnosticsMetrics.cs
index bce5c4b..0d89dae 100644
--- a/VigilCareRecordsAPI/Diagnostics/DiagnosticsMetrics.cs
+++ b/VigilCareRecordsAPI/Diagnostics/DiagnosticsMetrics.cs
@@ -94,4 +94,16 @@ public static class DiagnosticsMetrics
         {
             LabelNames = new[] { "outcome" }
         });
+
+    public static readonly Gauge PromotionPendingRetries = Metrics.CreateGauge(
+        "digitization_promotion_pending_retries",
+        "Count of promotion attempts with a scheduled retry that have not yet succeeded.");
+
+    public static readonly Gauge PromotionExhaustedTotal = Metrics.CreateGauge(
+        "digitization_promotion_exhausted_total",
+        "Count of APPROVED batches where all retry attempts are exhausted.");
+
+    public static readonly Gauge ApprovalQueueAgeSeconds = Metrics.CreateGauge(
+        "digitization_approval_queue_age_seconds",
+        "Age in seconds of the oldest batch in APPROVED status awaiting promotion retry.");
 }
\ No newline at end of file
diff --git a/docs/vigilcare-records-gap-analysis.md b/docs/vigilcare-records-gap-analysis.md
index d4a13d6..2e4c95d 100644
--- a/docs/vigilcare-records-gap-analysis.md
+++ b/docs/vigilcare-records-gap-analysis.md
@@ -819,7 +819,7 @@ A batch stuck in `APPROVED` with exhausted retries is invisible in Prometheus da
 | 22 | No frontend tests | P4 | E | Open |
 | 23 | No field-level draft audit | P5 | F | Open |
 | 24 | Integration test gaps | P5 | F | Done |
-| 25 | MetricsCollector missing retry gauges | P5 | F | Open |
+| 25 | MetricsCollector missing retry gauges | P5 | F | Done |
 
 ---