fix: MetricsCollectorService does not track APPROVED or retry-pending batches

This commit is contained in:
voltsrage
2026-06-27 20:58:57 +08:00
parent 903cd54a51
commit 5db60f46eb
3 changed files with 47 additions and 4 deletions
@@ -39,7 +39,8 @@ public class MetricsCollectorService : BackgroundService
BatchStatus.Verified, BatchStatus.Verified,
BatchStatus.AwaitingClinicalApproval, BatchStatus.AwaitingClinicalApproval,
BatchStatus.Approved, BatchStatus.Approved,
BatchStatus.Promoted BatchStatus.Promoted,
BatchStatus.Cancelled
}; };
public MetricsCollectorService( public MetricsCollectorService(
@@ -121,11 +122,41 @@ public class MetricsCollectorService : BackgroundService
DiagnosticsMetrics.QueueAgeSeconds.Set(0); DiagnosticsMetrics.QueueAgeSeconds.Set(0);
} }
// --- Promotion retry gauges ---
var pendingRetries = await db.PromotionAttempts
.AsNoTracking()
.CountAsync(a => !a.Succeeded && a.NextRetryAt != null, ct);
DiagnosticsMetrics.PromotionPendingRetries.Set(pendingRetries);
var exhaustedCount = await db.DigitizationBatches
.AsNoTracking()
.Where(b => b.Status == BatchStatus.Approved)
.Where(b => db.PromotionAttempts
.Any(a => a.BatchId == b.Id && !a.Succeeded && a.NextRetryAt == null))
.CountAsync(ct);
DiagnosticsMetrics.PromotionExhaustedTotal.Set(exhaustedCount);
// --- Approval queue age: oldest APPROVED batch ---
var oldestApprovedUpdatedAt = await db.DigitizationBatches
.AsNoTracking()
.Where(b => b.Status == BatchStatus.Approved)
.OrderBy(b => b.UpdatedAt)
.Select(b => (DateTimeOffset?)b.UpdatedAt)
.FirstOrDefaultAsync(ct);
DiagnosticsMetrics.ApprovalQueueAgeSeconds.Set(
oldestApprovedUpdatedAt.HasValue
? (DateTimeOffset.UtcNow - oldestApprovedUpdatedAt.Value).TotalSeconds
: 0);
_logger.LogDebug( _logger.LogDebug(
"Metrics collected: {StatusCount} status groups, queue age {QueueAge}s", "Metrics collected: {StatusCount} status groups, queue age {QueueAge}s, " +
"pending retries {PendingRetries}, exhausted {Exhausted}",
statusCounts.Count, statusCounts.Count,
oldestPendingUpdatedAt.HasValue oldestPendingUpdatedAt.HasValue
? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds ? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds
: 0); : 0,
pendingRetries,
exhaustedCount);
} }
} }
@@ -94,4 +94,16 @@ public static class DiagnosticsMetrics
{ {
LabelNames = new[] { "outcome" } LabelNames = new[] { "outcome" }
}); });
public static readonly Gauge PromotionPendingRetries = Metrics.CreateGauge(
"digitization_promotion_pending_retries",
"Count of promotion attempts with a scheduled retry that have not yet succeeded.");
public static readonly Gauge PromotionExhaustedTotal = Metrics.CreateGauge(
"digitization_promotion_exhausted_total",
"Count of APPROVED batches where all retry attempts are exhausted.");
public static readonly Gauge ApprovalQueueAgeSeconds = Metrics.CreateGauge(
"digitization_approval_queue_age_seconds",
"Age in seconds of the oldest batch in APPROVED status awaiting promotion retry.");
} }
+1 -1
View File
@@ -819,7 +819,7 @@ A batch stuck in `APPROVED` with exhausted retries is invisible in Prometheus da
| 22 | No frontend tests | P4 | E | Open | | 22 | No frontend tests | P4 | E | Open |
| 23 | No field-level draft audit | P5 | F | Open | | 23 | No field-level draft audit | P5 | F | Open |
| 24 | Integration test gaps | P5 | F | Done | | 24 | Integration test gaps | P5 | F | Done |
| 25 | MetricsCollector missing retry gauges | P5 | F | Open | | 25 | MetricsCollector missing retry gauges | P5 | F | Done |
--- ---