fix: MetricsCollectorService does not track APPROVED or retry-pending batches
This commit is contained in:
@@ -39,7 +39,8 @@ public class MetricsCollectorService : BackgroundService
|
|||||||
BatchStatus.Verified,
|
BatchStatus.Verified,
|
||||||
BatchStatus.AwaitingClinicalApproval,
|
BatchStatus.AwaitingClinicalApproval,
|
||||||
BatchStatus.Approved,
|
BatchStatus.Approved,
|
||||||
BatchStatus.Promoted
|
BatchStatus.Promoted,
|
||||||
|
BatchStatus.Cancelled
|
||||||
};
|
};
|
||||||
|
|
||||||
public MetricsCollectorService(
|
public MetricsCollectorService(
|
||||||
@@ -121,11 +122,41 @@ public class MetricsCollectorService : BackgroundService
|
|||||||
DiagnosticsMetrics.QueueAgeSeconds.Set(0);
|
DiagnosticsMetrics.QueueAgeSeconds.Set(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- Promotion retry gauges ---
|
||||||
|
var pendingRetries = await db.PromotionAttempts
|
||||||
|
.AsNoTracking()
|
||||||
|
.CountAsync(a => !a.Succeeded && a.NextRetryAt != null, ct);
|
||||||
|
DiagnosticsMetrics.PromotionPendingRetries.Set(pendingRetries);
|
||||||
|
|
||||||
|
var exhaustedCount = await db.DigitizationBatches
|
||||||
|
.AsNoTracking()
|
||||||
|
.Where(b => b.Status == BatchStatus.Approved)
|
||||||
|
.Where(b => db.PromotionAttempts
|
||||||
|
.Any(a => a.BatchId == b.Id && !a.Succeeded && a.NextRetryAt == null))
|
||||||
|
.CountAsync(ct);
|
||||||
|
DiagnosticsMetrics.PromotionExhaustedTotal.Set(exhaustedCount);
|
||||||
|
|
||||||
|
// --- Approval queue age: oldest APPROVED batch ---
|
||||||
|
var oldestApprovedUpdatedAt = await db.DigitizationBatches
|
||||||
|
.AsNoTracking()
|
||||||
|
.Where(b => b.Status == BatchStatus.Approved)
|
||||||
|
.OrderBy(b => b.UpdatedAt)
|
||||||
|
.Select(b => (DateTimeOffset?)b.UpdatedAt)
|
||||||
|
.FirstOrDefaultAsync(ct);
|
||||||
|
|
||||||
|
DiagnosticsMetrics.ApprovalQueueAgeSeconds.Set(
|
||||||
|
oldestApprovedUpdatedAt.HasValue
|
||||||
|
? (DateTimeOffset.UtcNow - oldestApprovedUpdatedAt.Value).TotalSeconds
|
||||||
|
: 0);
|
||||||
|
|
||||||
_logger.LogDebug(
|
_logger.LogDebug(
|
||||||
"Metrics collected: {StatusCount} status groups, queue age {QueueAge}s",
|
"Metrics collected: {StatusCount} status groups, queue age {QueueAge}s, " +
|
||||||
|
"pending retries {PendingRetries}, exhausted {Exhausted}",
|
||||||
statusCounts.Count,
|
statusCounts.Count,
|
||||||
oldestPendingUpdatedAt.HasValue
|
oldestPendingUpdatedAt.HasValue
|
||||||
? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds
|
? (DateTimeOffset.UtcNow - oldestPendingUpdatedAt.Value).TotalSeconds
|
||||||
: 0);
|
: 0,
|
||||||
|
pendingRetries,
|
||||||
|
exhaustedCount);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -94,4 +94,16 @@ public static class DiagnosticsMetrics
|
|||||||
{
|
{
|
||||||
LabelNames = new[] { "outcome" }
|
LabelNames = new[] { "outcome" }
|
||||||
});
|
});
|
||||||
|
|
||||||
|
public static readonly Gauge PromotionPendingRetries = Metrics.CreateGauge(
|
||||||
|
"digitization_promotion_pending_retries",
|
||||||
|
"Count of promotion attempts with a scheduled retry that have not yet succeeded.");
|
||||||
|
|
||||||
|
public static readonly Gauge PromotionExhaustedTotal = Metrics.CreateGauge(
|
||||||
|
"digitization_promotion_exhausted_total",
|
||||||
|
"Count of APPROVED batches where all retry attempts are exhausted.");
|
||||||
|
|
||||||
|
public static readonly Gauge ApprovalQueueAgeSeconds = Metrics.CreateGauge(
|
||||||
|
"digitization_approval_queue_age_seconds",
|
||||||
|
"Age in seconds of the oldest batch in APPROVED status awaiting promotion retry.");
|
||||||
}
|
}
|
||||||
@@ -819,7 +819,7 @@ A batch stuck in `APPROVED` with exhausted retries is invisible in Prometheus da
|
|||||||
| 22 | No frontend tests | P4 | E | Open |
|
| 22 | No frontend tests | P4 | E | Open |
|
||||||
| 23 | No field-level draft audit | P5 | F | Open |
|
| 23 | No field-level draft audit | P5 | F | Open |
|
||||||
| 24 | Integration test gaps | P5 | F | Done |
|
| 24 | Integration test gaps | P5 | F | Done |
|
||||||
| 25 | MetricsCollector missing retry gauges | P5 | F | Open |
|
| 25 | MetricsCollector missing retry gauges | P5 | F | Done |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user