feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job

2026-06-27 13:29:04 +08:00
parent e22d33b654
commit 04bdb7e85c
31 changed files with 4482 additions and 259 deletions
@@ -0,0 +1,89 @@
+using Prometheus;
+
+/// <summary>
+/// Application-level Prometheus metrics for the digitization pipeline.
+/// All metrics are static singletons — safe for concurrent use across
+/// all services and background workers.
+///
+/// prometheus-net throws InvalidOperationException if a metric with the
+/// same name but different label configuration is registered twice.
+/// Static fields guarantee each metric is created exactly once.
+/// </summary>
+public static class DiagnosticsMetrics
+{
+    /// <summary>
+    /// Gauge: count of digitization batches per status.
+    /// Updated periodically by MetricsCollectorService.
+    /// Labels: status (UPLOADED, IN_ENTRY, PENDING_VERIFICATION, etc.)
+    ///
+    /// This is a gauge (not a counter) because statuses change — a batch
+    /// moves from UPLOADED to IN_ENTRY, decrementing one label and
+    /// incrementing another. The gauge is set to the current count
+    /// each collection cycle.
+    /// </summary>
+    public static readonly Gauge BatchesByStatus = Metrics.CreateGauge(
+        "digitization_batches_by_status",
+        "Number of digitization batches grouped by current status.",
+        new GaugeConfiguration
+        {
+            LabelNames = new[] { "status" }
+        });
+
+    /// <summary>
+    /// Histogram: how long a promotion operation takes in seconds.
+    /// Recorded in PromotionService when a batch transitions to Promoted.
+    /// Buckets tuned for typical promotion durations (50ms to 30s).
+    ///
+    /// The p50/p95/p99 can be derived from the bucket boundaries in
+    /// Grafana using histogram_quantile().
+    /// </summary>
+    public static readonly Histogram PromotionDuration = Metrics.CreateHistogram(
+        "digitization_promotion_duration_seconds",
+        "Duration of batch promotion operations in seconds.",
+        new HistogramConfiguration
+        {
+            Buckets = new[] { 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0 }
+        });
+
+    /// <summary>
+    /// Counter: total number of batch rejections.
+    /// Incremented in VerificationService on every rejection.
+    /// Labels: reason_category (verification_failed, clinical_rejected)
+    ///
+    /// Supervisors need to distinguish between verification-stage rejections
+    /// (data entry errors) and clinical-stage rejections (clinical judgment
+    /// issues). The label enables separate alerting thresholds.
+    /// </summary>
+    public static readonly Counter RejectionTotal = Metrics.CreateCounter(
+        "digitization_rejection_total",
+        "Total number of digitization batch rejections.",
+        new CounterConfiguration
+        {
+            LabelNames = new[] { "reason_category" }
+        });
+
+    private static readonly string[] RejectionReasonCategories =
+    {
+        "verification_failed",
+        "clinical_rejected"
+    };
+
+    static DiagnosticsMetrics()
+    {
+        // Expose all reason_category label combinations at 0 before any rejections occur.
+        foreach (var category in RejectionReasonCategories)
+            RejectionTotal.WithLabels(category).Inc(0);
+    }
+
+    /// <summary>
+    /// Gauge: age in seconds of the oldest batch in PendingVerification status.
+    /// Updated periodically by MetricsCollectorService.
+    /// A high value indicates the verification queue is backed up.
+    ///
+    /// This is a gauge because it reflects a point-in-time measurement —
+    /// the age of the oldest pending batch right now.
+    /// </summary>
+    public static readonly Gauge QueueAgeSeconds = Metrics.CreateGauge(
+        "digitization_queue_age_seconds",
+        "Age in seconds of the oldest batch in pending_verification status.");
+}