feature: Observability: Prometheus Metrics and Grafana

2026-06-17 16:25:27 +08:00
parent 101040f9d9
commit df99bf3c91
22 changed files with 1462 additions and 132 deletions
@@ -0,0 +1,68 @@
+using Prometheus;
+
+public sealed class ClinicalMetrics
+{
+    // --- Counters ---
+
+    // Labeled by observation_code and source so clinicians can see which device types
+    // and which codes dominate the ingest volume.
+    public readonly Counter ObservationsIngestedTotal = Metrics.CreateCounter(
+        "observations_ingested_total",
+        "Total observations ingested, labeled by observation code and source.",
+        labelNames: new[] { "observation_code", "source" });
+
+    // Labeled by alert_type (THRESHOLD_BREACH, SEPSIS_WARNING) and severity
+    // (Critical, Warning) so the dashboard can show Critical vs Warning rates separately.
+    public readonly Counter ClinicalAlertsTotal = Metrics.CreateCounter(
+        "clinical_alerts_total",
+        "Total clinical alerts generated, labeled by type and severity.",
+        labelNames: new[] { "alert_type", "severity" });
+
+    // Incremented only when INSERT WHERE NOT EXISTS succeeds — duplicate-suppressed
+    // SIRS detections do not count. This is the true detection rate, not the evaluation rate.
+    public readonly Counter SirsDetectionsTotal = Metrics.CreateCounter(
+        "sirs_detections_total",
+        "Total SEPSIS_WARNING alerts generated by the sepsis detection engine.");
+
+    // Incremented by EscalationWorkerService when it processes a message from
+    // alerts.escalation.queue. A rising escalations_total is the strongest operational
+    // signal that critical alerts are not being acknowledged by the attending physician.
+    public readonly Counter EscalationsTotal = Metrics.CreateCounter(
+        "escalations_total",
+        "Total alert escalations processed through the DLQ escalation path.");
+
+    // --- Histograms ---
+
+    // Measures the full ingest transaction: Redis cache lookup + alert evaluation +
+    // outbox write + COMMIT. The 99th percentile matters for patient safety —
+    // a slow ingest path delays the critical alert creation.
+    public readonly Histogram ObservationIngestDuration = Metrics.CreateHistogram(
+        "observation_ingest_duration_seconds",
+        "Ingest transaction duration from request receipt to COMMIT.",
+        new HistogramConfiguration
+        {
+            Buckets = new[] { 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0 }
+        });
+
+    // --- Gauges (set by background collectors, not incremented inline) ---
+
+    // The most clinically significant panel. A non-zero value means a patient's
+    // critical alert has gone unacknowledged for more than 5 minutes.
+    // In a real deployment this panel drives an on-call pager alert at the nurse station.
+    public readonly Gauge AlertsUnacknowledgedGauge = Metrics.CreateGauge(
+        "alerts_unacknowledged_gauge",
+        "Count of open CRITICAL alerts older than 5 minutes with no acknowledgment.");
+
+    // Per consumer group so the dashboard can show whether es-indexer, sepsis-engine,
+    // or data-lake-writer is falling behind the observation stream.
+    public readonly Gauge KafkaConsumerLag = Metrics.CreateGauge(
+        "kafka_consumer_lag",
+        "Approximate consumer group lag in messages, labeled by consumer group.",
+        labelNames: new[] { "consumer_group" });
+
+    // An outbox that is growing means the relay is not keeping up or Kafka is unavailable.
+    // In a patient safety system, a growing outbox delays alert delivery to all consumers.
+    public readonly Gauge OutboxPendingEvents = Metrics.CreateGauge(
+        "outbox_pending_events",
+        "Count of outbox events not yet relayed to Kafka.");
+}