feature: Observability: Prometheus Metrics and Grafana

2026-06-17 16:25:27 +08:00
parent 101040f9d9
commit df99bf3c91
22 changed files with 1462 additions and 132 deletions
@@ -0,0 +1,181 @@
+using System.Globalization;
+using System.Net.Http.Json;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using StackExchange.Redis;
+
+public class ObservabilityPhase8Tests : IClassFixture<ApiFixture>
+{
+    private static readonly string[] ExpectedMetrics =
+    {
+        "observations_ingested_total",
+        "observation_ingest_duration_seconds",
+        "clinical_alerts_total",
+        "alerts_unacknowledged_gauge",
+        "kafka_consumer_lag",
+        "outbox_pending_events",
+        "sirs_detections_total",
+        "escalations_total",
+    };
+
+    private readonly ApiFixture _fixture;
+    private readonly HttpClient _http;
+
+    public ObservabilityPhase8Tests(ApiFixture fixture)
+    {
+        _fixture = fixture;
+        _http = fixture.CreateClient();
+    }
+
+    [Fact]
+    public async Task MetricsEndpoint_ReturnsAllEightMetricFamilies()
+    {
+        var resp = await _http.GetAsync("/metrics");
+        resp.EnsureSuccessStatusCode();
+
+        Assert.Equal("text/plain", resp.Content.Headers.ContentType?.MediaType);
+
+        var body = await resp.Content.ReadAsStringAsync();
+
+        foreach (var metric in ExpectedMetrics)
+        {
+            Assert.Contains(metric, body);
+        }
+    }
+
+    [Fact]
+    public async Task CorrelationMiddleware_AddsXCorrelationIdHeader()
+    {
+        var resp = await _http.GetAsync("/api/v1/alert-thresholds");
+        resp.EnsureSuccessStatusCode();
+
+        Assert.True(resp.Headers.Contains("X-Correlation-Id"),
+            "X-Correlation-Id response header is missing.");
+        var value = resp.Headers.GetValues("X-Correlation-Id").First();
+        Assert.NotEmpty(value);
+    }
+
+    [Fact]
+    public async Task CorrelationMiddleware_EchoesIncomingCorrelationId()
+    {
+        var correlationId = "test-correlation-abc123";
+        var req = new HttpRequestMessage(HttpMethod.Get, "/api/v1/alert-thresholds");
+        req.Headers.Add("X-Correlation-Id", correlationId);
+
+        var resp = await _http.SendAsync(req);
+        resp.EnsureSuccessStatusCode();
+
+        var echoed = resp.Headers.GetValues("X-Correlation-Id").First();
+        Assert.Equal(correlationId, echoed);
+    }
+
+    [Fact]
+    public async Task ObservationIngest_IncrementsObservationsIngestedTotal()
+    {
+        var beforeBody = await (await _http.GetAsync("/metrics")).Content.ReadAsStringAsync();
+        var before = ParseCounterValue(beforeBody, "observations_ingested_total");
+
+        await EnsureHeartRateThresholdAsync();
+        var encounterId = await CreateActiveEncounterAsync();
+
+        var ingestResp = await _http.PostAsJsonAsync(
+            $"/api/v1/encounters/{encounterId}/observations",
+            new BatchIngestRequest(new List<IngestObservationRequest>
+            {
+                new("HEART_RATE", 72, "bpm", ObservationSource.Device, DateTimeOffset.UtcNow, null)
+            }));
+        ingestResp.EnsureSuccessStatusCode();
+
+        var afterBody = await (await _http.GetAsync("/metrics")).Content.ReadAsStringAsync();
+        var after = ParseCounterValue(afterBody, "observations_ingested_total");
+
+        Assert.True(after > before,
+            $"observations_ingested_total did not increase. Before={before} After={after}");
+    }
+
+    private async Task EnsureHeartRateThresholdAsync()
+    {
+        using var scope = _fixture.Services.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+
+        var existing = await db.AlertThresholds
+            .FirstOrDefaultAsync(t => t.ObservationCode == "HEART_RATE");
+
+        if (existing is null)
+        {
+            db.AlertThresholds.Add(new AlertThreshold
+            {
+                Id = Guid.NewGuid(),
+                ObservationCode = "HEART_RATE",
+                DisplayName = "Heart Rate",
+                Unit = "bpm",
+                CriticalLow = 30,
+                WarningLow = 50,
+                WarningHigh = 100,
+                CriticalHigh = 150,
+                CreatedAt = DateTimeOffset.UtcNow
+            });
+            await db.SaveChangesAsync();
+        }
+
+        var redis = scope.ServiceProvider.GetRequiredService<IConnectionMultiplexer>();
+        var cache = redis.GetDatabase(1);
+        await cache.StringSetAsync("threshold:HEART_RATE",
+            """{"ObservationCode":"HEART_RATE","CriticalLow":30,"WarningLow":50,"WarningHigh":100,"CriticalHigh":150}""");
+    }
+
+    // Parse the sum of all label combinations for a counter family.
+    private static double ParseCounterValue(string metricsBody, string metricName)
+    {
+        double total = 0;
+        foreach (var rawLine in metricsBody.Split('\n', StringSplitOptions.RemoveEmptyEntries))
+        {
+            var line = rawLine.Trim();
+            if (line.StartsWith('#')) continue;
+            if (!line.StartsWith(metricName, StringComparison.Ordinal)) continue;
+
+            var valueToken = line.Split(' ', StringSplitOptions.RemoveEmptyEntries).LastOrDefault();
+            if (valueToken is null) continue;
+            if (double.TryParse(valueToken, NumberStyles.Float, CultureInfo.InvariantCulture, out var value))
+            {
+                total += value;
+            }
+        }
+        return total;
+    }
+
+    private async Task<Guid> CreateActiveEncounterAsync()
+    {
+        using var scope = _fixture.Services.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+
+        var patient = new Patient
+        {
+            Id = Guid.NewGuid(),
+            Mrn = $"MRN-METRICS-{Guid.NewGuid():N}"[..18],
+            FirstName = "Metrics",
+            LastName = "Test",
+            DateOfBirth = new DateOnly(1975, 3, 20),
+            Gender = "Female",
+            CreatedAt = DateTimeOffset.UtcNow
+        };
+
+        var encounter = new Encounter
+        {
+            Id = Guid.NewGuid(),
+            PatientId = patient.Id,
+            EncounterType = EncounterType.Inpatient,
+            Status = EncounterStatus.Active,
+            Department = Department.Icu,
+            AttendingPhysician = "Dr. Osei",
+            AdmittedAt = DateTimeOffset.UtcNow,
+            CreatedAt = DateTimeOffset.UtcNow
+        };
+
+        db.Patients.Add(patient);
+        db.Encounters.Add(encounter);
+        await db.SaveChangesAsync();
+
+        return encounter.Id;
+    }
+}