feature: Observability: Prometheus Metrics and Grafana
This commit is contained in:
voltsrage
@@ -1,5 +1,7 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Prometheus;
|
||||
using Serilog.Context;
|
||||
using StackExchange.Redis;
|
||||
|
||||
public class ObservationService : IObservationService
|
||||
@@ -7,15 +9,18 @@ public class ObservationService : IObservationService
|
||||
private readonly AppDbContext _db;
|
||||
private readonly IConnectionMultiplexer _redis;
|
||||
private readonly ILogger<ObservationService> _logger;
|
||||
private readonly ClinicalMetrics _metrics;
|
||||
|
||||
public ObservationService(
|
||||
AppDbContext db,
|
||||
IConnectionMultiplexer redis,
|
||||
ILogger<ObservationService> logger)
|
||||
ILogger<ObservationService> logger,
|
||||
ClinicalMetrics metrics)
|
||||
{
|
||||
_db = db;
|
||||
_redis = redis;
|
||||
_logger = logger;
|
||||
_metrics = metrics;
|
||||
}
|
||||
|
||||
public async Task<IngestResult> IngestAsync(Guid encounterId, IngestObservationRequest req)
|
||||
@@ -56,114 +61,133 @@ public class ObservationService : IObservationService
|
||||
if (!PlausibilityValidator.IsPlausible(req.ObservationCode, req.Value, out var reason))
|
||||
throw new ValidationException(reason!, "OBSERVATION_OUT_OF_PLAUSIBLE_RANGE");
|
||||
|
||||
// Steps 4–8 are one atomic transaction
|
||||
await using var tx = await _db.Database.BeginTransactionAsync();
|
||||
try
|
||||
using (LogContext.PushProperty("EncounterId", encounterId))
|
||||
using (LogContext.PushProperty("PatientId", encounter.PatientId))
|
||||
{
|
||||
// Step 4 — insert observation
|
||||
var observation = new Observation
|
||||
using var timer = _metrics.ObservationIngestDuration.NewTimer();
|
||||
|
||||
await using var tx = await _db.Database.BeginTransactionAsync();
|
||||
try
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
EncounterId = encounterId,
|
||||
ObservationCode = req.ObservationCode,
|
||||
Value = req.Value,
|
||||
Unit = req.Unit,
|
||||
Source = req.Source,
|
||||
IdempotencyKey = req.IdempotencyKey,
|
||||
RecordedAt = req.RecordedAt,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
_db.Observations.Add(observation);
|
||||
|
||||
// Step 5 — load threshold from Redis; fall back to PostgreSQL on miss
|
||||
var threshold = await LoadThresholdAsync(req.ObservationCode);
|
||||
|
||||
if (threshold is null)
|
||||
throw new ValidationException(
|
||||
$"No alert threshold is configured for observation code '{req.ObservationCode}'. " +
|
||||
"Register a threshold before recording observations for this code.",
|
||||
"UNKNOWN_OBSERVATION_CODE");
|
||||
|
||||
ClinicalAlert? alert = null;
|
||||
|
||||
// Step 6 — critical threshold detection (synchronous)
|
||||
// WARNING detection is intentionally deferred to the Kafka consumer.
|
||||
// A critical potassium of 2.1 mEq/L is immediately life-threatening — the alert
|
||||
// must exist before this API call returns. A warning heart rate of 95 bpm warrants
|
||||
// attention but not an emergency page; the additional Kafka latency is clinically safe.
|
||||
if (IsCriticalBreach(req.Value, threshold))
|
||||
{
|
||||
alert = new ClinicalAlert
|
||||
// Step 4 — insert observation
|
||||
var observation = new Observation
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
EncounterId = encounterId,
|
||||
PatientId = encounter.PatientId,
|
||||
ObservationId = observation.Id,
|
||||
AlertType = AlertTypeExtensions.CriticalFor(req.ObservationCode),
|
||||
Severity = AlertSeverity.Critical,
|
||||
Details = BuildCriticalDetails(req, threshold),
|
||||
Status = AlertStatus.Open,
|
||||
TriggeredAt = DateTimeOffset.UtcNow
|
||||
ObservationCode = req.ObservationCode,
|
||||
Value = req.Value,
|
||||
Unit = req.Unit,
|
||||
Source = req.Source,
|
||||
IdempotencyKey = req.IdempotencyKey,
|
||||
RecordedAt = req.RecordedAt,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
_db.ClinicalAlerts.Add(alert);
|
||||
_db.Observations.Add(observation);
|
||||
|
||||
// Step 6b — outbox event for the alert (relay picks this up in Phase 3)
|
||||
_db.OutboxEvents.Add(BuildOutboxEvent("alert.generated", new
|
||||
// Step 5 — load threshold from Redis; fall back to PostgreSQL on miss
|
||||
var threshold = await LoadThresholdAsync(req.ObservationCode);
|
||||
|
||||
if (threshold is null)
|
||||
throw new ValidationException(
|
||||
$"No alert threshold is configured for observation code '{req.ObservationCode}'. " +
|
||||
"Register a threshold before recording observations for this code.",
|
||||
"UNKNOWN_OBSERVATION_CODE");
|
||||
|
||||
ClinicalAlert? alert = null;
|
||||
|
||||
// Step 6 — critical threshold detection (synchronous)
|
||||
// WARNING detection is intentionally deferred to the Kafka consumer.
|
||||
// A critical potassium of 2.1 mEq/L is immediately life-threatening — the alert
|
||||
// must exist before this API call returns. A warning heart rate of 95 bpm warrants
|
||||
// attention but not an emergency page; the additional Kafka latency is clinically safe.
|
||||
if (IsCriticalBreach(req.Value, threshold))
|
||||
{
|
||||
alertId = alert.Id,
|
||||
alert = new ClinicalAlert
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
EncounterId = encounterId,
|
||||
PatientId = encounter.PatientId,
|
||||
ObservationId = observation.Id,
|
||||
AlertType = AlertTypeExtensions.CriticalFor(req.ObservationCode),
|
||||
Severity = AlertSeverity.Critical,
|
||||
Details = BuildCriticalDetails(req, threshold),
|
||||
Status = AlertStatus.Open,
|
||||
TriggeredAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
_db.ClinicalAlerts.Add(alert);
|
||||
|
||||
// Step 6b — outbox event for the alert (relay picks this up in Phase 3)
|
||||
_db.OutboxEvents.Add(BuildOutboxEvent("alert.generated", new
|
||||
{
|
||||
alertId = alert.Id,
|
||||
encounterId,
|
||||
patientId = encounter.PatientId,
|
||||
department = encounter.Department.ToDbString(),
|
||||
alertType = alert.AlertType.ToDbString(),
|
||||
severity = alert.Severity.ToDbString(),
|
||||
details = alert.Details,
|
||||
attendingPhysician = encounter.AttendingPhysician,
|
||||
triggeredAt = alert.TriggeredAt,
|
||||
partitionKey = encounterId.ToString()
|
||||
}, encounterId.ToString()));
|
||||
}
|
||||
|
||||
// Step 7 — outbox event for the observation (always; Kafka consumer handles warnings)
|
||||
_db.OutboxEvents.Add(BuildOutboxEvent("observation.recorded", new
|
||||
{
|
||||
observationId = observation.Id,
|
||||
encounterId,
|
||||
patientId = encounter.PatientId,
|
||||
department = encounter.Department.ToDbString(),
|
||||
alertType = alert.AlertType.ToDbString(),
|
||||
severity = alert.Severity.ToDbString(),
|
||||
details = alert.Details,
|
||||
attendingPhysician = encounter.AttendingPhysician,
|
||||
triggeredAt = alert.TriggeredAt,
|
||||
mrn = encounter.Patient?.Mrn,
|
||||
observationCode = req.ObservationCode,
|
||||
value = req.Value,
|
||||
unit = req.Unit,
|
||||
source = req.Source.ToDbString(),
|
||||
recordedAt = req.RecordedAt,
|
||||
partitionKey = encounterId.ToString()
|
||||
}, encounterId.ToString()));
|
||||
|
||||
// Step 8 — COMMIT
|
||||
await _db.SaveChangesAsync();
|
||||
await tx.CommitAsync();
|
||||
|
||||
_metrics.ObservationsIngestedTotal
|
||||
.WithLabels(req.ObservationCode, req.Source.ToDbString())
|
||||
.Inc();
|
||||
|
||||
if (alert is not null)
|
||||
{
|
||||
_metrics.ClinicalAlertsTotal
|
||||
.WithLabels(alert.AlertType.ToDbString(), alert.Severity.ToDbString())
|
||||
.Inc();
|
||||
_logger.LogWarning(
|
||||
"Critical threshold breach. Code={Code} Value={Value} AlertId={AlertId}",
|
||||
req.ObservationCode, req.Value, alert.Id);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Observation {ObservationId} ingested for encounter {EncounterId}. AlertCreated={AlertCreated}",
|
||||
observation.Id, encounterId, alert is not null);
|
||||
|
||||
return IngestResult.Created(observation, alert);
|
||||
}
|
||||
|
||||
// Step 7 — outbox event for the observation (always; Kafka consumer handles warnings)
|
||||
_db.OutboxEvents.Add(BuildOutboxEvent("observation.recorded", new
|
||||
catch (DbUpdateException ex) when (IsUniqueViolation(ex))
|
||||
{
|
||||
observationId = observation.Id,
|
||||
encounterId,
|
||||
patientId = encounter.PatientId,
|
||||
mrn = encounter.Patient?.Mrn,
|
||||
observationCode = req.ObservationCode,
|
||||
value = req.Value,
|
||||
unit = req.Unit,
|
||||
source = req.Source.ToDbString(),
|
||||
recordedAt = req.RecordedAt,
|
||||
partitionKey = encounterId.ToString()
|
||||
}, encounterId.ToString()));
|
||||
|
||||
// Step 8 — COMMIT
|
||||
await _db.SaveChangesAsync();
|
||||
await tx.CommitAsync();
|
||||
|
||||
_logger.LogInformation(
|
||||
"Observation {ObservationId} ingested for encounter {EncounterId}. AlertCreated={AlertCreated}",
|
||||
observation.Id, encounterId, alert is not null);
|
||||
|
||||
return IngestResult.Created(observation, alert);
|
||||
}
|
||||
catch (DbUpdateException ex) when (IsUniqueViolation(ex))
|
||||
{
|
||||
// Race condition: two concurrent retries both passed the pre-check above.
|
||||
// The unique partial index caught it. Roll back and return the existing row.
|
||||
await tx.RollbackAsync();
|
||||
var existing = await _db.Observations
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(o => o.IdempotencyKey == req.IdempotencyKey);
|
||||
if (existing is not null)
|
||||
return IngestResult.Duplicate(existing);
|
||||
throw new ConflictException("Concurrent duplicate submission.", "CONCURRENT_DUPLICATE");
|
||||
}
|
||||
catch
|
||||
{
|
||||
await tx.RollbackAsync();
|
||||
throw;
|
||||
// Race condition: two concurrent retries both passed the pre-check above.
|
||||
// The unique partial index caught it. Roll back and return the existing row.
|
||||
await tx.RollbackAsync();
|
||||
var existing = await _db.Observations
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(o => o.IdempotencyKey == req.IdempotencyKey);
|
||||
if (existing is not null)
|
||||
return IngestResult.Duplicate(existing);
|
||||
throw new ConflictException("Concurrent duplicate submission.", "CONCURRENT_DUPLICATE");
|
||||
}
|
||||
catch
|
||||
{
|
||||
await tx.RollbackAsync();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user