Trent/vigilcare-clinical
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature: Observability: Prometheus Metrics and Grafana

Browse Source
This commit is contained in:
voltsrage
2026-06-17 16:25:27 +08:00
parent 101040f9d9
commit df99bf3c91
22 changed files with 1462 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VigilCareClinicalAPI/BackgroundServices/Metrics/AlertsUnacknowledgedCollector.cs
+54
View File
@@ -0,0 +1,54 @@
using Microsoft.EntityFrameworkCore;
public sealed class AlertsUnacknowledgedCollector : BackgroundService
{
// 5 minutes matches the DLQ TTL — an alert that survived escalation is still open.
private static readonly TimeSpan UnacknowledgedThreshold = TimeSpan.FromMinutes(5);
private readonly IServiceScopeFactory _scopes;
private readonly ClinicalMetrics _metrics;
private readonly ILogger<AlertsUnacknowledgedCollector> _logger;
public AlertsUnacknowledgedCollector(
IServiceScopeFactory scopes,
ClinicalMetrics metrics,
ILogger<AlertsUnacknowledgedCollector> logger)
{
_scopes = scopes;
_metrics = metrics;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken ct)
{
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(30));
while (await timer.WaitForNextTickAsync(ct))
await CollectAsync(ct);
}
private async Task CollectAsync(CancellationToken ct)
{
try
{
await using var scope = _scopes.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var cutoff = DateTimeOffset.UtcNow - UnacknowledgedThreshold;
var count = await db.ClinicalAlerts
.CountAsync(a => a.Severity == AlertSeverity.Critical
&& a.Status == AlertStatus.Open
&& a.TriggeredAt < cutoff, ct);
_metrics.AlertsUnacknowledgedGauge.Set(count);
if (count > 0)
_logger.LogWarning(
"[PATIENT-SAFETY] alerts_unacknowledged_gauge={Count} " +
"(CRITICAL alerts open > 5 min)", count);
}
catch (Exception ex)
{
_logger.LogError(ex, "AlertsUnacknowledgedCollector failed");
}
}
}
VigilCareClinicalAPI/BackgroundServices/Metrics/KafkaConsumerLagCollector.cs
+91
View File
@@ -0,0 +1,91 @@
using Confluent.Kafka;
using Confluent.Kafka.Admin;
using Microsoft.Extensions.Options;
public sealed class KafkaConsumerLagCollector : BackgroundService
{
private static readonly string[] Groups =
{
"es-indexer",
"sepsis-engine",
"notification-publisher",
"data-lake-writer",
};
private readonly KafkaOptions _kafkaOptions;
private readonly ClinicalMetrics _metrics;
private readonly ILogger<KafkaConsumerLagCollector> _logger;
public KafkaConsumerLagCollector(
IOptions<KafkaOptions> kafkaOptions,
ClinicalMetrics metrics,
ILogger<KafkaConsumerLagCollector> logger)
{
_kafkaOptions = kafkaOptions.Value;
_metrics = metrics;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken ct)
{
// Initial delay so Kafka is reachable before the first collection.
await Task.Delay(TimeSpan.FromSeconds(20), ct);
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(30));
while (await timer.WaitForNextTickAsync(ct))
{
foreach (var group in Groups)
{
try { await CollectGroupLagAsync(group, ct); }
catch (Exception ex)
{
_logger.LogDebug(ex, "KafkaConsumerLagCollector: failed for group {Group}", group);
}
}
}
}
private async Task CollectGroupLagAsync(string groupId, CancellationToken ct)
{
var adminConfig = new AdminClientConfig
{ BootstrapServers = _kafkaOptions.BootstrapServers };
using var admin = new AdminClientBuilder(adminConfig).Build();
// List the committed offsets for this consumer group across all its partitions.
var result = await admin.ListConsumerGroupOffsetsAsync(
new[] { new ConsumerGroupTopicPartitions(groupId, null) },
new ListConsumerGroupOffsetsOptions { RequireStableOffsets = false });
var partitions = result.FirstOrDefault()?.Partitions ?? [];
if (!partitions.Any())
{
_metrics.KafkaConsumerLag.WithLabels(groupId).Set(0);
return;
}
// Query high watermarks using a temporary consumer (does not join the group).
using var tempConsumer = new ConsumerBuilder<Ignore, Ignore>(new ConsumerConfig
{
BootstrapServers = _kafkaOptions.BootstrapServers,
GroupId = $"__lag-probe",
}).Build();
long totalLag = 0;
foreach (var tpo in partitions)
{
if (tpo.Error.IsError || tpo.Offset == Offset.Unset) continue;
var watermarks = tempConsumer.QueryWatermarkOffsets(
tpo.TopicPartition, TimeSpan.FromSeconds(5));
// Committed offset is the next offset the consumer will read.
// High watermark is the latest available offset + 1.
// Lag = messages the consumer has not yet read.
var lag = watermarks.High.Value - tpo.Offset.Value;
totalLag += Math.Max(0L, lag);
}
_metrics.KafkaConsumerLag.WithLabels(groupId).Set(totalLag);
}
}
VigilCareClinicalAPI/BackgroundServices/Metrics/OutboxPendingCollector.cs
+42
View File
@@ -0,0 +1,42 @@
using Microsoft.EntityFrameworkCore;
public sealed class OutboxPendingCollector : BackgroundService
{
private readonly IServiceScopeFactory _scopes;
private readonly ClinicalMetrics _metrics;
private readonly ILogger<OutboxPendingCollector> _logger;
public OutboxPendingCollector(
IServiceScopeFactory scopes,
ClinicalMetrics metrics,
ILogger<OutboxPendingCollector> logger)
{
_scopes = scopes;
_metrics = metrics;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken ct)
{
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(30));
while (await timer.WaitForNextTickAsync(ct))
await CollectAsync(ct);
}
private async Task CollectAsync(CancellationToken ct)
{
try
{
await using var scope = _scopes.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var count = await db.OutboxEvents
.CountAsync(e => e.ProcessedAt == null, ct);
_metrics.OutboxPendingEvents.Set(count);
}
catch (Exception ex)
{
_logger.LogError(ex, "OutboxPendingCollector failed");
}
}
}
VigilCareClinicalAPI/BackgroundServices/Notifications/EscalationWorkerService.cs
+21 -11
View File
@@ -3,21 +3,25 @@ using System.Text.Json;
using Microsoft.Extensions.Options;
using RabbitMQ.Client;
using RabbitMQ.Client.Events;
using Serilog.Context;
public sealed class EscalationWorkerService : BackgroundService
{
private readonly IOptions<RabbitMqOptions> _opts;
private readonly IServiceScopeFactory _scopes;
private readonly ClinicalMetrics _metrics;
private readonly ILogger<EscalationWorkerService> _logger;
public EscalationWorkerService(
IOptions<RabbitMqOptions> opts,
IServiceScopeFactory scopes,
ClinicalMetrics metrics,
ILogger<EscalationWorkerService> logger)
{
_opts = opts;
_scopes = scopes;
_logger = logger;
_opts = opts;
_scopes = scopes;
_metrics = metrics;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -57,15 +61,20 @@ public sealed class EscalationWorkerService : BackgroundService
var payload = Encoding.UTF8.GetString(ea.Body.Span);
var doc = JsonDocument.Parse(payload);
var alertId = Guid.Parse(doc.RootElement.GetProperty("alertId").GetString()!);
var encounterId = doc.RootElement.GetProperty("encounterId").GetString();
var encounterId = Guid.Parse(doc.RootElement.GetProperty("encounterId").GetString()!);
_logger.LogCritical(
"[ESCALATION] Paging on-call backup — AlertId={AlertId} EncounterId={EncounterId}",
alertId, encounterId);
using (LogContext.PushProperty("EncounterId", encounterId))
using (LogContext.PushProperty("AlertId", alertId))
{
_logger.LogCritical("[ESCALATION] Paging on-call backup for alert {AlertId}", alertId);
}
try
{
await UpdateAlertStatusEscalatedAsync(alertId, ct);
var escalated = await UpdateAlertStatusEscalatedAsync(alertId, ct);
if (escalated)
_metrics.EscalationsTotal.Inc();
channel.BasicAck(ea.DeliveryTag, multiple: false);
_logger.LogWarning(
@@ -78,18 +87,19 @@ public sealed class EscalationWorkerService : BackgroundService
}
}
private async Task UpdateAlertStatusEscalatedAsync(Guid alertId, CancellationToken ct)
private async Task<bool> UpdateAlertStatusEscalatedAsync(Guid alertId, CancellationToken ct)
{
await using var scope = _scopes.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var alert = await db.ClinicalAlerts.FindAsync(new object[] { alertId }, ct);
if (alert is null) return;
if (alert is null) return false;
// Only escalate if still open — if acknowledged between NACK and TTL expiry, leave it.
if (alert.Status != AlertStatus.Open) return;
if (alert.Status != AlertStatus.Open) return false;
alert.Status = AlertStatus.Escalated;
await db.SaveChangesAsync(ct);
return true;
}
}
VigilCareClinicalAPI/BackgroundServices/Notifications/PagingWorkerService.cs
+6
View File
@@ -51,6 +51,12 @@ public sealed class PagingWorkerService : BackgroundService
{
await HandlePageAsync(channel, ea, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// Host is stopping while we were waiting for ack — requeue so restart does not false-escalate.
_logger.LogInformation("PagingWorker stopping — requeueing in-flight page message");
channel.BasicNack(ea.DeliveryTag, multiple: false, requeue: true);
}
catch (Exception ex)
{
_logger.LogError(ex, "PagingWorker failed — NACKing to DLQ");
VigilCareClinicalAPI/Observability/Metrics/ClinicalMetrics.cs
+68
View File
@@ -0,0 +1,68 @@
using Prometheus;
public sealed class ClinicalMetrics
{
// --- Counters ---
// Labeled by observation_code and source so clinicians can see which device types
// and which codes dominate the ingest volume.
public readonly Counter ObservationsIngestedTotal = Metrics.CreateCounter(
"observations_ingested_total",
"Total observations ingested, labeled by observation code and source.",
labelNames: new[] { "observation_code", "source" });
// Labeled by alert_type (THRESHOLD_BREACH, SEPSIS_WARNING) and severity
// (Critical, Warning) so the dashboard can show Critical vs Warning rates separately.
public readonly Counter ClinicalAlertsTotal = Metrics.CreateCounter(
"clinical_alerts_total",
"Total clinical alerts generated, labeled by type and severity.",
labelNames: new[] { "alert_type", "severity" });
// Incremented only when INSERT WHERE NOT EXISTS succeeds — duplicate-suppressed
// SIRS detections do not count. This is the true detection rate, not the evaluation rate.
public readonly Counter SirsDetectionsTotal = Metrics.CreateCounter(
"sirs_detections_total",
"Total SEPSIS_WARNING alerts generated by the sepsis detection engine.");
// Incremented by EscalationWorkerService when it processes a message from
// alerts.escalation.queue. A rising escalations_total is the strongest operational
// signal that critical alerts are not being acknowledged by the attending physician.
public readonly Counter EscalationsTotal = Metrics.CreateCounter(
"escalations_total",
"Total alert escalations processed through the DLQ escalation path.");
// --- Histograms ---
// Measures the full ingest transaction: Redis cache lookup + alert evaluation +
// outbox write + COMMIT. The 99th percentile matters for patient safety —
// a slow ingest path delays the critical alert creation.
public readonly Histogram ObservationIngestDuration = Metrics.CreateHistogram(
"observation_ingest_duration_seconds",
"Ingest transaction duration from request receipt to COMMIT.",
new HistogramConfiguration
{
Buckets = new[] { 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0 }
});
// --- Gauges (set by background collectors, not incremented inline) ---
// The most clinically significant panel. A non-zero value means a patient's
// critical alert has gone unacknowledged for more than 5 minutes.
// In a real deployment this panel drives an on-call pager alert at the nurse station.
public readonly Gauge AlertsUnacknowledgedGauge = Metrics.CreateGauge(
"alerts_unacknowledged_gauge",
"Count of open CRITICAL alerts older than 5 minutes with no acknowledgment.");
// Per consumer group so the dashboard can show whether es-indexer, sepsis-engine,
// or data-lake-writer is falling behind the observation stream.
public readonly Gauge KafkaConsumerLag = Metrics.CreateGauge(
"kafka_consumer_lag",
"Approximate consumer group lag in messages, labeled by consumer group.",
labelNames: new[] { "consumer_group" });
// An outbox that is growing means the relay is not keeping up or Kafka is unavailable.
// In a patient safety system, a growing outbox delays alert delivery to all consumers.
public readonly Gauge OutboxPendingEvents = Metrics.CreateGauge(
"outbox_pending_events",
"Count of outbox events not yet relayed to Kafka.");
}
VigilCareClinicalAPI/Program.cs
+9 -1
View File
@@ -1,5 +1,6 @@
using Elastic.Clients.Elasticsearch;
using Microsoft.EntityFrameworkCore;
using Prometheus;
using Serilog;
using StackExchange.Redis;
using System.Text.Json.Serialization;
@@ -19,7 +20,9 @@ try
builder.Host.UseSerilog((ctx, services, config) =>
config.ReadFrom.Configuration(ctx.Configuration)
.ReadFrom.Services(services)
.Enrich.FromLogContext());
.Enrich.FromLogContext()
.Enrich.WithMachineName()
.Enrich.WithThreadId());
}
builder.Services.AddDbContext<AppDbContext>(opts =>
@@ -77,6 +80,10 @@ try
builder.Services.AddHostedService<EscalationWorkerService>();
builder.Services.AddHostedService<DischargeSummaryWorkerService>();
builder.Services.AddHostedService<ReconciliationScheduler>();
builder.Services.AddSingleton<ClinicalMetrics>();
builder.Services.AddHostedService<AlertsUnacknowledgedCollector>();
builder.Services.AddHostedService<OutboxPendingCollector>();
builder.Services.AddHostedService<KafkaConsumerLagCollector>();
builder.Services.AddControllers()
.AddJsonOptions(opts =>
@@ -118,6 +125,7 @@ try
app.UseSwaggerUI();
}
app.MapMetrics("/metrics");
app.MapControllers();
app.Run();
VigilCareClinicalAPI/Properties/launchSettings.json
+2 -2
View File
@@ -14,7 +14,7 @@
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "swagger",
"applicationUrl": "http://localhost:5270",
"applicationUrl": "http://0.0.0.0:5270",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
@@ -24,7 +24,7 @@
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "swagger",
"applicationUrl": "https://localhost:7146;http://localhost:5270",
"applicationUrl": "https://localhost:7146;http://0.0.0.0:5270",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
VigilCareClinicalAPI/Sepsis/SirsDetector.cs
+17 -5
View File
@@ -1,5 +1,6 @@
using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using Serilog.Context;
using StackExchange.Redis;
public class SirsDetector
@@ -12,15 +13,18 @@ public class SirsDetector
private readonly IConnectionMultiplexer _redis;
private readonly IServiceProvider _services;
private readonly ILogger<SirsDetector> _logger;
private readonly ClinicalMetrics _metrics;
public SirsDetector(
IConnectionMultiplexer redis,
IServiceProvider services,
ILogger<SirsDetector> logger)
ILogger<SirsDetector> logger,
ClinicalMetrics metrics)
{
_redis = redis;
_services = services;
_logger = logger;
_metrics = metrics;
}
public async Task<SirsResult> ProcessObservationAsync(
@@ -151,10 +155,18 @@ public class SirsDetector
await db.SaveChangesAsync(ct);
await tx.CommitAsync(ct);
_logger.LogWarning(
"SEPSIS_WARNING alert {AlertId} created for encounter {EncounterId} " +
"— {Active}/4 SIRS criteria active",
alertId, encounterId, activeCount);
_metrics.SirsDetectionsTotal.Inc();
_metrics.ClinicalAlertsTotal
.WithLabels(AlertType.SepsisWarning.ToDbString(), AlertSeverity.Critical.ToDbString())
.Inc();
using (LogContext.PushProperty("EncounterId", encounterId))
using (LogContext.PushProperty("PatientId", patientId))
{
_logger.LogWarning(
"SEPSIS_WARNING created. ActiveCriteriaCount={Count} AlertId={AlertId}",
activeCount, alertId);
}
return true;
}
VigilCareClinicalAPI/Services/ObservationService.cs
+119 -95
View File
@@ -1,5 +1,7 @@
using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using Prometheus;
using Serilog.Context;
using StackExchange.Redis;
public class ObservationService : IObservationService
@@ -7,15 +9,18 @@ public class ObservationService : IObservationService
private readonly AppDbContext _db;
private readonly IConnectionMultiplexer _redis;
private readonly ILogger<ObservationService> _logger;
private readonly ClinicalMetrics _metrics;
public ObservationService(
AppDbContext db,
IConnectionMultiplexer redis,
ILogger<ObservationService> logger)
ILogger<ObservationService> logger,
ClinicalMetrics metrics)
{
_db = db;
_redis = redis;
_logger = logger;
_metrics = metrics;
}
public async Task<IngestResult> IngestAsync(Guid encounterId, IngestObservationRequest req)
@@ -56,114 +61,133 @@ public class ObservationService : IObservationService
if (!PlausibilityValidator.IsPlausible(req.ObservationCode, req.Value, out var reason))
throw new ValidationException(reason!, "OBSERVATION_OUT_OF_PLAUSIBLE_RANGE");
// Steps 4–8 are one atomic transaction
await using var tx = await _db.Database.BeginTransactionAsync();
try
using (LogContext.PushProperty("EncounterId", encounterId))
using (LogContext.PushProperty("PatientId", encounter.PatientId))
{
// Step 4 — insert observation
var observation = new Observation
using var timer = _metrics.ObservationIngestDuration.NewTimer();
await using var tx = await _db.Database.BeginTransactionAsync();
try
{
Id = Guid.NewGuid(),
EncounterId = encounterId,
ObservationCode = req.ObservationCode,
Value = req.Value,
Unit = req.Unit,
Source = req.Source,
IdempotencyKey = req.IdempotencyKey,
RecordedAt = req.RecordedAt,
CreatedAt = DateTimeOffset.UtcNow
};
_db.Observations.Add(observation);
// Step 5 — load threshold from Redis; fall back to PostgreSQL on miss
var threshold = await LoadThresholdAsync(req.ObservationCode);
if (threshold is null)
throw new ValidationException(
$"No alert threshold is configured for observation code '{req.ObservationCode}'. " +
"Register a threshold before recording observations for this code.",
"UNKNOWN_OBSERVATION_CODE");
ClinicalAlert? alert = null;
// Step 6 — critical threshold detection (synchronous)
// WARNING detection is intentionally deferred to the Kafka consumer.
// A critical potassium of 2.1 mEq/L is immediately life-threatening — the alert
// must exist before this API call returns. A warning heart rate of 95 bpm warrants
// attention but not an emergency page; the additional Kafka latency is clinically safe.
if (IsCriticalBreach(req.Value, threshold))
{
alert = new ClinicalAlert
// Step 4 — insert observation
var observation = new Observation
{
Id = Guid.NewGuid(),
EncounterId = encounterId,
PatientId = encounter.PatientId,
ObservationId = observation.Id,
AlertType = AlertTypeExtensions.CriticalFor(req.ObservationCode),
Severity = AlertSeverity.Critical,
Details = BuildCriticalDetails(req, threshold),
Status = AlertStatus.Open,
TriggeredAt = DateTimeOffset.UtcNow
ObservationCode = req.ObservationCode,
Value = req.Value,
Unit = req.Unit,
Source = req.Source,
IdempotencyKey = req.IdempotencyKey,
RecordedAt = req.RecordedAt,
CreatedAt = DateTimeOffset.UtcNow
};
_db.ClinicalAlerts.Add(alert);
_db.Observations.Add(observation);
// Step 6b — outbox event for the alert (relay picks this up in Phase 3)
_db.OutboxEvents.Add(BuildOutboxEvent("alert.generated", new
// Step 5 — load threshold from Redis; fall back to PostgreSQL on miss
var threshold = await LoadThresholdAsync(req.ObservationCode);
if (threshold is null)
throw new ValidationException(
$"No alert threshold is configured for observation code '{req.ObservationCode}'. " +
"Register a threshold before recording observations for this code.",
"UNKNOWN_OBSERVATION_CODE");
ClinicalAlert? alert = null;
// Step 6 — critical threshold detection (synchronous)
// WARNING detection is intentionally deferred to the Kafka consumer.
// A critical potassium of 2.1 mEq/L is immediately life-threatening — the alert
// must exist before this API call returns. A warning heart rate of 95 bpm warrants
// attention but not an emergency page; the additional Kafka latency is clinically safe.
if (IsCriticalBreach(req.Value, threshold))
{
alertId = alert.Id,
alert = new ClinicalAlert
{
Id = Guid.NewGuid(),
EncounterId = encounterId,
PatientId = encounter.PatientId,
ObservationId = observation.Id,
AlertType = AlertTypeExtensions.CriticalFor(req.ObservationCode),
Severity = AlertSeverity.Critical,
Details = BuildCriticalDetails(req, threshold),
Status = AlertStatus.Open,
TriggeredAt = DateTimeOffset.UtcNow
};
_db.ClinicalAlerts.Add(alert);
// Step 6b — outbox event for the alert (relay picks this up in Phase 3)
_db.OutboxEvents.Add(BuildOutboxEvent("alert.generated", new
{
alertId = alert.Id,
encounterId,
patientId = encounter.PatientId,
department = encounter.Department.ToDbString(),
alertType = alert.AlertType.ToDbString(),
severity = alert.Severity.ToDbString(),
details = alert.Details,
attendingPhysician = encounter.AttendingPhysician,
triggeredAt = alert.TriggeredAt,
partitionKey = encounterId.ToString()
}, encounterId.ToString()));
}
// Step 7 — outbox event for the observation (always; Kafka consumer handles warnings)
_db.OutboxEvents.Add(BuildOutboxEvent("observation.recorded", new
{
observationId = observation.Id,
encounterId,
patientId = encounter.PatientId,
department = encounter.Department.ToDbString(),
alertType = alert.AlertType.ToDbString(),
severity = alert.Severity.ToDbString(),
details = alert.Details,
attendingPhysician = encounter.AttendingPhysician,
triggeredAt = alert.TriggeredAt,
mrn = encounter.Patient?.Mrn,
observationCode = req.ObservationCode,
value = req.Value,
unit = req.Unit,
source = req.Source.ToDbString(),
recordedAt = req.RecordedAt,
partitionKey = encounterId.ToString()
}, encounterId.ToString()));
// Step 8 — COMMIT
await _db.SaveChangesAsync();
await tx.CommitAsync();
_metrics.ObservationsIngestedTotal
.WithLabels(req.ObservationCode, req.Source.ToDbString())
.Inc();
if (alert is not null)
{
_metrics.ClinicalAlertsTotal
.WithLabels(alert.AlertType.ToDbString(), alert.Severity.ToDbString())
.Inc();
_logger.LogWarning(
"Critical threshold breach. Code={Code} Value={Value} AlertId={AlertId}",
req.ObservationCode, req.Value, alert.Id);
}
_logger.LogInformation(
"Observation {ObservationId} ingested for encounter {EncounterId}. AlertCreated={AlertCreated}",
observation.Id, encounterId, alert is not null);
return IngestResult.Created(observation, alert);
}
// Step 7 — outbox event for the observation (always; Kafka consumer handles warnings)
_db.OutboxEvents.Add(BuildOutboxEvent("observation.recorded", new
catch (DbUpdateException ex) when (IsUniqueViolation(ex))
{
observationId = observation.Id,
encounterId,
patientId = encounter.PatientId,
mrn = encounter.Patient?.Mrn,
observationCode = req.ObservationCode,
value = req.Value,
unit = req.Unit,
source = req.Source.ToDbString(),
recordedAt = req.RecordedAt,
partitionKey = encounterId.ToString()
}, encounterId.ToString()));
// Step 8 — COMMIT
await _db.SaveChangesAsync();
await tx.CommitAsync();
_logger.LogInformation(
"Observation {ObservationId} ingested for encounter {EncounterId}. AlertCreated={AlertCreated}",
observation.Id, encounterId, alert is not null);
return IngestResult.Created(observation, alert);
}
catch (DbUpdateException ex) when (IsUniqueViolation(ex))
{
// Race condition: two concurrent retries both passed the pre-check above.
// The unique partial index caught it. Roll back and return the existing row.
await tx.RollbackAsync();
var existing = await _db.Observations
.AsNoTracking()
.FirstOrDefaultAsync(o => o.IdempotencyKey == req.IdempotencyKey);
if (existing is not null)
return IngestResult.Duplicate(existing);
throw new ConflictException("Concurrent duplicate submission.", "CONCURRENT_DUPLICATE");
}
catch
{
await tx.RollbackAsync();
throw;
// Race condition: two concurrent retries both passed the pre-check above.
// The unique partial index caught it. Roll back and return the existing row.
await tx.RollbackAsync();
var existing = await _db.Observations
.AsNoTracking()
.FirstOrDefaultAsync(o => o.IdempotencyKey == req.IdempotencyKey);
if (existing is not null)
return IngestResult.Duplicate(existing);
throw new ConflictException("Concurrent duplicate submission.", "CONCURRENT_DUPLICATE");
}
catch
{
await tx.RollbackAsync();
throw;
}
}
}
VigilCareClinicalAPI/VigilCareClinicalAPI.csproj
+3
View File
@@ -18,8 +18,11 @@
</PackageReference>
<PackageReference Include="Minio" Version="6.0.3" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
<PackageReference Include="prometheus-net.AspNetCore" Version="8.2.1" />
<PackageReference Include="RabbitMQ.Client" Version="6.8.1" />
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
<PackageReference Include="Serilog.Enrichers.Environment" Version="2.3.0" />
<PackageReference Include="Serilog.Enrichers.Thread" Version="3.1.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1" />
<PackageReference Include="Serilog.Sinks.Seq" Version="9.1.0" />
<PackageReference Include="StackExchange.Redis" Version="3.0.0" />
VigilCareClinicalAPI/appsettings.json
+1 -1
View File
@@ -26,7 +26,7 @@
}
}
],
"Enrich": [ "FromLogContext" ]
"Enrich": [ "FromLogContext", "WithMachineName", "WithThreadId" ]
},
"Logging": {
"LogLevel": {