feature: Sepsis Early Warning Engine
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
using System.Text.Json;
|
||||
using Confluent.Kafka;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
public class SepsisEngineService : BackgroundService
|
||||
{
|
||||
private static readonly JsonSerializerOptions EventJsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly KafkaOptions _kafkaOptions;
|
||||
private readonly ILogger<SepsisEngineService> _logger;
|
||||
|
||||
public SepsisEngineService(
|
||||
IServiceProvider services,
|
||||
IOptions<KafkaOptions> kafkaOptions,
|
||||
ILogger<SepsisEngineService> logger)
|
||||
{
|
||||
_services = services;
|
||||
_kafkaOptions = kafkaOptions.Value;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var config = new ConsumerConfig
|
||||
{
|
||||
BootstrapServers = _kafkaOptions.BootstrapServers,
|
||||
GroupId = "sepsis-engine",
|
||||
AutoOffsetReset = AutoOffsetReset.Earliest,
|
||||
EnableAutoCommit = false
|
||||
};
|
||||
|
||||
using var consumer = new ConsumerBuilder<string, string>(config).Build();
|
||||
// Subscribes to observation.recorded only.
|
||||
// The es-indexer consumes all three topics; the sepsis engine only needs one.
|
||||
// Subscribing to a superset of needed topics would waste CPU deserializing
|
||||
// alert and encounter events that this engine discards immediately.
|
||||
consumer.Subscribe(_kafkaOptions.Topics.ObservationRecorded);
|
||||
|
||||
_logger.LogInformation("SepsisEngineService started — consumer group: sepsis-engine");
|
||||
|
||||
try
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
ConsumeResult<string, string>? result = null;
|
||||
try
|
||||
{
|
||||
result = consumer.Consume(stoppingToken);
|
||||
|
||||
var evt = JsonSerializer.Deserialize<SepsisObservationEvent>(
|
||||
result.Message.Value, EventJsonOptions)!;
|
||||
|
||||
// Create a scope per message — SirsDetector is scoped and
|
||||
// owns a fresh DbContext for each observation processed.
|
||||
using var scope = _services.CreateScope();
|
||||
var detector = scope.ServiceProvider.GetRequiredService<SirsDetector>();
|
||||
|
||||
var outcome = await detector.ProcessObservationAsync(
|
||||
evt.EncounterId,
|
||||
evt.PatientId,
|
||||
evt.ObservationCode,
|
||||
evt.Value,
|
||||
stoppingToken);
|
||||
|
||||
if (outcome.Outcome == SirsOutcome.AlertCreated)
|
||||
_logger.LogWarning(
|
||||
"SEPSIS_WARNING created via SepsisEngine " +
|
||||
"— encounter={EncounterId} code={Code} value={Value}",
|
||||
evt.EncounterId, evt.ObservationCode, evt.Value);
|
||||
|
||||
// Commit only after successful processing.
|
||||
consumer.Commit(result);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"SepsisEngine failed on topic={Topic} offset={Offset} — not committing",
|
||||
result?.Topic, result?.Offset.Value);
|
||||
// Back off before retrying so a persistent failure (e.g., Redis down)
|
||||
// does not spin the loop at maximum throughput.
|
||||
await Task.Delay(2000, stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
consumer.Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,20 @@ public static class DataSeeder
|
||||
Id = Guid.NewGuid(), ObservationCode = "SPO2", DisplayName = "Oxygen Saturation",
|
||||
Unit = "%", CriticalLow = 88, WarningLow = 92, WarningHigh = null, CriticalHigh = null,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
},
|
||||
new AlertThreshold
|
||||
{
|
||||
Id = Guid.NewGuid(), ObservationCode = "RESP_RATE",
|
||||
DisplayName = "Respiratory Rate", Unit = "breaths/min",
|
||||
CriticalLow = null, WarningLow = 12m, WarningHigh = 20m, CriticalHigh = 30m,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
},
|
||||
new AlertThreshold
|
||||
{
|
||||
Id = Guid.NewGuid(), ObservationCode = "WBC_K_UL",
|
||||
DisplayName = "White Blood Cell Count", Unit = "k/µL",
|
||||
CriticalLow = 2.0m, WarningLow = 4.0m, WarningHigh = 12.0m, CriticalHigh = 20.0m,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}
|
||||
};
|
||||
db.AlertThresholds.AddRange(thresholds);
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
// Subset of the observation.recorded payload — only the fields the sepsis engine needs.
|
||||
public record SepsisObservationEvent(
|
||||
Guid EncounterId,
|
||||
Guid PatientId,
|
||||
string ObservationCode,
|
||||
decimal Value);
|
||||
@@ -0,0 +1,7 @@
|
||||
public enum SirsOutcome
|
||||
{
|
||||
NotSirsCode,
|
||||
InsufficientCriteria,
|
||||
AlertCreated,
|
||||
AlertAlreadyOpen
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Discriminated result — allows tests and callers to assert the exact outcome
|
||||
// without inspecting PostgreSQL or Redis directly.
|
||||
public record SirsResult(SirsOutcome Outcome, int ActiveCount = 0)
|
||||
{
|
||||
public static readonly SirsResult NotSirsCode = new(SirsOutcome.NotSirsCode);
|
||||
public static readonly SirsResult AlertCreated = new(SirsOutcome.AlertCreated);
|
||||
public static readonly SirsResult AlertAlreadyOpen = new(SirsOutcome.AlertAlreadyOpen);
|
||||
|
||||
public static SirsResult InsufficientCriteria(int count) =>
|
||||
new(SirsOutcome.InsufficientCriteria, count);
|
||||
}
|
||||
@@ -12,10 +12,15 @@ try
|
||||
{
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
builder.Host.UseSerilog((ctx, services, config) =>
|
||||
config.ReadFrom.Configuration(ctx.Configuration)
|
||||
.ReadFrom.Services(services)
|
||||
.Enrich.FromLogContext());
|
||||
// Serilog's reloadable logger can only be frozen once per process; skip in
|
||||
// integration tests where WebApplicationFactory may build multiple hosts.
|
||||
if (!builder.Environment.IsEnvironment("Testing"))
|
||||
{
|
||||
builder.Host.UseSerilog((ctx, services, config) =>
|
||||
config.ReadFrom.Configuration(ctx.Configuration)
|
||||
.ReadFrom.Services(services)
|
||||
.Enrich.FromLogContext());
|
||||
}
|
||||
|
||||
builder.Services.AddDbContext<AppDbContext>(opts =>
|
||||
opts.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
|
||||
@@ -43,12 +48,14 @@ try
|
||||
builder.Services.AddScoped<IObservationQueryService, ObservationQueryService>();
|
||||
builder.Services.AddScoped<IAlertService, AlertService>();
|
||||
builder.Services.AddScoped<IAnalyticsService, AnalyticsService>();
|
||||
builder.Services.AddScoped<SirsDetector>();
|
||||
|
||||
builder.Services.AddHostedService<ThresholdCacheLoader>();
|
||||
builder.Services.AddHostedService<KafkaTopicProvisioner>();
|
||||
builder.Services.AddHostedService<OutboxRelayService>();
|
||||
builder.Services.AddHostedService<ElasticIndexProvisioner>();
|
||||
builder.Services.AddHostedService<EsIndexerService>();
|
||||
builder.Services.AddHostedService<SepsisEngineService>();
|
||||
|
||||
builder.Services.AddControllers()
|
||||
.AddJsonOptions(opts =>
|
||||
@@ -70,11 +77,14 @@ try
|
||||
await DataSeeder.SeedAsync(db, redis);
|
||||
}
|
||||
|
||||
app.UseSerilogRequestLogging(options =>
|
||||
if (!app.Environment.IsEnvironment("Testing"))
|
||||
{
|
||||
options.MessageTemplate =
|
||||
"HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.000}ms";
|
||||
});
|
||||
app.UseSerilogRequestLogging(options =>
|
||||
{
|
||||
options.MessageTemplate =
|
||||
"HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.000}ms";
|
||||
});
|
||||
}
|
||||
|
||||
app.UseMiddleware<CorrelationIdMiddleware>();
|
||||
app.UseMiddleware<ExceptionHandlerMiddleware>();
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using StackExchange.Redis;
|
||||
|
||||
public class SirsDetector
|
||||
{
|
||||
// 30 minutes in seconds. This is a clinical parameter: SIRS criteria evaluated
|
||||
// outside a 30-minute window are clinically stale. The TTL enforces the window
|
||||
// automatically — no cleanup job required.
|
||||
private const int SirsTtlSeconds = 1800;
|
||||
|
||||
private readonly IConnectionMultiplexer _redis;
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger<SirsDetector> _logger;
|
||||
|
||||
public SirsDetector(
|
||||
IConnectionMultiplexer redis,
|
||||
IServiceProvider services,
|
||||
ILogger<SirsDetector> logger)
|
||||
{
|
||||
_redis = redis;
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<SirsResult> ProcessObservationAsync(
|
||||
Guid encounterId,
|
||||
Guid patientId,
|
||||
string observationCode,
|
||||
decimal value,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
// Fast exit for non-SIRS codes. The sepsis engine subscribes to the full
|
||||
// observation.recorded stream — the majority of messages (SpO2, potassium, glucose)
|
||||
// are not SIRS-relevant and are discarded here without touching Redis or PostgreSQL.
|
||||
if (!SirsEvaluator.SirsCodes.Contains(observationCode))
|
||||
return SirsResult.NotSirsCode;
|
||||
|
||||
var cache = _redis.GetDatabase();
|
||||
var key = SirsEvaluator.CriterionKey(encounterId, observationCode);
|
||||
|
||||
if (SirsEvaluator.MeetsCriterion(observationCode, value))
|
||||
{
|
||||
// SET with EX refreshes the TTL on every qualifying observation.
|
||||
// A patient with tachycardia posting a reading every 60 seconds will keep
|
||||
// sirs:{id}:HEART_RATE alive for 30 minutes after the LAST qualifying reading,
|
||||
// not the first — the window slides forward with each new abnormal value.
|
||||
await cache.StringSetAsync(key, "1", TimeSpan.FromSeconds(SirsTtlSeconds));
|
||||
|
||||
_logger.LogDebug("SIRS criterion set: {Key} (TTL={Ttl}s)", key, SirsTtlSeconds);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Criterion no longer met — remove the key immediately rather than waiting
|
||||
// for TTL expiry. If a patient's temperature normalises at 37.0 °C, the
|
||||
// fever criterion must stop contributing to the count right away.
|
||||
// Without this DEL, a recovered criterion could persist for up to 30 minutes
|
||||
// and falsely sustain a SEPSIS_WARNING count.
|
||||
await cache.KeyDeleteAsync(key);
|
||||
|
||||
_logger.LogDebug("SIRS criterion cleared: {Key}", key);
|
||||
}
|
||||
|
||||
// Count active criteria in one MGET round-trip.
|
||||
// MGET is O(N) where N = number of keys requested (4 here, always).
|
||||
// Never use KEYS pattern for this check: KEYS scans the entire keyspace
|
||||
// and blocks all other Redis operations until the scan completes.
|
||||
var allKeys = SirsEvaluator.AllCriterionKeys(encounterId);
|
||||
var values = await cache.StringGetAsync(allKeys);
|
||||
var activeCount = values.Count(v => v.HasValue);
|
||||
|
||||
_logger.LogDebug(
|
||||
"SIRS state for encounter {Id}: {Active}/4 criteria active after {Code}={Value}",
|
||||
encounterId, activeCount, observationCode, value);
|
||||
|
||||
if (activeCount < 2)
|
||||
return SirsResult.InsufficientCriteria(activeCount);
|
||||
|
||||
// Two or more criteria are active — attempt to create the alert.
|
||||
var created = await TryCreateAlertAsync(encounterId, patientId, activeCount, ct);
|
||||
return created ? SirsResult.AlertCreated : SirsResult.AlertAlreadyOpen;
|
||||
}
|
||||
|
||||
// Creates the SEPSIS_WARNING alert and its outbox event in one atomic transaction.
|
||||
// The INSERT WHERE NOT EXISTS pattern makes this safe under at-least-once delivery:
|
||||
// if the consumer crashes after the INSERT but before committing the Kafka offset,
|
||||
// the observation is reprocessed on restart. The second run hits the WHERE NOT EXISTS
|
||||
// subquery, finds the existing open alert, inserts 0 rows, and returns false — no
|
||||
// duplicate alert, no duplicate outbox event.
|
||||
private async Task<bool> TryCreateAlertAsync(
|
||||
Guid encounterId,
|
||||
Guid patientId,
|
||||
int activeCount,
|
||||
CancellationToken ct)
|
||||
{
|
||||
using var scope = _services.CreateScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
|
||||
await using var tx = await db.Database.BeginTransactionAsync(ct);
|
||||
|
||||
var alertId = Guid.NewGuid();
|
||||
var triggeredAt = DateTimeOffset.UtcNow;
|
||||
var details =
|
||||
$"SIRS criteria met: {activeCount} of 4 criteria active within the 30-minute window.";
|
||||
|
||||
// One SQL round-trip: check + insert atomically.
|
||||
// status IN ('OPEN', 'ESCALATED') prevents re-creating an alert that has been
|
||||
// escalated but not yet resolved — the patient is still in danger.
|
||||
var affected = await db.Database.ExecuteSqlInterpolatedAsync($"""
|
||||
INSERT INTO clinical_alerts
|
||||
(id, encounter_id, patient_id, alert_type, severity, details, status, triggered_at)
|
||||
SELECT {alertId}, {encounterId}, {patientId},
|
||||
'SEPSIS_WARNING', 'CRITICAL', {details}, 'OPEN', {triggeredAt}
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM clinical_alerts
|
||||
WHERE encounter_id = {encounterId}
|
||||
AND alert_type = 'SEPSIS_WARNING'
|
||||
AND status IN ('OPEN', 'ESCALATED')
|
||||
)
|
||||
""", ct);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
await tx.RollbackAsync(ct);
|
||||
_logger.LogDebug(
|
||||
"SEPSIS_WARNING already open for encounter {Id} — no new alert", encounterId);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Alert was created — write the outbox event in the same transaction.
|
||||
// The relay (Phase 3) will publish to alert.generated, which Phase 6's
|
||||
// notification worker reads to page the attending physician via RabbitMQ.
|
||||
db.OutboxEvents.Add(new OutboxEvent
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Topic = "alert.generated",
|
||||
Payload = JsonSerializer.Serialize(new
|
||||
{
|
||||
alertId,
|
||||
encounterId,
|
||||
patientId,
|
||||
alertType = AlertType.SepsisWarning.ToDbString(),
|
||||
severity = "Critical",
|
||||
triggeredAt,
|
||||
partitionKey = encounterId.ToString()
|
||||
}),
|
||||
PartitionKey = encounterId.ToString(),
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
|
||||
await db.SaveChangesAsync(ct);
|
||||
await tx.CommitAsync(ct);
|
||||
|
||||
_logger.LogWarning(
|
||||
"SEPSIS_WARNING alert {AlertId} created for encounter {EncounterId} " +
|
||||
"— {Active}/4 SIRS criteria active",
|
||||
alertId, encounterId, activeCount);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using StackExchange.Redis;
|
||||
|
||||
public static class SirsEvaluator
|
||||
{
|
||||
// The four SIRS codes defined by this project's simplified SIRS criteria.
|
||||
// Observations for any other code are ignored by the sepsis engine entirely —
|
||||
// they pass through to the outbox and Elasticsearch but do not affect SIRS state.
|
||||
public static readonly IReadOnlySet<string> SirsCodes =
|
||||
new HashSet<string> { "TEMP_C", "HEART_RATE", "RESP_RATE", "WBC_K_UL" };
|
||||
|
||||
// Returns true if the observation value meets the SIRS criterion for its code.
|
||||
// These thresholds are clinical parameters, not configuration — changing them
|
||||
// requires clinical review, not a config file edit. They live here as named constants.
|
||||
public static bool MeetsCriterion(string observationCode, decimal value) =>
|
||||
observationCode switch
|
||||
{
|
||||
// Fever (> 38.3 °C) or hypothermia (< 36.0 °C)
|
||||
"TEMP_C" => value > 38.3m || value < 36.0m,
|
||||
// Tachycardia
|
||||
"HEART_RATE" => value > 90m,
|
||||
// Tachypnea
|
||||
"RESP_RATE" => value > 20m,
|
||||
// Leukocytosis or leukopenia
|
||||
"WBC_K_UL" => value > 12.0m || value < 4.0m,
|
||||
_ => false
|
||||
};
|
||||
|
||||
// Redis key for one SIRS criterion for one encounter.
|
||||
public static string CriterionKey(Guid encounterId, string code) =>
|
||||
$"sirs:{encounterId}:{code}";
|
||||
|
||||
// All four Redis keys for one encounter — used in MGET to count active criteria.
|
||||
// The order is stable so the MGET result array always maps to the same codes.
|
||||
public static RedisKey[] AllCriterionKeys(Guid encounterId) =>
|
||||
SirsCodes
|
||||
.Select(code => (RedisKey)CriterionKey(encounterId, code))
|
||||
.ToArray();
|
||||
}
|
||||
Reference in New Issue
Block a user