feature: Sepsis Early Warning Engine

This commit is contained in:
voltsrage
2026-06-17 01:05:15 +08:00
parent fde3d56484
commit ec780144c6
12 changed files with 1194 additions and 8 deletions
@@ -0,0 +1,98 @@
using System.Text.Json;
using Confluent.Kafka;
using Microsoft.Extensions.Options;
public class SepsisEngineService : BackgroundService
{
private static readonly JsonSerializerOptions EventJsonOptions = new()
{
PropertyNameCaseInsensitive = true
};
private readonly IServiceProvider _services;
private readonly KafkaOptions _kafkaOptions;
private readonly ILogger<SepsisEngineService> _logger;
public SepsisEngineService(
IServiceProvider services,
IOptions<KafkaOptions> kafkaOptions,
ILogger<SepsisEngineService> logger)
{
_services = services;
_kafkaOptions = kafkaOptions.Value;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var config = new ConsumerConfig
{
BootstrapServers = _kafkaOptions.BootstrapServers,
GroupId = "sepsis-engine",
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false
};
using var consumer = new ConsumerBuilder<string, string>(config).Build();
// Subscribes to observation.recorded only.
// The es-indexer consumes all three topics; the sepsis engine only needs one.
// Subscribing to a superset of needed topics would waste CPU deserializing
// alert and encounter events that this engine discards immediately.
consumer.Subscribe(_kafkaOptions.Topics.ObservationRecorded);
_logger.LogInformation("SepsisEngineService started — consumer group: sepsis-engine");
try
{
while (!stoppingToken.IsCancellationRequested)
{
ConsumeResult<string, string>? result = null;
try
{
result = consumer.Consume(stoppingToken);
var evt = JsonSerializer.Deserialize<SepsisObservationEvent>(
result.Message.Value, EventJsonOptions)!;
// Create a scope per message — SirsDetector is scoped and
// owns a fresh DbContext for each observation processed.
using var scope = _services.CreateScope();
var detector = scope.ServiceProvider.GetRequiredService<SirsDetector>();
var outcome = await detector.ProcessObservationAsync(
evt.EncounterId,
evt.PatientId,
evt.ObservationCode,
evt.Value,
stoppingToken);
if (outcome.Outcome == SirsOutcome.AlertCreated)
_logger.LogWarning(
"SEPSIS_WARNING created via SepsisEngine " +
"— encounter={EncounterId} code={Code} value={Value}",
evt.EncounterId, evt.ObservationCode, evt.Value);
// Commit only after successful processing.
consumer.Commit(result);
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex,
"SepsisEngine failed on topic={Topic} offset={Offset} — not committing",
result?.Topic, result?.Offset.Value);
// Back off before retrying so a persistent failure (e.g., Redis down)
// does not spin the loop at maximum throughput.
await Task.Delay(2000, stoppingToken);
}
}
}
finally
{
consumer.Close();
}
}
}
@@ -62,6 +62,20 @@ public static class DataSeeder
Id = Guid.NewGuid(), ObservationCode = "SPO2", DisplayName = "Oxygen Saturation",
Unit = "%", CriticalLow = 88, WarningLow = 92, WarningHigh = null, CriticalHigh = null,
CreatedAt = DateTimeOffset.UtcNow
},
new AlertThreshold
{
Id = Guid.NewGuid(), ObservationCode = "RESP_RATE",
DisplayName = "Respiratory Rate", Unit = "breaths/min",
CriticalLow = null, WarningLow = 12m, WarningHigh = 20m, CriticalHigh = 30m,
CreatedAt = DateTimeOffset.UtcNow
},
new AlertThreshold
{
Id = Guid.NewGuid(), ObservationCode = "WBC_K_UL",
DisplayName = "White Blood Cell Count", Unit = "k/µL",
CriticalLow = 2.0m, WarningLow = 4.0m, WarningHigh = 12.0m, CriticalHigh = 20.0m,
CreatedAt = DateTimeOffset.UtcNow
}
};
db.AlertThresholds.AddRange(thresholds);
@@ -0,0 +1,6 @@
// Subset of the observation.recorded payload — only the fields the sepsis engine needs.
public record SepsisObservationEvent(
Guid EncounterId,
Guid PatientId,
string ObservationCode,
decimal Value);
@@ -0,0 +1,7 @@
public enum SirsOutcome
{
NotSirsCode,
InsufficientCriteria,
AlertCreated,
AlertAlreadyOpen
}
@@ -0,0 +1,11 @@
// Discriminated result — allows tests and callers to assert the exact outcome
// without inspecting PostgreSQL or Redis directly.
public record SirsResult(SirsOutcome Outcome, int ActiveCount = 0)
{
public static readonly SirsResult NotSirsCode = new(SirsOutcome.NotSirsCode);
public static readonly SirsResult AlertCreated = new(SirsOutcome.AlertCreated);
public static readonly SirsResult AlertAlreadyOpen = new(SirsOutcome.AlertAlreadyOpen);
public static SirsResult InsufficientCriteria(int count) =>
new(SirsOutcome.InsufficientCriteria, count);
}
+18 -8
View File
@@ -12,10 +12,15 @@ try
{
var builder = WebApplication.CreateBuilder(args);
builder.Host.UseSerilog((ctx, services, config) =>
config.ReadFrom.Configuration(ctx.Configuration)
.ReadFrom.Services(services)
.Enrich.FromLogContext());
// Serilog's reloadable logger can only be frozen once per process; skip in
// integration tests where WebApplicationFactory may build multiple hosts.
if (!builder.Environment.IsEnvironment("Testing"))
{
builder.Host.UseSerilog((ctx, services, config) =>
config.ReadFrom.Configuration(ctx.Configuration)
.ReadFrom.Services(services)
.Enrich.FromLogContext());
}
builder.Services.AddDbContext<AppDbContext>(opts =>
opts.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
@@ -43,12 +48,14 @@ try
builder.Services.AddScoped<IObservationQueryService, ObservationQueryService>();
builder.Services.AddScoped<IAlertService, AlertService>();
builder.Services.AddScoped<IAnalyticsService, AnalyticsService>();
builder.Services.AddScoped<SirsDetector>();
builder.Services.AddHostedService<ThresholdCacheLoader>();
builder.Services.AddHostedService<KafkaTopicProvisioner>();
builder.Services.AddHostedService<OutboxRelayService>();
builder.Services.AddHostedService<ElasticIndexProvisioner>();
builder.Services.AddHostedService<EsIndexerService>();
builder.Services.AddHostedService<SepsisEngineService>();
builder.Services.AddControllers()
.AddJsonOptions(opts =>
@@ -70,11 +77,14 @@ try
await DataSeeder.SeedAsync(db, redis);
}
app.UseSerilogRequestLogging(options =>
if (!app.Environment.IsEnvironment("Testing"))
{
options.MessageTemplate =
"HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.000}ms";
});
app.UseSerilogRequestLogging(options =>
{
options.MessageTemplate =
"HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.000}ms";
});
}
app.UseMiddleware<CorrelationIdMiddleware>();
app.UseMiddleware<ExceptionHandlerMiddleware>();
+161
View File
@@ -0,0 +1,161 @@
using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using StackExchange.Redis;
public class SirsDetector
{
// 30 minutes in seconds. This is a clinical parameter: SIRS criteria evaluated
// outside a 30-minute window are clinically stale. The TTL enforces the window
// automatically — no cleanup job required.
private const int SirsTtlSeconds = 1800;
private readonly IConnectionMultiplexer _redis;
private readonly IServiceProvider _services;
private readonly ILogger<SirsDetector> _logger;
public SirsDetector(
IConnectionMultiplexer redis,
IServiceProvider services,
ILogger<SirsDetector> logger)
{
_redis = redis;
_services = services;
_logger = logger;
}
public async Task<SirsResult> ProcessObservationAsync(
Guid encounterId,
Guid patientId,
string observationCode,
decimal value,
CancellationToken ct = default)
{
// Fast exit for non-SIRS codes. The sepsis engine subscribes to the full
// observation.recorded stream — the majority of messages (SpO2, potassium, glucose)
// are not SIRS-relevant and are discarded here without touching Redis or PostgreSQL.
if (!SirsEvaluator.SirsCodes.Contains(observationCode))
return SirsResult.NotSirsCode;
var cache = _redis.GetDatabase();
var key = SirsEvaluator.CriterionKey(encounterId, observationCode);
if (SirsEvaluator.MeetsCriterion(observationCode, value))
{
// SET with EX refreshes the TTL on every qualifying observation.
// A patient with tachycardia posting a reading every 60 seconds will keep
// sirs:{id}:HEART_RATE alive for 30 minutes after the LAST qualifying reading,
// not the first — the window slides forward with each new abnormal value.
await cache.StringSetAsync(key, "1", TimeSpan.FromSeconds(SirsTtlSeconds));
_logger.LogDebug("SIRS criterion set: {Key} (TTL={Ttl}s)", key, SirsTtlSeconds);
}
else
{
// Criterion no longer met — remove the key immediately rather than waiting
// for TTL expiry. If a patient's temperature normalises at 37.0 °C, the
// fever criterion must stop contributing to the count right away.
// Without this DEL, a recovered criterion could persist for up to 30 minutes
// and falsely sustain a SEPSIS_WARNING count.
await cache.KeyDeleteAsync(key);
_logger.LogDebug("SIRS criterion cleared: {Key}", key);
}
// Count active criteria in one MGET round-trip.
// MGET is O(N) where N = number of keys requested (4 here, always).
// Never use KEYS pattern for this check: KEYS scans the entire keyspace
// and blocks all other Redis operations until the scan completes.
var allKeys = SirsEvaluator.AllCriterionKeys(encounterId);
var values = await cache.StringGetAsync(allKeys);
var activeCount = values.Count(v => v.HasValue);
_logger.LogDebug(
"SIRS state for encounter {Id}: {Active}/4 criteria active after {Code}={Value}",
encounterId, activeCount, observationCode, value);
if (activeCount < 2)
return SirsResult.InsufficientCriteria(activeCount);
// Two or more criteria are active — attempt to create the alert.
var created = await TryCreateAlertAsync(encounterId, patientId, activeCount, ct);
return created ? SirsResult.AlertCreated : SirsResult.AlertAlreadyOpen;
}
// Creates the SEPSIS_WARNING alert and its outbox event in one atomic transaction.
// The INSERT WHERE NOT EXISTS pattern makes this safe under at-least-once delivery:
// if the consumer crashes after the INSERT but before committing the Kafka offset,
// the observation is reprocessed on restart. The second run hits the WHERE NOT EXISTS
// subquery, finds the existing open alert, inserts 0 rows, and returns false — no
// duplicate alert, no duplicate outbox event.
private async Task<bool> TryCreateAlertAsync(
Guid encounterId,
Guid patientId,
int activeCount,
CancellationToken ct)
{
using var scope = _services.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
await using var tx = await db.Database.BeginTransactionAsync(ct);
var alertId = Guid.NewGuid();
var triggeredAt = DateTimeOffset.UtcNow;
var details =
$"SIRS criteria met: {activeCount} of 4 criteria active within the 30-minute window.";
// One SQL round-trip: check + insert atomically.
// status IN ('OPEN', 'ESCALATED') prevents re-creating an alert that has been
// escalated but not yet resolved — the patient is still in danger.
var affected = await db.Database.ExecuteSqlInterpolatedAsync($"""
INSERT INTO clinical_alerts
(id, encounter_id, patient_id, alert_type, severity, details, status, triggered_at)
SELECT {alertId}, {encounterId}, {patientId},
'SEPSIS_WARNING', 'CRITICAL', {details}, 'OPEN', {triggeredAt}
WHERE NOT EXISTS (
SELECT 1 FROM clinical_alerts
WHERE encounter_id = {encounterId}
AND alert_type = 'SEPSIS_WARNING'
AND status IN ('OPEN', 'ESCALATED')
)
""", ct);
if (affected == 0)
{
await tx.RollbackAsync(ct);
_logger.LogDebug(
"SEPSIS_WARNING already open for encounter {Id} — no new alert", encounterId);
return false;
}
// Alert was created — write the outbox event in the same transaction.
// The relay (Phase 3) will publish to alert.generated, which Phase 6's
// notification worker reads to page the attending physician via RabbitMQ.
db.OutboxEvents.Add(new OutboxEvent
{
Id = Guid.NewGuid(),
Topic = "alert.generated",
Payload = JsonSerializer.Serialize(new
{
alertId,
encounterId,
patientId,
alertType = AlertType.SepsisWarning.ToDbString(),
severity = "Critical",
triggeredAt,
partitionKey = encounterId.ToString()
}),
PartitionKey = encounterId.ToString(),
CreatedAt = DateTimeOffset.UtcNow
});
await db.SaveChangesAsync(ct);
await tx.CommitAsync(ct);
_logger.LogWarning(
"SEPSIS_WARNING alert {AlertId} created for encounter {EncounterId} " +
"— {Active}/4 SIRS criteria active",
alertId, encounterId, activeCount);
return true;
}
}
@@ -0,0 +1,38 @@
using StackExchange.Redis;
public static class SirsEvaluator
{
// The four SIRS codes defined by this project's simplified SIRS criteria.
// Observations for any other code are ignored by the sepsis engine entirely —
// they pass through to the outbox and Elasticsearch but do not affect SIRS state.
public static readonly IReadOnlySet<string> SirsCodes =
new HashSet<string> { "TEMP_C", "HEART_RATE", "RESP_RATE", "WBC_K_UL" };
// Returns true if the observation value meets the SIRS criterion for its code.
// These thresholds are clinical parameters, not configuration — changing them
// requires clinical review, not a config file edit. They live here as named constants.
public static bool MeetsCriterion(string observationCode, decimal value) =>
observationCode switch
{
// Fever (> 38.3 °C) or hypothermia (< 36.0 °C)
"TEMP_C" => value > 38.3m || value < 36.0m,
// Tachycardia
"HEART_RATE" => value > 90m,
// Tachypnea
"RESP_RATE" => value > 20m,
// Leukocytosis or leukopenia
"WBC_K_UL" => value > 12.0m || value < 4.0m,
_ => false
};
// Redis key for one SIRS criterion for one encounter.
public static string CriterionKey(Guid encounterId, string code) =>
$"sirs:{encounterId}:{code}";
// All four Redis keys for one encounter — used in MGET to count active criteria.
// The order is stable so the MGET result array always maps to the same codes.
public static RedisKey[] AllCriterionKeys(Guid encounterId) =>
SirsCodes
.Select(code => (RedisKey)CriterionKey(encounterId, code))
.ToArray();
}