feature: Reconciliation Jobs

2026-06-17 14:36:10 +08:00
parent 56d052aa4f
commit 101040f9d9
40 changed files with 2845 additions and 32 deletions
@@ -0,0 +1,96 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+
+public sealed class DisconnectedMonitorsCheck
+{
+    public static readonly ReconciliationCheckType CheckType = ReconciliationCheckType.ActiveInpatientNoObservation;
+
+    private readonly AppDbContext _db;
+    private readonly ReconciliationPublisher _publisher;
+    private readonly ReconciliationJobOptions _opts;
+    private readonly ILogger<DisconnectedMonitorsCheck> _logger;
+
+    public DisconnectedMonitorsCheck(
+        AppDbContext db,
+        ReconciliationPublisher publisher,
+        IOptions<ReconciliationJobOptions> opts,
+        ILogger<DisconnectedMonitorsCheck> logger)
+    {
+        _db        = db;
+        _publisher = publisher;
+        _opts      = opts.Value;
+        _logger    = logger;
+    }
+
+    public async Task<int> RunAsync(CancellationToken ct)
+    {
+        var cutoff = DateTimeOffset.UtcNow
+            .AddHours(-_opts.NoObservationThresholdHours);
+
+        // Limitation: this check detects absence of recorded observations, not absence
+        // of actual clinical monitoring. A nurse who took manual vitals but did not
+        // enter them into the system would still trigger a reconciliation alert.
+        // This is a known gap; the check errs on the side of false positives.
+        var candidates = await _db.Encounters
+            .Where(e => e.Status       == EncounterStatus.Active
+                     && e.EncounterType == EncounterType.Inpatient)
+            .Select(e => new
+            {
+                EncounterId       = e.Id,
+                e.PatientId,
+                LastObservationAt = e.Observations
+                    .OrderByDescending(o => o.RecordedAt)
+                    .Select(o => (DateTimeOffset?)o.RecordedAt)
+                    .FirstOrDefault(),
+            })
+            .ToListAsync(ct);
+
+        var stale = candidates
+            .Where(e => e.LastObservationAt == null || e.LastObservationAt < cutoff)
+            .ToList();
+
+        if (!stale.Any())
+        {
+            _logger.LogDebug("[RECONCILIATION] Check3 — all active inpatients have recent observations");
+            return 0;
+        }
+
+        var inserted = 0;
+        foreach (var enc in stale)
+        {
+            var alreadyOpen = await _db.ReconciliationAlerts
+                .AnyAsync(r => r.CheckType   == CheckType
+                            && r.EncounterId == enc.EncounterId
+                            && r.ResolvedAt  == null, ct);
+            if (alreadyOpen) continue;
+
+            var lastObs = enc.LastObservationAt.HasValue
+                ? $"last observation {enc.LastObservationAt:u}"
+                : "no observations ever recorded";
+            var details = $"Active inpatient encounter {enc.EncounterId} has no recent observations "
+                        + $"({lastObs}). Monitor may be disconnected.";
+
+            var rec = new ReconciliationAlert
+            {
+                Id          = Guid.NewGuid(),
+                CheckType   = CheckType,  // ReconciliationCheckType enum
+                EncounterId = enc.EncounterId,
+                PatientId   = enc.PatientId,
+                Details     = details,
+                CreatedAt   = DateTimeOffset.UtcNow,
+            };
+
+            _db.ReconciliationAlerts.Add(rec);
+            await _db.SaveChangesAsync(ct);
+            await _publisher.PublishAsync(rec, ct);
+
+            _logger.LogWarning(
+                "[RECONCILIATION] {CheckType} — encounter {EncounterId}: {LastObs}",
+                CheckType, enc.EncounterId, lastObs);
+
+            inserted++;
+        }
+
+        return inserted;
+    }
+}
@@ -0,0 +1,93 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+
+public sealed class PendingOrdersCheck
+{
+    public static readonly ReconciliationCheckType CheckType = ReconciliationCheckType.PendingOrderNoResult;
+
+    private readonly AppDbContext _db;
+    private readonly ReconciliationPublisher _publisher;
+    private readonly ReconciliationJobOptions _opts;
+    private readonly ILogger<PendingOrdersCheck> _logger;
+
+    public PendingOrdersCheck(
+        AppDbContext db,
+        ReconciliationPublisher publisher,
+        IOptions<ReconciliationJobOptions> opts,
+        ILogger<PendingOrdersCheck> logger)
+    {
+        _db        = db;
+        _publisher = publisher;
+        _opts      = opts.Value;
+        _logger    = logger;
+    }
+
+    public async Task<int> RunAsync(CancellationToken ct)
+    {
+        var cutoff = DateTimeOffset.UtcNow
+            .AddHours(-_opts.PendingOrderThresholdHours);
+
+        var staleOrders = await _db.Orders
+            .Where(o => (o.Status == OrderStatus.Pending || o.Status == OrderStatus.InProgress)
+                     && o.OrderedAt < cutoff
+                     && o.ResultedAt == null)
+            .Select(o => new
+            {
+                o.Id,
+                o.EncounterId,
+                o.OrderType,
+                o.Description,
+                o.OrderedAt,
+                PatientId = o.Encounter!.PatientId,
+            })
+            .ToListAsync(ct);
+
+        if (!staleOrders.Any())
+        {
+            _logger.LogDebug("[RECONCILIATION] Check2 — no stale pending orders");
+            return 0;
+        }
+
+        var byEncounter = staleOrders
+            .GroupBy(o => new { o.EncounterId, o.PatientId })
+            .ToList();
+
+        var inserted = 0;
+        foreach (var group in byEncounter)
+        {
+            var alreadyOpen = await _db.ReconciliationAlerts
+                .AnyAsync(r => r.CheckType   == CheckType
+                            && r.EncounterId == group.Key.EncounterId
+                            && r.ResolvedAt  == null, ct);
+            if (alreadyOpen) continue;
+
+            var lines = group.Select(o =>
+                $"  [{o.OrderType.ToDbString()}] {o.Description} (ordered {o.OrderedAt:u}, "
+              + $"pending {(DateTimeOffset.UtcNow - o.OrderedAt).TotalHours:F1}h) ID={o.Id}");
+            var details = $"{group.Count()} order(s) pending without result for encounter "
+                        + $"{group.Key.EncounterId}:\n{string.Join("\n", lines)}";
+
+            var rec = new ReconciliationAlert
+            {
+                Id          = Guid.NewGuid(),
+                CheckType   = CheckType,  // ReconciliationCheckType enum
+                EncounterId = group.Key.EncounterId,
+                PatientId   = group.Key.PatientId,
+                Details     = details,
+                CreatedAt   = DateTimeOffset.UtcNow,
+            };
+
+            _db.ReconciliationAlerts.Add(rec);
+            await _db.SaveChangesAsync(ct);
+            await _publisher.PublishAsync(rec, ct);
+
+            _logger.LogWarning(
+                "[RECONCILIATION] {CheckType} — encounter {EncounterId}: {Count} stale order(s)",
+                CheckType, group.Key.EncounterId, group.Count());
+
+            inserted++;
+        }
+
+        return inserted;
+    }
+}
@@ -0,0 +1,62 @@
+using System.Text;
+using System.Text.Json;
+using Microsoft.Extensions.Options;
+using RabbitMQ.Client;
+
+public interface IReconciliationPublisher
+{
+    Task PublishAsync(ReconciliationAlert alert, CancellationToken ct);
+}
+
+public sealed class ReconciliationPublisher : IReconciliationPublisher
+{
+    private readonly RabbitMqOptions _opts;
+    private readonly ILogger<ReconciliationPublisher> _logger;
+
+    public ReconciliationPublisher(
+        IOptions<RabbitMqOptions> opts,
+        ILogger<ReconciliationPublisher> logger)
+    {
+        _opts = opts.Value;
+        _logger = logger;
+    }
+
+    public Task PublishAsync(ReconciliationAlert alert, CancellationToken ct)
+    {
+        var factory = new ConnectionFactory
+        {
+            HostName = _opts.Host,
+            Port = _opts.Port,
+            UserName = _opts.Username,
+            Password = _opts.Password,
+        };
+
+        using var connection = factory.CreateConnection("reconciliation-publisher");
+        using var channel = connection.CreateModel();
+
+        var props = channel.CreateBasicProperties();
+        props.Persistent = true;
+
+        var payload = JsonSerializer.Serialize(new
+        {
+            reconciliationAlertId = alert.Id,
+            checkType = alert.CheckType.ToDbString(),
+            encounterId = alert.EncounterId,
+            patientId = alert.PatientId,
+            details = alert.Details,
+            createdAt = alert.CreatedAt,
+        });
+
+        channel.BasicPublish(
+            exchange: RabbitMqTopologyProvisioner.Exchange,
+            routingKey: RabbitMqTopologyProvisioner.ReconciliationKey,
+            basicProperties: props,
+            body: Encoding.UTF8.GetBytes(payload));
+
+        _logger.LogInformation(
+            "[RECONCILIATION-PUBLISHED] CheckType={CheckType} EncounterId={EncounterId}",
+            alert.CheckType.ToDbString(), alert.EncounterId);
+
+        return Task.CompletedTask;
+    }
+}
@@ -0,0 +1,62 @@
+using Microsoft.Extensions.Options;
+
+public sealed class ReconciliationScheduler : BackgroundService
+{
+    private readonly IServiceScopeFactory _scopes;
+    private readonly ReconciliationJobOptions _opts;
+    private readonly ILogger<ReconciliationScheduler> _logger;
+
+    public ReconciliationScheduler(
+        IServiceScopeFactory scopes,
+        IOptions<ReconciliationJobOptions> opts,
+        ILogger<ReconciliationScheduler> logger)
+    {
+        _scopes = scopes;
+        _opts   = opts.Value;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        // Wait for RabbitMQ topology and migrations before first cycle.
+        await Task.Delay(TimeSpan.FromSeconds(15), stoppingToken);
+
+        _logger.LogInformation(
+            "ReconciliationScheduler started — interval {IntervalMinutes} min", _opts.IntervalMinutes);
+
+        using var timer = new PeriodicTimer(TimeSpan.FromMinutes(_opts.IntervalMinutes));
+
+        while (await timer.WaitForNextTickAsync(stoppingToken))
+        {
+            _logger.LogInformation("[RECONCILIATION] Starting reconciliation cycle");
+
+            await RunCheckAsync<UnacknowledgedAlertsCheck>(
+                check => check.RunAsync(stoppingToken), stoppingToken);
+
+            await RunCheckAsync<PendingOrdersCheck>(
+                check => check.RunAsync(stoppingToken), stoppingToken);
+
+            await RunCheckAsync<DisconnectedMonitorsCheck>(
+                check => check.RunAsync(stoppingToken), stoppingToken);
+
+            _logger.LogInformation("[RECONCILIATION] Cycle complete");
+        }
+    }
+
+    private async Task RunCheckAsync<T>(Func<T, Task<int>> run, CancellationToken ct)
+        where T : notnull
+    {
+        try
+        {
+            await using var scope = _scopes.CreateAsyncScope();
+            var check  = scope.ServiceProvider.GetRequiredService<T>();
+            var count  = await run(check);
+            _logger.LogInformation(
+                "[RECONCILIATION] {Check} — new alerts: {Count}", typeof(T).Name, count);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "[RECONCILIATION] {Check} failed", typeof(T).Name);
+        }
+    }
+}
@@ -0,0 +1,89 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+
+public sealed class UnacknowledgedAlertsCheck
+{
+    public static readonly ReconciliationCheckType CheckType = ReconciliationCheckType.UnacknowledgedCriticalAlert;
+
+    private readonly AppDbContext _db;
+    private readonly ReconciliationPublisher _publisher;
+    private readonly ReconciliationJobOptions _opts;
+    private readonly ILogger<UnacknowledgedAlertsCheck> _logger;
+
+    public UnacknowledgedAlertsCheck(
+        AppDbContext db,
+        ReconciliationPublisher publisher,
+        IOptions<ReconciliationJobOptions> opts,
+        ILogger<UnacknowledgedAlertsCheck> logger)
+    {
+        _db        = db;
+        _publisher = publisher;
+        _opts      = opts.Value;
+        _logger    = logger;
+    }
+
+    public async Task<int> RunAsync(CancellationToken ct)
+    {
+        var cutoff = DateTimeOffset.UtcNow
+            .AddMinutes(-_opts.UnacknowledgedAlertThresholdMinutes);
+
+        // Load all stale critical alerts, grouped by encounter.
+        var staleAlerts = await _db.ClinicalAlerts
+            .Where(a => a.Severity == AlertSeverity.Critical
+                     && a.Status   == AlertStatus.Open
+                     && a.TriggeredAt < cutoff)
+            .Select(a => new { a.Id, a.EncounterId, a.PatientId, a.TriggeredAt, a.Details })
+            .ToListAsync(ct);
+
+        if (!staleAlerts.Any())
+        {
+            _logger.LogDebug("[RECONCILIATION] Check1 — no stale critical alerts");
+            return 0;
+        }
+
+        // Group by encounter so one reconciliation_alert covers all alerts in that encounter.
+        var byEncounter = staleAlerts
+            .GroupBy(a => new { a.EncounterId, a.PatientId })
+            .ToList();
+
+        var inserted = 0;
+        foreach (var group in byEncounter)
+        {
+            // Deduplication: skip if an open reconciliation_alert already exists
+            // for this check type and encounter (partial index makes this fast).
+            var alreadyOpen = await _db.ReconciliationAlerts
+                .AnyAsync(r => r.CheckType   == CheckType
+                            && r.EncounterId == group.Key.EncounterId
+                            && r.ResolvedAt  == null, ct);
+            if (alreadyOpen) continue;
+
+            var alertIds = group.Select(a => a.Id).ToList();
+            var oldest   = group.Min(a => a.TriggeredAt);
+            var details  = $"{alertIds.Count} unacknowledged CRITICAL alert(s) for encounter "
+                         + $"{group.Key.EncounterId}. Oldest triggered at {oldest:u}. "
+                         + $"Alert IDs: {string.Join(", ", alertIds)}.";
+
+            var rec = new ReconciliationAlert
+            {
+                Id          = Guid.NewGuid(),
+                CheckType   = CheckType,  // ReconciliationCheckType enum
+                EncounterId = group.Key.EncounterId,
+                PatientId   = group.Key.PatientId,
+                Details     = details,
+                CreatedAt   = DateTimeOffset.UtcNow,
+            };
+
+            _db.ReconciliationAlerts.Add(rec);
+            await _db.SaveChangesAsync(ct);
+            await _publisher.PublishAsync(rec, ct);
+
+            _logger.LogWarning(
+                "[RECONCILIATION] {CheckType} — encounter {EncounterId}: {Details}",
+                CheckType, group.Key.EncounterId, details);
+
+            inserted++;
+        }
+
+        return inserted;
+    }
+}