feature: RabbitMQ Notification Workers and DLQ Escalation

2026-06-17 02:37:30 +08:00
parent ec780144c6
commit 0246143815
17 changed files with 1410 additions and 2 deletions
@@ -0,0 +1,95 @@
+using System.Text;
+using System.Text.Json;
+using Microsoft.Extensions.Options;
+using RabbitMQ.Client;
+using RabbitMQ.Client.Events;
+
+public sealed class EscalationWorkerService : BackgroundService
+{
+    private readonly IOptions<RabbitMqOptions> _opts;
+    private readonly IServiceScopeFactory _scopes;
+    private readonly ILogger<EscalationWorkerService> _logger;
+
+    public EscalationWorkerService(
+        IOptions<RabbitMqOptions> opts,
+        IServiceScopeFactory scopes,
+        ILogger<EscalationWorkerService> logger)
+    {
+        _opts   = opts;
+        _scopes = scopes;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
+
+        var o       = _opts.Value;
+        var factory = new ConnectionFactory
+        {
+            HostName               = o.Host,
+            Port                   = o.Port,
+            UserName               = o.Username,
+            Password               = o.Password,
+            DispatchConsumersAsync = true,
+        };
+
+        using var connection = factory.CreateConnection("escalation-worker");
+        using var channel    = connection.CreateModel();
+
+        channel.BasicQos(0, prefetchCount: 5, global: false);
+
+        var consumer = new AsyncEventingBasicConsumer(channel);
+        consumer.Received += async (sender, ea) =>
+        {
+            await HandleEscalationAsync(channel, ea, stoppingToken);
+        };
+
+        channel.BasicConsume("alerts.escalation.queue", autoAck: false, consumer);
+
+        _logger.LogInformation("EscalationWorkerService consuming alerts.escalation.queue");
+
+        await Task.Delay(Timeout.Infinite, stoppingToken);
+    }
+
+    private async Task HandleEscalationAsync(IModel channel, BasicDeliverEventArgs ea, CancellationToken ct)
+    {
+        var payload     = Encoding.UTF8.GetString(ea.Body.Span);
+        var doc         = JsonDocument.Parse(payload);
+        var alertId     = Guid.Parse(doc.RootElement.GetProperty("alertId").GetString()!);
+        var encounterId = doc.RootElement.GetProperty("encounterId").GetString();
+
+        _logger.LogCritical(
+            "[ESCALATION] Paging on-call backup — AlertId={AlertId} EncounterId={EncounterId}",
+            alertId, encounterId);
+
+        try
+        {
+            await UpdateAlertStatusEscalatedAsync(alertId, ct);
+            channel.BasicAck(ea.DeliveryTag, multiple: false);
+
+            _logger.LogWarning(
+                "[ESCALATION-DONE] Alert {AlertId} status → Escalated in PostgreSQL", alertId);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "EscalationWorker failed for alert {AlertId}", alertId);
+            channel.BasicNack(ea.DeliveryTag, multiple: false, requeue: true);
+        }
+    }
+
+    private async Task UpdateAlertStatusEscalatedAsync(Guid alertId, CancellationToken ct)
+    {
+        await using var scope = _scopes.CreateAsyncScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+
+        var alert = await db.ClinicalAlerts.FindAsync(new object[] { alertId }, ct);
+        if (alert is null) return;
+
+        // Only escalate if still open — if acknowledged between NACK and TTL expiry, leave it.
+        if (alert.Status != AlertStatus.Open) return;
+
+        alert.Status = AlertStatus.Escalated;
+        await db.SaveChangesAsync(ct);
+    }
+}