feature: Degraded Operations Visibility

2026-06-23 23:18:31 +08:00
parent 940e27c0ac
commit 4399996448
81 changed files with 3949 additions and 323 deletions
@@ -0,0 +1,57 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+
+public sealed class GatewayStaleDetectorService : BackgroundService
+{
+    private readonly IServiceScopeFactory _scopes;
+    private readonly GatewayMonitoringOptions _opts;
+    private readonly ClinicalMetrics _metrics;
+    private readonly ILogger<GatewayStaleDetectorService> _logger;
+
+    public GatewayStaleDetectorService(
+        IServiceScopeFactory scopes,
+        IOptions<GatewayMonitoringOptions> opts,
+        ClinicalMetrics metrics,
+        ILogger<GatewayStaleDetectorService> logger)
+    {
+        _scopes = scopes;
+        _opts = opts.Value;
+        _metrics = metrics;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken ct)
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromMinutes(_opts.PollIntervalMinutes));
+        while (await timer.WaitForNextTickAsync(ct))
+            await DetectStaleAsync(ct);
+    }
+
+    private async Task DetectStaleAsync(CancellationToken ct)
+    {
+        await using var scope = _scopes.CreateAsyncScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+        var cutoff = DateTimeOffset.UtcNow.AddMinutes(-_opts.StaleThresholdMinutes);
+
+        var stale = await db.WardGateways
+            .Include(g => g.Site)
+            .Where(g => g.Status != GatewayStatus.Offline
+                     && (g.LastHeartbeatAt == null || g.LastHeartbeatAt < cutoff))
+            .ToListAsync(ct);
+
+        foreach (var gateway in stale)
+        {
+            gateway.MarkOffline();
+            _logger.LogWarning(
+                "Gateway {Code} ({Department}) marked OFFLINE — last heartbeat {LastHeartbeat}",
+                gateway.GatewayCode, gateway.Department, gateway.LastHeartbeatAt);
+
+            _metrics.WardGatewaysOffline
+                .WithLabels(gateway.Site.SiteCode)
+                .Inc();
+        }
+
+        if (stale.Count > 0)
+            await db.SaveChangesAsync(ct);
+    }
+}