Fix: Outbox relay has no dead-letter or max retry limit + DataLake writer partial commit inconsistency + ThresholdCacheLoader crashes startup on Redis failure

VigilCareClinicalAPI/BackgroundServices/OutboxRelayService.cs

@@ -68,9 +68,10 @@ public class OutboxRelayService : BackgroundService
 
         var events = await db.OutboxEvents
             .FromSqlRaw("""
-                SELECT id, topic, payload, partition_key, created_at, processed_at
+                SELECT id, topic, payload, partition_key, created_at, processed_at,
+                       retry_count, last_error, failed_at
                 FROM outbox_events
-                WHERE processed_at IS NULL
+                WHERE processed_at IS NULL AND failed_at IS NULL
                 ORDER BY created_at ASC
                 LIMIT {0}
                 FOR UPDATE SKIP LOCKED
@@ -84,7 +85,7 @@ public class OutboxRelayService : BackgroundService
         }
 
         var published = new List<Guid>();
-        var failed = false;
+        var hadFailure = false;
 
         foreach (var ev in events)
         {
@@ -94,16 +95,13 @@ public class OutboxRelayService : BackgroundService
                     ev.Topic,
                     new Message<string, string>
                     {
-                        // The partition key ensures all events for one encounter land on
-                        // the same partition. The sepsis engine depends on this — if two
-                        // observations for the same patient arrive on different partitions,
-                        // a single-partition consumer misses one and SIRS evaluation is wrong.
                         Key = ev.PartitionKey ?? string.Empty,
                         Value = ev.Payload
                     },
                     ct);
 
                 published.Add(ev.Id);
+                ev.ProcessedAt = DateTimeOffset.UtcNow;
 
                 _logger.LogDebug(
                     "Published {Topic} offset={Offset} partition={Partition} key={Key}",
@@ -111,31 +109,35 @@ public class OutboxRelayService : BackgroundService
             }
             catch (ProduceException<string, string> ex)
             {
-                _logger.LogWarning(ex,
-                    "Kafka produce failed for outbox event {Id} — leaving unprocessed for next cycle",
-                    ev.Id);
-                failed = true;
-                break;  // stop processing this batch; retry from this row next cycle
+                ev.RetryCount++;
+                ev.LastError = ex.Error.Reason;
+
+                if (ev.RetryCount >= _options.OutboxMaxRetries)
+                {
+                    ev.FailedAt = DateTimeOffset.UtcNow;
+                    _logger.LogError(ex,
+                        "Outbox event {Id} permanently failed after {Retries} retries — topic={Topic}",
+                        ev.Id, ev.RetryCount, ev.Topic);
+                }
+                else
+                {
+                    _logger.LogWarning(ex,
+                        "Kafka produce failed for outbox event {Id} — retry {Retry}/{Max}",
+                        ev.Id, ev.RetryCount, _options.OutboxMaxRetries);
+                }
+
+                hadFailure = true;
+                break;
             }
         }
 
-        if (published.Count > 0)
-        {
-            var now = DateTimeOffset.UtcNow;
-            foreach (var id in published)
-            {
-                var ev = events.First(e => e.Id == id);
-                ev.ProcessedAt = now;
-            }
-            await db.SaveChangesAsync(ct);
-        }
-
+        await db.SaveChangesAsync(ct);
         await tx.CommitAsync(ct);
 
         if (published.Count > 0)
             _logger.LogInformation(
                 "Outbox relay published {Count} events. Failed={Failed}",
-                published.Count, failed);
+                published.Count, hadFailure);
     }
 
     public override void Dispose()

VigilCareClinicalAPI/BackgroundServices/ThresholdCacheLoader.cs

@@ -22,26 +22,51 @@ public class ThresholdCacheLoader : IHostedService
     {
         using var scope = _services.CreateScope();
         var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
-        var cache = _redis.GetDatabase();
 
         var thresholds = await db.AlertThresholds.ToListAsync(cancellationToken);
-        var batch = cache.CreateBatch();
 
-        foreach (var t in thresholds)
+        const int maxAttempts = 3;
+        int[] backoffMs = [2000, 4000, 8000];
+
+        for (var attempt = 0; attempt < maxAttempts; attempt++)
         {
-            var json = JsonSerializer.Serialize(new
+            try
             {
-                t.ObservationCode,
-                t.CriticalLow,
-                t.WarningLow,
-                t.WarningHigh,
-                t.CriticalHigh
-            });
-            _ = batch.StringSetAsync($"threshold:{t.ObservationCode}", json);
+                var cache = _redis.GetDatabase();
+                var batch = cache.CreateBatch();
+
+                foreach (var t in thresholds)
+                {
+                    var json = JsonSerializer.Serialize(new
+                    {
+                        t.ObservationCode,
+                        t.CriticalLow,
+                        t.WarningLow,
+                        t.WarningHigh,
+                        t.CriticalHigh
+                    });
+                    _ = batch.StringSetAsync($"threshold:{t.ObservationCode}", json);
+                }
+
+                batch.Execute();
+                _logger.LogInformation("Loaded {Count} alert thresholds into Redis cache", thresholds.Count);
+                return;
+            }
+            catch (RedisException ex)
+            {
+                _logger.LogWarning(ex,
+                    "Redis unavailable during threshold cache load — attempt {Attempt}/{Max}",
+                    attempt + 1, maxAttempts);
+
+                if (attempt < maxAttempts - 1)
+                    await Task.Delay(backoffMs[attempt], cancellationToken);
+            }
         }
 
-        batch.Execute();
-        _logger.LogInformation("Loaded {Count} alert thresholds into Redis cache", thresholds.Count);
+        _logger.LogError(
+            "Failed to load thresholds into Redis after {Max} attempts — " +
+            "application will start without cache; observation ingest falls back to PostgreSQL",
+            maxAttempts);
     }
 
     public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;