Fix: Outbox relay has no dead-letter or max retry limit + DataLake writer partial commit inconsistency + ThresholdCacheLoader crashes startup on Redis failure

VigilCareClinicalAPI/BackgroundServices/OutboxRelayService.cs

@@ -68,9 +68,10 @@ public class OutboxRelayService : BackgroundService
 
         var events = await db.OutboxEvents
             .FromSqlRaw("""
-                SELECT id, topic, payload, partition_key, created_at, processed_at
+                SELECT id, topic, payload, partition_key, created_at, processed_at,
+                       retry_count, last_error, failed_at
                 FROM outbox_events
-                WHERE processed_at IS NULL
+                WHERE processed_at IS NULL AND failed_at IS NULL
                 ORDER BY created_at ASC
                 LIMIT {0}
                 FOR UPDATE SKIP LOCKED
@@ -84,7 +85,7 @@ public class OutboxRelayService : BackgroundService
         }
 
         var published = new List<Guid>();
-        var failed = false;
+        var hadFailure = false;
 
         foreach (var ev in events)
         {
@@ -94,16 +95,13 @@ public class OutboxRelayService : BackgroundService
                     ev.Topic,
                     new Message<string, string>
                     {
-                        // The partition key ensures all events for one encounter land on
-                        // the same partition. The sepsis engine depends on this — if two
-                        // observations for the same patient arrive on different partitions,
-                        // a single-partition consumer misses one and SIRS evaluation is wrong.
                         Key = ev.PartitionKey ?? string.Empty,
                         Value = ev.Payload
                     },
                     ct);
 
                 published.Add(ev.Id);
+                ev.ProcessedAt = DateTimeOffset.UtcNow;
 
                 _logger.LogDebug(
                     "Published {Topic} offset={Offset} partition={Partition} key={Key}",
@@ -111,31 +109,35 @@ public class OutboxRelayService : BackgroundService
             }
             catch (ProduceException<string, string> ex)
             {
-                _logger.LogWarning(ex,
-                    "Kafka produce failed for outbox event {Id} — leaving unprocessed for next cycle",
-                    ev.Id);
-                failed = true;
-                break;  // stop processing this batch; retry from this row next cycle
+                ev.RetryCount++;
+                ev.LastError = ex.Error.Reason;
+
+                if (ev.RetryCount >= _options.OutboxMaxRetries)
+                {
+                    ev.FailedAt = DateTimeOffset.UtcNow;
+                    _logger.LogError(ex,
+                        "Outbox event {Id} permanently failed after {Retries} retries — topic={Topic}",
+                        ev.Id, ev.RetryCount, ev.Topic);
+                }
+                else
+                {
+                    _logger.LogWarning(ex,
+                        "Kafka produce failed for outbox event {Id} — retry {Retry}/{Max}",
+                        ev.Id, ev.RetryCount, _options.OutboxMaxRetries);
+                }
+
+                hadFailure = true;
+                break;
             }
         }
 
-        if (published.Count > 0)
-        {
-            var now = DateTimeOffset.UtcNow;
-            foreach (var id in published)
-            {
-                var ev = events.First(e => e.Id == id);
-                ev.ProcessedAt = now;
-            }
-            await db.SaveChangesAsync(ct);
-        }
-
+        await db.SaveChangesAsync(ct);
         await tx.CommitAsync(ct);
 
         if (published.Count > 0)
             _logger.LogInformation(
                 "Outbox relay published {Count} events. Failed={Failed}",
-                published.Count, failed);
+                published.Count, hadFailure);
     }
 
     public override void Dispose()