Fix: Outbox relay has no dead-letter or max retry limit + DataLake writer partial commit inconsistency + ThresholdCacheLoader crashes startup on Redis failure

VigilCareClinicalAPI/DataLake/DataLakeWriterService.cs

@@ -122,7 +122,8 @@ public sealed class DataLakeWriterService : BackgroundService
 
     private async Task FlushAsync(IConsumer<string, string> consumer, CancellationToken ct)
     {
-        var filesWritten = 0;
+        var flushedKeys = new HashSet<BufferKey>();
+        var failedPartitions = new HashSet<int>();
 
         foreach (var (key, events) in _buffer)
         {
@@ -135,7 +136,7 @@ public sealed class DataLakeWriterService : BackgroundService
                 var bytes       = await BuildParquetAsync(key.Topic, events, key.Partition);
 
                 await UploadToMinioAsync(objectKey, bytes, ct);
-                filesWritten++;
+                flushedKeys.Add(key);
 
                 _logger.LogInformation(
                     "[DATA-LAKE] Wrote {Count} events → {ObjectKey} ({Bytes} bytes)",
@@ -143,8 +144,8 @@ public sealed class DataLakeWriterService : BackgroundService
             }
             catch (Exception ex)
             {
-                // Log and continue — a failed file for one key must not prevent other
-                // keys from flushing. The uncommitted offsets will cause reprocessing.
+                failedPartitions.Add(key.Partition);
+
                 if (ex is OperationCanceledException && ct.IsCancellationRequested)
                 {
                     _logger.LogInformation(
@@ -157,24 +158,37 @@ public sealed class DataLakeWriterService : BackgroundService
             }
         }
 
-        // Commit only after at least one file uploaded successfully.
-        // Events for any key that failed above will be re-read on next startup.
-        if (filesWritten > 0 && _highWatermarks.Any())
+        // Only commit offsets for topic-partitions that had no failures.
+        var safeOffsets = _highWatermarks
+            .Where(kv => !failedPartitions.Contains(kv.Key.Partition))
+            .Select(kv => kv.Value)
+            .ToList();
+
+        if (safeOffsets.Count > 0)
         {
-            consumer.Commit(_highWatermarks.Values);
+            consumer.Commit(safeOffsets);
             _logger.LogInformation(
                 "[DATA-LAKE] Committed offsets for {PartitionCount} partitions after flushing {FileCount} files",
-                _highWatermarks.Count, filesWritten);
-        }
-        else if (filesWritten == 0 && _highWatermarks.Any())
-        {
-            _logger.LogWarning(
-                "[DATA-LAKE] Skipping offset commit — no files were written ({BufferedPartitions} partitions buffered)",
-                _highWatermarks.Count);
+                safeOffsets.Count, flushedKeys.Count);
         }
 
-        _buffer.Clear();
-        _highWatermarks.Clear();
+        if (failedPartitions.Count > 0)
+        {
+            _logger.LogWarning(
+                "[DATA-LAKE] Retained buffers for {FailedCount} failed partitions — will retry next flush",
+                failedPartitions.Count);
+        }
+
+        // Clear only successfully flushed keys; retain failed ones for retry.
+        foreach (var key in flushedKeys)
+            _buffer.Remove(key);
+
+        // Clear watermarks only for partitions with no failures.
+        foreach (var tp in _highWatermarks.Keys.ToList())
+        {
+            if (!failedPartitions.Contains(tp.Partition))
+                _highWatermarks.Remove(tp);
+        }
     }
 
     private async Task<byte[]> BuildParquetAsync(