fix: Kafka consumer poison pill causes infinite retry

2026-06-21 17:36:00 +08:00
parent 0d0bba19e6
commit 2d22ff06b6
10 changed files with 175 additions and 10 deletions
@@ -50,6 +50,8 @@ public sealed class NotificationPublisherService : BackgroundService
        _logger.LogInformation("NotificationPublisherService started — consuming {Topics}",
            string.Join(", ", _kafkaOptions.Topics.AlertGenerated, _kafkaOptions.Topics.EncounterStatusChanged));

+        var guard = new PoisonPillGuard("notification-publisher", _kafkaOptions.MaxPoisonRetries, _logger);
+
        try
        {
            while (!stoppingToken.IsCancellationRequested)
@@ -75,11 +77,17 @@ public sealed class NotificationPublisherService : BackgroundService
                        await HandleEncounterStatusChangedAsync(chan, props, result.Message.Value, stoppingToken);

                    consumer.Commit(result);
+                    guard.OnSuccess();
                }
                catch (Exception ex)
                {
-                    _logger.LogError(ex, "NotificationPublisher failed to process message from {Topic}", result.Topic);
-                    // Do not commit — message will be reprocessed after consumer restart.
+                    if (guard.ShouldSkip(result, ex))
+                    {
+                        consumer.Commit(result);
+                        continue;
+                    }
+
+                    _logger.LogError(ex, "NotificationPublisher failed to process message from {Topic} — will retry", result.Topic);
                }
            }
        }