using System.Text.Json; using Confluent.Kafka; using FluentAssertions; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using StackExchange.Redis; [Collection("Integration")] public class BackgroundServiceTests : IAsyncLifetime { private readonly ApiFixture _fixture; public BackgroundServiceTests(ApiFixture fixture) => _fixture = fixture; public async Task InitializeAsync() { using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); await DbResetHelper.ResetAsync(db); var redis = scope.ServiceProvider.GetRequiredService(); var server = redis.GetServer(redis.GetEndPoints().First()); await server.FlushDatabaseAsync(1); } public Task DisposeAsync() => Task.CompletedTask; // ======================================================================= // PoisonPillGuard — unit tests (no infrastructure needed) // ======================================================================= [Fact] public void PoisonPillGuard_PermanentError_SkipsImmediately() { var guard = new PoisonPillGuard("test-group", maxRetries: 5, NullLogger.Instance); var result = BuildConsumeResult("test-topic", 0, 0, "bad payload"); var shouldSkip = guard.ShouldSkip(result, new JsonException("Invalid JSON")); shouldSkip.Should().BeTrue("permanent errors (JsonException) must be skipped immediately"); } [Fact] public void PoisonPillGuard_FormatException_SkipsImmediately() { var guard = new PoisonPillGuard("test-group", maxRetries: 5, NullLogger.Instance); var result = BuildConsumeResult("test-topic", 0, 0, "not-a-guid"); var shouldSkip = guard.ShouldSkip(result, new FormatException("Input string was not in a correct format")); shouldSkip.Should().BeTrue("FormatException is a permanent error and must be skipped immediately"); } [Fact] public void PoisonPillGuard_WrappedPermanentError_SkipsImmediately() { var guard = new PoisonPillGuard("test-group", maxRetries: 5, NullLogger.Instance); var result = BuildConsumeResult("test-topic", 0, 0, "bad"); var wrappedException = new InvalidOperationException( "Processing failed", new JsonException("Invalid JSON")); var shouldSkip = guard.ShouldSkip(result, wrappedException); shouldSkip.Should().BeTrue( "permanent errors wrapped in other exceptions must still be detected via root cause"); } [Fact] public void PoisonPillGuard_TransientError_RetriesUpToMax() { const int maxRetries = 3; var guard = new PoisonPillGuard("test-group", maxRetries, NullLogger.Instance); var result = BuildConsumeResult("test-topic", 0, 42, "payload"); var transientError = new TimeoutException("Connection timed out"); for (var i = 1; i < maxRetries; i++) { var shouldSkip = guard.ShouldSkip(result, transientError); shouldSkip.Should().BeFalse($"retry {i}/{maxRetries} should not skip"); } var finalSkip = guard.ShouldSkip(result, transientError); finalSkip.Should().BeTrue( $"after {maxRetries} retries, the message must be skipped"); } [Fact] public void PoisonPillGuard_SuccessResetsRetryCount() { const int maxRetries = 3; var guard = new PoisonPillGuard("test-group", maxRetries, NullLogger.Instance); var result = BuildConsumeResult("test-topic", 0, 42, "payload"); var transientError = new TimeoutException("Connection timed out"); guard.ShouldSkip(result, transientError); guard.ShouldSkip(result, transientError); guard.OnSuccess(); guard.ShouldSkip(result, transientError).Should().BeFalse( "after OnSuccess, retry count resets and first failure should not skip"); } [Fact] public void PoisonPillGuard_DifferentOffset_ResetsRetryCount() { const int maxRetries = 3; var guard = new PoisonPillGuard("test-group", maxRetries, NullLogger.Instance); var transientError = new TimeoutException("Connection timed out"); var result1 = BuildConsumeResult("test-topic", 0, 42, "payload-a"); guard.ShouldSkip(result1, transientError); guard.ShouldSkip(result1, transientError); var result2 = BuildConsumeResult("test-topic", 0, 43, "payload-b"); guard.ShouldSkip(result2, transientError).Should().BeFalse( "a new offset resets the retry counter"); } // ======================================================================= // Outbox relay — retry count and dead-letter behavior // ======================================================================= [Fact] public async Task OutboxEvent_UnprocessedEvents_ArePickedUpByRelay() { using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var encounterId = Guid.NewGuid(); db.OutboxEvents.Add(new OutboxEvent { Id = Guid.NewGuid(), Topic = "observation.recorded", Payload = JsonSerializer.Serialize(new { encounterId, code = "HEART_RATE", value = 80 }), PartitionKey = encounterId.ToString(), CreatedAt = DateTimeOffset.UtcNow }); await db.SaveChangesAsync(); var pendingBefore = await db.OutboxEvents .CountAsync(e => e.ProcessedAt == null && e.FailedAt == null); pendingBefore.Should().BeGreaterThanOrEqualTo(1); var deadline = DateTimeOffset.UtcNow.AddSeconds(15); while (DateTimeOffset.UtcNow < deadline) { await Task.Delay(500); var remaining = await db.OutboxEvents .AsNoTracking() .CountAsync(e => e.ProcessedAt == null && e.FailedAt == null); if (remaining == 0) break; } var unprocessed = await db.OutboxEvents .AsNoTracking() .CountAsync(e => e.ProcessedAt == null && e.FailedAt == null); unprocessed.Should().Be(0, "the outbox relay should process all pending events"); } [Fact] public async Task OutboxEvent_FailedEvents_HaveRetryCountAndError() { using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var failedEvent = new OutboxEvent { Id = Guid.NewGuid(), Topic = "nonexistent.topic.that.will.fail", Payload = "{}", PartitionKey = "test", CreatedAt = DateTimeOffset.UtcNow, RetryCount = 9, LastError = "Simulated prior failure" }; db.OutboxEvents.Add(failedEvent); await db.SaveChangesAsync(); failedEvent.RetryCount.Should().Be(9); failedEvent.LastError.Should().NotBeNullOrEmpty(); failedEvent.ProcessedAt.Should().BeNull(); } [Fact] public async Task OutboxEvent_FailedAtSet_EventIsNotReprocessed() { using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var deadLetteredEvent = new OutboxEvent { Id = Guid.NewGuid(), Topic = "test.topic", Payload = "{}", PartitionKey = "test", CreatedAt = DateTimeOffset.UtcNow, RetryCount = 10, LastError = "Exceeded max retries", FailedAt = DateTimeOffset.UtcNow.AddMinutes(-1) }; db.OutboxEvents.Add(deadLetteredEvent); await db.SaveChangesAsync(); await Task.Delay(3_000); var evt = await db.OutboxEvents .AsNoTracking() .SingleAsync(e => e.Id == deadLetteredEvent.Id); evt.ProcessedAt.Should().BeNull("dead-lettered events must not be reprocessed"); evt.FailedAt.Should().NotBeNull(); } // ======================================================================= // SepsisBundleMonitor — concurrent scan safety // ======================================================================= [Fact] public async Task SepsisBundleMonitor_ConcurrentScans_NoDuplicateUpdates() { var (_, encounterId) = await SeedPatientEncounterAndBundleAsync(); using (var scope = _fixture.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var bundle = await db.SepsisBundles.SingleAsync(b => b.EncounterId == encounterId); bundle.DeadlineAt = DateTimeOffset.UtcNow.AddHours(-1); await db.SaveChangesAsync(); } var tasks = Enumerable.Range(0, 5).Select(_ => { return Task.Run(async () => { var monitor = new SepsisBundleMonitorService( _fixture.Services, _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService>()); await monitor.ScanOverdueBundlesAsync(CancellationToken.None); }); }); await Task.WhenAll(tasks); using var verifyScope = _fixture.Services.CreateScope(); var verifyDb = verifyScope.ServiceProvider.GetRequiredService(); var bundles = await verifyDb.SepsisBundles .AsNoTracking() .Where(b => b.EncounterId == encounterId) .ToListAsync(); bundles.Should().HaveCount(1); bundles[0].ComplianceStatus.Should().Be(SepsisBundleComplianceStatus.NonCompliant); } [Fact] public async Task SepsisBundleMonitor_CompletedBundle_NotMarkedOverdue() { var (_, encounterId) = await SeedPatientEncounterAndBundleAsync(); using (var scope = _fixture.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); var bundle = await db.SepsisBundles.SingleAsync(b => b.EncounterId == encounterId); bundle.ComplianceStatus = SepsisBundleComplianceStatus.Compliant; bundle.CompletedAt = DateTimeOffset.UtcNow; bundle.DeadlineAt = DateTimeOffset.UtcNow.AddHours(-1); await db.SaveChangesAsync(); } var monitor = new SepsisBundleMonitorService( _fixture.Services, _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService>()); await monitor.ScanOverdueBundlesAsync(CancellationToken.None); using var verifyScope = _fixture.Services.CreateScope(); var verifyDb = verifyScope.ServiceProvider.GetRequiredService(); var bundle2 = await verifyDb.SepsisBundles .AsNoTracking() .SingleAsync(b => b.EncounterId == encounterId); bundle2.ComplianceStatus.Should().Be(SepsisBundleComplianceStatus.Compliant, "already-completed bundles must not be marked non-compliant by the monitor"); } // ======================================================================= // EscalationWorker — only escalates open alerts // ======================================================================= [Fact] public async Task EscalationWorker_AcknowledgedAlert_NotEscalated() { using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var (patientId, encounterId) = await SeedPatientAndEncounterAsync(); var alertId = Guid.NewGuid(); db.ClinicalAlerts.Add(new ClinicalAlert { Id = alertId, EncounterId = encounterId, PatientId = patientId, AlertType = AlertType.SofaSepsis, Severity = AlertSeverity.Critical, Details = "SOFA delta +2", Status = AlertStatus.Acknowledged, TriggeredAt = DateTimeOffset.UtcNow }); await db.SaveChangesAsync(); var escalationService = new EscalationWorkerService( _fixture.Services.GetRequiredService>(), _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService>()); var escalated = await InvokeUpdateAlertStatusEscalatedAsync(escalationService, alertId); escalated.Should().BeFalse("acknowledged alerts must not be escalated"); var alert = await db.ClinicalAlerts.AsNoTracking().SingleAsync(a => a.Id == alertId); alert.Status.Should().Be(AlertStatus.Acknowledged); } [Fact] public async Task EscalationWorker_ResolvedAlert_NotEscalated() { using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var (patientId, encounterId) = await SeedPatientAndEncounterAsync(); var alertId = Guid.NewGuid(); db.ClinicalAlerts.Add(new ClinicalAlert { Id = alertId, EncounterId = encounterId, PatientId = patientId, AlertType = AlertType.SofaSepsis, Severity = AlertSeverity.Critical, Details = "SOFA delta +2", Status = AlertStatus.Resolved, TriggeredAt = DateTimeOffset.UtcNow }); await db.SaveChangesAsync(); var escalationService = new EscalationWorkerService( _fixture.Services.GetRequiredService>(), _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService>()); var escalated = await InvokeUpdateAlertStatusEscalatedAsync(escalationService, alertId); escalated.Should().BeFalse("resolved alerts must not be escalated"); var alert = await db.ClinicalAlerts.AsNoTracking().SingleAsync(a => a.Id == alertId); alert.Status.Should().Be(AlertStatus.Resolved); } [Fact] public async Task EscalationWorker_OpenAlert_IsEscalated() { using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var (patientId, encounterId) = await SeedPatientAndEncounterAsync(); var alertId = Guid.NewGuid(); db.ClinicalAlerts.Add(new ClinicalAlert { Id = alertId, EncounterId = encounterId, PatientId = patientId, AlertType = AlertType.SofaSepsis, Severity = AlertSeverity.Critical, Details = "SOFA delta +2", Status = AlertStatus.Open, TriggeredAt = DateTimeOffset.UtcNow }); await db.SaveChangesAsync(); var escalationService = new EscalationWorkerService( _fixture.Services.GetRequiredService>(), _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService>()); var escalated = await InvokeUpdateAlertStatusEscalatedAsync(escalationService, alertId); escalated.Should().BeTrue("open alerts must be escalated"); var alert = await db.ClinicalAlerts.AsNoTracking().SingleAsync(a => a.Id == alertId); alert.Status.Should().Be(AlertStatus.Escalated); } [Fact] public async Task EscalationWorker_NonexistentAlert_ReturnsFalse() { var escalationService = new EscalationWorkerService( _fixture.Services.GetRequiredService>(), _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService>()); var escalated = await InvokeUpdateAlertStatusEscalatedAsync(escalationService, Guid.NewGuid()); escalated.Should().BeFalse("nonexistent alerts must not cause errors"); } // ======================================================================= // Cancellation — background services respect CancellationToken // ======================================================================= [Fact] public async Task SepsisBundleMonitor_CancellationRequested_StopsGracefully() { var monitor = new SepsisBundleMonitorService( _fixture.Services, _fixture.Services.GetRequiredService(), _fixture.Services.GetRequiredService>()); using var cts = new CancellationTokenSource(); cts.Cancel(); var act = () => monitor.ScanOverdueBundlesAsync(cts.Token); await act.Should().ThrowAsync(); } // ======================================================================= // Helpers // ======================================================================= private static ConsumeResult BuildConsumeResult( string topic, int partition, long offset, string value) { return new ConsumeResult { Topic = topic, Partition = new Partition(partition), Offset = new Offset(offset), Message = new Message { Key = "test-key", Value = value } }; } private async Task<(Guid PatientId, Guid EncounterId)> SeedPatientAndEncounterAsync() { using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var patient = new Patient { Id = Guid.NewGuid(), Mrn = $"MRN-BG-{Guid.NewGuid():N}"[..20], FirstName = "Background", LastName = "Test", DateOfBirth = new DateOnly(1970, 1, 1), Gender = "M", CreatedAt = DateTimeOffset.UtcNow }; var encounter = new Encounter { Id = Guid.NewGuid(), PatientId = patient.Id, EncounterType = EncounterType.Inpatient, Status = EncounterStatus.Active, Department = Department.Icu, AttendingPhysician = "Dr. Background", AdmittedAt = DateTimeOffset.UtcNow, CreatedAt = DateTimeOffset.UtcNow }; db.Patients.Add(patient); db.Encounters.Add(encounter); await db.SaveChangesAsync(); return (patient.Id, encounter.Id); } private async Task<(Guid PatientId, Guid EncounterId)> SeedPatientEncounterAndBundleAsync() { var (patientId, encounterId) = await SeedPatientAndEncounterAsync(); using var scope = _fixture.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var handler = scope.ServiceProvider.GetRequiredService(); var alertId = Guid.NewGuid(); db.ClinicalAlerts.Add(new ClinicalAlert { Id = alertId, EncounterId = encounterId, PatientId = patientId, AlertType = AlertType.SofaSepsis, Severity = AlertSeverity.Critical, Details = "SOFA delta +2", Status = AlertStatus.Open, TriggeredAt = DateTimeOffset.UtcNow }); await db.SaveChangesAsync(); await handler.OnSepsisAlertCreatedAsync( encounterId, alertId, AlertType.SofaSepsis, CancellationToken.None); return (patientId, encounterId); } private static async Task InvokeUpdateAlertStatusEscalatedAsync( EscalationWorkerService service, Guid alertId) { var method = typeof(EscalationWorkerService).GetMethod( "UpdateAlertStatusEscalatedAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); var task = (Task)method!.Invoke(service, new object[] { alertId, CancellationToken.None })!; return await task; } }