vigilcare-clinical/VigilCareClinicalAPI.Tests/BackgroundServiceTests.cs

using System.Text.Json;
using Confluent.Kafka;
using FluentAssertions;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StackExchange.Redis;

[Collection("Integration")]
public class BackgroundServiceTests : IAsyncLifetime
{
    private readonly ApiFixture _fixture;

    public BackgroundServiceTests(ApiFixture fixture) => _fixture = fixture;

    public async Task InitializeAsync()
    {
        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
        await DbResetHelper.ResetAsync(db);

        var redis = scope.ServiceProvider.GetRequiredService<IConnectionMultiplexer>();
        var server = redis.GetServer(redis.GetEndPoints().First());
        await server.FlushDatabaseAsync(1);
    }

    public Task DisposeAsync() => Task.CompletedTask;

    // =======================================================================
    //  PoisonPillGuard — unit tests (no infrastructure needed)
    // =======================================================================

    [Fact]
    public void PoisonPillGuard_PermanentError_SkipsImmediately()
    {
        var guard = new PoisonPillGuard("test-group", maxRetries: 5, NullLogger.Instance);
        var result = BuildConsumeResult("test-topic", 0, 0, "bad payload");

        var shouldSkip = guard.ShouldSkip(result, new JsonException("Invalid JSON"));

        shouldSkip.Should().BeTrue("permanent errors (JsonException) must be skipped immediately");
    }

    [Fact]
    public void PoisonPillGuard_FormatException_SkipsImmediately()
    {
        var guard = new PoisonPillGuard("test-group", maxRetries: 5, NullLogger.Instance);
        var result = BuildConsumeResult("test-topic", 0, 0, "not-a-guid");

        var shouldSkip = guard.ShouldSkip(result, new FormatException("Input string was not in a correct format"));

        shouldSkip.Should().BeTrue("FormatException is a permanent error and must be skipped immediately");
    }

    [Fact]
    public void PoisonPillGuard_WrappedPermanentError_SkipsImmediately()
    {
        var guard = new PoisonPillGuard("test-group", maxRetries: 5, NullLogger.Instance);
        var result = BuildConsumeResult("test-topic", 0, 0, "bad");

        var wrappedException = new InvalidOperationException(
            "Processing failed", new JsonException("Invalid JSON"));

        var shouldSkip = guard.ShouldSkip(result, wrappedException);

        shouldSkip.Should().BeTrue(
            "permanent errors wrapped in other exceptions must still be detected via root cause");
    }

    [Fact]
    public void PoisonPillGuard_TransientError_RetriesUpToMax()
    {
        const int maxRetries = 3;
        var guard = new PoisonPillGuard("test-group", maxRetries, NullLogger.Instance);
        var result = BuildConsumeResult("test-topic", 0, 42, "payload");
        var transientError = new TimeoutException("Connection timed out");

        for (var i = 1; i < maxRetries; i++)
        {
            var shouldSkip = guard.ShouldSkip(result, transientError);
            shouldSkip.Should().BeFalse($"retry {i}/{maxRetries} should not skip");
        }

        var finalSkip = guard.ShouldSkip(result, transientError);
        finalSkip.Should().BeTrue(
            $"after {maxRetries} retries, the message must be skipped");
    }

    [Fact]
    public void PoisonPillGuard_SuccessResetsRetryCount()
    {
        const int maxRetries = 3;
        var guard = new PoisonPillGuard("test-group", maxRetries, NullLogger.Instance);
        var result = BuildConsumeResult("test-topic", 0, 42, "payload");
        var transientError = new TimeoutException("Connection timed out");

        guard.ShouldSkip(result, transientError);
        guard.ShouldSkip(result, transientError);

        guard.OnSuccess();

        guard.ShouldSkip(result, transientError).Should().BeFalse(
            "after OnSuccess, retry count resets and first failure should not skip");
    }

    [Fact]
    public void PoisonPillGuard_DifferentOffset_ResetsRetryCount()
    {
        const int maxRetries = 3;
        var guard = new PoisonPillGuard("test-group", maxRetries, NullLogger.Instance);
        var transientError = new TimeoutException("Connection timed out");

        var result1 = BuildConsumeResult("test-topic", 0, 42, "payload-a");
        guard.ShouldSkip(result1, transientError);
        guard.ShouldSkip(result1, transientError);

        var result2 = BuildConsumeResult("test-topic", 0, 43, "payload-b");
        guard.ShouldSkip(result2, transientError).Should().BeFalse(
            "a new offset resets the retry counter");
    }

    // =======================================================================
    //  Outbox relay — retry count and dead-letter behavior
    // =======================================================================

    [Fact]
    public async Task OutboxEvent_UnprocessedEvents_ArePickedUpByRelay()
    {
        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

        var encounterId = Guid.NewGuid();
        db.OutboxEvents.Add(new OutboxEvent
        {
            Id = Guid.NewGuid(),
            Topic = "observation.recorded",
            Payload = JsonSerializer.Serialize(new { encounterId, code = "HEART_RATE", value = 80 }),
            PartitionKey = encounterId.ToString(),
            CreatedAt = DateTimeOffset.UtcNow
        });
        await db.SaveChangesAsync();

        var pendingBefore = await db.OutboxEvents
            .CountAsync(e => e.ProcessedAt == null && e.FailedAt == null);

        pendingBefore.Should().BeGreaterThanOrEqualTo(1);

        var deadline = DateTimeOffset.UtcNow.AddSeconds(15);
        while (DateTimeOffset.UtcNow < deadline)
        {
            await Task.Delay(500);
            var remaining = await db.OutboxEvents
                .AsNoTracking()
                .CountAsync(e => e.ProcessedAt == null && e.FailedAt == null);
            if (remaining == 0) break;
        }

        var unprocessed = await db.OutboxEvents
            .AsNoTracking()
            .CountAsync(e => e.ProcessedAt == null && e.FailedAt == null);

        unprocessed.Should().Be(0, "the outbox relay should process all pending events");
    }

    [Fact]
    public async Task OutboxEvent_FailedEvents_HaveRetryCountAndError()
    {
        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

        var failedEvent = new OutboxEvent
        {
            Id = Guid.NewGuid(),
            Topic = "nonexistent.topic.that.will.fail",
            Payload = "{}",
            PartitionKey = "test",
            CreatedAt = DateTimeOffset.UtcNow,
            RetryCount = 9,
            LastError = "Simulated prior failure"
        };
        db.OutboxEvents.Add(failedEvent);
        await db.SaveChangesAsync();

        failedEvent.RetryCount.Should().Be(9);
        failedEvent.LastError.Should().NotBeNullOrEmpty();
        failedEvent.ProcessedAt.Should().BeNull();
    }

    [Fact]
    public async Task OutboxEvent_FailedAtSet_EventIsNotReprocessed()
    {
        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

        var deadLetteredEvent = new OutboxEvent
        {
            Id = Guid.NewGuid(),
            Topic = "test.topic",
            Payload = "{}",
            PartitionKey = "test",
            CreatedAt = DateTimeOffset.UtcNow,
            RetryCount = 10,
            LastError = "Exceeded max retries",
            FailedAt = DateTimeOffset.UtcNow.AddMinutes(-1)
        };
        db.OutboxEvents.Add(deadLetteredEvent);
        await db.SaveChangesAsync();

        await Task.Delay(3_000);

        var evt = await db.OutboxEvents
            .AsNoTracking()
            .SingleAsync(e => e.Id == deadLetteredEvent.Id);

        evt.ProcessedAt.Should().BeNull("dead-lettered events must not be reprocessed");
        evt.FailedAt.Should().NotBeNull();
    }

    // =======================================================================
    //  SepsisBundleMonitor — concurrent scan safety
    // =======================================================================

    [Fact]
    public async Task SepsisBundleMonitor_ConcurrentScans_NoDuplicateUpdates()
    {
        var (_, encounterId) = await SeedPatientEncounterAndBundleAsync();

        using (var scope = _fixture.Services.CreateScope())
        {
            var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
            var bundle = await db.SepsisBundles.SingleAsync(b => b.EncounterId == encounterId);
            bundle.DeadlineAt = DateTimeOffset.UtcNow.AddHours(-1);
            await db.SaveChangesAsync();
        }

        var tasks = Enumerable.Range(0, 5).Select(_ =>
        {
            return Task.Run(async () =>
            {
                var monitor = new SepsisBundleMonitorService(
                    _fixture.Services,
                    _fixture.Services.GetRequiredService<ClinicalMetrics>(),
                    _fixture.Services.GetRequiredService<ILogger<SepsisBundleMonitorService>>());
                await monitor.ScanOverdueBundlesAsync(CancellationToken.None);
            });
        });

        await Task.WhenAll(tasks);

        using var verifyScope = _fixture.Services.CreateScope();
        var verifyDb = verifyScope.ServiceProvider.GetRequiredService<AppDbContext>();

        var bundles = await verifyDb.SepsisBundles
            .AsNoTracking()
            .Where(b => b.EncounterId == encounterId)
            .ToListAsync();

        bundles.Should().HaveCount(1);
        bundles[0].ComplianceStatus.Should().Be(SepsisBundleComplianceStatus.NonCompliant);
    }

    [Fact]
    public async Task SepsisBundleMonitor_CompletedBundle_NotMarkedOverdue()
    {
        var (_, encounterId) = await SeedPatientEncounterAndBundleAsync();

        using (var scope = _fixture.Services.CreateScope())
        {
            var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
            var bundle = await db.SepsisBundles.SingleAsync(b => b.EncounterId == encounterId);
            bundle.ComplianceStatus = SepsisBundleComplianceStatus.Compliant;
            bundle.CompletedAt = DateTimeOffset.UtcNow;
            bundle.DeadlineAt = DateTimeOffset.UtcNow.AddHours(-1);
            await db.SaveChangesAsync();
        }

        var monitor = new SepsisBundleMonitorService(
            _fixture.Services,
            _fixture.Services.GetRequiredService<ClinicalMetrics>(),
            _fixture.Services.GetRequiredService<ILogger<SepsisBundleMonitorService>>());

        await monitor.ScanOverdueBundlesAsync(CancellationToken.None);

        using var verifyScope = _fixture.Services.CreateScope();
        var verifyDb = verifyScope.ServiceProvider.GetRequiredService<AppDbContext>();

        var bundle2 = await verifyDb.SepsisBundles
            .AsNoTracking()
            .SingleAsync(b => b.EncounterId == encounterId);

        bundle2.ComplianceStatus.Should().Be(SepsisBundleComplianceStatus.Compliant,
            "already-completed bundles must not be marked non-compliant by the monitor");
    }

    // =======================================================================
    //  EscalationWorker — only escalates open alerts
    // =======================================================================

    [Fact]
    public async Task EscalationWorker_AcknowledgedAlert_NotEscalated()
    {
        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

        var (patientId, encounterId) = await SeedPatientAndEncounterAsync();

        var alertId = Guid.NewGuid();
        db.ClinicalAlerts.Add(new ClinicalAlert
        {
            Id = alertId,
            EncounterId = encounterId,
            PatientId = patientId,
            AlertType = AlertType.SofaSepsis,
            Severity = AlertSeverity.Critical,
            Details = "SOFA delta +2",
            Status = AlertStatus.Acknowledged,
            TriggeredAt = DateTimeOffset.UtcNow
        });
        await db.SaveChangesAsync();

        var escalationService = new EscalationWorkerService(
            _fixture.Services.GetRequiredService<Microsoft.Extensions.Options.IOptions<RabbitMqOptions>>(),
            _fixture.Services.GetRequiredService<IServiceScopeFactory>(),
            _fixture.Services.GetRequiredService<ClinicalMetrics>(),
            _fixture.Services.GetRequiredService<ILogger<EscalationWorkerService>>());

        var escalated = await InvokeUpdateAlertStatusEscalatedAsync(escalationService, alertId);

        escalated.Should().BeFalse("acknowledged alerts must not be escalated");

        var alert = await db.ClinicalAlerts.AsNoTracking().SingleAsync(a => a.Id == alertId);
        alert.Status.Should().Be(AlertStatus.Acknowledged);
    }

    [Fact]
    public async Task EscalationWorker_ResolvedAlert_NotEscalated()
    {
        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

        var (patientId, encounterId) = await SeedPatientAndEncounterAsync();

        var alertId = Guid.NewGuid();
        db.ClinicalAlerts.Add(new ClinicalAlert
        {
            Id = alertId,
            EncounterId = encounterId,
            PatientId = patientId,
            AlertType = AlertType.SofaSepsis,
            Severity = AlertSeverity.Critical,
            Details = "SOFA delta +2",
            Status = AlertStatus.Resolved,
            TriggeredAt = DateTimeOffset.UtcNow
        });
        await db.SaveChangesAsync();

        var escalationService = new EscalationWorkerService(
            _fixture.Services.GetRequiredService<Microsoft.Extensions.Options.IOptions<RabbitMqOptions>>(),
            _fixture.Services.GetRequiredService<IServiceScopeFactory>(),
            _fixture.Services.GetRequiredService<ClinicalMetrics>(),
            _fixture.Services.GetRequiredService<ILogger<EscalationWorkerService>>());

        var escalated = await InvokeUpdateAlertStatusEscalatedAsync(escalationService, alertId);

        escalated.Should().BeFalse("resolved alerts must not be escalated");

        var alert = await db.ClinicalAlerts.AsNoTracking().SingleAsync(a => a.Id == alertId);
        alert.Status.Should().Be(AlertStatus.Resolved);
    }

    [Fact]
    public async Task EscalationWorker_OpenAlert_IsEscalated()
    {
        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

        var (patientId, encounterId) = await SeedPatientAndEncounterAsync();

        var alertId = Guid.NewGuid();
        db.ClinicalAlerts.Add(new ClinicalAlert
        {
            Id = alertId,
            EncounterId = encounterId,
            PatientId = patientId,
            AlertType = AlertType.SofaSepsis,
            Severity = AlertSeverity.Critical,
            Details = "SOFA delta +2",
            Status = AlertStatus.Open,
            TriggeredAt = DateTimeOffset.UtcNow
        });
        await db.SaveChangesAsync();

        var escalationService = new EscalationWorkerService(
            _fixture.Services.GetRequiredService<Microsoft.Extensions.Options.IOptions<RabbitMqOptions>>(),
            _fixture.Services.GetRequiredService<IServiceScopeFactory>(),
            _fixture.Services.GetRequiredService<ClinicalMetrics>(),
            _fixture.Services.GetRequiredService<ILogger<EscalationWorkerService>>());

        var escalated = await InvokeUpdateAlertStatusEscalatedAsync(escalationService, alertId);

        escalated.Should().BeTrue("open alerts must be escalated");

        var alert = await db.ClinicalAlerts.AsNoTracking().SingleAsync(a => a.Id == alertId);
        alert.Status.Should().Be(AlertStatus.Escalated);
    }

    [Fact]
    public async Task EscalationWorker_NonexistentAlert_ReturnsFalse()
    {
        var escalationService = new EscalationWorkerService(
            _fixture.Services.GetRequiredService<Microsoft.Extensions.Options.IOptions<RabbitMqOptions>>(),
            _fixture.Services.GetRequiredService<IServiceScopeFactory>(),
            _fixture.Services.GetRequiredService<ClinicalMetrics>(),
            _fixture.Services.GetRequiredService<ILogger<EscalationWorkerService>>());

        var escalated = await InvokeUpdateAlertStatusEscalatedAsync(escalationService, Guid.NewGuid());

        escalated.Should().BeFalse("nonexistent alerts must not cause errors");
    }

    // =======================================================================
    //  Cancellation — background services respect CancellationToken
    // =======================================================================

    [Fact]
    public async Task SepsisBundleMonitor_CancellationRequested_StopsGracefully()
    {
        var monitor = new SepsisBundleMonitorService(
            _fixture.Services,
            _fixture.Services.GetRequiredService<ClinicalMetrics>(),
            _fixture.Services.GetRequiredService<ILogger<SepsisBundleMonitorService>>());

        using var cts = new CancellationTokenSource();
        cts.Cancel();

        var act = () => monitor.ScanOverdueBundlesAsync(cts.Token);

        await act.Should().ThrowAsync<OperationCanceledException>();
    }

    // =======================================================================
    //  Helpers
    // =======================================================================

    private static ConsumeResult<string, string> BuildConsumeResult(
        string topic, int partition, long offset, string value)
    {
        return new ConsumeResult<string, string>
        {
            Topic = topic,
            Partition = new Partition(partition),
            Offset = new Offset(offset),
            Message = new Message<string, string>
            {
                Key = "test-key",
                Value = value
            }
        };
    }

    private async Task<(Guid PatientId, Guid EncounterId)> SeedPatientAndEncounterAsync()
    {
        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

        var patient = new Patient
        {
            Id = Guid.NewGuid(),
            Mrn = $"MRN-BG-{Guid.NewGuid():N}"[..20],
            FirstName = "Background",
            LastName = "Test",
            DateOfBirth = new DateOnly(1970, 1, 1),
            Gender = "M",
            CreatedAt = DateTimeOffset.UtcNow
        };
        var encounter = new Encounter
        {
            Id = Guid.NewGuid(),
            PatientId = patient.Id,
            EncounterType = EncounterType.Inpatient,
            Status = EncounterStatus.Active,
            Department = Department.Icu,
            AttendingPhysician = "Dr. Background",
            AdmittedAt = DateTimeOffset.UtcNow,
            CreatedAt = DateTimeOffset.UtcNow
        };

        db.Patients.Add(patient);
        db.Encounters.Add(encounter);
        await db.SaveChangesAsync();

        return (patient.Id, encounter.Id);
    }

    private async Task<(Guid PatientId, Guid EncounterId)> SeedPatientEncounterAndBundleAsync()
    {
        var (patientId, encounterId) = await SeedPatientAndEncounterAsync();

        using var scope = _fixture.Services.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
        var handler = scope.ServiceProvider.GetRequiredService<SepsisAlertHandler>();

        var alertId = Guid.NewGuid();
        db.ClinicalAlerts.Add(new ClinicalAlert
        {
            Id = alertId,
            EncounterId = encounterId,
            PatientId = patientId,
            AlertType = AlertType.SofaSepsis,
            Severity = AlertSeverity.Critical,
            Details = "SOFA delta +2",
            Status = AlertStatus.Open,
            TriggeredAt = DateTimeOffset.UtcNow
        });
        await db.SaveChangesAsync();

        await handler.OnSepsisAlertCreatedAsync(
            encounterId, alertId, AlertType.SofaSepsis, CancellationToken.None);

        return (patientId, encounterId);
    }

    private static async Task<bool> InvokeUpdateAlertStatusEscalatedAsync(
        EscalationWorkerService service, Guid alertId)
    {
        var method = typeof(EscalationWorkerService).GetMethod(
            "UpdateAlertStatusEscalatedAsync",
            System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);

        var task = (Task<bool>)method!.Invoke(service, new object[] { alertId, CancellationToken.None })!;
        return await task;
    }
}