diff --git a/VigilCareClinicalAPI/BackgroundServices/KafkaTopicProvisioner.cs b/VigilCareClinicalAPI/BackgroundServices/KafkaTopicProvisioner.cs new file mode 100644 index 0000000..2c92cd2 --- /dev/null +++ b/VigilCareClinicalAPI/BackgroundServices/KafkaTopicProvisioner.cs @@ -0,0 +1,55 @@ +using Confluent.Kafka; +using Confluent.Kafka.Admin; +using Microsoft.Extensions.Options; + +public class KafkaTopicProvisioner : IHostedService +{ + private readonly KafkaOptions _options; + private readonly ILogger _logger; + + public KafkaTopicProvisioner(IOptions options, ILogger logger) + { + _options = options.Value; + _logger = logger; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + using var admin = new AdminClientBuilder(new AdminClientConfig + { + BootstrapServers = _options.BootstrapServers + }).Build(); + + var topicNames = new[] + { + _options.Topics.ObservationRecorded, + _options.Topics.AlertGenerated, + _options.Topics.AlertAcknowledged, + _options.Topics.EncounterStatusChanged + }; + + var specs = topicNames.Select(name => new TopicSpecification + { + Name = name, + NumPartitions = _options.NumPartitions, + ReplicationFactor = 1 + }).ToList(); + + try + { + await admin.CreateTopicsAsync(specs); + _logger.LogInformation("Kafka topics provisioned: {Topics}", string.Join(", ", topicNames)); + } + catch (CreateTopicsException ex) + { + var errors = ex.Results.Where(r => r.Error.Code != ErrorCode.TopicAlreadyExists).ToList(); + if (errors.Count > 0) + throw new InvalidOperationException( + $"Failed to create Kafka topics: {string.Join(", ", errors.Select(e => e.Error.Reason))}"); + + _logger.LogInformation("Kafka topics already exist — skipping creation"); + } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} \ No newline at end of file diff --git a/VigilCareClinicalAPI/BackgroundServices/OutboxRelayService.cs b/VigilCareClinicalAPI/BackgroundServices/OutboxRelayService.cs new file mode 100644 index 0000000..ade903f --- /dev/null +++ b/VigilCareClinicalAPI/BackgroundServices/OutboxRelayService.cs @@ -0,0 +1,146 @@ +using Confluent.Kafka; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; + +public class OutboxRelayService : BackgroundService +{ + private readonly IServiceProvider _services; + private readonly KafkaOptions _options; + private readonly ILogger _logger; + private IProducer? _producer; + + public OutboxRelayService( + IServiceProvider services, + IOptions options, + ILogger logger) + { + _services = services; + _options = options.Value; + _logger = logger; + } + + public override Task StartAsync(CancellationToken cancellationToken) + { + _producer = new ProducerBuilder(new ProducerConfig + { + BootstrapServers = _options.BootstrapServers, + Acks = Acks.All, + // Idempotent producer: the broker deduplicates in-flight retries using + // the producer ID + sequence number. Prevents duplicates from network + // timeouts that cause the client to retry a message the broker already accepted. + EnableIdempotence = true, + MessageSendMaxRetries = 3, + RetryBackoffMs = 100 + }).Build(); + + return base.StartAsync(cancellationToken); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Outbox relay started. PollInterval={Interval}ms", _options.OutboxPollIntervalMs); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await ProcessBatchAsync(stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogError(ex, "Outbox relay error — will retry on next poll cycle"); + } + + await Task.Delay(_options.OutboxPollIntervalMs, stoppingToken); + } + } + + private async Task ProcessBatchAsync(CancellationToken ct) + { + using var scope = _services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + // FOR UPDATE SKIP LOCKED: if a second relay instance is running concurrently, + // it skips rows already locked by this instance. Both instances make progress + // without blocking each other. A plain SELECT without locking would cause both + // to read the same rows and publish duplicates. + await using var tx = await db.Database.BeginTransactionAsync(ct); + + var events = await db.OutboxEvents + .FromSqlRaw(""" + SELECT id, topic, payload, partition_key, created_at, processed_at + FROM outbox_events + WHERE processed_at IS NULL + ORDER BY created_at ASC + LIMIT {0} + FOR UPDATE SKIP LOCKED + """, _options.OutboxBatchSize) + .ToListAsync(ct); + + if (events.Count == 0) + { + await tx.RollbackAsync(ct); + return; + } + + var published = new List(); + var failed = false; + + foreach (var ev in events) + { + try + { + var result = await _producer!.ProduceAsync( + ev.Topic, + new Message + { + // The partition key ensures all events for one encounter land on + // the same partition. The sepsis engine depends on this — if two + // observations for the same patient arrive on different partitions, + // a single-partition consumer misses one and SIRS evaluation is wrong. + Key = ev.PartitionKey ?? string.Empty, + Value = ev.Payload + }, + ct); + + published.Add(ev.Id); + + _logger.LogDebug( + "Published {Topic} offset={Offset} partition={Partition} key={Key}", + ev.Topic, result.Offset.Value, result.Partition.Value, ev.PartitionKey); + } + catch (ProduceException ex) + { + _logger.LogWarning(ex, + "Kafka produce failed for outbox event {Id} — leaving unprocessed for next cycle", + ev.Id); + failed = true; + break; // stop processing this batch; retry from this row next cycle + } + } + + if (published.Count > 0) + { + var now = DateTimeOffset.UtcNow; + foreach (var id in published) + { + var ev = events.First(e => e.Id == id); + ev.ProcessedAt = now; + } + await db.SaveChangesAsync(ct); + } + + await tx.CommitAsync(ct); + + if (published.Count > 0) + _logger.LogInformation( + "Outbox relay published {Count} events. Failed={Failed}", + published.Count, failed); + } + + public override void Dispose() + { + _producer?.Dispose(); + base.Dispose(); + } +} \ No newline at end of file diff --git a/VigilCareClinicalAPI/Configuration/KafkaOptions.cs b/VigilCareClinicalAPI/Configuration/KafkaOptions.cs new file mode 100644 index 0000000..53f481c --- /dev/null +++ b/VigilCareClinicalAPI/Configuration/KafkaOptions.cs @@ -0,0 +1,9 @@ +public class KafkaOptions +{ + public const string Section = "Kafka"; + public string BootstrapServers { get; set; } = null!; + public KafkaTopicOptions Topics { get; set; } = null!; + public int NumPartitions { get; set; } = 6; + public int OutboxBatchSize { get; set; } = 100; + public int OutboxPollIntervalMs { get; set; } = 500; +} diff --git a/VigilCareClinicalAPI/Configuration/KafkaTopicOptions.cs b/VigilCareClinicalAPI/Configuration/KafkaTopicOptions.cs new file mode 100644 index 0000000..091007f --- /dev/null +++ b/VigilCareClinicalAPI/Configuration/KafkaTopicOptions.cs @@ -0,0 +1,7 @@ +public class KafkaTopicOptions +{ + public string ObservationRecorded { get; set; } = "observation.recorded"; + public string AlertGenerated { get; set; } = "alert.generated"; + public string AlertAcknowledged { get; set; } = "alert.acknowledged"; + public string EncounterStatusChanged { get; set; } = "encounter.status.changed"; +} \ No newline at end of file diff --git a/VigilCareClinicalAPI/Data/Configurations/OutboxEventConfiguration.cs b/VigilCareClinicalAPI/Data/Configurations/OutboxEventConfiguration.cs index dc712b5..ac57e6b 100644 --- a/VigilCareClinicalAPI/Data/Configurations/OutboxEventConfiguration.cs +++ b/VigilCareClinicalAPI/Data/Configurations/OutboxEventConfiguration.cs @@ -10,6 +10,7 @@ public class OutboxEventConfiguration : IEntityTypeConfiguration builder.Property(o => o.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()"); builder.Property(o => o.Topic).HasColumnName("topic").HasMaxLength(200).IsRequired(); builder.Property(o => o.Payload).HasColumnName("payload").HasColumnType("jsonb").IsRequired(); + builder.Property(o => o.PartitionKey).HasColumnName("partition_key").HasMaxLength(36); builder.Property(o => o.CreatedAt).HasColumnName("created_at").HasDefaultValueSql("NOW()"); builder.Property(o => o.ProcessedAt).HasColumnName("processed_at"); diff --git a/VigilCareClinicalAPI/Domains/Entities/OutboxEvent.cs b/VigilCareClinicalAPI/Domains/Entities/OutboxEvent.cs index d4a1588..f9125b8 100644 --- a/VigilCareClinicalAPI/Domains/Entities/OutboxEvent.cs +++ b/VigilCareClinicalAPI/Domains/Entities/OutboxEvent.cs @@ -3,6 +3,7 @@ public class OutboxEvent public Guid Id { get; set; } public string Topic { get; set; } = null!; public string Payload { get; set; } = null!; // JSON string + public string? PartitionKey { get; set; } // encounter_id for all clinical events public DateTimeOffset CreatedAt { get; set; } public DateTimeOffset? ProcessedAt { get; set; } } \ No newline at end of file diff --git a/VigilCareClinicalAPI/Migrations/20260616132054_AddOutboxPartitionKey.Designer.cs b/VigilCareClinicalAPI/Migrations/20260616132054_AddOutboxPartitionKey.Designer.cs new file mode 100644 index 0000000..b6833c1 --- /dev/null +++ b/VigilCareClinicalAPI/Migrations/20260616132054_AddOutboxPartitionKey.Designer.cs @@ -0,0 +1,568 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace VigilCareClinicalAPI.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20260616132054_AddOutboxPartitionKey")] + partial class AddOutboxPartitionKey + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.4") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("AlertThreshold", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + b.Property("CriticalHigh") + .HasColumnType("decimal(10,3)") + .HasColumnName("critical_high"); + + b.Property("CriticalLow") + .HasColumnType("decimal(10,3)") + .HasColumnName("critical_low"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("display_name"); + + b.Property("ObservationCode") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("observation_code"); + + b.Property("Unit") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("unit"); + + b.Property("WarningHigh") + .HasColumnType("decimal(10,3)") + .HasColumnName("warning_high"); + + b.Property("WarningLow") + .HasColumnType("decimal(10,3)") + .HasColumnName("warning_low"); + + b.HasKey("Id"); + + b.HasIndex("ObservationCode") + .IsUnique(); + + b.ToTable("alert_thresholds", (string)null); + }); + + modelBuilder.Entity("ClinicalAlert", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("AcknowledgedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("acknowledged_at"); + + b.Property("AcknowledgedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("acknowledged_by"); + + b.Property("AlertType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("alert_type"); + + b.Property("Details") + .IsRequired() + .HasColumnType("text") + .HasColumnName("details"); + + b.Property("EncounterId") + .HasColumnType("uuid") + .HasColumnName("encounter_id"); + + b.Property("ObservationId") + .HasColumnType("uuid") + .HasColumnName("observation_id"); + + b.Property("PatientId") + .HasColumnType("uuid") + .HasColumnName("patient_id"); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("resolved_at"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("severity"); + + b.Property("Status") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("status") + .HasDefaultValueSql("'OPEN'"); + + b.Property("TriggeredAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("triggered_at") + .HasDefaultValueSql("NOW()"); + + b.HasKey("Id"); + + b.HasIndex("EncounterId", "TriggeredAt"); + + b.HasIndex("PatientId", "TriggeredAt"); + + b.HasIndex("Severity", "TriggeredAt") + .HasFilter("status = 'OPEN'"); + + b.ToTable("clinical_alerts", null, t => + { + t.HasCheckConstraint("chk_clinical_alerts_alert_type", "alert_type IN ('SEPSIS_WARNING', 'CRITICAL_HEART_RATE', 'CRITICAL_TEMP_C', 'CRITICAL_POTASSIUM_MEQ_L', 'CRITICAL_SPO2', 'CRITICAL_RESP_RATE', 'CRITICAL_WBC_K_UL')"); + + t.HasCheckConstraint("chk_clinical_alerts_severity", "severity IN ('WARNING', 'CRITICAL')"); + + t.HasCheckConstraint("chk_clinical_alerts_status", "status IN ('OPEN', 'ACKNOWLEDGED', 'RESOLVED', 'ESCALATED')"); + }); + }); + + modelBuilder.Entity("Encounter", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("AdmittedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("admitted_at") + .HasDefaultValueSql("NOW()"); + + b.Property("AttendingPhysician") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("attending_physician"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + b.Property("Department") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("department"); + + b.Property("DischargedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("discharged_at"); + + b.Property("EncounterType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("encounter_type"); + + b.Property("PatientId") + .HasColumnType("uuid") + .HasColumnName("patient_id"); + + b.Property("Status") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("status") + .HasDefaultValueSql("'SCHEDULED'"); + + b.HasKey("Id"); + + b.HasIndex("PatientId", "AdmittedAt"); + + b.HasIndex("Status", "AdmittedAt") + .HasFilter("status = 'ACTIVE'"); + + b.ToTable("encounters", null, t => + { + t.HasCheckConstraint("chk_encounters_encounter_type", "encounter_type IN ('INPATIENT', 'OUTPATIENT', 'EMERGENCY')"); + + t.HasCheckConstraint("chk_encounters_status", "status IN ('SCHEDULED', 'ACTIVE', 'DISCHARGED', 'CANCELLED')"); + }); + }); + + modelBuilder.Entity("Observation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + b.Property("EncounterId") + .HasColumnType("uuid") + .HasColumnName("encounter_id"); + + b.Property("IdempotencyKey") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("idempotency_key"); + + b.Property("ObservationCode") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("observation_code"); + + b.Property("RecordedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("recorded_at"); + + b.Property("Source") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("source") + .HasDefaultValueSql("'MANUAL'"); + + b.Property("Unit") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("unit"); + + b.Property("Value") + .HasColumnType("decimal(10,3)") + .HasColumnName("value"); + + b.HasKey("Id"); + + b.HasIndex("IdempotencyKey") + .IsUnique() + .HasFilter("idempotency_key IS NOT NULL"); + + b.HasIndex("EncounterId", "ObservationCode", "RecordedAt"); + + b.ToTable("observations", null, t => + { + t.HasCheckConstraint("chk_observations_source", "source IN ('MANUAL', 'DEVICE', 'LAB')"); + }); + }); + + modelBuilder.Entity("Order", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EncounterId") + .HasColumnType("uuid") + .HasColumnName("encounter_id"); + + b.Property("OrderType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("order_type"); + + b.Property("OrderedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("ordered_at") + .HasDefaultValueSql("NOW()"); + + b.Property("OrderedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("ordered_by"); + + b.Property("ResultedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("resulted_at"); + + b.Property("Status") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasDefaultValue("pending") + .HasColumnName("status"); + + b.HasKey("Id"); + + b.HasIndex("EncounterId", "OrderedAt"); + + b.HasIndex("Status", "OrderedAt") + .HasFilter("status IN ('pending', 'in_progress')"); + + b.ToTable("orders", null, t => + { + t.HasCheckConstraint("chk_orders_order_type", "order_type IN ('LAB', 'IMAGING', 'MEDICATION', 'PROCEDURE')"); + }); + }); + + modelBuilder.Entity("OutboxEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + b.Property("PartitionKey") + .HasMaxLength(36) + .HasColumnType("character varying(36)") + .HasColumnName("partition_key"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("ProcessedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("processed_at"); + + b.Property("Topic") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("topic"); + + b.HasKey("Id"); + + b.HasIndex("CreatedAt") + .HasFilter("processed_at IS NULL"); + + b.ToTable("outbox_events", (string)null); + }); + + modelBuilder.Entity("Patient", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + b.Property("DateOfBirth") + .HasColumnType("date") + .HasColumnName("date_of_birth"); + + b.Property("FirstName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("first_name"); + + b.Property("Gender") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("gender"); + + b.Property("LastName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("last_name"); + + b.Property("Mrn") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("mrn"); + + b.Property("Status") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasDefaultValue("active") + .HasColumnName("status"); + + b.HasKey("Id"); + + b.HasIndex("Mrn") + .IsUnique(); + + b.ToTable("patients", (string)null); + }); + + modelBuilder.Entity("ReconciliationAlert", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id") + .HasDefaultValueSql("gen_random_uuid()"); + + b.Property("CheckType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("check_type"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("NOW()"); + + b.Property("Details") + .IsRequired() + .HasColumnType("text") + .HasColumnName("details"); + + b.Property("EncounterId") + .HasColumnType("uuid") + .HasColumnName("encounter_id"); + + b.Property("PatientId") + .HasColumnType("uuid") + .HasColumnName("patient_id"); + + b.Property("ResolvedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("resolved_at"); + + b.HasKey("Id"); + + b.HasIndex("CheckType", "EncounterId") + .HasFilter("resolved_at IS NULL"); + + b.ToTable("reconciliation_alerts", null, t => + { + t.HasCheckConstraint("chk_reconciliation_alerts_check_type", "check_type IN ('UNACKNOWLEDGED_CRITICAL_ALERT', 'PENDING_ORDER_NO_RESULT', 'ACTIVE_INPATIENT_NO_OBSERVATION')"); + }); + }); + + modelBuilder.Entity("ClinicalAlert", b => + { + b.HasOne("Encounter", "Encounter") + .WithMany("Alerts") + .HasForeignKey("EncounterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Encounter"); + }); + + modelBuilder.Entity("Encounter", b => + { + b.HasOne("Patient", "Patient") + .WithMany("Encounters") + .HasForeignKey("PatientId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Patient"); + }); + + modelBuilder.Entity("Observation", b => + { + b.HasOne("Encounter", "Encounter") + .WithMany("Observations") + .HasForeignKey("EncounterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Encounter"); + }); + + modelBuilder.Entity("Order", b => + { + b.HasOne("Encounter", "Encounter") + .WithMany("Orders") + .HasForeignKey("EncounterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Encounter"); + }); + + modelBuilder.Entity("Encounter", b => + { + b.Navigation("Alerts"); + + b.Navigation("Observations"); + + b.Navigation("Orders"); + }); + + modelBuilder.Entity("Patient", b => + { + b.Navigation("Encounters"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/VigilCareClinicalAPI/Migrations/20260616132054_AddOutboxPartitionKey.cs b/VigilCareClinicalAPI/Migrations/20260616132054_AddOutboxPartitionKey.cs new file mode 100644 index 0000000..1446e4a --- /dev/null +++ b/VigilCareClinicalAPI/Migrations/20260616132054_AddOutboxPartitionKey.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace VigilCareClinicalAPI.Migrations +{ + /// + public partial class AddOutboxPartitionKey : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "partition_key", + table: "outbox_events", + type: "character varying(36)", + maxLength: 36, + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "partition_key", + table: "outbox_events"); + } + } +} diff --git a/VigilCareClinicalAPI/Migrations/AppDbContextModelSnapshot.cs b/VigilCareClinicalAPI/Migrations/AppDbContextModelSnapshot.cs index 3a48a7a..acf6840 100644 --- a/VigilCareClinicalAPI/Migrations/AppDbContextModelSnapshot.cs +++ b/VigilCareClinicalAPI/Migrations/AppDbContextModelSnapshot.cs @@ -368,6 +368,11 @@ namespace VigilCareClinicalAPI.Migrations .HasColumnName("created_at") .HasDefaultValueSql("NOW()"); + b.Property("PartitionKey") + .HasMaxLength(36) + .HasColumnType("character varying(36)") + .HasColumnName("partition_key"); + b.Property("Payload") .IsRequired() .HasColumnType("jsonb") diff --git a/VigilCareClinicalAPI/Program.cs b/VigilCareClinicalAPI/Program.cs index 81c11f1..fb0bc1a 100644 --- a/VigilCareClinicalAPI/Program.cs +++ b/VigilCareClinicalAPI/Program.cs @@ -22,6 +22,9 @@ try builder.Services.AddSingleton(sp => ConnectionMultiplexer.Connect(sp.GetRequiredService()["Redis:ConnectionString"]!)); + builder.Services.Configure( + builder.Configuration.GetSection(KafkaOptions.Section)); + builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -30,6 +33,8 @@ try builder.Services.AddScoped(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); + builder.Services.AddHostedService(); builder.Services.AddControllers() .AddJsonOptions(opts => diff --git a/VigilCareClinicalAPI/Services/AlertService.cs b/VigilCareClinicalAPI/Services/AlertService.cs index 2ef3afa..89cf7ad 100644 --- a/VigilCareClinicalAPI/Services/AlertService.cs +++ b/VigilCareClinicalAPI/Services/AlertService.cs @@ -96,6 +96,7 @@ public class AlertService : IAlertService acknowledgedAt = alert.AcknowledgedAt, note = req.Note }), + PartitionKey = alert.EncounterId.ToString(), CreatedAt = DateTimeOffset.UtcNow }); diff --git a/VigilCareClinicalAPI/Services/EncounterService.cs b/VigilCareClinicalAPI/Services/EncounterService.cs index bbc9fec..83aa37f 100644 --- a/VigilCareClinicalAPI/Services/EncounterService.cs +++ b/VigilCareClinicalAPI/Services/EncounterService.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using Microsoft.EntityFrameworkCore; public class EncounterService : IEncounterService @@ -41,11 +42,29 @@ public class EncounterService : IEncounterService throw new ConflictException( $"Transition to '{targetStatus}' is not permitted from the current status.", "ILLEGAL_STATUS_TRANSITION"); - + + var previousStatus = encounter.Status; encounter.Status = targetStatus; + if (targetStatus == EncounterStatus.Discharged) encounter.DischargedAt = DateTimeOffset.UtcNow; + _db.OutboxEvents.Add(new OutboxEvent + { + Id = Guid.NewGuid(), + Topic = "encounter.status.changed", + Payload = JsonSerializer.Serialize(new + { + encounterId, + patientId = encounter.PatientId, + previousStatus = previousStatus.ToDbString(), + newStatus = targetStatus.ToDbString(), + changedAt = DateTimeOffset.UtcNow + }), + PartitionKey = encounterId.ToString(), + CreatedAt = DateTimeOffset.UtcNow + }); + await _db.SaveChangesAsync(); return new EncounterStatusTransitionResult(encounterId, targetStatus); } diff --git a/VigilCareClinicalAPI/Services/ObservationService.cs b/VigilCareClinicalAPI/Services/ObservationService.cs index 621d1bf..fc52aa0 100644 --- a/VigilCareClinicalAPI/Services/ObservationService.cs +++ b/VigilCareClinicalAPI/Services/ObservationService.cs @@ -116,7 +116,7 @@ public class ObservationService : IObservationService severity = alert.Severity.ToDbString(), triggeredAt = alert.TriggeredAt, partitionKey = encounterId.ToString() - })); + }, encounterId.ToString())); } // Step 7 — outbox event for the observation (always; Kafka consumer handles warnings) @@ -131,7 +131,7 @@ public class ObservationService : IObservationService source = req.Source.ToDbString(), recordedAt = req.RecordedAt, partitionKey = encounterId.ToString() - })); + }, encounterId.ToString())); // Step 8 — COMMIT await _db.SaveChangesAsync(); @@ -202,11 +202,12 @@ public class ObservationService : IObservationService return $"{req.ObservationCode} value {req.Value} {req.Unit} is above critical high of {t.CriticalHigh} {req.Unit}."; } - private static OutboxEvent BuildOutboxEvent(string topic, object payload) => new() + private static OutboxEvent BuildOutboxEvent(string topic, object payload, string partitionKey) => new() { Id = Guid.NewGuid(), Topic = topic, Payload = JsonSerializer.Serialize(payload), + PartitionKey = partitionKey, CreatedAt = DateTimeOffset.UtcNow }; diff --git a/VigilCareClinicalAPI/VigilCareClinicalAPI.csproj b/VigilCareClinicalAPI/VigilCareClinicalAPI.csproj index 803c26c..dd9278d 100644 --- a/VigilCareClinicalAPI/VigilCareClinicalAPI.csproj +++ b/VigilCareClinicalAPI/VigilCareClinicalAPI.csproj @@ -9,6 +9,7 @@ + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/VigilCareClinicalAPI/appsettings.json b/VigilCareClinicalAPI/appsettings.json index 9d0e9d6..15a3aee 100644 --- a/VigilCareClinicalAPI/appsettings.json +++ b/VigilCareClinicalAPI/appsettings.json @@ -35,5 +35,16 @@ "Microsoft.EntityFrameworkCore.Database.Command": "Information" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "Kafka": { + "BootstrapServers": "localhost:9092", + "Topics": { + "ObservationRecorded": "observation.recorded", + "AlertGenerated": "alert.generated", + "EncounterStatusChanged": "encounter.status.changed" + }, + "NumPartitions": 6, + "OutboxBatchSize": 100, + "OutboxPollIntervalMs": 500 + } } diff --git a/docker-compose.yml b/docker-compose.yml index 666ec17..dbdef46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,11 +19,38 @@ services: image: datalust/seq:latest environment: ACCEPT_EULA: "Y" + SEQ_FIRSTRUN_ADMINPASSWORD: "admin" ports: - "5345:80" volumes: - seq_data:/data + kafka: + image: apache/kafka:3.7.0 + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + # Disable auto-creation — topics are provisioned explicitly with the correct + # partition count. Auto-creation would silently create a topic with 1 partition + # if the relay publishes before the provisioner runs. + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + # Fixed ID prevents the broker from reinitializing storage on restart. + # Generate once with: kafka-storage.sh random-uuid | base64 + CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk" + volumes: + - kafka_data:/var/lib/kafka/data + volumes: pg_data: - seq_data: \ No newline at end of file + seq_data: + kafka_data: \ No newline at end of file diff --git a/docs/vigilcare-clinical-api-prd.md b/docs/vigilcare-clinical-api-prd.md index baa2c8a..f53fc46 100644 --- a/docs/vigilcare-clinical-api-prd.md +++ b/docs/vigilcare-clinical-api-prd.md @@ -12,6 +12,25 @@ This project maps to `sd-mid-009` (Outbox Pattern), `sd-mid-013` (CQRS), `sd-mid --- +## Local Development Setup + +Start all infrastructure services with: + +```bash +docker compose up -d +``` + +| Service | Port | Notes | +|---|---|---| +| PostgreSQL | 5436 | Database: `vigilcare`, user: `postgres`, password: `password` | +| Redis | 6382 | No auth | +| Seq | 5345 | UI at `http://localhost:5345` — login: `admin` / `admin` | +| Kafka | 9092 | KRaft mode, no Zookeeper | + +**Seq first-run:** `SEQ_FIRSTRUN_ADMINPASSWORD=admin` is set in `docker-compose.yml`. This password is only applied on the very first container start (when `/data` volume is empty). After initialization, the password is stored in the volume and this env var is ignored. + +--- + ## Goals - Model the observe-alert-acknowledge lifecycle that sits at the center of any clinical monitoring system diff --git a/scripts/run-kafka-outbox-tests.sh b/scripts/run-kafka-outbox-tests.sh new file mode 100755 index 0000000..ac5facc --- /dev/null +++ b/scripts/run-kafka-outbox-tests.sh @@ -0,0 +1,366 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="${COMPOSE_FILE:-${SCRIPT_DIR}/../docker-compose.yml}" + +BASE_URL="${BASE_URL:-http://localhost:5270}" +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5436}" +PGDATABASE="${PGDATABASE:-vigilcare}" +PGUSER="${PGUSER:-postgres}" +PGPASSWORD="${PGPASSWORD:-password}" + +TOPIC_OBSERVATION="${TOPIC_OBSERVATION:-observation.recorded}" +TOPIC_ENCOUNTER="${TOPIC_ENCOUNTER:-encounter.status.changed}" +RELAY_WAIT_SECS="${RELAY_WAIT_SECS:-45}" +KAFKA_READY_WAIT_SECS="${KAFKA_READY_WAIT_SECS:-30}" + +RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")" + +TMP_FILES=() + +cleanup() { + local f + for f in "${TMP_FILES[@]}"; do + rm -f "${f}" "${f}.status" 2>/dev/null || true + done +} +trap cleanup EXIT + +if ! command -v curl >/dev/null 2>&1; then + echo "Missing dependency: curl" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "Missing dependency: jq" + exit 1 +fi + +if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then + echo "Missing dependency: docker compose (${COMPOSE_FILE})" + exit 1 +fi + +compose() { + docker compose -f "${COMPOSE_FILE}" "$@" +} + +kafka_exec() { + compose exec -T kafka "$@" +} + +psql_cmd() { + local sql="$1" + if command -v psql >/dev/null 2>&1; then + PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" + else + compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}" + fi +} + +request() { + local method="$1" + local url="$2" + local body="${3:-}" + local tmp_body + tmp_body="$(mktemp)" + TMP_FILES+=("${tmp_body}") + local status + + if [[ -n "${body}" ]]; then + status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}" \ + -H "Content-Type: application/json" -d "${body}")" + else + status="$(curl -sS -o "${tmp_body}" -w "%{http_code}" -X "${method}" "${url}")" + fi + + echo "${status}" > "${tmp_body}.status" + echo "${tmp_body}" +} + +assert_status() { + local expected="$1" + local body_file="$2" + local status + status="$(cat "${body_file}.status")" + if [[ "${status}" != "${expected}" ]]; then + echo "Expected HTTP ${expected}, got ${status}" + echo "Response body:" + cat "${body_file}" + echo + return 1 + fi +} + +wait_for_kafka() { + local elapsed=0 + while (( elapsed < KAFKA_READY_WAIT_SECS )); do + if kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "Kafka did not become ready within ${KAFKA_READY_WAIT_SECS}s" + return 1 +} + +wait_for_outbox_processed() { + local outbox_id="$1" + local elapsed=0 + while (( elapsed < RELAY_WAIT_SECS )); do + local processed + processed="$(psql_cmd "SELECT processed_at IS NOT NULL FROM outbox_events WHERE id = '${outbox_id}'")" + if [[ "${processed}" == "t" ]]; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + echo "Outbox row ${outbox_id} was not processed within ${RELAY_WAIT_SECS}s" + return 1 +} + +wait_for_pending_outbox_count() { + local expected="$1" + local elapsed=0 + while (( elapsed < RELAY_WAIT_SECS )); do + local pending + pending="$(psql_cmd "SELECT COUNT(*) FROM outbox_events WHERE processed_at IS NULL")" + if [[ "${pending}" == "${expected}" ]]; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + echo "Expected ${expected} pending outbox row(s), timed out after ${RELAY_WAIT_SECS}s" + return 1 +} + +consume_topic() { + local topic="$1" + local group="$2" + local max_messages="$3" + local tmp_out + tmp_out="$(mktemp)" + TMP_FILES+=("${tmp_out}") + + kafka_exec /opt/kafka/bin/kafka-console-consumer.sh \ + --bootstrap-server localhost:9092 \ + --topic "${topic}" \ + --group "${group}" \ + --from-beginning \ + --property print.key=true \ + --property print.partition=true \ + --timeout-ms 15000 \ + --max-messages "${max_messages}" > "${tmp_out}" 2>/dev/null || true + + echo "${tmp_out}" +} + +TOTAL_STEPS=11 + +echo "Running Kafka + outbox verification against ${BASE_URL}" +echo "Script run id: ${SCRIPT_RUN_ID}" + +echo "" +echo "[0/${TOTAL_STEPS}] Preflight — API, Postgres, and Kafka reachable" +preflight_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)" +if [[ "${preflight_status}" != "200" ]]; then + echo "API not reachable at ${BASE_URL} (HTTP ${preflight_status})." + echo "Start the API with: dotnet run --project VigilCareClinicalAPI" + exit 1 +fi + +if ! psql_cmd "SELECT 1" >/dev/null 2>&1; then + echo "Postgres not reachable on ${PGHOST}:${PGPORT}." + echo "Start the stack with: docker compose up -d" + exit 1 +fi + +if ! wait_for_kafka; then + exit 1 +fi +echo "OK: API, Postgres, and Kafka are up" + +echo "" +echo "[1/${TOTAL_STEPS}] Verifying Kafka topics were provisioned" +topic_list="$(kafka_exec /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list)" +for topic in alert.generated alert.acknowledged encounter.status.changed observation.recorded; do + if ! grep -qx "${topic}" <<< "${topic_list}"; then + echo "Missing Kafka topic: ${topic}" + echo "Topics found:" + echo "${topic_list}" + exit 1 + fi +done +echo "OK: observation.recorded, alert.generated, alert.acknowledged, encounter.status.changed exist" + +echo "" +echo "[2/${TOTAL_STEPS}] Creating patient and two active encounters" +patient_payload='{"firstName":"Kafka","lastName":"Verifier","dateOfBirth":"1990-06-01","gender":"M"}' +resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")" +assert_status "201" "${resp}" +patient_id="$(jq -r '.data.id' "${resp}")" + +enc_payload_a='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Kafka"}' +resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload_a}")" +assert_status "201" "${resp}" +encounter_a="$(jq -r '.data.id' "${resp}")" + +enc_payload_b='{"encounterType":"Outpatient","department":"Clinic","attendingPhysician":"Dr. Kafka"}' +resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload_b}")" +assert_status "201" "${resp}" +encounter_b="$(jq -r '.data.id' "${resp}")" + +enc_payload_discharge='{"encounterType":"Emergency","department":"ED","attendingPhysician":"Dr. Kafka"}' +resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload_discharge}")" +assert_status "201" "${resp}" +encounter_discharge="$(jq -r '.data.id' "${resp}")" + +echo "OK: encounters ${encounter_a}, ${encounter_b}, ${encounter_discharge}" + +echo "" +echo "[3/${TOTAL_STEPS}] Ingesting observations for partition-key verification" +for encounter_id in "${encounter_a}" "${encounter_b}"; do + for i in 1 2; do + obs_payload="$(jq -nc \ + --arg recordedAt "${RECORDED_AT}" \ + --arg key "kafka-script-${SCRIPT_RUN_ID}-${encounter_id}-${i}" \ + '{observations:[{observationCode:"HEART_RATE",value:80,unit:"bpm",source:"DEVICE",recordedAt:$recordedAt,idempotencyKey:$key}]}')" + resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${obs_payload}")" + assert_status "201" "${resp}" + done +done +echo "OK: four observation.recorded outbox rows written" + +echo "" +echo "[4/${TOTAL_STEPS}] Waiting for relay to publish observation events" +latest_outbox="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = '${TOPIC_OBSERVATION}' AND partition_key = '${encounter_a}' ORDER BY created_at DESC LIMIT 1")" +if [[ -z "${latest_outbox}" ]]; then + echo "Could not find outbox row for encounter ${encounter_a}" + exit 1 +fi +wait_for_outbox_processed "${latest_outbox}" +echo "OK: relay marked observation outbox rows processed" + +echo "" +echo "[5/${TOTAL_STEPS}] Verifying partition ordering (same encounter key → same partition)" +consumer_out="$(consume_topic "${TOPIC_OBSERVATION}" "partition-check-${SCRIPT_RUN_ID}" 50)" +for encounter_id in "${encounter_a}" "${encounter_b}"; do + partitions="$(grep -F "${encounter_id}" "${consumer_out}" | sed -n 's/^Partition:\([0-9]*\).*/\1/p' | sort -u)" + partition_count="$(echo "${partitions}" | grep -c . || true)" + if [[ "${partition_count}" -lt 1 ]]; then + echo "No Kafka messages found for encounter ${encounter_id}" + exit 1 + fi + if [[ "${partition_count}" -gt 1 ]]; then + echo "Encounter ${encounter_id} landed on multiple partitions: ${partitions}" + exit 1 + fi +done +echo "OK: each encounter's messages share a single partition" + +echo "" +echo "[6/${TOTAL_STEPS}] Verifying consumer group independence" +group_a="es-indexer-${SCRIPT_RUN_ID}" +group_b="sepsis-engine-${SCRIPT_RUN_ID}" +out_a="$(consume_topic "${TOPIC_OBSERVATION}" "${group_a}" 20)" +out_b="$(consume_topic "${TOPIC_OBSERVATION}" "${group_b}" 20)" +count_a="$(grep -c '^Partition:' "${out_a}" || true)" +count_b="$(grep -c '^Partition:' "${out_b}" || true)" +if [[ "${count_a}" -lt 4 || "${count_b}" -lt 4 ]]; then + echo "Expected each consumer group to read at least 4 messages, got ${count_a} and ${count_b}" + exit 1 +fi + +offset_a="$(kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \ + --bootstrap-server localhost:9092 \ + --describe --group "${group_a}" 2>/dev/null | awk '/observation.recorded/ {sum += $4} END {print sum+0}')" +offset_b="$(kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \ + --bootstrap-server localhost:9092 \ + --describe --group "${group_b}" 2>/dev/null | awk '/observation.recorded/ {sum += $4} END {print sum+0}')" +if [[ "${offset_a}" -lt 4 || "${offset_b}" -lt 4 ]]; then + echo "Consumer groups did not commit expected offsets (es-indexer=${offset_a}, sepsis-engine=${offset_b})" + exit 1 +fi +echo "OK: two consumer groups read independently (offsets es-indexer=${offset_a}, sepsis-engine=${offset_b})" + +echo "" +echo "[7/${TOTAL_STEPS}] Verifying encounter.status.changed outbox write and Kafka publish" +discharge_payload='{"status":"Discharged"}' +resp="$(request PATCH "${BASE_URL}/api/v1/encounters/${encounter_discharge}/status" "${discharge_payload}")" +assert_status "200" "${resp}" + +outbox_status="$(psql_cmd "SELECT payload->>'newStatus' FROM outbox_events WHERE topic = '${TOPIC_ENCOUNTER}' AND partition_key = '${encounter_discharge}' ORDER BY created_at DESC LIMIT 1")" +if [[ "${outbox_status}" != "DISCHARGED" ]]; then + echo "Expected encounter.status.changed outbox row with newStatus=DISCHARGED, got '${outbox_status}'" + exit 1 +fi + +discharge_outbox_id="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = '${TOPIC_ENCOUNTER}' AND partition_key = '${encounter_discharge}' ORDER BY created_at DESC LIMIT 1")" +wait_for_outbox_processed "${discharge_outbox_id}" + +encounter_consumer_out="$(consume_topic "${TOPIC_ENCOUNTER}" "encounter-check-${SCRIPT_RUN_ID}" 5)" +if ! grep -Fq "${encounter_discharge}" "${encounter_consumer_out}"; then + echo "encounter.status.changed message not found on Kafka for ${encounter_discharge}" + exit 1 +fi +echo "OK: discharge produced outbox row and Kafka message" + +echo "" +echo "[8/${TOTAL_STEPS}] Verifying relay survives Kafka restart (no rows lost)" +compose stop kafka >/dev/null + +restart_obs_payload="$(jq -nc \ + --arg recordedAt "${RECORDED_AT}" \ + --arg key "kafka-restart-${SCRIPT_RUN_ID}" \ + '{observations:[{observationCode:"HEART_RATE",value:83,unit:"bpm",source:"DEVICE",recordedAt:$recordedAt,idempotencyKey:$key}]}')" +resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_a}/observations" "${restart_obs_payload}")" +assert_status "201" "${resp}" + +restart_outbox_id="$(psql_cmd "SELECT id FROM outbox_events WHERE topic = '${TOPIC_OBSERVATION}' AND partition_key = '${encounter_a}' ORDER BY created_at DESC LIMIT 1")" +restart_pending="$(psql_cmd "SELECT processed_at IS NULL FROM outbox_events WHERE id = '${restart_outbox_id}'")" +if [[ "${restart_pending}" != "t" ]]; then + echo "Expected outbox row ${restart_outbox_id} to remain pending while Kafka is down" + exit 1 +fi +echo "OK: outbox row pending while Kafka is stopped" + +compose start kafka >/dev/null +wait_for_kafka +wait_for_outbox_processed "${restart_outbox_id}" +echo "OK: relay caught up after Kafka restart" + +echo "" +echo "[9/${TOTAL_STEPS}] Verifying partial index and relay poll plan" +index_count="$(psql_cmd "SELECT COUNT(*) FROM pg_indexes WHERE tablename = 'outbox_events' AND indexdef LIKE '%processed_at IS NULL%'")" +if [[ "${index_count}" -lt 1 ]]; then + echo "Expected partial index on outbox_events (processed_at IS NULL), found ${index_count}" + exit 1 +fi + +psql_cmd "ANALYZE outbox_events" >/dev/null +explain_out="$(psql_cmd "BEGIN; SET LOCAL enable_seqscan = off; EXPLAIN SELECT id, topic, payload, partition_key FROM outbox_events WHERE processed_at IS NULL ORDER BY created_at ASC LIMIT 100 FOR UPDATE SKIP LOCKED; ROLLBACK;")" +if ! grep -qi 'Index Scan' <<< "${explain_out}"; then + echo "Expected Index Scan when seqscan is disabled, got:" + echo "${explain_out}" + exit 1 +fi +echo "OK: partial index exists and backs the relay poll query" + +echo "" +echo "[10/${TOTAL_STEPS}] Verifying partition_key is set on clinical outbox rows" +missing_keys="$(psql_cmd "SELECT COUNT(*) FROM outbox_events WHERE created_at >= NOW() - INTERVAL '10 minutes' AND topic IN ('${TOPIC_OBSERVATION}', '${TOPIC_ENCOUNTER}', 'alert.generated', 'alert.acknowledged') AND (partition_key IS NULL OR partition_key = '')")" +if [[ "${missing_keys}" != "0" ]]; then + echo "Found ${missing_keys} clinical outbox row(s) without partition_key" + exit 1 +fi +echo "OK: all clinical outbox rows have partition_key" + +echo "" +echo "All ${TOTAL_STEPS} Kafka + outbox checks passed."