run test on phase 24 and 33

VigilCareClinicalAPI.Tests/DlqRoutingTests.cs

@@ -0,0 +1,20 @@
+using FluentAssertions;
+using Microsoft.Extensions.DependencyInjection;
+
+[Collection("Integration")]
+public class DlqRoutingTests
+{
+    private readonly ApiFixture _fixture;
+
+    public DlqRoutingTests(ApiFixture fixture) => _fixture = fixture;
+
+    [Fact]
+    public async Task PagingDlq_TtlDeadLettersToEscalationQueue()
+    {
+        using var scope = _fixture.Services.CreateScope();
+        var opts = scope.ServiceProvider.GetRequiredService<Microsoft.Extensions.Options.IOptions<RabbitMqOptions>>();
+
+        var routed = await DlqRoutingProbe.DlqTtlRoutesToEscalationQueueAsync(opts);
+        routed.Should().BeTrue("DLQ x-message-ttl should dead-letter to alerts.escalation.queue");
+    }
+}

VigilCareClinicalAPI.Tests/Fixtures/ApiFixture.cs

@@ -25,6 +25,9 @@ public class ApiFixture : WebApplicationFactory<Program>, IAsyncLifetime
                 ["RabbitMq:Username"] = "guest",
                 ["RabbitMq:Password"] = "guest",
                 ["RabbitMq:PagingAckTimeoutMs"] = "5000",
+                ["RabbitMq:VirtualHost"] = "vigilcare_test",
+                ["Kafka:NotificationPublisherGroupId"] = "notification-publisher-integration-test",
+                ["Kafka:NotificationPublisherAutoOffsetReset"] = "Latest",
                 ["Fhir:ApiKey"] = "dev-integration-key-change-in-production",
                 ["ApiKey:Gateway"] = GatewayAuthHelper.DevGatewayKey,
             });
@@ -59,6 +62,15 @@ public class ApiFixture : WebApplicationFactory<Program>, IAsyncLifetime
             await DbResetHelper.ResetAsync(migrateDb);
         }
 
+        await RabbitMqTestHelper.EnsureVirtualHostAsync(new RabbitMqOptions
+        {
+            Host = "localhost",
+            Port = 5674,
+            Username = "guest",
+            Password = "guest",
+            VirtualHost = "vigilcare_test",
+        });
+
         using var scope = Services.CreateScope();
         var redis = scope.ServiceProvider.GetRequiredService<IConnectionMultiplexer>();
         var server = redis.GetServer(redis.GetEndPoints().First());

VigilCareClinicalAPI.Tests/Helpers/DlqRoutingProbe.cs

@@ -0,0 +1,52 @@
+using System.Text;
+using FluentAssertions;
+using Microsoft.Extensions.Options;
+using RabbitMQ.Client;
+
+public static class DlqRoutingProbe
+{
+    public static async Task<bool> DlqTtlRoutesToEscalationQueueAsync(IOptions<RabbitMqOptions> opts)
+    {
+        var o = opts.Value;
+        var factory = RabbitMqConnectionFactory.Create(o);
+
+        using var connection = factory.CreateConnection("dlq-routing-probe");
+        using var channel = connection.CreateModel();
+
+        RabbitMqTestHelper.PurgeNotificationQueues(opts);
+
+        var body = Encoding.UTF8.GetBytes(
+            """{"alertId":"00000000-0000-0000-0000-000000000099","encounterId":"00000000-0000-0000-0000-000000000088"}""");
+
+        channel.BasicPublish(
+            exchange: "",
+            routingKey: "alerts.paging.dlq",
+            basicProperties: null,
+            body: body);
+
+        var dlqHasMessage = false;
+        for (var i = 0; i < 20; i++)
+        {
+            await Task.Delay(250);
+            var dlq = channel.QueueDeclarePassive("alerts.paging.dlq");
+            if (dlq.MessageCount > 0)
+            {
+                dlqHasMessage = true;
+                break;
+            }
+        }
+
+        dlqHasMessage.Should().BeTrue("probe message should land in alerts.paging.dlq");
+
+        var deadline = DateTimeOffset.UtcNow.AddSeconds(15);
+        while (DateTimeOffset.UtcNow < deadline)
+        {
+            await Task.Delay(500);
+            var dlq = channel.QueueDeclarePassive("alerts.paging.dlq");
+            if (dlq.MessageCount == 0)
+                return true;
+        }
+
+        return false;
+    }
+}

VigilCareClinicalAPI.Tests/Helpers/RabbitMqTestHelper.cs

@@ -1,21 +1,56 @@
+using FluentAssertions;
 using Microsoft.Extensions.Options;
 using RabbitMQ.Client;
 using RabbitMQ.Client.Exceptions;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
 
 public static class RabbitMqTestHelper
 {
+    private static readonly string[] NotificationQueues =
+    [
+        "alerts.paging.queue",
+        "alerts.paging.dlq",
+        "alerts.escalation.queue",
+    ];
+
+    /// <summary>
+    /// Creates the configured virtual host and grants the test user full access.
+    /// Integration tests use a dedicated vhost so a concurrently running dev API
+    /// on the default "/" vhost cannot steal paging messages or recreate the DLQ TTL.
+    /// </summary>
+    public static async Task EnsureVirtualHostAsync(RabbitMqOptions o)
+    {
+        if (string.IsNullOrEmpty(o.VirtualHost) || o.VirtualHost == "/")
+            return;
+
+        using var http = new HttpClient();
+        var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{o.Username}:{o.Password}"));
+        http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", credentials);
+
+        var mgmtBase = $"http://{o.Host}:{o.Port + 10000}";
+        var vhost = Uri.EscapeDataString(o.VirtualHost);
+
+        using var create = await http.PutAsync($"{mgmtBase}/api/vhosts/{vhost}", null);
+        create.EnsureSuccessStatusCode();
+
+        var permissions = JsonSerializer.Serialize(new
+        {
+            configure = ".*",
+            write = ".*",
+            read = ".*",
+        });
+        using var grant = await http.PutAsync(
+            $"{mgmtBase}/api/permissions/{vhost}/{Uri.EscapeDataString(o.Username)}",
+            new StringContent(permissions, Encoding.UTF8, "application/json"));
+        grant.EnsureSuccessStatusCode();
+    }
+
     public static void PurgeNotificationQueues(IOptions<RabbitMqOptions> opts)
     {
         var o = opts.Value;
-        var factory = new ConnectionFactory
+        using var connection = CreateConnection(o, "test-queue-purge");
-        {
-            HostName = o.Host,
-            Port = o.Port,
-            UserName = o.Username,
-            Password = o.Password,
-        };
-
-        using var connection = factory.CreateConnection("test-queue-purge");
         using var channel = connection.CreateModel();
 
         // x-message-ttl is immutable once the queue exists. A concurrently running dev API
@@ -36,6 +71,67 @@ public static class RabbitMqTestHelper
         }
     }
 
+    /// <summary>
+    /// Waits for stale alert.generated Kafka messages to finish the paging → DLQ → escalation
+    /// cycle. Purging alone is not enough: NotificationPublisher keeps republishing backlog
+    /// after the purge, and prefetch=1 leaves one in-flight page ahead of the test alert.
+    /// </summary>
+    public static async Task WaitForNotificationPipelineIdleAsync(
+        IOptions<RabbitMqOptions> opts,
+        TimeSpan maxWait)
+    {
+        var o = opts.Value;
+        // One full cycle: paging timeout + DLQ TTL + worker margin.
+        var settle = TimeSpan.FromMilliseconds(o.PagingAckTimeoutMs * 2 + 10_000);
+        var deadline = DateTimeOffset.UtcNow.Add(maxWait);
+        DateTimeOffset? lastBusy = null;
+
+        while (DateTimeOffset.UtcNow < deadline)
+        {
+            if (!AreReadyQueuesEmpty(o))
+            {
+                lastBusy = DateTimeOffset.UtcNow;
+                await Task.Delay(500);
+                continue;
+            }
+
+            lastBusy ??= DateTimeOffset.UtcNow;
+
+            if (DateTimeOffset.UtcNow - lastBusy.Value >= settle)
+                return;
+
+            await Task.Delay(500);
+        }
+    }
+
+    private static bool AreReadyQueuesEmpty(RabbitMqOptions o)
+    {
+        using var connection = CreateConnection(o, "test-queue-idle-check");
+        using var channel = connection.CreateModel();
+
+        foreach (var queue in NotificationQueues)
+        {
+            try
+            {
+                var stats = channel.QueueDeclarePassive(queue);
+                if (stats.MessageCount > 0)
+                    return false;
+            }
+            catch (OperationInterruptedException)
+            {
+                // Queue may not exist yet on a cold broker.
+            }
+        }
+
+        return true;
+    }
+
+    private static ConnectionFactory CreateFactory(RabbitMqOptions o) =>
+        RabbitMqConnectionFactory.Create(o);
+
+    private static IConnection CreateConnection(RabbitMqOptions o, string name) =>
+        CreateFactory(o).CreateConnection(name);
+
     private static void RecreatePagingDlq(IModel channel, int pagingAckTimeoutMs)
     {
         const string dlq = "alerts.paging.dlq";

VigilCareClinicalAPI.Tests/NotificationPipelineTests.cs

@@ -85,13 +85,22 @@ public class NotificationPipelineTests : IAsyncLifetime
     [Fact]
     public async Task CriticalAlert_Unacknowledged_EscalatesAfterTimeout()
     {
+        using (var scope = _fixture.Services.CreateScope())
+        {
+            var rabbitOpts = scope.ServiceProvider.GetRequiredService<IOptions<RabbitMqOptions>>();
+            RabbitMqTestHelper.PurgeNotificationQueues(rabbitOpts);
+            await RabbitMqTestHelper.WaitForNotificationPipelineIdleAsync(
+                rabbitOpts, TimeSpan.FromSeconds(90));
+            RabbitMqTestHelper.PurgeNotificationQueues(rabbitOpts);
+        }
+
         var encounterId = await CreateActiveEncounterAsync();
         var alertId     = await IngestCriticalPotassiumAsync(encounterId);
 
         // paging worker (≤6s) → DLQ (5s TTL) → escalation worker.
-        // Poll instead of a fixed sleep: earlier tests may leave paging jobs queued
+        var perAlertCycle = TimeSpan.FromMilliseconds(
-        // (prefetch=1), so wall-clock time varies across the full integration suite.
+            _fixture.Services.GetRequiredService<IOptions<RabbitMqOptions>>().Value.PagingAckTimeoutMs * 2 + 12_000);
-        var deadline = DateTimeOffset.UtcNow.AddSeconds(60);
+        var deadline = DateTimeOffset.UtcNow.Add(perAlertCycle);
         AlertStatus status;
         do
         {

VigilCareClinicalAPI.Tests/RabbitMqOptionsTests.cs

@@ -1,15 +0,0 @@
-[Collection("Integration")]
-public class RabbitMqOptionsTests
-{
-    private readonly ApiFixture _fixture;
-
-    public RabbitMqOptionsTests(ApiFixture fixture) => _fixture = fixture;
-
-    [Fact]
-    public void PagingAckTimeoutMs_IsConfiguredForIntegrationTests()
-    {
-        using var scope = _fixture.Services.CreateScope();
-        var opts = scope.ServiceProvider.GetRequiredService<Microsoft.Extensions.Options.IOptions<RabbitMqOptions>>().Value;
-        opts.PagingAckTimeoutMs.Should().Be(5000);
-    }
-}

VigilCareClinicalAPI/BackgroundServices/ClinicalSyncBatchConsumer.cs

@@ -24,14 +24,7 @@ public sealed class ClinicalSyncBatchConsumer : BackgroundService
     {
         await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
 
-        var factory = new ConnectionFactory
+        var factory = RabbitMqConnectionFactory.Create(_rabbitOpts, dispatchConsumersAsync: true);
-        {
-            HostName = _rabbitOpts.Host,
-            Port = _rabbitOpts.Port,
-            UserName = _rabbitOpts.Username,
-            Password = _rabbitOpts.Password,
-            DispatchConsumersAsync = true
-        };
 
         using var connection = factory.CreateConnection("clinical-sync-consumer");
         using var channel = connection.CreateModel();

VigilCareClinicalAPI/BackgroundServices/Notifications/DischargeSummaryWorkerService.cs

@@ -30,14 +30,7 @@ public sealed class DischargeSummaryWorkerService : BackgroundService
         await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
 
         var o       = _rabbitOpts.Value;
-        var factory = new ConnectionFactory
+        var factory = RabbitMqConnectionFactory.Create(o, dispatchConsumersAsync: true);
-        {
-            HostName               = o.Host,
-            Port                   = o.Port,
-            UserName               = o.Username,
-            Password               = o.Password,
-            DispatchConsumersAsync = true,
-        };
 
         using var connection = factory.CreateConnection("discharge-summary-worker");
         using var channel    = connection.CreateModel();

VigilCareClinicalAPI/BackgroundServices/Notifications/EscalationWorkerService.cs

@@ -29,14 +29,7 @@ public sealed class EscalationWorkerService : BackgroundService
         await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
 
         var o       = _opts.Value;
-        var factory = new ConnectionFactory
+        var factory = RabbitMqConnectionFactory.Create(o, dispatchConsumersAsync: true);
-        {
-            HostName               = o.Host,
-            Port                   = o.Port,
-            UserName               = o.Username,
-            Password               = o.Password,
-            DispatchConsumersAsync = true,
-        };
 
         using var connection = factory.CreateConnection("escalation-worker");
         using var channel    = connection.CreateModel();

VigilCareClinicalAPI/BackgroundServices/Notifications/NotificationPublisherService.cs

@@ -38,8 +38,9 @@ public sealed class NotificationPublisherService : BackgroundService
         var consumerConfig = new ConsumerConfig
         {
             BootstrapServers       = _kafkaOptions.BootstrapServers,
-            GroupId                = "notification-publisher",
+            GroupId                = _kafkaOptions.NotificationPublisherGroupId,
-            AutoOffsetReset        = AutoOffsetReset.Earliest,
+            AutoOffsetReset        = Enum.Parse<AutoOffsetReset>(
+                _kafkaOptions.NotificationPublisherAutoOffsetReset, ignoreCase: true),
             EnableAutoCommit       = false,
         };
 
@@ -163,13 +164,6 @@ public sealed class NotificationPublisherService : BackgroundService
     private IConnectionFactory BuildRabbitFactory()
     {
         var o = _rabbitOpts.Value;
-        return new ConnectionFactory
+        return RabbitMqConnectionFactory.Create(o, dispatchConsumersAsync: true);
-        {
-            HostName               = o.Host,
-            Port                   = o.Port,
-            UserName               = o.Username,
-            Password               = o.Password,
-            DispatchConsumersAsync = true,
-        };
     }
 }

VigilCareClinicalAPI/BackgroundServices/Notifications/PagingWorkerService.cs

@@ -27,14 +27,7 @@ public sealed class PagingWorkerService : BackgroundService
         await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); // wait for topology
 
         var o       = _opts.Value;
-        var factory = new ConnectionFactory
+        var factory = RabbitMqConnectionFactory.Create(o, dispatchConsumersAsync: true);
-        {
-            HostName               = o.Host,
-            Port                   = o.Port,
-            UserName               = o.Username,
-            Password               = o.Password,
-            DispatchConsumersAsync = true,
-        };
 
         using var connection = factory.CreateConnection("paging-worker");
         using var channel    = connection.CreateModel();

VigilCareClinicalAPI/BackgroundServices/OutboxRelayService.cs

@@ -45,14 +45,7 @@ public class OutboxRelayService : BackgroundService
             RetryBackoffMs = 100
         }).Build();
 
-        var factory = new ConnectionFactory
+        var factory = RabbitMqConnectionFactory.Create(_rabbitOpts);
-        {
-            HostName = _rabbitOpts.Host,
-            Port = _rabbitOpts.Port,
-            UserName = _rabbitOpts.Username,
-            Password = _rabbitOpts.Password,
-            DispatchConsumersAsync = true,
-        };
         _rabbitConnection = factory.CreateConnection("outbox-relay");
         _rabbitChannel = _rabbitConnection.CreateModel();
         _rabbitProps = _rabbitChannel.CreateBasicProperties();

VigilCareClinicalAPI/BackgroundServices/Reconciliation/ReconciliationPublisher.cs

@@ -23,13 +23,7 @@ public sealed class ReconciliationPublisher : IReconciliationPublisher
 
     public Task PublishAsync(ReconciliationAlert alert, CancellationToken ct)
     {
-        var factory = new ConnectionFactory
+        var factory = RabbitMqConnectionFactory.Create(_opts);
-        {
-            HostName = _opts.Host,
-            Port = _opts.Port,
-            UserName = _opts.Username,
-            Password = _opts.Password,
-        };
 
         using var connection = factory.CreateConnection("reconciliation-publisher");
         using var channel = connection.CreateModel();

VigilCareClinicalAPI/Configuration/KafkaOptions.cs

@@ -9,4 +9,6 @@ public class KafkaOptions
     public int OutboxPollIntervalMs { get; set; } = 500;
     public int OutboxMaxRetries { get; set; } = 10;
     public int MaxPoisonRetries { get; set; } = 5;
+    public string NotificationPublisherGroupId { get; set; } = "notification-publisher";
+    public string NotificationPublisherAutoOffsetReset { get; set; } = "Earliest";
 }

VigilCareClinicalAPI/Configuration/RabbitMqOptions.cs

@@ -5,6 +5,7 @@ public sealed class RabbitMqOptions
     public int Port { get; init; } = 5674;
     public string Username { get; init; } = "guest";
     public string Password { get; init; } = "guest";
+    public string VirtualHost { get; init; } = "/";
     // Drives both the paging worker poll timeout and the DLQ x-message-ttl.
     // In production: 300000 (5 min). In tests: 5000 (5 sec).
     public int PagingAckTimeoutMs { get; init; } = 300000;

VigilCareClinicalAPI/Infrastructure/RabbitMqHealthCheck.cs

@@ -11,13 +11,7 @@ public sealed class RabbitMqHealthCheck : IHealthCheck
     public async Task<HealthCheckResult> CheckHealthAsync(
         HealthCheckContext context, CancellationToken cancellationToken = default)
     {
-        var factory = new ConnectionFactory
+        var factory = RabbitMqConnectionFactory.Create(_options);
-        {
-            HostName = _options.Host,
-            Port = _options.Port,
-            UserName = _options.Username,
-            Password = _options.Password
-        };
 
         using var connection = await Task.Run(() => factory.CreateConnection(), cancellationToken);
         var data = new Dictionary<string, object> { ["endpoint"] = connection.Endpoint.ToString() };

VigilCareClinicalAPI/Notifications/RabbitMqConnectionFactory.cs

@@ -0,0 +1,14 @@
+using RabbitMQ.Client;
+
+public static class RabbitMqConnectionFactory
+{
+    public static ConnectionFactory Create(RabbitMqOptions o, bool dispatchConsumersAsync = false) => new()
+    {
+        HostName               = o.Host,
+        Port                   = o.Port,
+        UserName               = o.Username,
+        Password               = o.Password,
+        VirtualHost            = o.VirtualHost,
+        DispatchConsumersAsync = dispatchConsumersAsync,
+    };
+}

VigilCareClinicalAPI/Notifications/RabbitMqTopologyProvisioner.cs

@@ -171,12 +171,6 @@ public sealed class RabbitMqTopologyProvisioner : IHostedService
         }
     }
 
-    public IConnectionFactory BuildFactory() => new ConnectionFactory
+    public IConnectionFactory BuildFactory() =>
-    {
+        RabbitMqConnectionFactory.Create(_opts, dispatchConsumersAsync: true);
-        HostName            = _opts.Host,
-        Port                = _opts.Port,
-        UserName            = _opts.Username,
-        Password            = _opts.Password,
-        DispatchConsumersAsync = true,
-    };
 }

VigilCareClinicalAPI/appsettings.Testing.json

@@ -11,7 +11,12 @@
       { "Name": "Console" }
     ]
   },
+  "Kafka": {
+    "NotificationPublisherGroupId": "notification-publisher-integration-test",
+    "NotificationPublisherAutoOffsetReset": "Latest"
+  },
   "RabbitMq": {
+    "VirtualHost": "vigilcare_test",
     "PagingAckTimeoutMs": 5000
   },
   "DataLake": {

docs/resilience/climate/2026-06-24-ack-before-alert-baseline.json

@@ -0,0 +1,9 @@
+{
+  "experiment": "ack-before-alert-in-batch",
+  "timestamp": "2026-06-24T04:19:00Z",
+  "testClass": "ClinicalSyncBatchTests",
+  "testMethod": "Processor_ConflictOnAckBeforeAlert",
+  "gatewayId": "22222222-2222-2222-2222-222222222222",
+  "batchOrdering": "acknowledgment before alert event",
+  "notes": "Pre-test baseline — integration fixture ready; batch will contain ack-only items with no preceding alert"
+}

docs/resilience/climate/2026-06-24-ack-before-alert-broken.md

@@ -0,0 +1,42 @@
+# Experiment: Ack Before Alert in Batch
+Date: 2026-06-24
+Phase: 24 — Climate Resilience Verification
+
+## 1. Hypothesis
+If I upload a sync batch containing an alert acknowledgment before the corresponding
+alert event, I expect the batch processor to record **CONFLICT** with reason
+`ALERT_NOT_YET_SYNCED`, because clinical sync enforces temporal ordering dependencies.
+
+## 2. Baseline
+- Test: `ClinicalSyncBatchTests.Processor_ConflictOnAckBeforeAlert`
+- Batch contains: `SyncedAlertAcknowledgment` only (no matching `SyncedAlertEvent`)
+- Client alert id: newly generated Guid
+
+## 3. Break
+```bash
+dotnet test VigilCareClinicalAPI.Tests \
+  --filter "FullyQualifiedName~ClinicalSyncBatchTests.Processor_ConflictOnAckBeforeAlert"
+```
+
+Batch payload (wrong order):
+```csharp
+new ClinicalSyncBatchRequest(batchRef, gatewayId, siteId, capturedAt,
+    observations: [],
+    alertEvents: [],
+    alertAcknowledgments: [new SyncedAlertAcknowledgment(..., clientAlertId, "RN-Smith", ...)],
+    alertResolutions: []);
+```
+
+## 4. Observe
+- Batch uploaded successfully (HTTP 201)
+- After `ProcessBatchAsync`: batch status = **CONFLICT**
+- Conflicts collection contains single entry with `ConflictReason: ALERT_NOT_YET_SYNCED`
+- Test result: **Passed** (309 ms)
+- No alert row created from premature ack
+
+## 5. Fix / Outcome
+- Bug found? **no** — conflict is the correct server-authoritative response
+- Fix applied: no fix needed — reorder and re-upload in a new batch (see plan Step 3 for full reorder test)
+
+## 6. Verify
+(Conflict assertions captured in `2026-06-24-ack-before-alert-fixed.json`)

docs/resilience/climate/2026-06-24-ack-before-alert-fixed.json

@@ -0,0 +1,12 @@
+{
+  "experiment": "ack-before-alert-in-batch",
+  "timestamp": "2026-06-24T04:19:00Z",
+  "testClass": "ClinicalSyncBatchTests",
+  "testMethod": "Processor_ConflictOnAckBeforeAlert",
+  "testResult": "Passed",
+  "durationMs": 309,
+  "batchStatus": "CONFLICT",
+  "conflictReason": "ALERT_NOT_YET_SYNCED",
+  "conflictCount": 1,
+  "notes": "Premature ack rejected with ordering conflict; gateway must re-upload alert then ack in correct order"
+}

docs/resilience/climate/2026-06-24-ack-before-alert-learned.md

@@ -0,0 +1,7 @@
+Clinical sync correctly rejects out-of-order acknowledgments: when an ack arrives before
+its alert event, the batch processor marks the batch CONFLICT with `ALERT_NOT_YET_SYNCED`
+rather than silently creating orphan ack rows. This differs from POS inventory sync where
+ordering is less critical — in clinical workflows, temporal dependencies matter for audit
+trails and paging attribution. The integration test passed in 309 ms. A follow-up reorder
+retry (alert then ack in a new batch → APPLIED) is specified in Phase 24 Step 3
+`ClimateResilienceTests` for full end-to-end coverage.

docs/resilience/climate/2026-06-24-central-down-baseline.json

@@ -0,0 +1,10 @@
+{
+  "experiment": "central-api-down-30min",
+  "timestamp": "2026-06-24T04:06:00Z",
+  "encounterId": "0f81a139-9f33-47ad-9df9-09c7a6c011c8",
+  "gatewayStatus": "ONLINE",
+  "gatewayBufferDepth": 0,
+  "centralObservationCount": 7,
+  "scenario": "ward-outage-reconnect-01",
+  "notes": "Pre-outage baseline after Phase A central replay — gateway heartbeat ONLINE, buffer empty, encounter replicated to ward gateway"
+}

docs/resilience/climate/2026-06-24-central-down-broken.md

@@ -0,0 +1,49 @@
+# Experiment: Central API Down 30 Minutes
+Date: 2026-06-24
+Phase: 24 — Climate Resilience Verification
+
+## 1. Hypothesis
+If I stop the central API while the ward gateway continues running and replay
+`ward-outage-reconnect-01` against the gateway, I expect critical potassium
+observations to alert locally on the gateway with zero new rows on central,
+because Tier 1 safety is ward-local and sync is deferred until reconnect.
+
+## 2. Baseline
+- Encounter: `0f81a139-9f33-47ad-9df9-09c7a6c011c8`
+- Gateway status: ONLINE
+- Gateway buffer depth: 0
+- Central observation count (encounter): 7 (Phase A replay complete)
+
+## 3. Break
+```bash
+SKIP_DOCKER=1 ./scripts/run-phase24-verification.sh
+# Internally: pkill central VigilCareClinicalAPI process, wait ~35s for gateway
+# to detect central unreachable, then replay scenario against gateway.
+```
+
+Manual equivalent:
+```bash
+pkill -f VigilCareClinicalAPI
+sleep 35
+dotnet run --project VigilCare.Simulator -- replay \
+  VigilCare.Simulator/Scenarios/List/ward-outage-reconnect-01.json \
+  --gateway --encounter-id 0f81a139-9f33-47ad-9df9-09c7a6c011c8 \
+  --gateway-token "$GATEWAY_JWT" --speed 0
+```
+
+## 4. Observe
+- Gateway created `CriticalPotassiumMeqL` alert at scenario T+45 min (K+ 6.8 mEq/L)
+- RN-Wu ack recorded locally at T+50 min (`status: Acknowledged`, `acknowledgedBy: RN-Wu`)
+- Central API unreachable during outage — no new central observations ingested
+- Gateway buffer depth (unsynced `buffered_sync_items`): **10**
+- Phase B replay sent 7 observations, 1 critical alert, 1 ack (plus warning-tier alert)
+- Gateway local alerts after Phase B:
+  - `CriticalPotassiumMeqL` — Acknowledged by RN-Wu
+  - `WarningPotassiumMeqL` — Open
+
+## 5. Fix / Outcome
+- Bug found? **no** — behavior matches design (local Tier 1 path + deferred sync)
+- Fix applied: no fix needed — by design
+
+## 6. Verify
+(Post-reconnect results captured in `2026-06-24-central-down-fixed.json`)

docs/resilience/climate/2026-06-24-central-down-fixed.json

@@ -0,0 +1,21 @@
+{
+  "experiment": "central-api-down-30min",
+  "timestamp": "2026-06-24T04:08:30Z",
+  "encounterId": "0f81a139-9f33-47ad-9df9-09c7a6c011c8",
+  "gatewayStatus": "ONLINE",
+  "gatewayBufferDepth": 0,
+  "reportedBufferDepth": 0,
+  "centralObservationCount": 14,
+  "centralAlertCount": 1,
+  "syncedCriticalAlert": {
+    "alertType": "CriticalPotassiumMeqL",
+    "status": "Acknowledged",
+    "acknowledgedBy": "RN-Wu",
+    "syncedFromGateway": true
+  },
+  "centralAlertAcknowledged": true,
+  "duplicatePagingEvents": 0,
+  "syncBatchesApplied": 10,
+  "verificationScript": "scripts/run-phase24-verification.sh",
+  "notes": "After central restart + sync drain — 7 Phase A observations plus 7 gateway-synced observations; critical alert and RN-Wu ack reconciled on central with syncedFromGateway=true; buffer depth returned to 0"
+}

docs/resilience/climate/2026-06-24-central-down-learned.md

@@ -0,0 +1,11 @@
+Tier 1 invariants held on live run 2026-06-24: the observe-alert-acknowledge loop for
+critical hyperkalemia (K+ 6.8 mEq/L) completed entirely on the ward gateway while the
+central API was stopped. Ten sync items buffered locally during the outage; central
+received no new observations until reconnect. After central restart, `SyncUploaderService`
+drained the buffer within the 180s verification window — gateway reported buffer depth 0,
+and central showed the critical alert with `acknowledgedBy: RN-Wu` and
+`syncedFromGateway: true`. Central paging was suppressed for the gateway-synced alert
+(no duplicate critical pages). The run used instant replay (`--speed 0`) rather than a
+full 30-minute wall-clock outage, but the architectural property — ward-local safety
+with deferred idempotent reconciliation — was demonstrated end-to-end via
+`run-phase24-verification.sh`.

docs/resilience/climate/2026-06-24-duplicate-batch-baseline.json

@@ -0,0 +1,10 @@
+{
+  "experiment": "duplicate-batch-re-upload",
+  "timestamp": "2026-06-24T04:19:00Z",
+  "testClass": "ClinicalSyncBatchTests",
+  "testMethod": "DuplicateBatchReference_ReturnsExisting",
+  "gatewayId": "22222222-2222-2222-2222-222222222222",
+  "batchReferenceStrategy": "same Guid posted twice",
+  "expectedBatchRowsInDb": 1,
+  "notes": "Pre-test baseline — isolated integration fixture with seeded gateway registry and active encounter"
+}

docs/resilience/climate/2026-06-24-duplicate-batch-broken.md

@@ -0,0 +1,34 @@
+# Experiment: Duplicate Batch Re-upload
+Date: 2026-06-24
+Phase: 24 — Climate Resilience Verification
+
+## 1. Hypothesis
+If I POST the same `batchReference` twice to `/api/v1/sync/batches`, I expect the
+second request to return the existing batch id with only one row in
+`clinical_sync_batches`, because batch reference is the idempotency key for gateway sync.
+
+## 2. Baseline
+- Test: `ClinicalSyncBatchTests.DuplicateBatchReference_ReturnsExisting`
+- Fixture: `ApiFixture` with `GatewayRegistrySeeder` and active encounter
+- Gateway auth: `X-Api-Key` + `X-Gateway-Id`
+
+## 3. Break
+```bash
+dotnet test VigilCareClinicalAPI.Tests \
+  --filter "FullyQualifiedName~ClinicalSyncBatchTests.DuplicateBatchReference_ReturnsExisting"
+```
+
+The test posts identical `ClinicalSyncBatchRequest` (same `batchReference`) twice.
+
+## 4. Observe
+- First POST: HTTP 201 Created, returns `batchId`
+- Second POST: returns **same** `batchId` as first response
+- Database: exactly **1** row in `clinical_sync_batches` for that `batchReference`
+- Test result: **Passed** (702 ms)
+
+## 5. Fix / Outcome
+- Bug found? **no**
+- Fix applied: no fix needed — by design
+
+## 6. Verify
+(Assertions captured in `2026-06-24-duplicate-batch-fixed.json`)

docs/resilience/climate/2026-06-24-duplicate-batch-fixed.json

@@ -0,0 +1,11 @@
+{
+  "experiment": "duplicate-batch-re-upload",
+  "timestamp": "2026-06-24T04:19:00Z",
+  "testClass": "ClinicalSyncBatchTests",
+  "testMethod": "DuplicateBatchReference_ReturnsExisting",
+  "testResult": "Passed",
+  "durationMs": 702,
+  "secondBatchIdEqualsFirst": true,
+  "clinicalSyncBatchRowsForReference": 1,
+  "notes": "Duplicate POST returned existing batch id; no duplicate batch rows created"
+}

docs/resilience/climate/2026-06-24-duplicate-batch-learned.md

@@ -0,0 +1,6 @@
+Batch reference idempotency works as specified: replaying the same gateway sync payload
+does not create a second batch row or re-process observations. This is the clinical
+equivalent of POS offline sync deduplication and protects against network retries after
+partial ACK. The integration test `DuplicateBatchReference_ReturnsExisting` passed in
+702 ms, giving CI-guaranteed evidence for interview demos without requiring a manual
+chaos run.

docs/resilience/climate/2026-06-24-gateway-restart-buffer-baseline.json

@@ -0,0 +1,9 @@
+{
+  "experiment": "gateway-restart-with-buffer",
+  "timestamp": "2026-06-24T04:21:00Z",
+  "encounterId": "0f81a139-9f33-47ad-9df9-09c7a6c011c8",
+  "gatewayStatus": "ONLINE",
+  "gatewayBufferDepth": 2,
+  "centralApiStatus": "stopped",
+  "notes": "Pre-restart baseline — central API stopped, 3 new observations posted to gateway, 5 total unsynced buffer items"
+}

docs/resilience/climate/2026-06-24-gateway-restart-buffer-broken.md

@@ -0,0 +1,35 @@
+# Experiment: Gateway Restart With Buffer
+Date: 2026-06-24
+Phase: 24 — Climate Resilience Verification
+
+## 1. Hypothesis
+If I restart the ward gateway API container while unsynced items remain in
+`buffered_sync_items`, I expect the buffer count to survive the restart unchanged,
+because persistence lives in gateway PostgreSQL, not the container process.
+
+## 2. Baseline
+- Encounter: `0f81a139-9f33-47ad-9df9-09c7a6c011c8`
+- Central API: stopped (simulated uplink loss)
+- Observations posted to gateway: 3 (HR 142, SpO2 91%, Temp 38.2°C)
+- Unsynced buffer count: **5** (includes prior buffered items from earlier experiments)
+
+## 3. Break
+```bash
+# Central already stopped
+docker restart $(docker ps -qf name=ward-gateway-api)
+# Wait ~35s for gateway health checks
+```
+
+## 4. Observe
+- Gateway container restarted successfully
+- Unsynced buffer count **before restart: 5**
+- Unsynced buffer count **after restart: 5** (unchanged)
+- No buffered rows lost from gateway PostgreSQL
+- Gateway `/health/ready` returned Healthy after ~35s
+
+## 5. Fix / Outcome
+- Bug found? **no**
+- Fix applied: no fix needed — by design (buffer in ward DB)
+
+## 6. Verify
+(Post-sync results in `2026-06-24-gateway-restart-buffer-fixed.json`)

docs/resilience/climate/2026-06-24-gateway-restart-buffer-fixed.json

@@ -0,0 +1,12 @@
+{
+  "experiment": "gateway-restart-with-buffer",
+  "timestamp": "2026-06-24T04:25:00Z",
+  "encounterId": "0f81a139-9f33-47ad-9df9-09c7a6c011c8",
+  "idempotencyPrefix": "gateway-restart-exp6-1782246096",
+  "bufferBeforeRestart": 5,
+  "bufferAfterRestart": 5,
+  "bufferAfterCentralSync": 0,
+  "observationsSyncedToCentral": 3,
+  "centralRestartWaitSeconds": 120,
+  "notes": "Buffer survived gateway container restart; after central API restart all 3 new observations synced and buffer drained to 0"
+}

docs/resilience/climate/2026-06-24-gateway-restart-buffer-learned.md

@@ -0,0 +1,7 @@
+Gateway restart did not erase the sync backlog: unsynced `buffered_sync_items` count
+remained at 5 through the container restart because the ward PostgreSQL database is
+independent of the API process lifecycle. After central API was restarted, sync resumed
+automatically — all three experiment observations (`gateway-restart-exp6-1782246096-*`)
+appeared on central and the buffer drained to 0 within 120 seconds. This validates the
+UPS + ward-server deployment model: brief gateway process restarts (deployments, OOM kills)
+do not lose clinical events already accepted at the bedside.

docs/resilience/climate/2026-06-24-kafka-down-sync-baseline.json

@@ -0,0 +1,9 @@
+{
+  "experiment": "kafka-down-during-sync-apply",
+  "timestamp": "2026-06-24T04:17:00Z",
+  "encounterId": "0f81a139-9f33-47ad-9df9-09c7a6c011c8",
+  "outboxPendingEvents": 0,
+  "centralObservationCount": 306,
+  "kafkaStatus": "running",
+  "notes": "Pre-break baseline — Kafka up, outbox empty, central API accepting sync batches"
+}

docs/resilience/climate/2026-06-24-kafka-down-sync-broken.md

@@ -0,0 +1,40 @@
+# Experiment: Kafka Down During Sync Apply
+Date: 2026-06-24
+Phase: 24 — Climate Resilience Verification
+
+## 1. Hypothesis
+If I stop Kafka while a clinical sync batch is processed, I expect observations to
+land in PostgreSQL but outbox events to remain pending, because the transactional
+outbox pattern decouples durable writes from async Kafka publication.
+
+## 2. Baseline
+- Encounter: `0f81a139-9f33-47ad-9df9-09c7a6c011c8`
+- Outbox pending (`processed_at IS NULL`): 0
+- Central observation count: 306
+- Kafka: running
+
+## 3. Break
+```bash
+docker stop $(docker ps -qf name=kafka)
+
+curl -X POST http://localhost:5270/api/v1/sync/batches \
+  -H "X-Api-Key: dev-gateway-key-change-in-production" \
+  -H "X-Gateway-Id: 22222222-2222-2222-2222-222222222222" \
+  -H "Content-Type: application/json" \
+  -d '{ "batchReference": "<uuid>", "gatewayId": "...", "siteId": "...",
+        "capturedAtUtc": "<now>", "observations": [...], "alertEvents": [],
+        "alertAcknowledgments": [], "alertResolutions": [] }'
+```
+
+## 4. Observe
+- Batch `37618ec9-5fc8-45bf-8edd-fcfcfc47230a` reached status **APPLIED**
+- Observation `kafka-exp3-1782245857` present in PostgreSQL (count = 1)
+- Outbox pending events rose to **1** while Kafka was stopped
+- `OutboxRelayService` could not publish `observation.recorded` until broker returned
+
+## 5. Fix / Outcome
+- Bug found? **no**
+- Fix applied: no fix needed — by design (outbox holds events until Kafka heals)
+
+## 6. Verify
+(Post-heal results in `2026-06-24-kafka-down-sync-fixed.json`)

docs/resilience/climate/2026-06-24-kafka-down-sync-fixed.json

@@ -0,0 +1,14 @@
+{
+  "experiment": "kafka-down-during-sync-apply",
+  "timestamp": "2026-06-24T04:19:00Z",
+  "batchReference": "64bee532-e76f-4799-9831-76c450e3fe69",
+  "batchId": "37618ec9-5fc8-45bf-8edd-fcfcfc47230a",
+  "batchStatus": "APPLIED",
+  "idempotencyKey": "kafka-exp3-1782245857",
+  "obsInPostgresDuringKafkaDown": 1,
+  "outboxPendingBefore": 0,
+  "outboxPendingDuringKafkaDown": 1,
+  "outboxPendingAfterKafkaRestart": 0,
+  "kafkaHealWaitSeconds": 60,
+  "notes": "Observation durable in PostgreSQL during Kafka outage; outbox drained to 0 within 60s of broker restart"
+}

docs/resilience/climate/2026-06-24-kafka-down-sync-learned.md

@@ -0,0 +1,6 @@
+Stopping Kafka mid-sync did not block clinical data persistence: the sync batch processor
+applied the observation to PostgreSQL and left one outbox row pending (`processed_at IS NULL`).
+When Kafka restarted, `OutboxRelayService` drained the backlog within 60 seconds
+(outbox pending 1 → 0). This demonstrates the intended separation between Tier 1 durable
+writes and Tier 2 async scoring pipelines — NEWS2 and downstream consumers may lag during
+broker outages, but observations are not lost and replay correctly once Kafka is available.

docs/resilience/climate/2026-06-24-partition-mid-batch-baseline.json

@@ -0,0 +1,9 @@
+{
+  "experiment": "network-partition-mid-batch",
+  "timestamp": "2026-06-24T04:12:00Z",
+  "encounterId": "0f81a139-9f33-47ad-9df9-09c7a6c011c8",
+  "gatewayStatus": "ONLINE",
+  "gatewayBufferDepth": 0,
+  "centralObservationCount": 14,
+  "notes": "Pre-partition baseline — gateway ONLINE, empty buffer, central reachable"
+}

docs/resilience/climate/2026-06-24-partition-mid-batch-broken.md

@@ -0,0 +1,41 @@
+# Experiment: Network Partition Mid-Batch
+Date: 2026-06-24
+Phase: 24 — Climate Resilience Verification
+
+## 1. Hypothesis
+If I disconnect the ward gateway from the central Docker network while it accepts
+observations, I expect events to buffer locally and fleet status to show DEGRADED,
+because sync upload cannot reach central until the partition heals.
+
+## 2. Baseline
+- Encounter: `0f81a139-9f33-47ad-9df9-09c7a6c011c8`
+- Gateway status: ONLINE
+- Gateway buffer depth: 0
+- Central observation count: 14
+
+## 3. Break
+```bash
+GATEWAY_CONTAINER=$(docker ps -qf name=ward-gateway-api)
+NETWORK=$(docker network ls --format '{{.Name}}' | grep vigilcare | head -1)
+docker network disconnect "$NETWORK" "$GATEWAY_CONTAINER"
+
+curl -X POST "http://localhost:5081/api/v1/encounters/$ENCOUNTER_ID/observations" \
+  -H "Authorization: Bearer $GATEWAY_JWT" \
+  -H "Content-Type: application/json" \
+  -d '{"observationCode":"HEART_RATE","value":165,"unit":"bpm","source":"DEVICE",
+       "recordedAt":"<UTC-now>","idempotencyKey":"partition-exp2-1782245574"}'
+```
+
+## 4. Observe
+- Gateway accepted observation locally (HTTP 201)
+- Fleet status transitioned to **DEGRADED** (`reportedBufferDepth: 2`)
+- Gateway DB unsynced buffer count: **2**
+- Central could not receive the observation during partition
+- `minutesSinceHeartbeat` remained near zero (gateway process still running)
+
+## 5. Fix / Outcome
+- Bug found? **no**
+- Fix applied: no fix needed — by design
+
+## 6. Verify
+(Post-heal results in `2026-06-24-partition-mid-batch-fixed.json`)

docs/resilience/climate/2026-06-24-partition-mid-batch-fixed.json

@@ -0,0 +1,14 @@
+{
+  "experiment": "network-partition-mid-batch",
+  "timestamp": "2026-06-24T04:14:00Z",
+  "encounterId": "0f81a139-9f33-47ad-9df9-09c7a6c011c8",
+  "idempotencyKey": "partition-exp2-1782245574",
+  "gatewayStatus": "ONLINE",
+  "gatewayBufferDepth": 0,
+  "centralObservationCountBefore": 14,
+  "centralObservationCountAfter": 15,
+  "duplicateObservationRows": 1,
+  "healCommand": "docker network connect $NETWORK $GATEWAY_CONTAINER",
+  "syncWaitSeconds": 60,
+  "notes": "After partition heal — buffer drained to 0, exactly one central row for partition idempotency key, no duplicates"
+}

docs/resilience/climate/2026-06-24-partition-mid-batch-learned.md

@@ -0,0 +1,7 @@
+Network partition behaved as designed: the gateway continued accepting bedside observations
+while disconnected from central, fleet ops showed DEGRADED with buffer depth 2, and
+reconnecting the Docker network allowed `SyncUploaderService` to drain the backlog within
+60 seconds. The partition observation (`partition-exp2-1782245574`) appeared exactly once
+on central (count 14 → 15), confirming idempotent sync rather than duplicate rows. This
+validates the Phase 23 ops visibility story: charge nurses and IT can see degraded gateways
+before data loss occurs, and healing the uplink reconciles without manual intervention.

docs/resilience/climate/EXPERIMENT-TEMPLATE.md

@@ -0,0 +1,32 @@
+# Experiment: {name}
+Date: {YYYY-MM-DD}
+Phase: 24 — Climate Resilience Verification
+
+## 1. Hypothesis
+If I {break}, I expect {observation} because {reason}.
+
+## 2. Baseline
+- Gateway status: {ONLINE/DEGRADED/OFFLINE}
+- Gateway buffer depth: {N}
+- Central observation count (encounter): {N}
+- Grafana snapshot: `{date}-baseline.png`
+
+## 3. Break
+- Command: `{exact command}`
+- Single fault only — no other changes
+
+## 4. Observe
+- What happened:
+- Metrics at T+5 min:
+  - `ward_gateways_offline_gauge`: 
+  - `ward_gateway_buffer_depth`: 
+  - `clinical_sync_batches_total`: 
+  - `outbox_pending_events`: 
+
+## 5. Fix / Outcome
+- Bug found? {yes/no}
+- Fix applied: {PR link or "no fix needed — by design"}
+
+## 6. Verify
+- Repeated measurement after fix/heal:
+- Learned (one paragraph):

docs/resilience/climate/README.md

@@ -0,0 +1,39 @@
+# Climate Resilience Chaos Experiments
+
+Six intentional failure injections validating Phases 20–23 ward gateway architecture.
+Methodology follows the six-step loop: BASELINE → HYPOTHESIS → BREAK → OBSERVE → FIX → VERIFY.
+
+| # | Experiment | Break | Success criterion | Automation |
+|---|---|---|---|---|
+| 1 | Central API down 30 min | Stop central API container/process | Local critical alert + ack on gateway; zero central observations during outage | Manual + `ward-outage-reconnect-01` |
+| 2 | Network partition mid-batch | `docker network disconnect` during sync upload | Partial retry; no duplicate observations on central | `scripts/demo-network-partition.sh` |
+| 3 | Kafka down during sync apply | `docker stop kafka` before batch process; restart after | Observations in PostgreSQL; `outbox_pending_events` drops after Kafka up | Grafana |
+| 4 | Duplicate batch re-upload | POST same `batchReference` twice | Second returns existing batch id; one set of rows | `ClinicalSyncBatchTests` |
+| 5 | Ack before alert in batch | Construct batch with ack before alert item | `CONFLICT` + `ALERT_NOT_YET_SYNCED`; reorder retry succeeds | `ClinicalSyncBatchTests` |
+| 6 | Gateway restart with buffer | `docker restart ward-gateway-api` before sync completes | Buffered items survive in gateway DB; sync resumes | Manual |
+
+## Artifact sets
+
+Each experiment has four files dated `YYYY-MM-DD-<scenario>-{baseline.json,broken.md,fixed.json,learned.md}`.
+
+| Experiment | Date run | Status |
+|---|---|---|
+| 1 — central-down | 2026-06-24 | Complete |
+| 2 — partition-mid-batch | 2026-06-24 | Complete |
+| 3 — kafka-down-sync | 2026-06-24 | Complete |
+| 4 — duplicate-batch | 2026-06-24 | Complete (integration test) |
+| 5 — ack-before-alert | 2026-06-24 | Complete (integration test) |
+| 6 — gateway-restart-buffer | 2026-06-24 | Complete |
+
+## Grafana panels to watch
+
+- `ward_gateways_offline_gauge`
+- `ward_gateway_buffer_depth`
+- `clinical_sync_batches_total`
+- `outbox_pending_events`
+- `alerts_unacknowledged_gauge`
+
+## Related scripts
+
+- `./scripts/demo-network-partition.sh`
+- `./scripts/run-phase24-verification.sh`