fix issues with phase-32

2026-06-23 00:16:39 +08:00
parent 1824e4eef1
commit 90b8baa2a1
12 changed files with 237 additions and 14 deletions
@@ -58,7 +58,11 @@ public class GcsScoringService : BackgroundService
                        evt.Value,
                        stoppingToken);

-                    if (outcome.Outcome == GcsOutcome.ScoreComputed)
+                    if (outcome.Outcome == GcsOutcome.EncounterNotFound)
+                        _logger.LogWarning(
+                            "Skipping stale observation.recorded event — encounter={Id} code={Code} offset={Offset}",
+                            evt.EncounterId, evt.ObservationCode, result.Offset.Value);
+                    else if (outcome.Outcome == GcsOutcome.ScoreComputed)
                        _logger.LogInformation(
                            "GCS scored via consumer — encounter={Id} total={Total} class={Class}",
                            evt.EncounterId, outcome.TotalScore, outcome.Classification);
@@ -58,7 +58,11 @@ public class News2ScoringService : BackgroundService
                        evt.Value,
                        stoppingToken);

-                    if (outcome.Outcome == News2Outcome.ScoreComputed)
+                    if (outcome.Outcome == News2Outcome.EncounterNotFound)
+                        _logger.LogWarning(
+                            "Skipping stale observation.recorded event — encounter={Id} code={Code} offset={Offset}",
+                            evt.EncounterId, evt.ObservationCode, result.Offset.Value);
+                    else if (outcome.Outcome == News2Outcome.ScoreComputed)
                        _logger.LogInformation(
                            "NEWS2 scored via consumer — encounter={Id} score={Score} risk={Risk}",
                            evt.EncounterId, outcome.TotalScore, outcome.RiskLevel);
@@ -2,5 +2,6 @@ public enum GcsOutcome
 {
    NotGcsCode,
    IncompleteComponents,
+    EncounterNotFound,
    ScoreComputed
 }
@@ -2,5 +2,6 @@ public enum News2Outcome
 {
    NotNews2Code,
    IncompleteParameters,
+    EncounterNotFound,
    ScoreComputed
 }
@@ -60,6 +60,17 @@ public class GcsDetector
        var classification = GcsCalculator.ClassifyGcs(total);
        var calculatedAt = DateTimeOffset.UtcNow;

+        using (var checkScope = _services.CreateScope())
+        {
+            var db = checkScope.ServiceProvider.GetRequiredService<AppDbContext>();
+            if (!await db.Encounters.AnyAsync(e => e.Id == encounterId, ct))
+            {
+                _logger.LogWarning(
+                    "Skipping GCS score for unknown encounter {EncounterId}", encounterId);
+                return GcsResult.EncounterNotFound;
+            }
+        }
+
        await PersistScoreAsync(
            encounterId, patientId, (int)eye, (int)verbal, (int)motor,
            total, classification, calculatedAt, ct);
@@ -43,13 +43,8 @@ namespace VigilCareClinicalAPI.Migrations
                name: "name_search_token",
                table: "patients");

-            migrationBuilder.AlterColumn<DateOnly>(
-                name: "date_of_birth",
-                table: "patients",
-                type: "date",
-                nullable: false,
-                oldClrType: typeof(string),
-                oldType: "text");
+            migrationBuilder.Sql(
+                "ALTER TABLE patients ALTER COLUMN date_of_birth TYPE date USING date_of_birth::date;");
        }
    }
 }
@@ -6,6 +6,7 @@ public record GcsResult(
    int PresentComponents = 0)
 {
    public static readonly GcsResult NotGcsCode = new(GcsOutcome.NotGcsCode);
+    public static readonly GcsResult EncounterNotFound = new(GcsOutcome.EncounterNotFound);

    public static GcsResult IncompleteComponents(int presentCount) =>
        new(GcsOutcome.IncompleteComponents, PresentComponents: presentCount);
@@ -7,6 +7,7 @@ public record News2Result(
    bool HasSingleParamThree = false)
 {
    public static readonly News2Result NotNews2Code = new(News2Outcome.NotNews2Code);
+    public static readonly News2Result EncounterNotFound = new(News2Outcome.EncounterNotFound);

    public static News2Result IncompleteParameters(int presentCount) =>
        new(News2Outcome.IncompleteParameters, PresentParameters: presentCount);
@@ -103,6 +103,17 @@ public class News2Detector
        var hasSingleParamThree = paramScores.Any(s => s == 3);
        var riskLevel = News2Calculator.DetermineRiskLevel(totalScore, hasSingleParamThree);

+        using (var scope = _services.CreateScope())
+        {
+            var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+            if (!await db.Encounters.AnyAsync(e => e.Id == encounterId, ct))
+            {
+                _logger.LogWarning(
+                    "Skipping NEWS2 score for unknown encounter {EncounterId}", encounterId);
+                return News2Result.EncounterNotFound;
+            }
+        }
+
        await PersistScoreAsync(
            encounterId, patientId, totalScore, riskLevel,
            paramScores, hasSingleParamThree, ct);
@@ -0,0 +1,198 @@
+# PHI Encryption & Access Logging — Operations Runbook
+
+Phase 32 encrypts sensitive patient demographics at rest and records who accessed PHI. This runbook covers key management, migration, rotation, compliance, and troubleshooting.
+
+---
+
+## Architecture summary
+
+| Layer | Mechanism |
+|---|---|
+| Encryption at rest | ASP.NET Data Protection API (AES-256-GCM) via EF Core value converters |
+| Encrypted columns | `first_name`, `last_name`, `date_of_birth`, `allergies`, `emergency_contact_name`, `emergency_contact_phone` |
+| Plaintext (by design) | `mrn` — exact-match lookup only |
+| Name search | HMAC-SHA256 `name_search_token` index — search without decrypting all rows |
+| Access audit | Append-only `phi_access_logs` table; written by `PhiAccessLogService` on patient Get/List/Register/Update |
+
+Configuration lives in `appsettings.json` under `PhiEncryption` and `DataProtection`. Implementation: `PhiEncryptionService`, `PatientPhiConverterConfigurator`, `PatientService`.
+
+---
+
+## Secrets and key storage
+
+**Never commit:**
+
+- `data-protection-keys/` (Data Protection key ring)
+- Production `PhiEncryption:SearchTokenKey`
+- Any Key Vault / KMS credentials
+
+`data-protection-keys/` is listed in `.gitignore`. Treat loss of this directory as **unrecoverable data loss** for encrypted PHI columns.
+
+| Environment | Data Protection key ring | Search HMAC key (`SearchTokenKey`) |
+|---|---|---|
+| Development | `./data-protection-keys/` on disk (`DataProtection:KeyPath`) | `appsettings.json` (dev placeholder only) |
+| Production | Azure Key Vault XML blob or AWS KMS-backed store | Key Vault / Secrets Manager secret — **not** appsettings |
+
+---
+
+## Initial deployment / encrypting existing rows
+
+New patients are encrypted automatically on save. Existing plaintext rows need a one-time re-save.
+
+### Option A — CLI (recommended)
+
+```bash
+./scripts/encrypt-existing-patient-phi.sh
+```
+
+Or directly:
+
+```bash
+dotnet run --project VigilCareClinicalAPI -- encrypt-phi
+```
+
+Loads every patient through EF, applies value converters, and recomputes `name_search_token`.
+
+### Option B — Startup hosted service
+
+`PatientPhiMigrationService` runs on application start and backfills search tokens for rows that are not yet encrypted. It is registered in `Program.cs`. Disable or remove after the first successful production deploy to avoid redundant work on every restart.
+
+---
+
+## Verifying encryption
+
+### Automated tests
+
+```bash
+dotnet test VigilCareClinicalAPI.Tests --filter "FullyQualifiedName~PhiEncryption"
+```
+
+### Live stack (API + Postgres running)
+
+```bash
+./scripts/run-phase32-verification.sh
+```
+
+### Manual DB check
+
+Encrypted `first_name` values will **not** equal the patient's plaintext name:
+
+```sql
+SELECT id, mrn, left(first_name, 30) AS encrypted_prefix
+FROM patients
+LIMIT 5;
+```
+
+Prefix often starts with `CfDJ8` (Data Protection default).
+
+### PHI access logs
+
+```bash
+# Admin token required (AuditRead permission)
+curl -H "Authorization: Bearer $TOKEN" \
+  "http://localhost:5270/api/v1/phi-access-logs?patientId=<uuid>"
+```
+
+Prometheus metric: `phi_access_logs_total{access_type="VIEW|LIST|SEARCH|CREATE|UPDATE"}`.
+
+---
+
+## Key rotation
+
+**Always back up the current key ring before rotating.**
+
+### Encryption keys (Data Protection)
+
+1. Snapshot `data-protection-keys/` (or export Key Vault blob).
+2. Deploy new key ring **alongside** the old one (Data Protection supports multiple keys; newest encrypts, all decrypt).
+3. Change `PhiEncryption:ProtectorPurpose` to a new version string (e.g. `VigilCare.PatientPhi.v2`).
+4. Re-run encrypt command to re-encrypt all patient rows with the new protector:
+   ```bash
+   dotnet run --project VigilCareClinicalAPI -- encrypt-phi
+   ```
+5. Verify API reads and raw DB ciphertext look correct.
+6. Retire old keys only after confirming all rows decrypt successfully.
+
+### Search HMAC key (`SearchTokenKey`)
+
+1. Store new key in secrets manager.
+2. Update `PhiEncryption:SearchTokenKey` in configuration.
+3. Re-run `encrypt-phi` (recomputes all `name_search_token` values).
+4. Confirm name search still works (`GET /api/v1/patients?q=First+Last`).
+
+Rotating one key without the other does not require touching the other, but both rotations need a full patient re-save.
+
+---
+
+## PHI access log retention (HIPAA)
+
+`phi_access_logs` records who accessed patient demographics, when, from which path, and (hashed) search terms. HIPAA requires audit controls; retention guidance is **minimum 6 years**.
+
+This phase creates the table and query API (`GET /api/v1/phi-access-logs`). Automated retention/archival policy is **out of scope** — plan for Phase 33+ (partitioning, cold storage, or purge job with legal review).
+
+Query endpoints:
+
+| Endpoint | Permission |
+|---|---|
+| `GET /api/v1/phi-access-logs` | `AuditRead` |
+| `GET /api/v1/phi-access-logs/patients/{id}` | `PatientsRead` |
+
+---
+
+## Cross-system PHI (out of scope)
+
+These paths are **not** covered by column encryption in PostgreSQL:
+
+| System | Risk | Action |
+|---|---|---|
+| Elasticsearch patient documents | May index decrypted names at index time | Exclude PHI fields or accept decrypted-at-index policy |
+| Kafka / data lake Parquet events | `patientName` in outbox payloads (e.g. encounter events) | Review lake schemas; redact or tokenize in a future phase |
+| Application logs | Serilog must not log PHI fields | Audit log configuration |
+
+---
+
+## Troubleshooting
+
+### `CryptographicException` / cannot decrypt patient
+
+- Key ring missing or wrong `DataProtection:KeyPath`
+- App deployed to new host without copying `data-protection-keys/`
+- `ProtectorPurpose` changed without re-running `encrypt-phi`
+
+**Fix:** Restore key ring from backup. Do not delete old keys until all data is re-encrypted.
+
+### Name search returns no results
+
+- `name_search_token` is null on older rows → run `encrypt-phi`
+- `SearchTokenKey` changed without token recompute
+- Query format: two-term search uses `First Last` (space-separated); single term matches first or last name only
+
+### PHI access logs empty
+
+- Caller not authenticated (`PhiAccessLogService` skips unauthenticated requests)
+- `PhiEncryption:LogListAccess` is `false` (list/search aggregate logs suppressed)
+- Integration/FHIR paths must still authenticate (Phase 31 Integration role)
+
+### Column length errors on encrypt
+
+Migration `WidenPhiEncryptedColumns` widens `first_name`, `last_name`, and emergency contact columns to `text`. Ciphertext is longer than plaintext. If new columns are added to encryption, ensure DB column types accommodate protected payload size.
+
+### Tests fail with 500 on patient register
+
+- Confirm migrations applied (`AddPatientNameSearchToken`, `AddPhiAccessLogs`, `WidenPhiEncryptedColumns`)
+- Confirm `PhiEncryption:SearchTokenKey` is set in configuration
+- Confirm `data-protection-keys/` is writable in the API working directory
+
+---
+
+## Related files
+
+| File | Purpose |
+|---|---|
+| `VigilCareClinicalAPI/Services/PhiEncryptionService.cs` | Encrypt/decrypt + HMAC tokens |
+| `VigilCareClinicalAPI/Services/PhiAccessLogService.cs` | Access audit writes |
+| `VigilCareClinicalAPI/Commands/EncryptPhiCommand.cs` | Bulk re-save CLI |
+| `VigilCareClinicalAPI/BackgroundServices/PatientPhiMigrationService.cs` | Startup token backfill |
+| `scripts/encrypt-existing-patient-phi.sh` | Wrapper for encrypt CLI |
+| `scripts/run-phase32-verification.sh` | End-to-end verification |
+| `docs/plans/phase-32-plan.md` | Implementation plan and design rationale |
@@ -5,7 +5,3 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

 echo "Re-save all patients through EF to apply encryption converters..."
 dotnet run --project "${ROOT_DIR}/VigilCareClinicalAPI" --no-build -- encrypt-phi
-```
-
-```bash
-chmod +x scripts/encrypt-existing-patient-phi.sh
@@ -27,7 +27,7 @@ curl -sf "${BASE_URL}/api/v1/phi-access-logs?patientId=${PATIENT_ID}" \
  -H "Authorization: Bearer ${TOKEN}" | jq -e '.data.totalCount >= 1'

 echo "Verify raw DB encryption (requires psql)"
-docker compose exec -T postgres psql -U vigilcare -d vigilcare -c \
+docker compose exec -T postgres psql -U postgres -d vigilcare -c \
  "SELECT id, left(first_name, 20) AS encrypted_prefix FROM patients WHERE id = '${PATIENT_ID}';"

 echo "Phase 32 verification complete."