feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job

This commit is contained in:
voltsrage
2026-06-27 13:29:04 +08:00
parent e22d33b654
commit 04bdb7e85c
31 changed files with 4482 additions and 259 deletions
+265 -236
View File
@@ -1,3 +1,4 @@
using System.Diagnostics;
using System.Text.Json;
using Microsoft.EntityFrameworkCore;
@@ -39,167 +40,181 @@ public class PromotionService : IPromotionService
}
}
// --- Load batch with all draft data ---
var batch = await _db.DigitizationBatches
.Include(b => b.DraftPatient)
.Include(b => b.DraftEncounter)
.Include(b => b.DraftObservations)
.FirstOrDefaultAsync(b => b.Id == batchId);
if (batch is null)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
// --- Status validation ---
if (batch.Status != BatchStatus.Verified && batch.Status != BatchStatus.AwaitingClinicalApproval)
throw new ConflictException(
$"Batch must be in 'verified' or 'awaiting_clinical_approval' status to approve. Current status: '{batch.Status.ToDbString()}'.",
"ILLEGAL_STATUS_TRANSITION");
// --- Separation of duties: approver cannot be the entry clerk ---
if (batch.EnteredByUserId == approverUserId)
throw new ConflictException(
"Separation of duties: the entry clerk cannot approve their own batch.",
"SEPARATION_OF_DUTIES_VIOLATION");
// --- Also cannot be the verifier ---
if (batch.VerifiedByUserId == approverUserId)
throw new ConflictException(
"Separation of duties: the verifier cannot also approve the same batch.",
"SEPARATION_OF_DUTIES_VIOLATION");
// --- Validate draft data completeness (corrections reuse the linked live patient) ---
if (!batch.SupersedesBatchId.HasValue)
{
if (batch.DraftPatient is null)
throw new ValidationException("Batch has no draft patient data.", "MISSING_DRAFT_PATIENT");
if (string.IsNullOrWhiteSpace(batch.DraftPatient.FullName))
throw new ValidationException("Draft patient must have a full name.", "INVALID_DRAFT_PATIENT");
}
else if (!batch.PatientId.HasValue)
{
throw new ValidationException(
"Correction batch has no linked patient.",
"MISSING_PATIENT");
}
// --- Begin atomic transaction ---
await using var transaction = await _db.Database.BeginTransactionAsync();
// --- Start timing the promotion ---
var stopwatch = Stopwatch.StartNew();
try
{
var now = DateTimeOffset.UtcNow;
// --- Load batch with all draft data ---
var batch = await _db.DigitizationBatches
.Include(b => b.DraftPatient)
.Include(b => b.DraftEncounter)
.Include(b => b.DraftObservations)
.FirstOrDefaultAsync(b => b.Id == batchId);
// === Step 1: Create or update Patient ===
Patient patient;
if (batch.SupersedesBatchId.HasValue)
if (batch is null)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
// --- Status validation ---
if (batch.Status != BatchStatus.Verified && batch.Status != BatchStatus.AwaitingClinicalApproval)
throw new ConflictException(
$"Batch must be in 'verified' or 'awaiting_clinical_approval' status to approve. Current status: '{batch.Status.ToDbString()}'.",
"ILLEGAL_STATUS_TRANSITION");
// --- Separation of duties: approver cannot be the entry clerk ---
if (batch.EnteredByUserId == approverUserId)
throw new ConflictException(
"Separation of duties: the entry clerk cannot approve their own batch.",
"SEPARATION_OF_DUTIES_VIOLATION");
// --- Also cannot be the verifier ---
if (batch.VerifiedByUserId == approverUserId)
throw new ConflictException(
"Separation of duties: the verifier cannot also approve the same batch.",
"SEPARATION_OF_DUTIES_VIOLATION");
// --- Validate draft data completeness (corrections reuse the linked live patient) ---
if (!batch.SupersedesBatchId.HasValue)
{
patient = await _db.Patients.FirstOrDefaultAsync(p => p.Id == batch.PatientId!.Value)
?? throw new NotFoundException(
$"Patient {batch.PatientId.Value} not found.",
"PATIENT_NOT_FOUND");
if (batch.DraftPatient is null)
throw new ValidationException("Batch has no draft patient data.", "MISSING_DRAFT_PATIENT");
if (string.IsNullOrWhiteSpace(batch.DraftPatient.FullName))
throw new ValidationException("Draft patient must have a full name.", "INVALID_DRAFT_PATIENT");
}
else
else if (!batch.PatientId.HasValue)
{
patient = await CreateOrUpdatePatientAsync(batch.DraftPatient!, now);
throw new ValidationException(
"Correction batch has no linked patient.",
"MISSING_PATIENT");
}
// === Step 2: Create or match Encounter (reuse original encounter for corrections) ===
var encounter = await ResolveClinicalEncounterAsync(batch, patient.Id, now);
await EnsureLiveEncounterAsync(encounter, now);
// --- Begin atomic transaction ---
await using var transaction = await _db.Database.BeginTransactionAsync();
// === Step 3: Insert each DraftObservation as live Observation ===
var (observationIds, outboxCount) = await PromoteObservationsAsync(
batch, patient.Id, encounter.Id, enableRetroactiveAlerts, now);
// === Step 3b: Mirror to live_observations for supersession / digitization history ===
PromoteLiveObservationsAsync(batch, patient.Id, encounter.Id, now);
SupersessionResult? supersessionResult = null;
if (batch.SupersedesBatchId.HasValue)
try
{
supersessionResult = await SupersedeOriginalBatchAsync(
batch.SupersedesBatchId.Value, batchId, approverUserId, now);
}
var now = DateTimeOffset.UtcNow;
// === Step 4: Update batch status to Promoted ===
batch.Status = BatchStatus.Promoted;
batch.ApprovedByUserId = approverUserId;
batch.PatientId = patient.Id;
batch.PromotedAt = now;
batch.PromotionEncounterId = encounter.Id;
batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
batch.UpdatedAt = now;
// === Step 5: Write DigitizationEvent ===
var promotionMetadata = new Dictionary<string, object>
{
["patientId"] = patient.Id,
["mrn"] = patient.Mrn,
["encounterId"] = encounter.Id,
["observationCount"] = observationIds.Length,
["outboxEventsWritten"] = outboxCount,
["enableRetroactiveAlerts"] = enableRetroactiveAlerts,
["track"] = batch.Track.ToDbString(),
["isCorrection"] = batch.SupersedesBatchId.HasValue
};
if (supersessionResult is not null)
{
promotionMetadata["supersession"] = new
// === Step 1: Create or update Patient ===
Patient patient;
if (batch.SupersedesBatchId.HasValue)
{
originalBatchId = supersessionResult.OriginalBatchId,
observationsSuperseded = supersessionResult.ObservationsSuperseded,
observationsReplaced = supersessionResult.ObservationsReplaced
patient = await _db.Patients.FirstOrDefaultAsync(p => p.Id == batch.PatientId!.Value)
?? throw new NotFoundException(
$"Patient {batch.PatientId.Value} not found.",
"PATIENT_NOT_FOUND");
}
else
{
patient = await CreateOrUpdatePatientAsync(batch.DraftPatient!, now);
}
// === Step 2: Create or match Encounter (reuse original encounter for corrections) ===
var encounter = await ResolveClinicalEncounterAsync(batch, patient.Id, now);
await EnsureLiveEncounterAsync(encounter, now);
// === Step 3: Insert each DraftObservation as live Observation ===
var (observationIds, outboxCount) = await PromoteObservationsAsync(
batch, patient.Id, encounter.Id, enableRetroactiveAlerts, now);
// === Step 3b: Mirror to live_observations for supersession / digitization history ===
PromoteLiveObservationsAsync(batch, patient.Id, encounter.Id, now);
SupersessionResult? supersessionResult = null;
if (batch.SupersedesBatchId.HasValue)
{
supersessionResult = await SupersedeOriginalBatchAsync(
batch.SupersedesBatchId.Value, batchId, approverUserId, now);
}
// === Step 4: Update batch status to Promoted ===
batch.Status = BatchStatus.Promoted;
batch.ApprovedByUserId = approverUserId;
batch.PatientId = patient.Id;
batch.PromotedAt = now;
batch.PromotionEncounterId = encounter.Id;
batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
batch.UpdatedAt = now;
// === Step 5: Write DigitizationEvent ===
var promotionMetadata = new Dictionary<string, object>
{
["patientId"] = patient.Id,
["mrn"] = patient.Mrn,
["encounterId"] = encounter.Id,
["observationCount"] = observationIds.Length,
["outboxEventsWritten"] = outboxCount,
["enableRetroactiveAlerts"] = enableRetroactiveAlerts,
["track"] = batch.Track.ToDbString(),
["isCorrection"] = batch.SupersedesBatchId.HasValue
};
if (supersessionResult is not null)
{
promotionMetadata["supersession"] = new
{
originalBatchId = supersessionResult.OriginalBatchId,
observationsSuperseded = supersessionResult.ObservationsSuperseded,
observationsReplaced = supersessionResult.ObservationsReplaced
};
}
_db.DigitizationEvents.Add(new DigitizationEvent
{
Id = Guid.NewGuid(),
BatchId = batchId,
EventType = batch.SupersedesBatchId.HasValue
? DigitizationEventType.CorrectionPromoted
: DigitizationEventType.Promoted,
ActorUserId = approverUserId,
OccurredAt = now,
MetadataJson = JsonSerializer.Serialize(promotionMetadata)
});
// === Step 6: Store idempotency record (within same transaction) ===
var result = new PromotionResultResponse(
BatchId: batchId,
Status: BatchStatus.Promoted.ToDbString(),
PatientId: patient.Id,
Mrn: patient.Mrn,
EncounterId: encounter.Id,
ObservationIds: observationIds,
PromotedAt: now,
OutboxEventsWritten: outboxCount
);
if (!string.IsNullOrWhiteSpace(idempotencyKey))
{
await _idempotency.SaveAsync(
idempotencyKey, "batch_promote", batchId,
200, result, TimeSpan.FromHours(24));
}
// === Step 7: Commit ===
await _db.SaveChangesAsync();
await transaction.CommitAsync();
stopwatch.Stop();
DiagnosticsMetrics.PromotionDuration.Observe(stopwatch.Elapsed.TotalSeconds);
_logger.LogInformation(
"Batch {BatchId} promoted in {ElapsedMs}ms: Patient {PatientId} (MRN {Mrn}), " +
"Encounter {EncounterId}, {ObsCount} observations, {OutboxCount} outbox events",
batchId, stopwatch.ElapsedMilliseconds, patient.Id, patient.Mrn, encounter.Id,
observationIds.Length, outboxCount);
return result;
}
_db.DigitizationEvents.Add(new DigitizationEvent
catch
{
Id = Guid.NewGuid(),
BatchId = batchId,
EventType = batch.SupersedesBatchId.HasValue
? DigitizationEventType.CorrectionPromoted
: DigitizationEventType.Promoted,
ActorUserId = approverUserId,
OccurredAt = now,
MetadataJson = JsonSerializer.Serialize(promotionMetadata)
});
// === Step 6: Store idempotency record (within same transaction) ===
var result = new PromotionResultResponse(
BatchId: batchId,
Status: BatchStatus.Promoted.ToDbString(),
PatientId: patient.Id,
Mrn: patient.Mrn,
EncounterId: encounter.Id,
ObservationIds: observationIds,
PromotedAt: now,
OutboxEventsWritten: outboxCount
);
if (!string.IsNullOrWhiteSpace(idempotencyKey))
{
await _idempotency.SaveAsync(
idempotencyKey, "batch_promote", batchId,
200, result, TimeSpan.FromHours(24));
await transaction.RollbackAsync();
throw;
}
// === Step 7: Commit ===
await _db.SaveChangesAsync();
await transaction.CommitAsync();
_logger.LogInformation(
"Batch {BatchId} promoted: Patient {PatientId} (MRN {Mrn}), " +
"Encounter {EncounterId}, {ObsCount} observations, {OutboxCount} outbox events",
batchId, patient.Id, patient.Mrn, encounter.Id,
observationIds.Length, outboxCount);
return result;
}
catch
{
await transaction.RollbackAsync();
stopwatch.Stop();
throw;
}
}
@@ -513,117 +528,131 @@ public class PromotionService : IPromotionService
public async Task<PromotionResult> PromoteAsync(Guid batchId, Guid actorUserId)
{
await using var transaction = await _db.Database.BeginTransactionAsync();
var stopwatch = Stopwatch.StartNew();
try
{
// Load the batch with all draft data
var batch = await _db.DigitizationBatches
.Include(b => b.DraftPatient)
.Include(b => b.DraftEncounter)
.Include(b => b.DraftObservations)
.FirstOrDefaultAsync(b => b.Id == batchId);
await using var transaction = await _db.Database.BeginTransactionAsync();
if (batch is null)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
if (batch.Status != BatchStatus.Approved)
throw new ConflictException(
$"Only approved batches can be promoted. Batch is in '{batch.Status.ToDbString()}' status.",
"ILLEGAL_STATUS_TRANSITION");
// Resolve or create the live encounter
var encounterId = await ResolveEncounterAsync(batch);
// Promote draft observations to live observations
var now = DateTimeOffset.UtcNow;
var liveObservations = batch.DraftObservations.Select(draft => new LiveObservation
try
{
Id = Guid.NewGuid(),
EncounterId = encounterId,
PatientId = batch.PatientId,
SourceBatchId = batch.Id,
ObservationCode = draft.ObservationCode,
Value = draft.Value,
Unit = draft.Unit,
RecordedAt = draft.RecordedAt,
Note = draft.Note,
CreatedAt = now,
IsSuperseded = false,
SupersededByBatchId = null,
SupersededAt = null
}).ToList();
// Load the batch with all draft data
var batch = await _db.DigitizationBatches
.Include(b => b.DraftPatient)
.Include(b => b.DraftEncounter)
.Include(b => b.DraftObservations)
.FirstOrDefaultAsync(b => b.Id == batchId);
_db.LiveObservations.AddRange(liveObservations);
if (batch is null)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
// Handle supersession if this is a correction batch
SupersessionResult? supersessionResult = null;
if (batch.SupersedesBatchId.HasValue)
{
supersessionResult = await SupersedeOriginalBatchAsync(
batch.SupersedesBatchId.Value,
batch.Id,
actorUserId,
now);
}
if (batch.Status != BatchStatus.Approved)
throw new ConflictException(
$"Only approved batches can be promoted. Batch is in '{batch.Status.ToDbString()}' status.",
"ILLEGAL_STATUS_TRANSITION");
// Update batch status to Promoted
batch.Status = BatchStatus.Promoted;
batch.PromotedAt = now;
batch.PromotionEncounterId = encounterId;
batch.UpdatedAt = now;
// Resolve or create the live encounter
var encounterId = await ResolveEncounterAsync(batch);
// Record promotion event on the correction batch
var promotionMetadata = new Dictionary<string, object>
{
["encounterId"] = encounterId,
["observationsPromoted"] = liveObservations.Count,
["isCorrection"] = batch.SupersedesBatchId.HasValue
};
if (supersessionResult is not null)
{
promotionMetadata["supersession"] = new
// Promote draft observations to live observations
var now = DateTimeOffset.UtcNow;
var liveObservations = batch.DraftObservations.Select(draft => new LiveObservation
{
originalBatchId = supersessionResult.OriginalBatchId,
observationsSuperseded = supersessionResult.ObservationsSuperseded,
observationsReplaced = supersessionResult.ObservationsReplaced
Id = Guid.NewGuid(),
EncounterId = encounterId,
PatientId = batch.PatientId,
SourceBatchId = batch.Id,
ObservationCode = draft.ObservationCode,
Value = draft.Value,
Unit = draft.Unit,
RecordedAt = draft.RecordedAt,
Note = draft.Note,
CreatedAt = now,
IsSuperseded = false,
SupersededByBatchId = null,
SupersededAt = null
}).ToList();
_db.LiveObservations.AddRange(liveObservations);
// Handle supersession if this is a correction batch
SupersessionResult? supersessionResult = null;
if (batch.SupersedesBatchId.HasValue)
{
supersessionResult = await SupersedeOriginalBatchAsync(
batch.SupersedesBatchId.Value,
batch.Id,
actorUserId,
now);
}
// Update batch status to Promoted
batch.Status = BatchStatus.Promoted;
batch.PromotedAt = now;
batch.PromotionEncounterId = encounterId;
batch.UpdatedAt = now;
// Record promotion event on the correction batch
var promotionMetadata = new Dictionary<string, object>
{
["encounterId"] = encounterId,
["observationsPromoted"] = liveObservations.Count,
["isCorrection"] = batch.SupersedesBatchId.HasValue
};
if (supersessionResult is not null)
{
promotionMetadata["supersession"] = new
{
originalBatchId = supersessionResult.OriginalBatchId,
observationsSuperseded = supersessionResult.ObservationsSuperseded,
observationsReplaced = supersessionResult.ObservationsReplaced
};
}
_db.DigitizationEvents.Add(new DigitizationEvent
{
Id = Guid.NewGuid(),
BatchId = batch.Id,
EventType = batch.SupersedesBatchId.HasValue
? DigitizationEventType.CorrectionPromoted
: DigitizationEventType.Promoted,
ActorUserId = actorUserId,
OccurredAt = now,
MetadataJson = JsonSerializer.Serialize(promotionMetadata)
});
await _db.SaveChangesAsync();
await transaction.CommitAsync();
stopwatch.Stop();
DiagnosticsMetrics.PromotionDuration.Observe(stopwatch.Elapsed.TotalSeconds);
_logger.LogInformation(
"Batch {BatchId} promoted in {ElapsedMs}ms (correction={IsCorrection}, " +
"observations={ObservationCount}, superseded={SupersededCount})",
batchId,
stopwatch.ElapsedMilliseconds,
batch.SupersedesBatchId.HasValue,
liveObservations.Count,
supersessionResult?.ObservationsSuperseded ?? 0);
return new PromotionResult(
batch.Id,
encounterId,
liveObservations.Count,
batch.SupersedesBatchId.HasValue,
supersessionResult);
}
_db.DigitizationEvents.Add(new DigitizationEvent
catch
{
Id = Guid.NewGuid(),
BatchId = batch.Id,
EventType = batch.SupersedesBatchId.HasValue
? DigitizationEventType.CorrectionPromoted
: DigitizationEventType.Promoted,
ActorUserId = actorUserId,
OccurredAt = now,
MetadataJson = JsonSerializer.Serialize(promotionMetadata)
});
await _db.SaveChangesAsync();
await transaction.CommitAsync();
_logger.LogInformation(
"Batch {BatchId} promoted (correction={IsCorrection}, " +
"observations={ObservationCount}, superseded={SupersededCount})",
batchId,
batch.SupersedesBatchId.HasValue,
liveObservations.Count,
supersessionResult?.ObservationsSuperseded ?? 0);
return new PromotionResult(
batch.Id,
encounterId,
liveObservations.Count,
batch.SupersedesBatchId.HasValue,
supersessionResult);
await transaction.RollbackAsync();
throw;
}
}
catch
{
await transaction.RollbackAsync();
stopwatch.Stop();
throw;
}
}