feature: Prometheus Metrics, Supervisor Dashboard, and Promotion Retry Job

This commit is contained in:
voltsrage
2026-06-27 13:29:04 +08:00
parent e22d33b654
commit 04bdb7e85c
31 changed files with 4482 additions and 259 deletions
@@ -0,0 +1,78 @@
using Microsoft.EntityFrameworkCore;
/// <summary>
/// Provides cursor-paginated access to the audit trail of digitization events
/// for a given batch. Events are ordered by occurred_at ascending with id
/// as a tie-breaker for events at the same timestamp.
/// </summary>
public class BatchEventService : IBatchEventService
{
private readonly AppDbContext _db;
private const int MaxPageSize = 200;
private const int DefaultPageSize = 50;
public BatchEventService(AppDbContext db)
{
_db = db;
}
public async Task<CursorPagedResult<BatchEventResponse>> GetEventsAsync(
Guid batchId, DateTimeOffset? after, int pageSize)
{
// Validate batch exists
var batchExists = await _db.DigitizationBatches
.AnyAsync(b => b.Id == batchId);
if (!batchExists)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
// Clamp page size
pageSize = Math.Clamp(pageSize, 1, MaxPageSize);
// Build query
var query = _db.DigitizationEvents
.AsNoTracking()
.Include(e => e.Actor)
.Where(e => e.BatchId == batchId);
// Apply cursor filter — only events strictly after the cursor timestamp
if (after.HasValue)
{
query = query.Where(e => e.OccurredAt > after.Value);
}
// Fetch pageSize+1 rows to determine HasMore without a COUNT query
var events = await query
.OrderBy(e => e.OccurredAt)
.ThenBy(e => e.Id) // tie-breaker for events at the same timestamp
.Take(pageSize + 1)
.Select(e => new BatchEventResponse(
e.Id,
e.BatchId,
e.EventType.ToDbString(),
e.ActorUserId,
e.Actor.Username,
e.Actor.FullName,
e.OccurredAt,
e.MetadataJson))
.ToListAsync();
var hasMore = events.Count > pageSize;
var page = hasMore ? events.Take(pageSize).ToList() : events;
// Build next cursor from the last item's OccurredAt
string? nextCursor = null;
if (hasMore && page.Count > 0)
{
var lastEvent = page[^1];
// ISO-8601 round-trip format preserves full precision
nextCursor = lastEvent.OccurredAt.ToString("o");
}
return new CursorPagedResult<BatchEventResponse>(
Items: page,
PageSize: pageSize,
NextCursor: nextCursor,
HasMore: hasMore);
}
}
@@ -0,0 +1,12 @@
public interface IBatchEventService
{
/// <summary>
/// Returns cursor-paginated audit trail events for a batch.
/// Events are ordered by OccurredAt ascending (oldest first).
/// </summary>
/// <param name="batchId">The batch to query events for.</param>
/// <param name="after">Cursor: ISO-8601 timestamp. Only events after this timestamp are returned.</param>
/// <param name="pageSize">Number of events per page. Default 50, max 200.</param>
Task<CursorPagedResult<BatchEventResponse>> GetEventsAsync(
Guid batchId, DateTimeOffset? after, int pageSize);
}
+265 -236
View File
@@ -1,3 +1,4 @@
using System.Diagnostics;
using System.Text.Json;
using Microsoft.EntityFrameworkCore;
@@ -39,167 +40,181 @@ public class PromotionService : IPromotionService
}
}
// --- Load batch with all draft data ---
var batch = await _db.DigitizationBatches
.Include(b => b.DraftPatient)
.Include(b => b.DraftEncounter)
.Include(b => b.DraftObservations)
.FirstOrDefaultAsync(b => b.Id == batchId);
if (batch is null)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
// --- Status validation ---
if (batch.Status != BatchStatus.Verified && batch.Status != BatchStatus.AwaitingClinicalApproval)
throw new ConflictException(
$"Batch must be in 'verified' or 'awaiting_clinical_approval' status to approve. Current status: '{batch.Status.ToDbString()}'.",
"ILLEGAL_STATUS_TRANSITION");
// --- Separation of duties: approver cannot be the entry clerk ---
if (batch.EnteredByUserId == approverUserId)
throw new ConflictException(
"Separation of duties: the entry clerk cannot approve their own batch.",
"SEPARATION_OF_DUTIES_VIOLATION");
// --- Also cannot be the verifier ---
if (batch.VerifiedByUserId == approverUserId)
throw new ConflictException(
"Separation of duties: the verifier cannot also approve the same batch.",
"SEPARATION_OF_DUTIES_VIOLATION");
// --- Validate draft data completeness (corrections reuse the linked live patient) ---
if (!batch.SupersedesBatchId.HasValue)
{
if (batch.DraftPatient is null)
throw new ValidationException("Batch has no draft patient data.", "MISSING_DRAFT_PATIENT");
if (string.IsNullOrWhiteSpace(batch.DraftPatient.FullName))
throw new ValidationException("Draft patient must have a full name.", "INVALID_DRAFT_PATIENT");
}
else if (!batch.PatientId.HasValue)
{
throw new ValidationException(
"Correction batch has no linked patient.",
"MISSING_PATIENT");
}
// --- Begin atomic transaction ---
await using var transaction = await _db.Database.BeginTransactionAsync();
// --- Start timing the promotion ---
var stopwatch = Stopwatch.StartNew();
try
{
var now = DateTimeOffset.UtcNow;
// --- Load batch with all draft data ---
var batch = await _db.DigitizationBatches
.Include(b => b.DraftPatient)
.Include(b => b.DraftEncounter)
.Include(b => b.DraftObservations)
.FirstOrDefaultAsync(b => b.Id == batchId);
// === Step 1: Create or update Patient ===
Patient patient;
if (batch.SupersedesBatchId.HasValue)
if (batch is null)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
// --- Status validation ---
if (batch.Status != BatchStatus.Verified && batch.Status != BatchStatus.AwaitingClinicalApproval)
throw new ConflictException(
$"Batch must be in 'verified' or 'awaiting_clinical_approval' status to approve. Current status: '{batch.Status.ToDbString()}'.",
"ILLEGAL_STATUS_TRANSITION");
// --- Separation of duties: approver cannot be the entry clerk ---
if (batch.EnteredByUserId == approverUserId)
throw new ConflictException(
"Separation of duties: the entry clerk cannot approve their own batch.",
"SEPARATION_OF_DUTIES_VIOLATION");
// --- Also cannot be the verifier ---
if (batch.VerifiedByUserId == approverUserId)
throw new ConflictException(
"Separation of duties: the verifier cannot also approve the same batch.",
"SEPARATION_OF_DUTIES_VIOLATION");
// --- Validate draft data completeness (corrections reuse the linked live patient) ---
if (!batch.SupersedesBatchId.HasValue)
{
patient = await _db.Patients.FirstOrDefaultAsync(p => p.Id == batch.PatientId!.Value)
?? throw new NotFoundException(
$"Patient {batch.PatientId.Value} not found.",
"PATIENT_NOT_FOUND");
if (batch.DraftPatient is null)
throw new ValidationException("Batch has no draft patient data.", "MISSING_DRAFT_PATIENT");
if (string.IsNullOrWhiteSpace(batch.DraftPatient.FullName))
throw new ValidationException("Draft patient must have a full name.", "INVALID_DRAFT_PATIENT");
}
else
else if (!batch.PatientId.HasValue)
{
patient = await CreateOrUpdatePatientAsync(batch.DraftPatient!, now);
throw new ValidationException(
"Correction batch has no linked patient.",
"MISSING_PATIENT");
}
// === Step 2: Create or match Encounter (reuse original encounter for corrections) ===
var encounter = await ResolveClinicalEncounterAsync(batch, patient.Id, now);
await EnsureLiveEncounterAsync(encounter, now);
// --- Begin atomic transaction ---
await using var transaction = await _db.Database.BeginTransactionAsync();
// === Step 3: Insert each DraftObservation as live Observation ===
var (observationIds, outboxCount) = await PromoteObservationsAsync(
batch, patient.Id, encounter.Id, enableRetroactiveAlerts, now);
// === Step 3b: Mirror to live_observations for supersession / digitization history ===
PromoteLiveObservationsAsync(batch, patient.Id, encounter.Id, now);
SupersessionResult? supersessionResult = null;
if (batch.SupersedesBatchId.HasValue)
try
{
supersessionResult = await SupersedeOriginalBatchAsync(
batch.SupersedesBatchId.Value, batchId, approverUserId, now);
}
var now = DateTimeOffset.UtcNow;
// === Step 4: Update batch status to Promoted ===
batch.Status = BatchStatus.Promoted;
batch.ApprovedByUserId = approverUserId;
batch.PatientId = patient.Id;
batch.PromotedAt = now;
batch.PromotionEncounterId = encounter.Id;
batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
batch.UpdatedAt = now;
// === Step 5: Write DigitizationEvent ===
var promotionMetadata = new Dictionary<string, object>
{
["patientId"] = patient.Id,
["mrn"] = patient.Mrn,
["encounterId"] = encounter.Id,
["observationCount"] = observationIds.Length,
["outboxEventsWritten"] = outboxCount,
["enableRetroactiveAlerts"] = enableRetroactiveAlerts,
["track"] = batch.Track.ToDbString(),
["isCorrection"] = batch.SupersedesBatchId.HasValue
};
if (supersessionResult is not null)
{
promotionMetadata["supersession"] = new
// === Step 1: Create or update Patient ===
Patient patient;
if (batch.SupersedesBatchId.HasValue)
{
originalBatchId = supersessionResult.OriginalBatchId,
observationsSuperseded = supersessionResult.ObservationsSuperseded,
observationsReplaced = supersessionResult.ObservationsReplaced
patient = await _db.Patients.FirstOrDefaultAsync(p => p.Id == batch.PatientId!.Value)
?? throw new NotFoundException(
$"Patient {batch.PatientId.Value} not found.",
"PATIENT_NOT_FOUND");
}
else
{
patient = await CreateOrUpdatePatientAsync(batch.DraftPatient!, now);
}
// === Step 2: Create or match Encounter (reuse original encounter for corrections) ===
var encounter = await ResolveClinicalEncounterAsync(batch, patient.Id, now);
await EnsureLiveEncounterAsync(encounter, now);
// === Step 3: Insert each DraftObservation as live Observation ===
var (observationIds, outboxCount) = await PromoteObservationsAsync(
batch, patient.Id, encounter.Id, enableRetroactiveAlerts, now);
// === Step 3b: Mirror to live_observations for supersession / digitization history ===
PromoteLiveObservationsAsync(batch, patient.Id, encounter.Id, now);
SupersessionResult? supersessionResult = null;
if (batch.SupersedesBatchId.HasValue)
{
supersessionResult = await SupersedeOriginalBatchAsync(
batch.SupersedesBatchId.Value, batchId, approverUserId, now);
}
// === Step 4: Update batch status to Promoted ===
batch.Status = BatchStatus.Promoted;
batch.ApprovedByUserId = approverUserId;
batch.PatientId = patient.Id;
batch.PromotedAt = now;
batch.PromotionEncounterId = encounter.Id;
batch.EnableRetroactiveAlerts = enableRetroactiveAlerts;
batch.UpdatedAt = now;
// === Step 5: Write DigitizationEvent ===
var promotionMetadata = new Dictionary<string, object>
{
["patientId"] = patient.Id,
["mrn"] = patient.Mrn,
["encounterId"] = encounter.Id,
["observationCount"] = observationIds.Length,
["outboxEventsWritten"] = outboxCount,
["enableRetroactiveAlerts"] = enableRetroactiveAlerts,
["track"] = batch.Track.ToDbString(),
["isCorrection"] = batch.SupersedesBatchId.HasValue
};
if (supersessionResult is not null)
{
promotionMetadata["supersession"] = new
{
originalBatchId = supersessionResult.OriginalBatchId,
observationsSuperseded = supersessionResult.ObservationsSuperseded,
observationsReplaced = supersessionResult.ObservationsReplaced
};
}
_db.DigitizationEvents.Add(new DigitizationEvent
{
Id = Guid.NewGuid(),
BatchId = batchId,
EventType = batch.SupersedesBatchId.HasValue
? DigitizationEventType.CorrectionPromoted
: DigitizationEventType.Promoted,
ActorUserId = approverUserId,
OccurredAt = now,
MetadataJson = JsonSerializer.Serialize(promotionMetadata)
});
// === Step 6: Store idempotency record (within same transaction) ===
var result = new PromotionResultResponse(
BatchId: batchId,
Status: BatchStatus.Promoted.ToDbString(),
PatientId: patient.Id,
Mrn: patient.Mrn,
EncounterId: encounter.Id,
ObservationIds: observationIds,
PromotedAt: now,
OutboxEventsWritten: outboxCount
);
if (!string.IsNullOrWhiteSpace(idempotencyKey))
{
await _idempotency.SaveAsync(
idempotencyKey, "batch_promote", batchId,
200, result, TimeSpan.FromHours(24));
}
// === Step 7: Commit ===
await _db.SaveChangesAsync();
await transaction.CommitAsync();
stopwatch.Stop();
DiagnosticsMetrics.PromotionDuration.Observe(stopwatch.Elapsed.TotalSeconds);
_logger.LogInformation(
"Batch {BatchId} promoted in {ElapsedMs}ms: Patient {PatientId} (MRN {Mrn}), " +
"Encounter {EncounterId}, {ObsCount} observations, {OutboxCount} outbox events",
batchId, stopwatch.ElapsedMilliseconds, patient.Id, patient.Mrn, encounter.Id,
observationIds.Length, outboxCount);
return result;
}
_db.DigitizationEvents.Add(new DigitizationEvent
catch
{
Id = Guid.NewGuid(),
BatchId = batchId,
EventType = batch.SupersedesBatchId.HasValue
? DigitizationEventType.CorrectionPromoted
: DigitizationEventType.Promoted,
ActorUserId = approverUserId,
OccurredAt = now,
MetadataJson = JsonSerializer.Serialize(promotionMetadata)
});
// === Step 6: Store idempotency record (within same transaction) ===
var result = new PromotionResultResponse(
BatchId: batchId,
Status: BatchStatus.Promoted.ToDbString(),
PatientId: patient.Id,
Mrn: patient.Mrn,
EncounterId: encounter.Id,
ObservationIds: observationIds,
PromotedAt: now,
OutboxEventsWritten: outboxCount
);
if (!string.IsNullOrWhiteSpace(idempotencyKey))
{
await _idempotency.SaveAsync(
idempotencyKey, "batch_promote", batchId,
200, result, TimeSpan.FromHours(24));
await transaction.RollbackAsync();
throw;
}
// === Step 7: Commit ===
await _db.SaveChangesAsync();
await transaction.CommitAsync();
_logger.LogInformation(
"Batch {BatchId} promoted: Patient {PatientId} (MRN {Mrn}), " +
"Encounter {EncounterId}, {ObsCount} observations, {OutboxCount} outbox events",
batchId, patient.Id, patient.Mrn, encounter.Id,
observationIds.Length, outboxCount);
return result;
}
catch
{
await transaction.RollbackAsync();
stopwatch.Stop();
throw;
}
}
@@ -513,117 +528,131 @@ public class PromotionService : IPromotionService
public async Task<PromotionResult> PromoteAsync(Guid batchId, Guid actorUserId)
{
await using var transaction = await _db.Database.BeginTransactionAsync();
var stopwatch = Stopwatch.StartNew();
try
{
// Load the batch with all draft data
var batch = await _db.DigitizationBatches
.Include(b => b.DraftPatient)
.Include(b => b.DraftEncounter)
.Include(b => b.DraftObservations)
.FirstOrDefaultAsync(b => b.Id == batchId);
await using var transaction = await _db.Database.BeginTransactionAsync();
if (batch is null)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
if (batch.Status != BatchStatus.Approved)
throw new ConflictException(
$"Only approved batches can be promoted. Batch is in '{batch.Status.ToDbString()}' status.",
"ILLEGAL_STATUS_TRANSITION");
// Resolve or create the live encounter
var encounterId = await ResolveEncounterAsync(batch);
// Promote draft observations to live observations
var now = DateTimeOffset.UtcNow;
var liveObservations = batch.DraftObservations.Select(draft => new LiveObservation
try
{
Id = Guid.NewGuid(),
EncounterId = encounterId,
PatientId = batch.PatientId,
SourceBatchId = batch.Id,
ObservationCode = draft.ObservationCode,
Value = draft.Value,
Unit = draft.Unit,
RecordedAt = draft.RecordedAt,
Note = draft.Note,
CreatedAt = now,
IsSuperseded = false,
SupersededByBatchId = null,
SupersededAt = null
}).ToList();
// Load the batch with all draft data
var batch = await _db.DigitizationBatches
.Include(b => b.DraftPatient)
.Include(b => b.DraftEncounter)
.Include(b => b.DraftObservations)
.FirstOrDefaultAsync(b => b.Id == batchId);
_db.LiveObservations.AddRange(liveObservations);
if (batch is null)
throw new NotFoundException("Batch not found.", "BATCH_NOT_FOUND");
// Handle supersession if this is a correction batch
SupersessionResult? supersessionResult = null;
if (batch.SupersedesBatchId.HasValue)
{
supersessionResult = await SupersedeOriginalBatchAsync(
batch.SupersedesBatchId.Value,
batch.Id,
actorUserId,
now);
}
if (batch.Status != BatchStatus.Approved)
throw new ConflictException(
$"Only approved batches can be promoted. Batch is in '{batch.Status.ToDbString()}' status.",
"ILLEGAL_STATUS_TRANSITION");
// Update batch status to Promoted
batch.Status = BatchStatus.Promoted;
batch.PromotedAt = now;
batch.PromotionEncounterId = encounterId;
batch.UpdatedAt = now;
// Resolve or create the live encounter
var encounterId = await ResolveEncounterAsync(batch);
// Record promotion event on the correction batch
var promotionMetadata = new Dictionary<string, object>
{
["encounterId"] = encounterId,
["observationsPromoted"] = liveObservations.Count,
["isCorrection"] = batch.SupersedesBatchId.HasValue
};
if (supersessionResult is not null)
{
promotionMetadata["supersession"] = new
// Promote draft observations to live observations
var now = DateTimeOffset.UtcNow;
var liveObservations = batch.DraftObservations.Select(draft => new LiveObservation
{
originalBatchId = supersessionResult.OriginalBatchId,
observationsSuperseded = supersessionResult.ObservationsSuperseded,
observationsReplaced = supersessionResult.ObservationsReplaced
Id = Guid.NewGuid(),
EncounterId = encounterId,
PatientId = batch.PatientId,
SourceBatchId = batch.Id,
ObservationCode = draft.ObservationCode,
Value = draft.Value,
Unit = draft.Unit,
RecordedAt = draft.RecordedAt,
Note = draft.Note,
CreatedAt = now,
IsSuperseded = false,
SupersededByBatchId = null,
SupersededAt = null
}).ToList();
_db.LiveObservations.AddRange(liveObservations);
// Handle supersession if this is a correction batch
SupersessionResult? supersessionResult = null;
if (batch.SupersedesBatchId.HasValue)
{
supersessionResult = await SupersedeOriginalBatchAsync(
batch.SupersedesBatchId.Value,
batch.Id,
actorUserId,
now);
}
// Update batch status to Promoted
batch.Status = BatchStatus.Promoted;
batch.PromotedAt = now;
batch.PromotionEncounterId = encounterId;
batch.UpdatedAt = now;
// Record promotion event on the correction batch
var promotionMetadata = new Dictionary<string, object>
{
["encounterId"] = encounterId,
["observationsPromoted"] = liveObservations.Count,
["isCorrection"] = batch.SupersedesBatchId.HasValue
};
if (supersessionResult is not null)
{
promotionMetadata["supersession"] = new
{
originalBatchId = supersessionResult.OriginalBatchId,
observationsSuperseded = supersessionResult.ObservationsSuperseded,
observationsReplaced = supersessionResult.ObservationsReplaced
};
}
_db.DigitizationEvents.Add(new DigitizationEvent
{
Id = Guid.NewGuid(),
BatchId = batch.Id,
EventType = batch.SupersedesBatchId.HasValue
? DigitizationEventType.CorrectionPromoted
: DigitizationEventType.Promoted,
ActorUserId = actorUserId,
OccurredAt = now,
MetadataJson = JsonSerializer.Serialize(promotionMetadata)
});
await _db.SaveChangesAsync();
await transaction.CommitAsync();
stopwatch.Stop();
DiagnosticsMetrics.PromotionDuration.Observe(stopwatch.Elapsed.TotalSeconds);
_logger.LogInformation(
"Batch {BatchId} promoted in {ElapsedMs}ms (correction={IsCorrection}, " +
"observations={ObservationCount}, superseded={SupersededCount})",
batchId,
stopwatch.ElapsedMilliseconds,
batch.SupersedesBatchId.HasValue,
liveObservations.Count,
supersessionResult?.ObservationsSuperseded ?? 0);
return new PromotionResult(
batch.Id,
encounterId,
liveObservations.Count,
batch.SupersedesBatchId.HasValue,
supersessionResult);
}
_db.DigitizationEvents.Add(new DigitizationEvent
catch
{
Id = Guid.NewGuid(),
BatchId = batch.Id,
EventType = batch.SupersedesBatchId.HasValue
? DigitizationEventType.CorrectionPromoted
: DigitizationEventType.Promoted,
ActorUserId = actorUserId,
OccurredAt = now,
MetadataJson = JsonSerializer.Serialize(promotionMetadata)
});
await _db.SaveChangesAsync();
await transaction.CommitAsync();
_logger.LogInformation(
"Batch {BatchId} promoted (correction={IsCorrection}, " +
"observations={ObservationCount}, superseded={SupersededCount})",
batchId,
batch.SupersedesBatchId.HasValue,
liveObservations.Count,
supersessionResult?.ObservationsSuperseded ?? 0);
return new PromotionResult(
batch.Id,
encounterId,
liveObservations.Count,
batch.SupersedesBatchId.HasValue,
supersessionResult);
await transaction.RollbackAsync();
throw;
}
}
catch
{
await transaction.RollbackAsync();
stopwatch.Stop();
throw;
}
}
@@ -138,6 +138,10 @@ public class VerificationService : IVerificationService
await _db.SaveChangesAsync();
DiagnosticsMetrics.RejectionTotal
.WithLabels("verification_failed")
.Inc();
_logger.LogInformation(
"Batch {BatchId} verification failed by {VerifierUserId}, rejected",
batchId, verifierUserId);
@@ -206,6 +210,13 @@ public class VerificationService : IVerificationService
await _db.SaveChangesAsync();
var category = previousStatus == BatchStatus.AwaitingClinicalApproval
? "clinical_rejected"
: "verification_failed";
DiagnosticsMetrics.RejectionTotal
.WithLabels(category)
.Inc();
_logger.LogInformation(
"Batch {BatchId} rejected by {ActorUserId} from {PreviousStatus}: {Reason}",
batchId, actorUserId, previousStatus.ToDbString(), request.Reason);
@@ -125,6 +125,7 @@ public class WorkQueueService : IWorkQueueService
.AsNoTracking()
.Where(e => e.OccurredAt >= cutoff)
.Where(e => e.EventType == DigitizationEventType.Rejected
|| e.EventType == DigitizationEventType.VerificationFailed
|| e.EventType == DigitizationEventType.Verified
|| e.EventType == DigitizationEventType.VerifiedPendingClinical)
.GroupBy(e => e.EventType)
@@ -132,7 +133,8 @@ public class WorkQueueService : IWorkQueueService
.ToListAsync();
var rejections = recentEvents
.Where(e => e.EventType == DigitizationEventType.Rejected)
.Where(e => e.EventType == DigitizationEventType.Rejected
|| e.EventType == DigitizationEventType.VerificationFailed)
.Sum(e => e.Count);
var verifications = recentEvents