Trent/vigilcare-clinical
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature: MinIO Data Lake Writer (Parquet, Partitioned)

Browse Source
This commit is contained in:
voltsrage
2026-06-17 18:15:43 +08:00
parent df99bf3c91
commit 65be22fd1c
13 changed files with 818 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VigilCareClinicalAPI/DataLake/DataLakeOptions.cs
+17
View File
@@ -0,0 +1,17 @@
public sealed class DataLakeOptions
{
public const string Section = "DataLake";
// Maximum events buffered before a forced flush. At ~17 obs/sec steady state
// this fires approximately once per minute. Set low in tests (3–5) to avoid
// waiting for real traffic.
public int FlushCount { get; init; } = 1_000;
// Maximum age of the oldest buffered event before a time-based flush is forced.
// Production: 300 seconds (5 minutes). Tests: 10 seconds.
public int FlushIntervalSeconds { get; init; } = 300;
// MinIO bucket. The discharge summary worker from Phase 6 uses the same bucket
// under a different prefix — all VigilCare data stays in one bucket.
public string BucketName { get; init; } = "vigilcare";
}
VigilCareClinicalAPI/DataLake/DataLakeWriterService.cs
+302
View File
@@ -0,0 +1,302 @@
using System.Text.Json;
using Confluent.Kafka;
using Microsoft.Extensions.Options;
using Minio.DataModel.Args;
public sealed class DataLakeWriterService : BackgroundService
{
private readonly KafkaOptions _kafkaOptions;
private readonly DataLakeOptions _opts;
private readonly MinioOptions _minioOpts;
private readonly ILogger<DataLakeWriterService> _logger;
// Buffer key: identifies one Parquet file-to-be.
// Events sharing a topic, date, and Kafka partition land in the same file.
private record BufferKey(string Topic, string DatePath, int Partition);
private record BufferedEvent(string Payload, long Offset);
private readonly Dictionary<BufferKey, List<BufferedEvent>> _buffer = new();
// Track the highest offset per topic-partition for post-flush commit.
private readonly Dictionary<TopicPartition, TopicPartitionOffset> _highWatermarks = new();
public DataLakeWriterService(
IOptions<KafkaOptions> kafkaOptions,
IOptions<DataLakeOptions> opts,
IOptions<MinioOptions> minioOpts,
ILogger<DataLakeWriterService> logger)
{
_kafkaOptions = kafkaOptions.Value;
_opts = opts.Value;
_minioOpts = minioOpts.Value;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken ct)
{
var consumerConfig = new ConsumerConfig
{
BootstrapServers = _kafkaOptions.BootstrapServers,
GroupId = "data-lake-writer",
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false,
};
using var consumer = new ConsumerBuilder<string, string>(consumerConfig).Build();
consumer.Subscribe(new[]
{
_kafkaOptions.Topics.ObservationRecorded,
_kafkaOptions.Topics.AlertGenerated,
_kafkaOptions.Topics.EncounterStatusChanged,
});
_logger.LogInformation(
"DataLakeWriterService started. FlushCount={FlushCount} FlushIntervalSeconds={FlushInterval}",
_opts.FlushCount, _opts.FlushIntervalSeconds);
var lastFlush = DateTimeOffset.UtcNow;
try
{
while (!ct.IsCancellationRequested)
{
ConsumeResult<string, string>? result;
try
{
result = consumer.Consume(TimeSpan.FromMilliseconds(500));
}
catch (ConsumeException ex)
{
_logger.LogError(ex, "DataLakeWriter consume error");
continue;
}
if (result is not null)
AddToBuffer(result);
var totalBuffered = _buffer.Values.Sum(v => v.Count);
var shouldFlushCount = totalBuffered >= _opts.FlushCount;
var shouldFlushTime = DateTimeOffset.UtcNow - lastFlush
>= TimeSpan.FromSeconds(_opts.FlushIntervalSeconds);
if ((shouldFlushCount || shouldFlushTime) && totalBuffered > 0)
{
await FlushAsync(consumer, ct);
lastFlush = DateTimeOffset.UtcNow;
}
}
}
finally
{
// Final flush on shutdown so buffered events are not lost.
if (_buffer.Values.Sum(v => v.Count) > 0)
{
try { await FlushAsync(consumer, ct); }
catch (Exception ex)
{
_logger.LogError(ex, "DataLakeWriter shutdown flush failed — some events may be re-read on next start");
}
}
consumer.Close();
}
}
private void AddToBuffer(ConsumeResult<string, string> result)
{
var datePath = ExtractDatePath(result.Topic, result.Message.Value);
var key = new BufferKey(result.Topic, datePath, result.Partition.Value);
if (!_buffer.TryGetValue(key, out var list))
{
list = new List<BufferedEvent>();
_buffer[key] = list;
}
list.Add(new BufferedEvent(result.Message.Value, result.Offset.Value));
// Track highest offset per topic-partition for post-flush commit.
var tp = new TopicPartition(result.Topic, result.Partition);
_highWatermarks[tp] = new TopicPartitionOffset(tp, result.Offset + 1);
}
private async Task FlushAsync(IConsumer<string, string> consumer, CancellationToken ct)
{
var filesWritten = 0;
foreach (var (key, events) in _buffer)
{
if (events.Count == 0) continue;
try
{
var firstOffset = events.Min(e => e.Offset);
var objectKey = BuildObjectKey(key, firstOffset);
var bytes = await BuildParquetAsync(key.Topic, events, key.Partition);
await UploadToMinioAsync(objectKey, bytes, ct);
filesWritten++;
_logger.LogInformation(
"[DATA-LAKE] Wrote {Count} events → {ObjectKey} ({Bytes} bytes)",
events.Count, objectKey, bytes.Length);
}
catch (Exception ex)
{
// Log and continue — a failed file for one key must not prevent other
// keys from flushing. The uncommitted offsets will cause reprocessing.
_logger.LogError(ex, "[DATA-LAKE] Failed to write file for key {Key}", key);
}
}
// Commit only after all files are uploaded.
// Events for any key that failed above will be re-read on next startup.
if (_highWatermarks.Any())
{
consumer.Commit(_highWatermarks.Values);
_logger.LogInformation(
"[DATA-LAKE] Committed offsets for {PartitionCount} partitions after flushing {FileCount} files",
_highWatermarks.Count, filesWritten);
}
_buffer.Clear();
_highWatermarks.Clear();
}
private async Task<byte[]> BuildParquetAsync(
string topic, List<BufferedEvent> events, int partition)
{
if (topic == _kafkaOptions.Topics.ObservationRecorded)
{
var rows = events.Select(e => ParseObservationRow(e, partition)).ToList();
return await ParquetFileBuilder.BuildObservationsAsync(rows);
}
if (topic == _kafkaOptions.Topics.AlertGenerated)
{
var rows = events.Select(e => ParseAlertRow(e, partition)).ToList();
return await ParquetFileBuilder.BuildAlertsAsync(rows);
}
if (topic == _kafkaOptions.Topics.EncounterStatusChanged)
{
var rows = events.Select(e => ParseEncounterRow(e, partition)).ToList();
return await ParquetFileBuilder.BuildEncountersAsync(rows);
}
throw new InvalidOperationException($"Unknown topic: {topic}");
}
// ---------------------------------------------------------------------------
// Object key and date partition helpers
// ---------------------------------------------------------------------------
// File path: observations/2025/01/15/partition-0-offset-0000001000.parquet
// The date comes from the event timestamp, not the wall clock.
// Events from the same encounter that cross midnight are written into the date
// bucket matching their recorded_at timestamp — consistent with how Athena and
// Spark partition-prune by event time, not ingest time.
private string BuildObjectKey(BufferKey key, long firstOffset)
{
var folder = key.Topic switch
{
var t when t == _kafkaOptions.Topics.ObservationRecorded => "observations",
var t when t == _kafkaOptions.Topics.AlertGenerated => "alerts",
var t when t == _kafkaOptions.Topics.EncounterStatusChanged => "encounters",
_ => "unknown",
};
return $"{folder}/{key.DatePath}/partition-{key.Partition}-offset-{firstOffset:D10}.parquet";
}
private string ExtractDatePath(string topic, string payload)
{
try
{
var doc = JsonDocument.Parse(payload);
var ts = topic switch
{
var t when t == _kafkaOptions.Topics.ObservationRecorded => doc.RootElement.GetProperty("recordedAt").GetDateTimeOffset(),
var t when t == _kafkaOptions.Topics.AlertGenerated => doc.RootElement.GetProperty("triggeredAt").GetDateTimeOffset(),
var t when t == _kafkaOptions.Topics.EncounterStatusChanged => doc.RootElement.GetProperty("changedAt").GetDateTimeOffset(),
_ => DateTimeOffset.UtcNow,
};
return $"{ts.Year:D4}/{ts.Month:D2}/{ts.Day:D2}";
}
catch
{
// Malformed payload: use today so the event is not lost.
var now = DateTimeOffset.UtcNow;
return $"{now.Year:D4}/{now.Month:D2}/{now.Day:D2}";
}
}
// ---------------------------------------------------------------------------
// Payload deserializers — each reads only the fields needed for the Parquet row.
// ---------------------------------------------------------------------------
private static ObservationRow ParseObservationRow(BufferedEvent e, int partition)
{
var d = JsonDocument.Parse(e.Payload).RootElement;
return new ObservationRow(
ObservationId : d.GetProperty("observationId").GetString() ?? "",
EncounterId : d.GetProperty("encounterId").GetString() ?? "",
PatientId : d.GetProperty("patientId").GetString() ?? "",
Mrn : d.TryGetProperty("mrn", out var mrn) ? mrn.GetString() ?? "" : "",
ObservationCode : d.GetProperty("observationCode").GetString() ?? "",
Value : d.GetProperty("value").GetDouble(),
Unit : d.GetProperty("unit").GetString() ?? "",
Source : d.GetProperty("source").GetString() ?? "",
RecordedAt : d.GetProperty("recordedAt").GetString() ?? "",
KafkaPartition : partition,
KafkaOffset : e.Offset
);
}
private static AlertRow ParseAlertRow(BufferedEvent e, int partition)
{
var d = JsonDocument.Parse(e.Payload).RootElement;
return new AlertRow(
AlertId : d.GetProperty("alertId").GetString() ?? "",
EncounterId : d.GetProperty("encounterId").GetString() ?? "",
PatientId : d.GetProperty("patientId").GetString() ?? "",
AlertType : d.GetProperty("alertType").GetString() ?? "",
Severity : d.GetProperty("severity").GetString() ?? "",
Details : d.GetProperty("details").GetString() ?? "",
TriggeredAt : d.GetProperty("triggeredAt").GetString() ?? "",
KafkaPartition : partition,
KafkaOffset : e.Offset
);
}
private static EncounterStatusRow ParseEncounterRow(BufferedEvent e, int partition)
{
var d = JsonDocument.Parse(e.Payload).RootElement;
return new EncounterStatusRow(
EncounterId : d.GetProperty("encounterId").GetString() ?? "",
PatientId : d.GetProperty("patientId").GetString() ?? "",
PreviousStatus : d.GetProperty("previousStatus").GetString() ?? "",
NewStatus : d.GetProperty("newStatus").GetString() ?? "",
ChangedAt : d.GetProperty("changedAt").GetString() ?? "",
KafkaPartition : partition,
KafkaOffset : e.Offset
);
}
// ---------------------------------------------------------------------------
// MinIO upload
// ---------------------------------------------------------------------------
private async Task UploadToMinioAsync(string objectKey, byte[] bytes, CancellationToken ct)
{
var client = MinioClientFactory.Build(_minioOpts);
var bucket = _opts.BucketName;
var exists = await client.BucketExistsAsync(
new BucketExistsArgs().WithBucket(bucket), ct);
if (!exists)
await client.MakeBucketAsync(new MakeBucketArgs().WithBucket(bucket), ct);
await client.PutObjectAsync(new PutObjectArgs()
.WithBucket(bucket)
.WithObject(objectKey)
.WithStreamData(new MemoryStream(bytes))
.WithObjectSize(bytes.Length)
.WithContentType("application/octet-stream"),
ct);
}
}
VigilCareClinicalAPI/DataLake/ParquetFileBuilder.cs
+102
View File
@@ -0,0 +1,102 @@
using Parquet;
using Parquet.Data;
using Parquet.Schema;
public static class ParquetFileBuilder
{
public static async Task<byte[]> BuildObservationsAsync(IReadOnlyList<ObservationRow> rows)
{
var schema = new ParquetSchema(
new DataField<string>("observation_id"),
new DataField<string>("encounter_id"),
new DataField<string>("patient_id"),
new DataField<string>("mrn"),
new DataField<string>("observation_code"),
new DataField<double>("value"),
new DataField<string>("unit"),
new DataField<string>("source"),
new DataField<string>("recorded_at"),
new DataField<int>("kafka_partition"),
new DataField<long>("kafka_offset")
);
using var ms = new MemoryStream();
using (var writer = await ParquetWriter.CreateAsync(schema, ms))
using (var rg = writer.CreateRowGroup())
{
var f = schema.DataFields;
await rg.WriteColumnAsync(new DataColumn(f[0], rows.Select(r => r.ObservationId).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[1], rows.Select(r => r.EncounterId).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[2], rows.Select(r => r.PatientId).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[3], rows.Select(r => r.Mrn).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[4], rows.Select(r => r.ObservationCode).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[5], rows.Select(r => r.Value).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[6], rows.Select(r => r.Unit).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[7], rows.Select(r => r.Source).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[8], rows.Select(r => r.RecordedAt).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[9], rows.Select(r => r.KafkaPartition).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[10], rows.Select(r => r.KafkaOffset).ToArray()));
}
return ms.ToArray();
}
public static async Task<byte[]> BuildAlertsAsync(IReadOnlyList<AlertRow> rows)
{
var schema = new ParquetSchema(
new DataField<string>("alert_id"),
new DataField<string>("encounter_id"),
new DataField<string>("patient_id"),
new DataField<string>("alert_type"),
new DataField<string>("severity"),
new DataField<string>("details"),
new DataField<string>("triggered_at"),
new DataField<int>("kafka_partition"),
new DataField<long>("kafka_offset")
);
using var ms = new MemoryStream();
using (var writer = await ParquetWriter.CreateAsync(schema, ms))
using (var rg = writer.CreateRowGroup())
{
var f = schema.DataFields;
await rg.WriteColumnAsync(new DataColumn(f[0], rows.Select(r => r.AlertId).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[1], rows.Select(r => r.EncounterId).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[2], rows.Select(r => r.PatientId).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[3], rows.Select(r => r.AlertType).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[4], rows.Select(r => r.Severity).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[5], rows.Select(r => r.Details).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[6], rows.Select(r => r.TriggeredAt).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[7], rows.Select(r => r.KafkaPartition).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[8], rows.Select(r => r.KafkaOffset).ToArray()));
}
return ms.ToArray();
}
public static async Task<byte[]> BuildEncountersAsync(IReadOnlyList<EncounterStatusRow> rows)
{
var schema = new ParquetSchema(
new DataField<string>("encounter_id"),
new DataField<string>("patient_id"),
new DataField<string>("previous_status"),
new DataField<string>("new_status"),
new DataField<string>("changed_at"),
new DataField<int>("kafka_partition"),
new DataField<long>("kafka_offset")
);
using var ms = new MemoryStream();
using (var writer = await ParquetWriter.CreateAsync(schema, ms))
using (var rg = writer.CreateRowGroup())
{
var f = schema.DataFields;
await rg.WriteColumnAsync(new DataColumn(f[0], rows.Select(r => r.EncounterId).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[1], rows.Select(r => r.PatientId).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[2], rows.Select(r => r.PreviousStatus).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[3], rows.Select(r => r.NewStatus).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[4], rows.Select(r => r.ChangedAt).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[5], rows.Select(r => r.KafkaPartition).ToArray()));
await rg.WriteColumnAsync(new DataColumn(f[6], rows.Select(r => r.KafkaOffset).ToArray()));
}
return ms.ToArray();
}
}
VigilCareClinicalAPI/Models/Records/Alert/AlertRow.cs
+11
View File
@@ -0,0 +1,11 @@
public sealed record AlertRow(
string AlertId,
string EncounterId,
string PatientId,
string AlertType,
string Severity,
string Details,
string TriggeredAt,
int KafkaPartition,
long KafkaOffset
);
VigilCareClinicalAPI/Models/Records/Encounter/EncounterStatusRow.cs
+9
View File
@@ -0,0 +1,9 @@
public sealed record EncounterStatusRow(
string EncounterId,
string PatientId,
string PreviousStatus,
string NewStatus,
string ChangedAt,
int KafkaPartition,
long KafkaOffset
);
VigilCareClinicalAPI/Models/Records/Observation/ObservationRow.cs
+13
View File
@@ -0,0 +1,13 @@
public sealed record ObservationRow(
string ObservationId,
string EncounterId,
string PatientId,
string Mrn,
string ObservationCode,
double Value,
string Unit,
string Source,
string RecordedAt,
int KafkaPartition,
long KafkaOffset
);
VigilCareClinicalAPI/Program.cs
+4
View File
@@ -55,6 +55,9 @@ try
builder.Services.Configure<ReconciliationJobOptions>(
builder.Configuration.GetSection(ReconciliationJobOptions.Section));
builder.Services.Configure<DataLakeOptions>(
builder.Configuration.GetSection(DataLakeOptions.Section));
builder.Services.AddScoped<IPatientService, PatientService>();
builder.Services.AddScoped<IEncounterService, EncounterService>();
builder.Services.AddScoped<IAlertThresholdService, AlertThresholdService>();
@@ -84,6 +87,7 @@ try
builder.Services.AddHostedService<AlertsUnacknowledgedCollector>();
builder.Services.AddHostedService<OutboxPendingCollector>();
builder.Services.AddHostedService<KafkaConsumerLagCollector>();
builder.Services.AddHostedService<DataLakeWriterService>();
builder.Services.AddControllers()
.AddJsonOptions(opts =>
VigilCareClinicalAPI/VigilCareClinicalAPI.csproj
+1
View File
@@ -18,6 +18,7 @@
</PackageReference>
<PackageReference Include="Minio" Version="6.0.3" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
<PackageReference Include="Parquet.Net" Version="4.24.0" />
<PackageReference Include="prometheus-net.AspNetCore" Version="8.2.1" />
<PackageReference Include="RabbitMQ.Client" Version="6.8.1" />
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
VigilCareClinicalAPI/appsettings.Testing.json
+4
View File
@@ -13,5 +13,9 @@
},
"RabbitMq": {
"PagingAckTimeoutMs": 5000
},
"DataLake": {
"FlushCount": 3,
"FlushIntervalSeconds": 10
}
}
VigilCareClinicalAPI/appsettings.json
+4
View File
@@ -74,5 +74,9 @@
"UnacknowledgedAlertThresholdMinutes": 30,
"PendingOrderThresholdHours": 4,
"NoObservationThresholdHours": 2
},
"DataLake": {
"FlushCount": 1000,
"FlushIntervalSeconds": 300
}
}