tests: MinIO Data Lake Writer (Parquet, Partitioned) test

scripts/run-phase9-verification.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}"
+
+TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}"
+TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~DataLakePhase9Tests}"
+BASE_URL="${BASE_URL:-http://localhost:5270}"
+MINIO_ALIAS="${MINIO_ALIAS:-localvc}"
+MINIO_URL="${MINIO_URL:-http://localhost:9005}"
+MINIO_BUCKET="${MINIO_BUCKET:-vigilcare}"
+MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-minioadmin}"
+MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-minioadmin}"
+KAFKA_GROUP="${KAFKA_GROUP:-data-lake-writer}"
+KAFKA_SERVICE="${KAFKA_SERVICE:-kafka}"
+KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:-localhost:9092}"
+
+need() {
+  command -v "$1" >/dev/null 2>&1 || {
+    echo "Missing dependency: $1"
+    exit 1
+  }
+}
+
+optional() {
+  command -v "$1" >/dev/null 2>&1
+}
+
+# Prefer MC_BIN, then user-local install, then first executable on PATH.
+resolve_mc() {
+  if [[ -n "${MC_BIN:-}" && -x "${MC_BIN}" ]]; then
+    echo "${MC_BIN}"
+    return 0
+  fi
+  local candidate="${HOME}/.local/bin/mc"
+  if [[ -x "${candidate}" ]]; then
+    echo "${candidate}"
+    return 0
+  fi
+  if command -v mc >/dev/null 2>&1; then
+    local path_mc
+    path_mc="$(command -v mc)"
+    if [[ -x "${path_mc}" ]]; then
+      echo "${path_mc}"
+      return 0
+    fi
+  fi
+  return 1
+}
+
+# Prefer DUCKDB_BIN, then official install location, then PATH.
+resolve_duckdb() {
+  if [[ -n "${DUCKDB_BIN:-}" && -x "${DUCKDB_BIN}" ]]; then
+    echo "${DUCKDB_BIN}"
+    return 0
+  fi
+  local candidate="${HOME}/.duckdb/cli/latest/duckdb"
+  if [[ -x "${candidate}" ]]; then
+    echo "${candidate}"
+    return 0
+  fi
+  if command -v duckdb >/dev/null 2>&1; then
+    local path_duckdb
+    path_duckdb="$(command -v duckdb)"
+    if [[ -x "${path_duckdb}" ]]; then
+      echo "${path_duckdb}"
+      return 0
+    fi
+  fi
+  return 1
+}
+
+need curl
+need dotnet
+need docker
+
+echo "Phase 9 verification starting..."
+echo "Repo root: ${ROOT_DIR}"
+
+echo "[1/6] Preflight API"
+api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/metrics" || true)"
+[[ "${api_status}" == "200" ]] || {
+  echo "API /metrics is not reachable at ${BASE_URL} (status=${api_status})"
+  exit 1
+}
+
+echo "[2/6] Run Phase 9 integration tests"
+dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}"
+
+echo "[3/6] Verify Kafka consumer group appears"
+group_list="$(docker compose -f "${COMPOSE_FILE}" exec -T "${KAFKA_SERVICE}" \
+  /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server "${KAFKA_BOOTSTRAP}" --list)"
+if ! grep -q "^${KAFKA_GROUP}$" <<< "${group_list}"; then
+  echo "Consumer group '${KAFKA_GROUP}' not found."
+  echo "Known groups:"
+  echo "${group_list}"
+  exit 1
+fi
+
+echo "[4/6] Describe Kafka consumer group offsets"
+docker compose -f "${COMPOSE_FILE}" exec -T "${KAFKA_SERVICE}" \
+  /opt/kafka/bin/kafka-consumer-groups.sh \
+  --bootstrap-server "${KAFKA_BOOTSTRAP}" \
+  --group "${KAFKA_GROUP}" \
+  --describe
+
+MC="$(resolve_mc || true)"
+if [[ -z "${MC}" ]]; then
+  echo "[5/6] Skipping MinIO checks: 'mc' not installed."
+  echo "Install without sudo:"
+  echo "  mkdir -p ~/.local/bin"
+  echo "  curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o ~/.local/bin/mc"
+  echo "  chmod +x ~/.local/bin/mc"
+  echo "  export PATH=\"\$HOME/.local/bin:\$PATH\""
+  echo
+  echo "Phase 9 verification completed (tests + Kafka checks)."
+  exit 0
+fi
+
+echo "[5/6] Verify Parquet objects exist in MinIO (using ${MC})"
+"${MC}" alias set "${MINIO_ALIAS}" "${MINIO_URL}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null
+
+check_prefix() {
+  local prefix="$1"
+  local count
+  count="$("${MC}" ls --recursive "${MINIO_ALIAS}/${MINIO_BUCKET}/${prefix}" | awk 'END{print NR+0}')"
+  if [[ "${count}" -le 0 ]]; then
+    echo "No objects found under ${prefix}"
+    exit 1
+  fi
+  echo "Found ${count} object(s) under ${prefix}"
+}
+
+check_prefix "observations/"
+check_prefix "alerts/"
+check_prefix "encounters/"
+
+DUCKDB="$(resolve_duckdb || true)"
+if [[ -z "${DUCKDB}" ]]; then
+  echo "[6/6] Skipping DuckDB schema check: 'duckdb' not installed."
+  echo "Install without sudo: curl https://install.duckdb.org | sh"
+  echo "Then add to PATH: export PATH=\"\$HOME/.duckdb/cli/latest:\$PATH\""
+  echo
+  echo "Phase 9 verification completed (tests + Kafka + MinIO checks)."
+  exit 0
+fi
+
+echo "[6/6] Verify Parquet schema with DuckDB (using ${DUCKDB})"
+first_obs_rel="$("${MC}" ls --recursive "${MINIO_ALIAS}/${MINIO_BUCKET}/observations/" | awk '/STANDARD/{print $NF; exit}')"
+[[ -n "${first_obs_rel}" ]] || {
+  echo "Could not resolve an observations Parquet key."
+  exit 1
+}
+first_obs_key="observations/${first_obs_rel}"
+
+tmp_parquet="$(mktemp /tmp/vc-phase9-obs-XXXXXX.parquet)"
+trap 'rm -f "${tmp_parquet}"' EXIT
+"${MC}" cp "${MINIO_ALIAS}/${MINIO_BUCKET}/${first_obs_key}" "${tmp_parquet}" >/dev/null
+
+"${DUCKDB}" -csv -c "DESCRIBE SELECT * FROM '${tmp_parquet}';" | tee /tmp/vc-phase9-duckdb-schema.txt >/dev/null
+
+for col in observation_id encounter_id kafka_partition kafka_offset; do
+  if ! grep -qE "^${col}," /tmp/vc-phase9-duckdb-schema.txt; then
+    echo "Expected column '${col}' not found in Parquet schema."
+    exit 1
+  fi
+done
+
+echo
+echo "Phase 9 verification checks passed."