#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}" TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}" TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~DataLakePhase9Tests}" BASE_URL="${BASE_URL:-http://localhost:5270}" MINIO_ALIAS="${MINIO_ALIAS:-localvc}" MINIO_URL="${MINIO_URL:-http://localhost:9005}" MINIO_BUCKET="${MINIO_BUCKET:-vigilcare}" MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-minioadmin}" MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-minioadmin}" KAFKA_GROUP="${KAFKA_GROUP:-data-lake-writer}" KAFKA_SERVICE="${KAFKA_SERVICE:-kafka}" KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:-localhost:9092}" need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing dependency: $1" exit 1 } } optional() { command -v "$1" >/dev/null 2>&1 } # Prefer MC_BIN, then user-local install, then first executable on PATH. resolve_mc() { if [[ -n "${MC_BIN:-}" && -x "${MC_BIN}" ]]; then echo "${MC_BIN}" return 0 fi local candidate="${HOME}/.local/bin/mc" if [[ -x "${candidate}" ]]; then echo "${candidate}" return 0 fi if command -v mc >/dev/null 2>&1; then local path_mc path_mc="$(command -v mc)" if [[ -x "${path_mc}" ]]; then echo "${path_mc}" return 0 fi fi return 1 } # Prefer DUCKDB_BIN, then official install location, then PATH. resolve_duckdb() { if [[ -n "${DUCKDB_BIN:-}" && -x "${DUCKDB_BIN}" ]]; then echo "${DUCKDB_BIN}" return 0 fi local candidate="${HOME}/.duckdb/cli/latest/duckdb" if [[ -x "${candidate}" ]]; then echo "${candidate}" return 0 fi if command -v duckdb >/dev/null 2>&1; then local path_duckdb path_duckdb="$(command -v duckdb)" if [[ -x "${path_duckdb}" ]]; then echo "${path_duckdb}" return 0 fi fi return 1 } need curl need dotnet need docker echo "Phase 9 verification starting..." echo "Repo root: ${ROOT_DIR}" echo "[1/6] Preflight API" api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/metrics" || true)" [[ "${api_status}" == "200" ]] || { echo "API /metrics is not reachable at ${BASE_URL} (status=${api_status})" exit 1 } echo "[2/6] Run Phase 9 integration tests" dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}" echo "[3/6] Verify Kafka consumer group appears" group_list="$(docker compose -f "${COMPOSE_FILE}" exec -T "${KAFKA_SERVICE}" \ /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server "${KAFKA_BOOTSTRAP}" --list)" if ! grep -q "^${KAFKA_GROUP}$" <<< "${group_list}"; then echo "Consumer group '${KAFKA_GROUP}' not found." echo "Known groups:" echo "${group_list}" exit 1 fi echo "[4/6] Describe Kafka consumer group offsets" docker compose -f "${COMPOSE_FILE}" exec -T "${KAFKA_SERVICE}" \ /opt/kafka/bin/kafka-consumer-groups.sh \ --bootstrap-server "${KAFKA_BOOTSTRAP}" \ --group "${KAFKA_GROUP}" \ --describe MC="$(resolve_mc || true)" if [[ -z "${MC}" ]]; then echo "[5/6] Skipping MinIO checks: 'mc' not installed." echo "Install without sudo:" echo " mkdir -p ~/.local/bin" echo " curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o ~/.local/bin/mc" echo " chmod +x ~/.local/bin/mc" echo " export PATH=\"\$HOME/.local/bin:\$PATH\"" echo echo "Phase 9 verification completed (tests + Kafka checks)." exit 0 fi echo "[5/6] Verify Parquet objects exist in MinIO (using ${MC})" "${MC}" alias set "${MINIO_ALIAS}" "${MINIO_URL}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null check_prefix() { local prefix="$1" local count count="$("${MC}" ls --recursive "${MINIO_ALIAS}/${MINIO_BUCKET}/${prefix}" | awk 'END{print NR+0}')" if [[ "${count}" -le 0 ]]; then echo "No objects found under ${prefix}" exit 1 fi echo "Found ${count} object(s) under ${prefix}" } check_prefix "observations/" check_prefix "alerts/" check_prefix "encounters/" DUCKDB="$(resolve_duckdb || true)" if [[ -z "${DUCKDB}" ]]; then echo "[6/6] Skipping DuckDB schema check: 'duckdb' not installed." echo "Install without sudo: curl https://install.duckdb.org | sh" echo "Then add to PATH: export PATH=\"\$HOME/.duckdb/cli/latest:\$PATH\"" echo echo "Phase 9 verification completed (tests + Kafka + MinIO checks)." exit 0 fi echo "[6/6] Verify Parquet schema with DuckDB (using ${DUCKDB})" first_obs_rel="$("${MC}" ls --recursive "${MINIO_ALIAS}/${MINIO_BUCKET}/observations/" | awk '/STANDARD/{print $NF; exit}')" [[ -n "${first_obs_rel}" ]] || { echo "Could not resolve an observations Parquet key." exit 1 } first_obs_key="observations/${first_obs_rel}" tmp_parquet="$(mktemp /tmp/vc-phase9-obs-XXXXXX.parquet)" trap 'rm -f "${tmp_parquet}"' EXIT "${MC}" cp "${MINIO_ALIAS}/${MINIO_BUCKET}/${first_obs_key}" "${tmp_parquet}" >/dev/null "${DUCKDB}" -csv -c "DESCRIBE SELECT * FROM '${tmp_parquet}';" | tee /tmp/vc-phase9-duckdb-schema.txt >/dev/null for col in observation_id encounter_id kafka_partition kafka_offset; do if ! grep -qE "^${col}," /tmp/vc-phase9-duckdb-schema.txt; then echo "Expected column '${col}' not found in Parquet schema." exit 1 fi done echo echo "Phase 9 verification checks passed."