vigilcare-clinical/scripts/run-phase9-verification.sh

#!/usr/bin/env bash

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}"

TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}"
TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~DataLakePhase9Tests}"
BASE_URL="${BASE_URL:-http://localhost:5270}"
MINIO_ALIAS="${MINIO_ALIAS:-localvc}"
MINIO_URL="${MINIO_URL:-http://localhost:9005}"
MINIO_BUCKET="${MINIO_BUCKET:-vigilcare}"
MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-minioadmin}"
MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-minioadmin}"
KAFKA_GROUP="${KAFKA_GROUP:-data-lake-writer}"
KAFKA_SERVICE="${KAFKA_SERVICE:-kafka}"
KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:-localhost:9092}"

need() {
  command -v "$1" >/dev/null 2>&1 || {
    echo "Missing dependency: $1"
    exit 1
  }
}

optional() {
  command -v "$1" >/dev/null 2>&1
}

# Prefer MC_BIN, then user-local install, then first executable on PATH.
resolve_mc() {
  if [[ -n "${MC_BIN:-}" && -x "${MC_BIN}" ]]; then
    echo "${MC_BIN}"
    return 0
  fi
  local candidate="${HOME}/.local/bin/mc"
  if [[ -x "${candidate}" ]]; then
    echo "${candidate}"
    return 0
  fi
  if command -v mc >/dev/null 2>&1; then
    local path_mc
    path_mc="$(command -v mc)"
    if [[ -x "${path_mc}" ]]; then
      echo "${path_mc}"
      return 0
    fi
  fi
  return 1
}

# Prefer DUCKDB_BIN, then official install location, then PATH.
resolve_duckdb() {
  if [[ -n "${DUCKDB_BIN:-}" && -x "${DUCKDB_BIN}" ]]; then
    echo "${DUCKDB_BIN}"
    return 0
  fi
  local candidate="${HOME}/.duckdb/cli/latest/duckdb"
  if [[ -x "${candidate}" ]]; then
    echo "${candidate}"
    return 0
  fi
  if command -v duckdb >/dev/null 2>&1; then
    local path_duckdb
    path_duckdb="$(command -v duckdb)"
    if [[ -x "${path_duckdb}" ]]; then
      echo "${path_duckdb}"
      return 0
    fi
  fi
  return 1
}

need curl
need dotnet
need docker

echo "Phase 9 verification starting..."
echo "Repo root: ${ROOT_DIR}"

echo "[1/6] Preflight API"
api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/metrics" || true)"
[[ "${api_status}" == "200" ]] || {
  echo "API /metrics is not reachable at ${BASE_URL} (status=${api_status})"
  exit 1
}

echo "[2/6] Run Phase 9 integration tests"
dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}"

echo "[3/6] Verify Kafka consumer group appears"
group_list="$(docker compose -f "${COMPOSE_FILE}" exec -T "${KAFKA_SERVICE}" \
  /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server "${KAFKA_BOOTSTRAP}" --list)"
if ! grep -q "^${KAFKA_GROUP}$" <<< "${group_list}"; then
  echo "Consumer group '${KAFKA_GROUP}' not found."
  echo "Known groups:"
  echo "${group_list}"
  exit 1
fi

echo "[4/6] Describe Kafka consumer group offsets"
docker compose -f "${COMPOSE_FILE}" exec -T "${KAFKA_SERVICE}" \
  /opt/kafka/bin/kafka-consumer-groups.sh \
  --bootstrap-server "${KAFKA_BOOTSTRAP}" \
  --group "${KAFKA_GROUP}" \
  --describe

MC="$(resolve_mc || true)"
if [[ -z "${MC}" ]]; then
  echo "[5/6] Skipping MinIO checks: 'mc' not installed."
  echo "Install without sudo:"
  echo "  mkdir -p ~/.local/bin"
  echo "  curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o ~/.local/bin/mc"
  echo "  chmod +x ~/.local/bin/mc"
  echo "  export PATH=\"\$HOME/.local/bin:\$PATH\""
  echo
  echo "Phase 9 verification completed (tests + Kafka checks)."
  exit 0
fi

echo "[5/6] Verify Parquet objects exist in MinIO (using ${MC})"
"${MC}" alias set "${MINIO_ALIAS}" "${MINIO_URL}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null

check_prefix() {
  local prefix="$1"
  local count
  count="$("${MC}" ls --recursive "${MINIO_ALIAS}/${MINIO_BUCKET}/${prefix}" | awk 'END{print NR+0}')"
  if [[ "${count}" -le 0 ]]; then
    echo "No objects found under ${prefix}"
    exit 1
  fi
  echo "Found ${count} object(s) under ${prefix}"
}

check_prefix "observations/"
check_prefix "alerts/"
check_prefix "encounters/"

DUCKDB="$(resolve_duckdb || true)"
if [[ -z "${DUCKDB}" ]]; then
  echo "[6/6] Skipping DuckDB schema check: 'duckdb' not installed."
  echo "Install without sudo: curl https://install.duckdb.org | sh"
  echo "Then add to PATH: export PATH=\"\$HOME/.duckdb/cli/latest:\$PATH\""
  echo
  echo "Phase 9 verification completed (tests + Kafka + MinIO checks)."
  exit 0
fi

echo "[6/6] Verify Parquet schema with DuckDB (using ${DUCKDB})"
first_obs_rel="$("${MC}" ls --recursive "${MINIO_ALIAS}/${MINIO_BUCKET}/observations/" | awk '/STANDARD/{print $NF; exit}')"
[[ -n "${first_obs_rel}" ]] || {
  echo "Could not resolve an observations Parquet key."
  exit 1
}
first_obs_key="observations/${first_obs_rel}"

tmp_parquet="$(mktemp /tmp/vc-phase9-obs-XXXXXX.parquet)"
trap 'rm -f "${tmp_parquet}"' EXIT
"${MC}" cp "${MINIO_ALIAS}/${MINIO_BUCKET}/${first_obs_key}" "${tmp_parquet}" >/dev/null

"${DUCKDB}" -csv -c "DESCRIBE SELECT * FROM '${tmp_parquet}';" | tee /tmp/vc-phase9-duckdb-schema.txt >/dev/null

for col in observation_id encounter_id kafka_partition kafka_offset; do
  if ! grep -qE "^${col}," /tmp/vc-phase9-duckdb-schema.txt; then
    echo "Expected column '${col}' not found in Parquet schema."
    exit 1
  fi
done

echo
echo "Phase 9 verification checks passed."