335 lines
12 KiB
Bash
335 lines
12 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
COMPOSE_FILE="${COMPOSE_FILE:-${ROOT_DIR}/docker-compose.yml}"
|
|
|
|
BASE_URL="${BASE_URL:-http://localhost:5270}"
|
|
ES_URL="${ES_URL:-http://localhost:9200}"
|
|
RABBITMQ_MGMT_URL="${RABBITMQ_MGMT_URL:-http://localhost:15674}"
|
|
RABBITMQ_USER="${RABBITMQ_USER:-guest}"
|
|
RABBITMQ_PASS="${RABBITMQ_PASS:-guest}"
|
|
REDIS_PORT="${REDIS_PORT:-6382}"
|
|
|
|
PGHOST="${PGHOST:-localhost}"
|
|
PGPORT="${PGPORT:-5436}"
|
|
PGDATABASE="${PGDATABASE:-vigilcare}"
|
|
PGUSER="${PGUSER:-postgres}"
|
|
PGPASSWORD="${PGPASSWORD:-password}"
|
|
|
|
TEST_PROJECT="${TEST_PROJECT:-${ROOT_DIR}/VigilCareClinicalAPI.Tests/VigilCareClinicalAPI.Tests.csproj}"
|
|
TEST_FILTER="${TEST_FILTER:-FullyQualifiedName~WarningAlertTests|FullyQualifiedName~OrderLifecycleTests|FullyQualifiedName~ValidationTests}"
|
|
FULL_TEST="${FULL_TEST:-0}"
|
|
|
|
WARNING_WAIT_SECS="${WARNING_WAIT_SECS:-45}"
|
|
INDEX_WAIT_SECS="${INDEX_WAIT_SECS:-60}"
|
|
WARNING_CONSUMER_GROUP="${WARNING_CONSUMER_GROUP:-warning-evaluator}"
|
|
ES_CONSUMER_GROUP="${ES_CONSUMER_GROUP:-es-indexer}"
|
|
|
|
SCRIPT_RUN_ID="$(date -u +"%Y%m%d%H%M%S")"
|
|
RECORDED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
|
|
|
TMP_FILES=()
|
|
cleanup() {
|
|
local f
|
|
for f in "${TMP_FILES[@]}"; do
|
|
rm -f "${f}" "${f}.status" 2>/dev/null || true
|
|
done
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
need() {
|
|
command -v "$1" >/dev/null 2>&1 || {
|
|
echo "Missing dependency: $1"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
need curl
|
|
need jq
|
|
need dotnet
|
|
|
|
if ! command -v docker >/dev/null 2>&1 || [[ ! -f "${COMPOSE_FILE}" ]]; then
|
|
echo "Missing dependency: docker compose (${COMPOSE_FILE})"
|
|
exit 1
|
|
fi
|
|
|
|
compose() {
|
|
docker compose -f "${COMPOSE_FILE}" "$@"
|
|
}
|
|
|
|
kafka_exec() {
|
|
compose exec -T kafka "$@"
|
|
}
|
|
|
|
redis_cmd() {
|
|
if command -v redis-cli >/dev/null 2>&1; then
|
|
redis-cli -p "${REDIS_PORT}" "$@"
|
|
else
|
|
compose exec -T redis redis-cli "$@"
|
|
fi
|
|
}
|
|
|
|
psql_cmd() {
|
|
local sql="$1"
|
|
if command -v psql >/dev/null 2>&1; then
|
|
PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
|
|
else
|
|
compose exec -T postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -tAc "${sql}"
|
|
fi
|
|
}
|
|
|
|
request() {
|
|
local method="$1"
|
|
local url="$2"
|
|
local body="${3:-}"
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
TMP_FILES+=("${tmp}")
|
|
local status
|
|
|
|
if [[ -n "${body}" ]]; then
|
|
status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}" \
|
|
-H "Content-Type: application/json" -d "${body}")"
|
|
else
|
|
status="$(curl -sS -o "${tmp}" -w "%{http_code}" -X "${method}" "${url}")"
|
|
fi
|
|
|
|
echo "${status}" > "${tmp}.status"
|
|
echo "${tmp}"
|
|
}
|
|
|
|
assert_status() {
|
|
local expected="$1"
|
|
local body_file="$2"
|
|
local status
|
|
status="$(<"${body_file}.status")"
|
|
if [[ "${status}" != "${expected}" ]]; then
|
|
echo "Expected HTTP ${expected}, got ${status}"
|
|
cat "${body_file}"
|
|
echo
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
rabbit_api() {
|
|
curl -sS -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" "${RABBITMQ_MGMT_URL}/api/${1}"
|
|
}
|
|
|
|
queue_stat() {
|
|
local queue="$1"
|
|
local field="$2"
|
|
rabbit_api "queues/%2F/${queue}" | jq -r "${field} // 0"
|
|
}
|
|
|
|
consumer_group_lag() {
|
|
local group="$1"
|
|
kafka_exec /opt/kafka/bin/kafka-consumer-groups.sh \
|
|
--bootstrap-server localhost:9092 \
|
|
--describe \
|
|
--group "${group}" 2>/dev/null | \
|
|
awk 'NR > 1 && $1 != "" { sum += $6 } END { print sum + 0 }'
|
|
}
|
|
|
|
wait_for_consumer_lag_zero() {
|
|
local group="$1"
|
|
local max_secs="$2"
|
|
local elapsed=0
|
|
local lag="unknown"
|
|
while (( elapsed < max_secs )); do
|
|
lag="$(consumer_group_lag "${group}")"
|
|
if [[ "${lag}" == "0" ]]; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
echo "Consumer group ${group} lag did not reach zero within ${max_secs}s (lag=${lag})"
|
|
exit 1
|
|
}
|
|
|
|
es_alert_hits() {
|
|
local alert_type="$1"
|
|
local encounter_id="$2"
|
|
local payload
|
|
payload="$(jq -nc \
|
|
--arg type "${alert_type}" \
|
|
--arg enc "${encounter_id}" \
|
|
'{query:{bool:{must:[{term:{alertType:$type}},{term:{encounterId:$enc}}]}},size:0,track_total_hits:true}')"
|
|
local body
|
|
body="$(curl -sS "${ES_URL}/clinical_alerts/_search" \
|
|
-H "Content-Type: application/json" \
|
|
-d "${payload}")"
|
|
jq -r '.hits.total.value // .hits.total // 0' <<< "${body}"
|
|
}
|
|
|
|
echo "Phase 11 verification starting..."
|
|
echo "Repo root: ${ROOT_DIR}"
|
|
echo "API: ${BASE_URL}"
|
|
|
|
echo "[1/10] Preflight API, Elasticsearch, PostgreSQL, Redis, RabbitMQ, Kafka"
|
|
api_status="$(curl -sS -o /dev/null -w "%{http_code}" "${BASE_URL}/api/v1/alert-thresholds" || true)"
|
|
es_status="$(curl -sS -o /dev/null -w "%{http_code}" "${ES_URL}/_cluster/health" || true)"
|
|
rabbit_status="$(curl -sS -o /dev/null -w "%{http_code}" -u "${RABBITMQ_USER}:${RABBITMQ_PASS}" "${RABBITMQ_MGMT_URL}/api/overview" || true)"
|
|
[[ "${api_status}" == "200" ]] || { echo "API not ready (${api_status}) — run docker compose up -d and dotnet run"; exit 1; }
|
|
[[ "${es_status}" == "200" ]] || { echo "Elasticsearch not ready (${es_status})"; exit 1; }
|
|
[[ "${rabbit_status}" == "200" ]] || { echo "RabbitMQ management API not ready (${rabbit_status})"; exit 1; }
|
|
redis_cmd PING >/dev/null || { echo "Redis not reachable on port ${REDIS_PORT}"; exit 1; }
|
|
psql_cmd "SELECT 1" >/dev/null || { echo "PostgreSQL not reachable"; exit 1; }
|
|
kafka_exec /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null || {
|
|
echo "Kafka not ready"
|
|
exit 1
|
|
}
|
|
redis_cmd EXISTS "threshold:HEART_RATE" | grep -q '^1$' || {
|
|
echo "Missing Redis threshold:HEART_RATE — restart API to run ThresholdCacheLoader"
|
|
exit 1
|
|
}
|
|
echo "OK: infrastructure preflight passed"
|
|
|
|
echo "[2/10] Create patient and active encounter"
|
|
patient_payload="$(jq -nc \
|
|
--arg fn "Phase11" \
|
|
--arg ln "Verify${SCRIPT_RUN_ID}" \
|
|
'{firstName:$fn,lastName:$ln,dateOfBirth:"1985-06-01",gender:"M"}')"
|
|
patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${patient_payload}")"
|
|
assert_status "201" "${patient_resp}"
|
|
patient_id="$(jq -r '.data.id' "${patient_resp}")"
|
|
|
|
enc_payload='{"encounterType":"Inpatient","department":"ICU","attendingPhysician":"Dr. Phase11"}'
|
|
enc_resp="$(request POST "${BASE_URL}/api/v1/patients/${patient_id}/encounters" "${enc_payload}")"
|
|
assert_status "201" "${enc_resp}"
|
|
encounter_id="$(jq -r '.data.id' "${enc_resp}")"
|
|
echo "OK: patient=${patient_id} encounter=${encounter_id}"
|
|
|
|
echo "[3/10] Warning-range heart rate — synchronous ingest must not create critical alert"
|
|
paging_publish_before="$(queue_stat "alerts.paging.queue" '.message_stats.publish')"
|
|
|
|
hr_payload="$(jq -nc \
|
|
--arg recordedAt "${RECORDED_AT}" \
|
|
--arg key "phase11-warning-hr-${encounter_id}" \
|
|
'{observations:[{observationCode:"HEART_RATE",value:105,unit:"bpm",source:"DEVICE",recordedAt:$recordedAt,idempotencyKey:$key}]}')"
|
|
hr_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/observations" "${hr_payload}")"
|
|
assert_status "201" "${hr_resp}"
|
|
[[ "$(jq -r '.data.alertGenerated' "${hr_resp}")" == "false" ]] || {
|
|
echo "Expected alertGenerated=false for warning-range heart rate 105"
|
|
cat "${hr_resp}"
|
|
exit 1
|
|
}
|
|
echo "OK: ingest returned alertGenerated=false"
|
|
|
|
echo "[4/10] Wait for warning-evaluator and verify WARNING_HEART_RATE in PostgreSQL"
|
|
wait_for_consumer_lag_zero "${WARNING_CONSUMER_GROUP}" "${WARNING_WAIT_SECS}"
|
|
|
|
warning_count="$(psql_cmd "SELECT COUNT(*) FROM clinical_alerts WHERE encounter_id = '${encounter_id}' AND severity = 'WARNING' AND alert_type = 'WARNING_HEART_RATE' AND status = 'OPEN'")"
|
|
[[ "${warning_count}" == "1" ]] || {
|
|
echo "Expected 1 WARNING_HEART_RATE alert in PostgreSQL, found ${warning_count}"
|
|
psql_cmd "SELECT alert_type, severity, status FROM clinical_alerts WHERE encounter_id = '${encounter_id}'" || true
|
|
exit 1
|
|
}
|
|
echo "OK: WARNING_HEART_RATE alert exists in PostgreSQL"
|
|
|
|
echo "[5/10] Verify warning alert indexed in Elasticsearch"
|
|
wait_for_consumer_lag_zero "${ES_CONSUMER_GROUP}" "${INDEX_WAIT_SECS}"
|
|
es_hits="$(es_alert_hits "WARNING_HEART_RATE" "${encounter_id}")"
|
|
[[ "${es_hits}" -ge 1 ]] || {
|
|
echo "Expected Elasticsearch hit for WARNING_HEART_RATE on encounter ${encounter_id}, found ${es_hits}"
|
|
exit 1
|
|
}
|
|
echo "OK: clinical_alerts index contains WARNING_HEART_RATE"
|
|
|
|
echo "[6/10] Verify warning alert was not published to alerts.paging.queue"
|
|
paging_publish_after="$(queue_stat "alerts.paging.queue" '.message_stats.publish')"
|
|
[[ "${paging_publish_after}" == "${paging_publish_before}" ]] || {
|
|
echo "Expected no new paging publishes for WARNING alert (before=${paging_publish_before}, after=${paging_publish_after})"
|
|
exit 1
|
|
}
|
|
echo "OK: alerts.paging.queue publish count unchanged"
|
|
|
|
echo "[7/10] Orders API — create, list, record result, illegal cancel"
|
|
order_payload='{"orderType":"Lab","description":"CBC with differential","orderedBy":"Dr. Phase11"}'
|
|
order_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/orders" "${order_payload}")"
|
|
assert_status "201" "${order_resp}"
|
|
order_id="$(jq -r '.data.id' "${order_resp}")"
|
|
[[ "$(jq -r '.data.status' "${order_resp}")" == "Pending" ]] || {
|
|
echo "Expected order status Pending"
|
|
cat "${order_resp}"
|
|
exit 1
|
|
}
|
|
|
|
list_resp="$(request GET "${BASE_URL}/api/v1/encounters/${encounter_id}/orders")"
|
|
assert_status "200" "${list_resp}"
|
|
[[ "$(jq -r '.data.totalCount' "${list_resp}")" -ge 1 ]] || {
|
|
echo "Expected totalCount >= 1 on order list"
|
|
cat "${list_resp}"
|
|
exit 1
|
|
}
|
|
|
|
result_payload='{"resultSummary":"WBC 8.5, Hgb 14.2, Plt 210 — all within normal limits"}'
|
|
result_resp="$(request PATCH "${BASE_URL}/api/v1/orders/${order_id}/result" "${result_payload}")"
|
|
assert_status "200" "${result_resp}"
|
|
[[ "$(jq -r '.data.status' "${result_resp}")" == "Resulted" ]] || {
|
|
echo "Expected order status Resulted after recording result"
|
|
cat "${result_resp}"
|
|
exit 1
|
|
}
|
|
[[ "$(jq -r '.data.resultSummary' "${result_resp}")" == *"8.5"* ]] || {
|
|
echo "Expected resultSummary to contain lab values"
|
|
cat "${result_resp}"
|
|
exit 1
|
|
}
|
|
|
|
cancel_payload='{"status":"Cancelled"}'
|
|
cancel_resp="$(request PATCH "${BASE_URL}/api/v1/orders/${order_id}/status" "${cancel_payload}")"
|
|
assert_status "409" "${cancel_resp}"
|
|
[[ "$(jq -r '.error.code' "${cancel_resp}")" == "ILLEGAL_ORDER_STATUS_TRANSITION" ]] || {
|
|
echo "Expected ILLEGAL_ORDER_STATUS_TRANSITION on cancel of resulted order"
|
|
cat "${cancel_resp}"
|
|
exit 1
|
|
}
|
|
echo "OK: order lifecycle and illegal transition guard verified"
|
|
|
|
echo "[8/10] Input validation — FluentValidation 400 responses"
|
|
invalid_patient="$(jq -nc '{firstName:"",lastName:"Test",dateOfBirth:"1990-01-01",gender:"M"}')"
|
|
val_patient_resp="$(request POST "${BASE_URL}/api/v1/patients" "${invalid_patient}")"
|
|
assert_status "400" "${val_patient_resp}"
|
|
[[ "$(jq -r '.error.code' "${val_patient_resp}")" == "VALIDATION_ERROR" ]] || {
|
|
echo "Expected VALIDATION_ERROR for empty firstName"
|
|
cat "${val_patient_resp}"
|
|
exit 1
|
|
}
|
|
|
|
invalid_threshold='{"observationCode":"TEST_CODE","displayName":"Test","unit":"units","criticalLow":50,"warningLow":30}'
|
|
val_threshold_resp="$(request POST "${BASE_URL}/api/v1/alert-thresholds" "${invalid_threshold}")"
|
|
assert_status "400" "${val_threshold_resp}"
|
|
threshold_details="$(jq -c '.error.details // []' "${val_threshold_resp}")"
|
|
echo "${threshold_details}" | grep -qi "CriticalLow must be less than WarningLow" || {
|
|
echo "Expected threshold ordering validation message"
|
|
cat "${val_threshold_resp}"
|
|
exit 1
|
|
}
|
|
|
|
invalid_order='{"orderType":"Lab","description":"","orderedBy":"Dr. Test"}'
|
|
val_order_resp="$(request POST "${BASE_URL}/api/v1/encounters/${encounter_id}/orders" "${invalid_order}")"
|
|
assert_status "400" "${val_order_resp}"
|
|
[[ "$(jq -r '.error.code' "${val_order_resp}")" == "VALIDATION_ERROR" ]] || {
|
|
echo "Expected VALIDATION_ERROR for empty order description"
|
|
cat "${val_order_resp}"
|
|
exit 1
|
|
}
|
|
echo "OK: validation rejects empty firstName, invalid threshold order, empty order description"
|
|
|
|
echo "[9/10] Run Phase 11 integration tests"
|
|
dotnet test "${TEST_PROJECT}" --filter "${TEST_FILTER}"
|
|
|
|
if [[ "${FULL_TEST}" == "1" ]]; then
|
|
echo "[10/10] Run full test suite (FULL_TEST=1)"
|
|
dotnet test "${ROOT_DIR}/VigilCareClinicalAPI.sln" 2>/dev/null || dotnet test "${ROOT_DIR}"
|
|
else
|
|
echo "[10/10] Skipping full suite (set FULL_TEST=1 to run all 64 tests)"
|
|
fi
|
|
|
|
echo
|
|
echo "Phase 11 verification checks passed."
|