72 lines
2.9 KiB
YAML
72 lines
2.9 KiB
YAML
# Production alert rules for VigilCare Clinical.
|
|
# Load via Prometheus rule_files / alerting config on the external Prometheus host.
|
|
# Metric names match VigilCareClinicalAPI/Observability/Metrics/ClinicalMetrics.cs.
|
|
|
|
groups:
|
|
- name: vigilcare_production
|
|
rules:
|
|
- alert: VigilCareApiDown
|
|
expr: up{job="vigilcare_api_prod"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
service: clinical-api
|
|
annotations:
|
|
summary: "VigilCare Clinical API is down"
|
|
description: "Prometheus cannot scrape vigilcare_api_prod for more than 2 minutes."
|
|
|
|
- alert: VigilCareOutboxBacklog
|
|
expr: outbox_pending_events{job="vigilcare_api_prod"} > 1000
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
service: clinical-api
|
|
annotations:
|
|
summary: "Outbox backlog is growing"
|
|
description: "outbox_pending_events is {{ $value }} (threshold 1000). Alert delivery to Kafka consumers is delayed."
|
|
|
|
- alert: VigilCareKafkaConsumerLag
|
|
expr: kafka_consumer_lag{job="vigilcare_api_prod"} > 10000
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
service: clinical-api
|
|
annotations:
|
|
summary: "Kafka consumer lag high for {{ $labels.consumer_group }}"
|
|
description: "consumer_group={{ $labels.consumer_group }} lag={{ $value }} (threshold 10000)."
|
|
|
|
# Prefer the blackbox probe job if configured; otherwise treat scrape failure
|
|
# of the API as covering liveness. This rule fires when the ready probe exists
|
|
# and reports failure.
|
|
- alert: VigilCareReadyCheckFailing
|
|
expr: probe_success{job="vigilcare_api_ready_probe"} == 0
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
service: clinical-api
|
|
annotations:
|
|
summary: "API /health/ready is failing"
|
|
description: "Blackbox probe of /health/ready has been failing for 3 minutes."
|
|
|
|
# GatewayStaleDetectorService marks stale gateways OFFLINE; the collector
|
|
# exposes them as ward_gateways_offline_gauge (site_code label).
|
|
- alert: VigilCareGatewayStale
|
|
expr: sum(ward_gateways_offline_gauge{job="vigilcare_api_prod"}) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
service: clinical-api
|
|
annotations:
|
|
summary: "One or more ward gateways are offline/degraded"
|
|
description: "{{ $value }} gateway(s) offline. Stale threshold is GatewayMonitoring:StaleThresholdMinutes (default 10)."
|
|
|
|
- alert: VigilCareCriticalAlertsUnacked
|
|
expr: alerts_unacknowledged_gauge{job="vigilcare_api_prod"} > 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: clinical-api
|
|
annotations:
|
|
summary: "Unacknowledged CRITICAL alerts"
|
|
description: "{{ $value }} CRITICAL alert(s) older than 5 minutes with no acknowledgment."
|