# Production alert rules for VigilCare Clinical.
# Load via Prometheus rule_files / alerting config on the external Prometheus host.
# Metric names match VigilCareClinicalAPI/Observability/Metrics/ClinicalMetrics.cs.

groups:
  - name: vigilcare_production
    rules:
      - alert: VigilCareApiDown
        expr: up{job="vigilcare_api_prod"} == 0
        for: 2m
        labels:
          severity: critical
          service: clinical-api
        annotations:
          summary: "VigilCare Clinical API is down"
          description: "Prometheus cannot scrape vigilcare_api_prod for more than 2 minutes."

      - alert: VigilCareOutboxBacklog
        expr: outbox_pending_events{job="vigilcare_api_prod"} > 1000
        for: 5m
        labels:
          severity: warning
          service: clinical-api
        annotations:
          summary: "Outbox backlog is growing"
          description: "outbox_pending_events is {{ $value }} (threshold 1000). Alert delivery to Kafka consumers is delayed."

      - alert: VigilCareKafkaConsumerLag
        expr: kafka_consumer_lag{job="vigilcare_api_prod"} > 10000
        for: 10m
        labels:
          severity: warning
          service: clinical-api
        annotations:
          summary: "Kafka consumer lag high for {{ $labels.consumer_group }}"
          description: "consumer_group={{ $labels.consumer_group }} lag={{ $value }} (threshold 10000)."

      # Prefer the blackbox probe job if configured; otherwise treat scrape failure
      # of the API as covering liveness. This rule fires when the ready probe exists
      # and reports failure.
      - alert: VigilCareReadyCheckFailing
        expr: probe_success{job="vigilcare_api_ready_probe"} == 0
        for: 3m
        labels:
          severity: critical
          service: clinical-api
        annotations:
          summary: "API /health/ready is failing"
          description: "Blackbox probe of /health/ready has been failing for 3 minutes."

      # GatewayStaleDetectorService marks stale gateways OFFLINE; the collector
      # exposes them as ward_gateways_offline_gauge (site_code label).
      - alert: VigilCareGatewayStale
        expr: sum(ward_gateways_offline_gauge{job="vigilcare_api_prod"}) > 0
        for: 5m
        labels:
          severity: warning
          service: clinical-api
        annotations:
          summary: "One or more ward gateways are offline/degraded"
          description: "{{ $value }} gateway(s) offline. Stale threshold is GatewayMonitoring:StaleThresholdMinutes (default 10)."

      - alert: VigilCareCriticalAlertsUnacked
        expr: alerts_unacknowledged_gauge{job="vigilcare_api_prod"} > 0
        for: 5m
        labels:
          severity: critical
          service: clinical-api
        annotations:
          summary: "Unacknowledged CRITICAL alerts"
          description: "{{ $value }} CRITICAL alert(s) older than 5 minutes with no acknowledgment."