groups: - name: backup rules: - alert: BackupJobFailed expr: increase(manage_backup_runs_total{status="failed"}[1h]) > 0 for: 0m labels: severity: critical category: backup annotations: summary: "Backup job {{ $labels.job_name }} failed" description: "At least one failed backup run was recorded for job {{ $labels.job_name }} within the last hour." - alert: BackupJobStuck expr: time() - manage_backup_runs_last_success_timestamp > 86400 for: 5m labels: severity: warning category: backup annotations: summary: "Backup job {{ $labels.job_name }} has not succeeded in 24h" description: "No successful backup run has been recorded for job {{ $labels.job_name }} in the last 24 hours." - name: observability_health rules: - alert: PrometheusTargetMissing expr: up == 0 for: 2m labels: severity: warning category: observability annotations: summary: "Prometheus target {{ $labels.job }} / {{ $labels.instance }} is down" description: "The Prometheus scrape target for {{ $labels.job }} on {{ $labels.instance }} has been unreachable for more than 2 minutes." - alert: AlertmanagerDown expr: up{job="alertmanager"} == 0 for: 2m labels: severity: critical category: observability annotations: summary: "Alertmanager is down" description: "Alertmanager has been unreachable for more than 2 minutes. Alerts may not be delivered." - alert: GrafanaDown expr: up{job="grafana"} == 0 for: 2m labels: severity: warning category: observability annotations: summary: "Grafana is down" description: "Grafana has been unreachable for more than 2 minutes. Dashboards are unavailable."