feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

2026-06-16 11:08:32 +00:00
parent 615e02e970
commit e2ad731b5f
80 changed files with 5020 additions and 3665 deletions
@@ -0,0 +1,53 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    monitor: manage
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets:
+          - localhost:9090
+
+  - job_name: manage-backend
+    static_configs:
+      - targets:
+          - backend:8000
+    metrics_path: /metrics
+    scrape_interval: 15s
+
+  - job_name: node-exporter
+    static_configs:
+      - targets:
+          - node-exporter:9100
+
+  - job_name: loki
+    static_configs:
+      - targets:
+          - loki:3100
+
+  - job_name: alertmanager
+    static_configs:
+      - targets:
+          - alertmanager:9093
+
+  - job_name: grafana
+    static_configs:
+      - targets:
+          - grafana:3000
+
+  - job_name: node-exporter-remote
+    file_sd_configs:
+      - files:
+          - /etc/prometheus/file-sd/node_exporter_targets.json
+        refresh_interval: 30s
@@ -0,0 +1,54 @@
+groups:
+  - name: backup
+    rules:
+      - alert: BackupJobFailed
+        expr: increase(manage_backup_runs_total{status="failed"}[1h]) > 0
+        for: 0m
+        labels:
+          severity: critical
+          category: backup
+        annotations:
+          summary: "Backup job {{ $labels.job_name }} failed"
+          description: "At least one failed backup run was recorded for job {{ $labels.job_name }} within the last hour."
+
+      - alert: BackupJobStuck
+        expr: time() - manage_backup_runs_last_success_timestamp > 86400
+        for: 5m
+        labels:
+          severity: warning
+          category: backup
+        annotations:
+          summary: "Backup job {{ $labels.job_name }} has not succeeded in 24h"
+          description: "No successful backup run has been recorded for job {{ $labels.job_name }} in the last 24 hours."
+
+  - name: observability_health
+    rules:
+      - alert: PrometheusTargetMissing
+        expr: up == 0
+        for: 2m
+        labels:
+          severity: warning
+          category: observability
+        annotations:
+          summary: "Prometheus target {{ $labels.job }} / {{ $labels.instance }} is down"
+          description: "The Prometheus scrape target for {{ $labels.job }} on {{ $labels.instance }} has been unreachable for more than 2 minutes."
+
+      - alert: AlertmanagerDown
+        expr: up{job="alertmanager"} == 0
+        for: 2m
+        labels:
+          severity: critical
+          category: observability
+        annotations:
+          summary: "Alertmanager is down"
+          description: "Alertmanager has been unreachable for more than 2 minutes. Alerts may not be delivered."
+
+      - alert: GrafanaDown
+        expr: up{job="grafana"} == 0
+        for: 2m
+        labels:
+          severity: warning
+          category: observability
+        annotations:
+          summary: "Grafana is down"
+          description: "Grafana has been unreachable for more than 2 minutes. Dashboards are unavailable."