feat: add backup alert generation engine

backend/src/media_library_viewer_api/services/backup_alert_engine.py

@@ -0,0 +1,111 @@
+import statistics
+from typing import Any
+
+
+def generate_alerts_for_run(
+    run: dict[str, Any],
+    previous_runs: list[dict[str, Any]],
+    job: dict[str, Any] | None,
+) -> list[dict[str, Any]]:
+    alerts = []
+    job_id = run["job_id"]
+    job_name = job["name"] if job else job_id
+
+    # 1. Failed status alert
+    if run["status"] == "failure":
+        alerts.append({
+            "job_id": job_id,
+            "run_id": run["id"],
+            "alert_type": "failed_status",
+            "severity": "critical",
+            "message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
+        })
+
+    # 2. Anomaly size alert
+    bytes_transferred = run.get("bytes_transferred")
+    if bytes_transferred is not None and previous_runs:
+        successful_runs = [r for r in previous_runs if r["status"] == "success" and r.get("bytes_transferred") is not None]
+        if len(successful_runs) >= 3:
+            sizes = [r["bytes_transferred"] for r in successful_runs[-7:]]
+            median_size = statistics.median(sizes)
+            if median_size > 0:
+                ratio = bytes_transferred / median_size
+                if bytes_transferred == 0:
+                    alerts.append({
+                        "job_id": job_id,
+                        "run_id": run["id"],
+                        "alert_type": "anomaly_size",
+                        "severity": "warning",
+                        "message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
+                    })
+                elif ratio < 0.1 or ratio > 3.0:
+                    alerts.append({
+                        "job_id": job_id,
+                        "run_id": run["id"],
+                        "alert_type": "anomaly_size",
+                        "severity": "warning",
+                        "message": f"Backup job '{job_name}' size anomaly: {bytes_transferred} bytes (median: {median_size})",
+                    })
+
+    # 3. Anomaly duration alert
+    duration_ms = run.get("duration_ms")
+    if duration_ms is not None and previous_runs:
+        successful_runs = [r for r in previous_runs if r["status"] == "success" and r.get("duration_ms") is not None]
+        if len(successful_runs) >= 3:
+            durations = [r["duration_ms"] for r in successful_runs[-7:]]
+            median_duration = statistics.median(durations)
+            if median_duration > 0 and duration_ms / median_duration > 3.0:
+                alerts.append({
+                    "job_id": job_id,
+                    "run_id": run["id"],
+                    "alert_type": "anomaly_duration",
+                    "severity": "warning",
+                    "message": f"Backup job '{job_name}' duration anomaly: {duration_ms}ms (median: {median_duration}ms)",
+                })
+
+    return alerts
+
+
+def check_missed_schedules(
+    jobs: list[dict[str, Any]],
+    get_latest_run: callable,
+    existing_alerts: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    alerts = []
+    import time
+    now = int(time.time())
+
+    for job in jobs:
+        interval = job.get("schedule_interval_seconds")
+        if not interval:
+            continue
+
+        latest_run = get_latest_run(job["id"])
+        if not latest_run:
+            # No runs ever — alert if job is older than interval * 1.5
+            if now - job["created_at"] > interval * 1.5:
+                alerts.append({
+                    "job_id": job["id"],
+                    "run_id": None,
+                    "alert_type": "missed_schedule",
+                    "severity": "warning",
+                    "message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
+                })
+        else:
+            last_run_time = latest_run["started_at"]
+            if now - last_run_time > interval * 1.5:
+                # Check if there's already an unresolved missed_schedule alert
+                has_open_alert = any(
+                    a["alert_type"] == "missed_schedule" and a["resolved_at"] is None
+                    for a in existing_alerts if a["job_id"] == job["id"]
+                )
+                if not has_open_alert:
+                    alerts.append({
+                        "job_id": job["id"],
+                        "run_id": None,
+                        "alert_type": "missed_schedule",
+                        "severity": "warning",
+                        "message": f"Backup job '{job['name']}' missed schedule: last run at {last_run_time} (expected every {interval}s)",
+                    })
+
+    return alerts

backend/tests/test_backups.py

@@ -55,3 +55,20 @@ def test_create_job_and_run():
         })
         assert run["job_id"] == job["id"]
         assert run["status"] == "success"
+
+
+def test_alert_failed_status():
+    from media_library_viewer_api.services.backup_alert_engine import generate_alerts_for_run
+
+    run = {
+        "id": "run-1",
+        "job_id": "job-1",
+        "status": "failure",
+        "bytes_transferred": 0,
+        "duration_ms": 1000,
+        "started_at": 1715392800,
+    }
+    alerts = generate_alerts_for_run(run, [], None)
+    assert len(alerts) == 1
+    assert alerts[0]["alert_type"] == "failed_status"
+    assert alerts[0]["severity"] == "critical"