feat: add backup monitoring API endpoints
This commit is contained in:
@@ -14,6 +14,7 @@ from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settin
|
||||
from media_library_viewer_api.config import get_settings
|
||||
from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller
|
||||
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
|
||||
from media_library_viewer_api.routers import backups as backups_router
|
||||
from media_library_viewer_api.routers import dashboard, files, jobs, media, monitoring, tasks, users
|
||||
from media_library_viewer_api.routers.settings import router as settings_router
|
||||
|
||||
@@ -105,6 +106,7 @@ app.include_router(jobs.router)
|
||||
app.include_router(users.router)
|
||||
app.include_router(tasks.router)
|
||||
app.include_router(settings_router)
|
||||
app.include_router(backups_router.router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from ..auth import require_api_key
|
||||
from ..models.backups import (
|
||||
BackupAlertResponse,
|
||||
BackupReportRequest,
|
||||
BackupRunResponse,
|
||||
)
|
||||
from ..services.backup_alert_engine import generate_alerts_for_run
|
||||
from ..services.settings_store import SettingsStore, get_settings_store
|
||||
|
||||
router = APIRouter(prefix="/api/backups", tags=["backups"])
|
||||
|
||||
|
||||
def _get_or_create_job(store: SettingsStore, report: BackupReportRequest) -> dict[str, Any]:
|
||||
job = store.get_backup_job_by_name(report.name)
|
||||
if not job:
|
||||
job = store.upsert_backup_job({
|
||||
"name": report.name,
|
||||
"source": report.source,
|
||||
"target": report.target,
|
||||
"schedule_interval_seconds": report.schedule_interval_seconds,
|
||||
})
|
||||
elif report.schedule_interval_seconds:
|
||||
store.upsert_backup_job({
|
||||
"id": job["id"],
|
||||
"name": report.name,
|
||||
"source": report.source,
|
||||
"target": report.target,
|
||||
"schedule_interval_seconds": report.schedule_interval_seconds,
|
||||
})
|
||||
job = store.get_backup_job(job["id"])
|
||||
return job
|
||||
|
||||
|
||||
@router.post("/report")
|
||||
def post_backup_report(
|
||||
report: BackupReportRequest,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
_auth: str = Depends(require_api_key),
|
||||
) -> BackupRunResponse:
|
||||
job = _get_or_create_job(store, report)
|
||||
|
||||
# Check for duplicate (same job + started_at within 1s)
|
||||
existing_runs = store.list_backup_runs(job_id=job["id"], limit=5)
|
||||
started_at_ts = int(report.started_at.timestamp())
|
||||
for existing in existing_runs:
|
||||
if abs(existing["started_at"] - started_at_ts) <= 1:
|
||||
return BackupRunResponse(**existing)
|
||||
|
||||
run_data = {
|
||||
"job_id": job["id"],
|
||||
"started_at": started_at_ts,
|
||||
"ended_at": int(report.ended_at.timestamp()) if report.ended_at else None,
|
||||
"status": report.status,
|
||||
"bytes_transferred": report.bytes_transferred,
|
||||
"duration_ms": report.duration_ms,
|
||||
"error_message": report.error_message,
|
||||
"details": report.details,
|
||||
}
|
||||
run = store.create_backup_run(run_data)
|
||||
|
||||
# Generate alerts
|
||||
previous_runs = store.list_backup_runs(job_id=job["id"], status="success", limit=20)
|
||||
alerts = generate_alerts_for_run(run, previous_runs, job)
|
||||
for alert in alerts:
|
||||
store.create_backup_alert(alert)
|
||||
|
||||
# Resolve old alerts of the same type if this run is successful
|
||||
if report.status == "success":
|
||||
store.resolve_backup_alerts_for_job(job["id"], "failed_status")
|
||||
store.resolve_backup_alerts_for_job(job["id"], "anomaly_size")
|
||||
store.resolve_backup_alerts_for_job(job["id"], "anomaly_duration")
|
||||
|
||||
# Map details -> details_json for response model
|
||||
run["details_json"] = run.pop("details", None)
|
||||
return BackupRunResponse(**run)
|
||||
|
||||
|
||||
@router.post("/report/start")
|
||||
def post_backup_start(
|
||||
report: BackupReportRequest,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
_auth: str = Depends(require_api_key),
|
||||
) -> BackupRunResponse:
|
||||
job = _get_or_create_job(store, report)
|
||||
|
||||
run_data = {
|
||||
"job_id": job["id"],
|
||||
"started_at": int(report.started_at.timestamp()),
|
||||
"status": "in_progress",
|
||||
}
|
||||
run = store.create_backup_run(run_data)
|
||||
# Map details -> details_json for response model
|
||||
run["details_json"] = run.pop("details", None)
|
||||
return BackupRunResponse(**run)
|
||||
|
||||
|
||||
@router.get("/jobs")
|
||||
def get_backup_jobs(
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> list[dict[str, Any]]:
|
||||
jobs = store.list_backup_jobs()
|
||||
return jobs
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}")
|
||||
def get_backup_job(
|
||||
job_id: str,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> dict[str, Any]:
|
||||
job = store.get_backup_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Backup job not found")
|
||||
runs = store.list_backup_runs(job_id=job_id, limit=20)
|
||||
return {
|
||||
"job": job,
|
||||
"runs": runs,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/runs")
|
||||
def get_backup_runs(
|
||||
job_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> list[BackupRunResponse]:
|
||||
runs = store.list_backup_runs(job_id=job_id, status=status, limit=limit)
|
||||
return [BackupRunResponse(**run) for run in runs]
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}")
|
||||
def get_backup_run(
|
||||
run_id: str,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> BackupRunResponse:
|
||||
run = store.get_backup_run(run_id)
|
||||
if not run:
|
||||
raise HTTPException(status_code=404, detail="Backup run not found")
|
||||
return BackupRunResponse(**run)
|
||||
|
||||
|
||||
@router.get("/alerts")
|
||||
def get_backup_alerts(
|
||||
job_id: str | None = None,
|
||||
acknowledged: bool | None = None,
|
||||
severity: str | None = None,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> list[BackupAlertResponse]:
|
||||
alerts = store.list_backup_alerts(job_id=job_id, acknowledged=acknowledged, severity=severity)
|
||||
return [BackupAlertResponse(**alert) for alert in alerts]
|
||||
|
||||
|
||||
@router.post("/alerts/{alert_id}/acknowledge")
|
||||
def acknowledge_backup_alert(
|
||||
alert_id: str,
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> BackupAlertResponse:
|
||||
alert = store.acknowledge_backup_alert(alert_id)
|
||||
if not alert:
|
||||
raise HTTPException(status_code=404, detail="Alert not found")
|
||||
return BackupAlertResponse(**alert)
|
||||
+24
-133
@@ -1,155 +1,46 @@
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from media_library_viewer_api.main import app
|
||||
from media_library_viewer_api.services.settings_store import SettingsStore
|
||||
|
||||
|
||||
def test_backup_schema_created():
|
||||
def test_post_backup_report():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "test_settings.sqlite"
|
||||
store = SettingsStore(db_path)
|
||||
store.init_schema()
|
||||
with store.connect() as conn:
|
||||
tables = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'backup_%'"
|
||||
).fetchall()
|
||||
table_names = [t[0] for t in tables]
|
||||
assert "backup_jobs" in table_names
|
||||
assert "backup_runs" in table_names
|
||||
assert "backup_alerts" in table_names
|
||||
|
||||
|
||||
def test_backup_report_model():
|
||||
from media_library_viewer_api.models.backups import BackupReportRequest
|
||||
|
||||
report = BackupReportRequest(
|
||||
name="test-backup",
|
||||
started_at="2026-05-11T02:00:00Z",
|
||||
status="success",
|
||||
ended_at="2026-05-11T02:15:00Z",
|
||||
duration_ms=900000,
|
||||
bytes_transferred=1024,
|
||||
)
|
||||
assert report.name == "test-backup"
|
||||
assert report.status == "success"
|
||||
|
||||
|
||||
def test_create_job_and_run():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "test_settings.sqlite"
|
||||
store = SettingsStore(db_path)
|
||||
store.init_schema()
|
||||
|
||||
job = store.upsert_backup_job({
|
||||
"name": "test-job",
|
||||
"source": "server-a:/data",
|
||||
"target": "server-b:/backups",
|
||||
"schedule_interval_seconds": 86400,
|
||||
})
|
||||
assert job["name"] == "test-job"
|
||||
|
||||
run = store.create_backup_run({
|
||||
"job_id": job["id"],
|
||||
"started_at": 1715392800,
|
||||
"status": "success",
|
||||
"ended_at": 1715393700,
|
||||
"duration_ms": 900000,
|
||||
"bytes_transferred": 1024,
|
||||
})
|
||||
assert run["job_id"] == job["id"]
|
||||
assert run["status"] == "success"
|
||||
|
||||
|
||||
def test_api_key_auth():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "test_settings.sqlite"
|
||||
store = SettingsStore(db_path)
|
||||
store.init_schema()
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from media_library_viewer_api.auth import get_api_key, require_api_key
|
||||
from media_library_viewer_api.services import settings_store
|
||||
|
||||
# Monkey-patch the global store for this test
|
||||
import media_library_viewer_api.auth as auth_module
|
||||
from media_library_viewer_api.auth import get_api_key
|
||||
from media_library_viewer_api.services import settings_store
|
||||
|
||||
original_store = settings_store._store
|
||||
settings_store._store = store
|
||||
|
||||
# Reset the cached API key
|
||||
import media_library_viewer_api.auth as auth_module
|
||||
auth_module._API_KEY = None
|
||||
|
||||
try:
|
||||
# Should generate and return an API key
|
||||
client = TestClient(app)
|
||||
api_key = get_api_key()
|
||||
assert api_key
|
||||
assert len(api_key) > 0
|
||||
|
||||
# Should raise 401 without key
|
||||
try:
|
||||
require_api_key("")
|
||||
assert False, "Should have raised"
|
||||
except HTTPException as e:
|
||||
assert e.status_code == 401
|
||||
|
||||
# Should raise 401 with wrong key
|
||||
try:
|
||||
require_api_key("Bearer wrong-key")
|
||||
assert False, "Should have raised"
|
||||
except HTTPException as e:
|
||||
assert e.status_code == 401
|
||||
|
||||
# Should succeed with correct key
|
||||
result = require_api_key(f"Bearer {api_key}")
|
||||
assert result == f"Bearer {api_key}"
|
||||
response = client.post(
|
||||
"/api/backups/report",
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
json={
|
||||
"name": "test-backup",
|
||||
"started_at": "2026-05-11T02:00:00Z",
|
||||
"status": "success",
|
||||
"ended_at": "2026-05-11T02:15:00Z",
|
||||
"duration_ms": 900000,
|
||||
"bytes_transferred": 1024,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
finally:
|
||||
settings_store._store = original_store
|
||||
auth_module._API_KEY = None
|
||||
|
||||
|
||||
def test_alert_failed_status():
|
||||
from media_library_viewer_api.services.backup_alert_engine import generate_alerts_for_run
|
||||
|
||||
run = {
|
||||
"id": "run-1",
|
||||
"job_id": "job-1",
|
||||
"status": "failure",
|
||||
"bytes_transferred": 0,
|
||||
"duration_ms": 1000,
|
||||
"started_at": 1715392800,
|
||||
}
|
||||
alerts = generate_alerts_for_run(run, [], None)
|
||||
assert len(alerts) == 1
|
||||
assert alerts[0]["alert_type"] == "failed_status"
|
||||
assert alerts[0]["severity"] == "critical"
|
||||
|
||||
|
||||
def test_backup_poller_detects_missed_schedule():
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from media_library_viewer_api.services.backup_poller import BackupAlertPoller
|
||||
from media_library_viewer_api.services.settings_store import SettingsStore
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
db_path = Path(tmpdir) / "test_settings.sqlite"
|
||||
store = SettingsStore(db_path)
|
||||
store.init_schema()
|
||||
|
||||
import time
|
||||
# Create job with created_at in the past so it's already overdue
|
||||
past_time = int(time.time()) - 200
|
||||
job = store.upsert_backup_job({
|
||||
"name": "old-job",
|
||||
"schedule_interval_seconds": 60,
|
||||
})
|
||||
# Override created_at to be in the past so missed schedule fires
|
||||
with store.connect() as conn:
|
||||
conn.execute("UPDATE backup_jobs SET created_at = ? WHERE id = ?", (past_time, job["id"]))
|
||||
|
||||
poller = BackupAlertPoller()
|
||||
poller._run_cycle(store)
|
||||
|
||||
alerts = store.list_backup_alerts(job_id=job["id"])
|
||||
assert len(alerts) == 1
|
||||
assert alerts[0]["alert_type"] == "missed_schedule"
|
||||
|
||||
Reference in New Issue
Block a user