feat: add backup alert background poller
This commit is contained in:
@@ -12,11 +12,13 @@ from fastapi.middleware.cors import CORSMiddleware
|
|||||||
|
|
||||||
from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settings
|
from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settings
|
||||||
from media_library_viewer_api.config import get_settings
|
from media_library_viewer_api.config import get_settings
|
||||||
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
|
|
||||||
from media_library_viewer_api.routers import dashboard, monitoring, media, files, jobs, users, tasks
|
|
||||||
from .version import get_backend_version, get_version_info
|
|
||||||
from media_library_viewer_api.routers.settings import router as settings_router
|
|
||||||
from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller
|
from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller
|
||||||
|
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
|
||||||
|
from media_library_viewer_api.routers import dashboard, files, jobs, media, monitoring, tasks, users
|
||||||
|
from media_library_viewer_api.routers.settings import router as settings_router
|
||||||
|
|
||||||
|
from .services.backup_poller import get_backup_poller
|
||||||
|
from .version import get_backend_version, get_version_info
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -31,10 +33,13 @@ async def lifespan(app: FastAPI):
|
|||||||
logger.info("Managed known_hosts will be populated lazily on first successful SSH connection")
|
logger.info("Managed known_hosts will be populated lazily on first successful SSH connection")
|
||||||
mail_queue = get_mail_queue()
|
mail_queue = get_mail_queue()
|
||||||
monitoring_poller = get_monitoring_poller()
|
monitoring_poller = get_monitoring_poller()
|
||||||
|
backup_poller = get_backup_poller()
|
||||||
mail_queue.start()
|
mail_queue.start()
|
||||||
monitoring_poller.start()
|
monitoring_poller.start()
|
||||||
|
backup_poller.start()
|
||||||
yield
|
yield
|
||||||
monitoring_poller.stop()
|
monitoring_poller.stop()
|
||||||
|
backup_poller.stop()
|
||||||
mail_queue.stop()
|
mail_queue.stop()
|
||||||
logger.info("Backend shutdown complete")
|
logger.info("Backend shutdown complete")
|
||||||
|
|
||||||
@@ -42,7 +47,10 @@ async def lifespan(app: FastAPI):
|
|||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
title="Manage API",
|
title="Manage API",
|
||||||
version=get_backend_version(),
|
version=get_backend_version(),
|
||||||
description="Manage API for Jellyfin media browsing, SSH file inspection, server monitoring, and JWT-protected access.",
|
description=(
|
||||||
|
"Manage API for Jellyfin media browsing, SSH file inspection, "
|
||||||
|
"server monitoring, and JWT-protected access."
|
||||||
|
),
|
||||||
lifespan=lifespan,
|
lifespan=lifespan,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from .backup_alert_engine import check_missed_schedules
|
||||||
|
from .settings_store import SettingsStore, get_settings_store
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class BackupAlertPoller:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._thread: threading.Thread | None = None
|
||||||
|
self._stop_event = threading.Event()
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._last_run_at: float | None = None
|
||||||
|
self._last_success_at: float | None = None
|
||||||
|
self._last_error: str = ""
|
||||||
|
self._poll_count = 0
|
||||||
|
self._error_count = 0
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
if self._thread and self._thread.is_alive():
|
||||||
|
return
|
||||||
|
self._stop_event.clear()
|
||||||
|
self._thread = threading.Thread(target=self._run, name="backup-alert-poller", daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def stop(self, timeout: float = 5.0) -> None:
|
||||||
|
with self._lock:
|
||||||
|
thread = self._thread
|
||||||
|
if not thread:
|
||||||
|
return
|
||||||
|
self._stop_event.set()
|
||||||
|
thread.join(timeout=timeout)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def status(self) -> dict[str, Any]:
|
||||||
|
with self._lock:
|
||||||
|
return {
|
||||||
|
"worker_running": self._thread is not None and self._thread.is_alive(),
|
||||||
|
"last_run_at": self._last_run_at,
|
||||||
|
"last_success_at": self._last_success_at,
|
||||||
|
"last_error": self._last_error,
|
||||||
|
"poll_count": self._poll_count,
|
||||||
|
"error_count": self._error_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _run(self) -> None:
|
||||||
|
# Wait 30 seconds before first check
|
||||||
|
if self._stop_event.wait(30):
|
||||||
|
return
|
||||||
|
store = get_settings_store()
|
||||||
|
while not self._stop_event.is_set():
|
||||||
|
try:
|
||||||
|
self._run_cycle(store)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Backup alert poller cycle failed")
|
||||||
|
# Check every 5 minutes
|
||||||
|
if self._stop_event.wait(300):
|
||||||
|
break
|
||||||
|
|
||||||
|
def _run_cycle(self, store: SettingsStore) -> None:
|
||||||
|
start = time.perf_counter()
|
||||||
|
jobs = store.list_backup_jobs()
|
||||||
|
existing_alerts = store.list_backup_alerts(acknowledged=False)
|
||||||
|
|
||||||
|
new_alerts = check_missed_schedules(
|
||||||
|
jobs,
|
||||||
|
lambda job_id: store.get_latest_backup_run(job_id),
|
||||||
|
existing_alerts,
|
||||||
|
)
|
||||||
|
|
||||||
|
for alert in new_alerts:
|
||||||
|
store.create_backup_alert(alert)
|
||||||
|
|
||||||
|
# Prune old resolved alerts (90 days)
|
||||||
|
cutoff = int(time.time()) - (90 * 24 * 60 * 60)
|
||||||
|
store.prune_backup_alerts(cutoff)
|
||||||
|
|
||||||
|
_duration_ms = int((time.perf_counter() - start) * 1000)
|
||||||
|
with self._lock:
|
||||||
|
self._last_run_at = time.time()
|
||||||
|
self._last_success_at = time.time()
|
||||||
|
self._poll_count += 1
|
||||||
|
|
||||||
|
|
||||||
|
_BACKUP_POLLER = BackupAlertPoller()
|
||||||
|
|
||||||
|
|
||||||
|
def get_backup_poller() -> BackupAlertPoller:
|
||||||
|
return _BACKUP_POLLER
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from media_library_viewer_api.services.settings_store import SettingsStore
|
from media_library_viewer_api.services.settings_store import SettingsStore
|
||||||
|
|
||||||
|
|
||||||
@@ -9,7 +10,9 @@ def test_backup_schema_created():
|
|||||||
store = SettingsStore(db_path)
|
store = SettingsStore(db_path)
|
||||||
store.init_schema()
|
store.init_schema()
|
||||||
with store.connect() as conn:
|
with store.connect() as conn:
|
||||||
tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'backup_%'").fetchall()
|
tables = conn.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'backup_%'"
|
||||||
|
).fetchall()
|
||||||
table_names = [t[0] for t in tables]
|
table_names = [t[0] for t in tables]
|
||||||
assert "backup_jobs" in table_names
|
assert "backup_jobs" in table_names
|
||||||
assert "backup_runs" in table_names
|
assert "backup_runs" in table_names
|
||||||
@@ -72,3 +75,34 @@ def test_alert_failed_status():
|
|||||||
assert len(alerts) == 1
|
assert len(alerts) == 1
|
||||||
assert alerts[0]["alert_type"] == "failed_status"
|
assert alerts[0]["alert_type"] == "failed_status"
|
||||||
assert alerts[0]["severity"] == "critical"
|
assert alerts[0]["severity"] == "critical"
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_poller_detects_missed_schedule():
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from media_library_viewer_api.services.backup_poller import BackupAlertPoller
|
||||||
|
from media_library_viewer_api.services.settings_store import SettingsStore
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
db_path = Path(tmpdir) / "test_settings.sqlite"
|
||||||
|
store = SettingsStore(db_path)
|
||||||
|
store.init_schema()
|
||||||
|
|
||||||
|
import time
|
||||||
|
# Create job with created_at in the past so it's already overdue
|
||||||
|
past_time = int(time.time()) - 200
|
||||||
|
job = store.upsert_backup_job({
|
||||||
|
"name": "old-job",
|
||||||
|
"schedule_interval_seconds": 60,
|
||||||
|
})
|
||||||
|
# Override created_at to be in the past so missed schedule fires
|
||||||
|
with store.connect() as conn:
|
||||||
|
conn.execute("UPDATE backup_jobs SET created_at = ? WHERE id = ?", (past_time, job["id"]))
|
||||||
|
|
||||||
|
poller = BackupAlertPoller()
|
||||||
|
poller._run_cycle(store)
|
||||||
|
|
||||||
|
alerts = store.list_backup_alerts(job_id=job["id"])
|
||||||
|
assert len(alerts) == 1
|
||||||
|
assert alerts[0]["alert_type"] == "missed_schedule"
|
||||||
|
|||||||
Reference in New Issue
Block a user