feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

backend/src/media_library_viewer_api/routers/settings.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import logging
 from io import StringIO
 from typing import Any
 
@@ -12,11 +13,13 @@ from pydantic import BaseModel, Field
 from media_library_viewer_api.clients.ssh import RemoteSSHClient
 from media_library_viewer_api.config import get_settings
 from media_library_viewer_api.dependencies import get_monitoring_poller, get_settings_store
-from media_library_viewer_api.services.monitoring_actions import start_collector
 from media_library_viewer_api.services.db_maintenance import remove_sqlite_database
 from media_library_viewer_api.services.known_hosts import has_known_host
 from media_library_viewer_api.services.media_index import MediaIndex
 from media_library_viewer_api.services.settings_store import SettingsStore
+from media_library_viewer_api.services.targets import write_prometheus_targets
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/settings", tags=["settings"])
 
@@ -95,9 +98,7 @@ def _raise_ssh_validation_error(host: str, port: int, exc: Exception) -> None:
     if "protocol banner" in lowered:
         raise HTTPException(
             status_code=status.HTTP_502_BAD_GATEWAY,
-            detail=(
-                f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."
-            ),
+            detail=(f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."),
         ) from exc
     if "no authentication methods available" in lowered or "authentication failed" in lowered:
         raise HTTPException(
@@ -125,10 +126,12 @@ def _validate_saved_machine_ssh(machine: MonitoringMachineInput, store: Settings
         client.close()
 
 
-def _start_machine_collector(machine: MonitoringMachineInput, store: SettingsStore) -> None:
-    if "monitoring" not in {str(service).strip().lower() for service in machine.services}:
-        return
-    start_collector(machine.model_dump(exclude_none=True), store)
+def _write_prometheus_targets(store: SettingsStore) -> None:
+    """Regenerate Prometheus file-SD targets after machine changes."""
+    try:
+        write_prometheus_targets(store)
+    except Exception:
+        logger.exception("Failed to write Prometheus file-SD targets")
 
 
 @router.post("/machines/test-ssh")
@@ -137,7 +140,9 @@ def test_machine_ssh(
     store: SettingsStore = Depends(get_settings_store),
 ) -> dict[str, Any]:
     if str(machine.mode or "").strip().lower() != "ssh":
-        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines"
+        )
 
     client, host, port = _resolve_ssh_client(machine, store)
     settings = get_settings()
@@ -174,7 +179,8 @@ def test_machine_ssh(
     return {
         "status": "ok",
         "message": (
-            f"SSH connection succeeded for {host}:{port}; host key {'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
+            f"SSH connection succeeded for {host}:{port}; host key "
+            f"{'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
         ),
         "host": host,
         "port": port,
@@ -188,11 +194,11 @@ def post_machine(
     store: SettingsStore = Depends(get_settings_store),
 ) -> dict[str, Any]:
     saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine.id)
+    _write_prometheus_targets(store)
     poller = get_monitoring_poller()
     try:
         saved_machine = MonitoringMachineInput.model_validate(saved)
         _validate_saved_machine_ssh(saved_machine, store)
-        _start_machine_collector(saved_machine, store)
     finally:
         poller.start()
         poller.kick()
@@ -208,11 +214,11 @@ def put_machine(
     if not store.get_machine(machine_id):
         raise HTTPException(status_code=404, detail="Machine not found")
     saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine_id)
+    _write_prometheus_targets(store)
     poller = get_monitoring_poller()
     try:
         saved_machine = MonitoringMachineInput.model_validate(saved)
         _validate_saved_machine_ssh(saved_machine, store)
-        _start_machine_collector(saved_machine, store)
     finally:
         poller.start()
         poller.kick()
@@ -224,6 +230,7 @@ def delete_machine(machine_id: str, store: SettingsStore = Depends(get_settings_
     if not store.get_machine(machine_id):
         raise HTTPException(status_code=404, detail="Machine not found")
     store.delete_machine(machine_id)
+    _write_prometheus_targets(store)
     return {"status": "deleted"}
 
 
@@ -311,8 +318,12 @@ def reset_local_database(
     expected = "RESET LOCAL DATABASE"
     if payload.confirm_phrase.strip().upper() != expected:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Confirmation phrase does not match")
-    if not (payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible):
-        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected")
+    if not (
+        payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected"
+        )
 
     settings_removed = remove_sqlite_database(store.db_path)
     media_index = MediaIndex()