now supports multi machine monitoring

backend/README.md

@@ -22,17 +22,20 @@ backend/
 │       │   ├── monitoring.py
 │       │   ├── media.py
 │       │   ├── users.py
+│       │   ├── settings.py
 │       │   ├── files.py
 │       │   └── jobs.py
 │       ├── clients/
 │       │   ├── jellyfin.py
 │       │   ├── jellyseerr.py
+│       │   ├── local.py
 │       │   ├── resources.py
 │       │   └── ssh.py
 │       ├── domain/
 │       │   └── media.py
 │       └── services/
-│           └── media_index.py
+│           ├── media_index.py
+│           └── settings_store.py
 └── tests/
 ```
 
@@ -47,7 +50,9 @@ pip install -e '.[dev]'
 
 ## Configuration
 
-Create a `.env` file in the project root (or set environment variables):
+Set environment variables directly in your shell or a wrapper script before running the app or Compose. The Docker Compose files use interpolation and do not require an `env_file` entry.
+
+For local `.env` development, you can still create one if you prefer, but it is optional.
 
 ```bash
 JELLYFIN_URL=https://jellyfin.example.com
@@ -101,17 +106,51 @@ API docs available at: http://localhost:8000/docs
 
 The repository root includes a production `docker-compose.yml` and a development `docker-compose.dev.yml`.
 The backend media index is stored in the `backend_cache` Docker volume so it survives container restarts and image rebuilds.
+The production compose file expects required environment variables to be supplied via interpolation (shell exports or inline `VAR=value docker compose ...`).
+
+### Configuration workflow examples
+
+1. Export your runtime variables before launching Compose:
+
+```bash
+export JELLYFIN_URL=https://jellyfin.example.com
+export JELLYFIN_API_KEY=your-api-key
+export SSH_HOST=media-server.example.com
+export SSH_USERNAME=username
+export SSH_KEY_HOST_DIR=$HOME/.ssh
+export SSH_KEY_NAME=id_ed25519
+export BACKEND_APP_HOST=manage.example.com
+export FRONTEND_APP_HOST=manage.example.com
+export CERT_RESOLVER=letsencrypt
+export VITE_OIDC_ISSUER=https://authentik.example/application/o/manage/
+export VITE_OIDC_CLIENT_ID=manage
+export VITE_OIDC_REDIRECT_URI=https://manage.example.com/
+export VITE_OIDC_POST_LOGOUT_REDIRECT_URI=https://manage.example.com/
+
+docker compose up --build
+```
+
+2. After the API is running, open the app, go to **Settings**, and add machine entries:
+   - **Local**: monitors the API host itself without SSH.
+   - **SSH**: monitors another machine using a host, username, key directory, and key name.
+
+3. Open **Monitoring** to see one section per configured machine. Each section uses its own collector state, disk path, metrics queries, and recent action history, which are populated automatically by the backend poller.
 
 ## API Endpoints
 
 - `GET /api/dashboard/counts` — Movie/series/episode totals
 - `GET /api/dashboard/libraries` — Per-library breakdown
 - `GET /api/dashboard/now-playing` — Active playback sessions
-- `GET /api/monitoring/status` — Collector status
-- `GET /api/monitoring/metrics` — Resource samples (last hour)
-- `GET /api/monitoring/disk` — Disk space
-- `POST /api/monitoring/start|stop|restart` — Collector controls
-- `GET /api/monitoring/diagnostics` — Collector debug info
+- `GET /api/monitoring/machines` — Persistent monitoring machine definitions
+- `GET /api/monitoring/status?machine_id=` — Collector status for a machine
+- `GET /api/monitoring/metrics?machine_id=` — Resource samples (last hour)
+- `GET /api/monitoring/disk?machine_id=` — Disk space
+- `POST /api/monitoring/start|stop|restart?machine_id=` — Collector controls
+- `GET /api/monitoring/diagnostics?machine_id=` — Collector debug info
+- `GET /api/monitoring/poller` — Backend poller status and configuration
+- `GET /api/monitoring/machines/{machine_id}/actions` — Recent machine action history
+- `GET /api/dashboard/monitoring` — Dashboard-wide per-machine monitoring summary table with 10-minute averages and min/max subtext
+- `GET /api/settings/machines` — Manage machine definitions
 - `GET /api/media/status` — Index status
 - `POST /api/media/build` — Rebuild index
 - `GET /api/media/query` — Query with filters/sort/pagination

backend/src/media_library_viewer_api/config.py

@@ -59,6 +59,11 @@ class Settings(BaseSettings):
     ssh_key_name: str = ""
     ssh_password: str = ""
 
+    # Monitoring poller
+    monitoring_poll_interval_seconds: int = 300
+    monitoring_poll_initial_delay_seconds: int = 20
+    monitoring_action_retention_days: int = 30
+
     # Remote paths
     remote_media_root: str = ""
     remote_path_prefix: str = ""

backend/src/media_library_viewer_api/dependencies.py

@@ -14,6 +14,11 @@ from media_library_viewer_api.clients.jellyseerr import JellyseerrClient
 from media_library_viewer_api.clients.ssh import RemoteSSHClient
 from media_library_viewer_api.config import get_settings
 from media_library_viewer_api.services.mail_queue import MailQueue, get_mail_queue as _get_mail_queue
+from media_library_viewer_api.services.monitoring_poller import (
+    MonitoringPoller,
+    get_monitoring_poller as _get_monitoring_poller,
+)
+from media_library_viewer_api.services.settings_store import SettingsStore, get_settings_store as _get_settings_store
 
 logger = logging.getLogger(__name__)
 
@@ -76,6 +81,16 @@ def get_mail_queue() -> MailQueue:
     return _get_mail_queue()
 
 
+def get_monitoring_poller() -> MonitoringPoller:
+    """Return the singleton background monitoring poller."""
+    return _get_monitoring_poller()
+
+
+def get_settings_store() -> SettingsStore:
+    """Return the singleton persistent settings store."""
+    return _get_settings_store()
+
+
 def get_user_id() -> str:
     """Return the configured Jellyfin user ID, or discover the first available user."""
     settings = get_settings()

backend/src/media_library_viewer_api/main.py

@@ -14,7 +14,8 @@ from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settin
 from media_library_viewer_api.config import get_settings
 from media_library_viewer_api.logging_utils import configure_logging, describe_settings
 from media_library_viewer_api.routers import dashboard, monitoring, media, files, jobs, users
-from media_library_viewer_api.dependencies import get_mail_queue
+from media_library_viewer_api.routers.settings import router as settings_router
+from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller
 
 logger = logging.getLogger(__name__)
 
@@ -27,8 +28,11 @@ async def lifespan(app: FastAPI):
     validate_auth_settings(settings)
     logger.info("Backend startup complete: %s", describe_settings(settings))
     mail_queue = get_mail_queue()
+    monitoring_poller = get_monitoring_poller()
     mail_queue.start()
+    monitoring_poller.start()
     yield
+    monitoring_poller.stop()
     mail_queue.stop()
     logger.info("Backend shutdown complete")
 
@@ -87,6 +91,7 @@ app.include_router(media.router)
 app.include_router(files.router)
 app.include_router(jobs.router)
 app.include_router(users.router)
+app.include_router(settings_router)
 
 
 @app.get("/api/health")

backend/src/media_library_viewer_api/routers/dashboard.py

@@ -7,8 +7,9 @@ from typing import Any
 
 from fastapi import APIRouter, Depends
 
-from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id
 from media_library_viewer_api.clients.jellyfin import JellyfinClient
+from media_library_viewer_api.dependencies import get_jellyfin_client, get_monitoring_poller, get_settings_store, get_user_id
+from media_library_viewer_api.services.monitoring_actions import collect_machine_overview
 
 logger = logging.getLogger(__name__)
 
@@ -37,6 +38,24 @@ def get_library_counts(
     return client.library_item_counts(user_id, libraries)
 
 
+@router.get("/monitoring")
+def get_monitoring_overview(
+    store=Depends(get_settings_store),
+) -> dict[str, Any]:
+    """Return one lightweight monitoring row per configured machine."""
+    machines = store.list_machines()
+    rows = [collect_machine_overview(machine) for machine in machines]
+    poller = get_monitoring_poller().snapshot()
+    enabled_count = sum(1 for machine in machines if machine.get("enabled"))
+    logger.info("Dashboard monitoring machines=%s enabled=%s", len(machines), enabled_count)
+    return {
+        "poller": poller,
+        "machines": rows,
+        "total": len(machines),
+        "enabled": enabled_count,
+    }
+
+
 def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[str, Any]]:
     """Normalize Jellyfin sessions into dashboard activity rows."""
     results: list[dict[str, Any]] = []

backend/src/media_library_viewer_api/routers/monitoring.py

@@ -1,4 +1,4 @@
-"""Monitoring router — metrics, collector controls, disk space."""
+"""Monitoring router — metrics, collector controls, and per-machine status."""
 
 from __future__ import annotations
 
@@ -6,31 +6,90 @@ import logging
 import time
 from typing import Any
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, HTTPException, Query
 
-from media_library_viewer_api.clients.ssh import RemoteSSHClient
-from media_library_viewer_api.dependencies import get_ssh_client
-
-logger = logging.getLogger(__name__)
 from media_library_viewer_api.clients.resources import (
     disk_space,
     read_resource_metrics,
-    resource_collector_debug_info,
     resource_collector_status,
-    restart_resource_collector,
-    start_resource_collector,
-    stop_resource_collector,
 )
 from media_library_viewer_api.config import get_settings
+from media_library_viewer_api.dependencies import get_settings_store
+from media_library_viewer_api.services.monitoring_actions import (
+    poll_machine_diagnostics,
+    run_machine_operation,
+    start_collector,
+    stop_collector,
+    restart_collector,
+)
+from media_library_viewer_api.services.settings_store import SettingsStore
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/monitoring", tags=["monitoring"])
 
 
+def _resolve_machine(store: SettingsStore, machine_id: str | None) -> dict[str, Any]:
+    """Return the requested machine or the first enabled machine.
+
+    Monitoring is treated as a machine-by-machine view. If a machine is
+    explicitly requested but disabled, we surface that as a user-facing error so
+    the Settings tab can be used to re-enable it.
+    """
+    machines = store.list_machines()
+    if machine_id:
+        machine = next((item for item in machines if item["id"] == machine_id), None)
+        if not machine:
+            raise HTTPException(status_code=404, detail="Monitoring machine not found")
+        if not machine.get("enabled"):
+            raise HTTPException(status_code=409, detail=f"Monitoring machine '{machine['name']}' is disabled")
+        return machine
+
+    for machine in machines:
+        if machine.get("enabled"):
+            return machine
+    raise HTTPException(status_code=404, detail="No enabled monitoring machines configured")
+
+
+@router.get("/machines")
+def get_machines(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
+    """Return monitoring machines for the UI."""
+    return store.list_machines()
+
+
+@router.get("/poller")
+def get_poller_status() -> dict[str, Any]:
+    """Return the backend poller status and configuration."""
+    from media_library_viewer_api.dependencies import get_monitoring_poller
+
+    poller = get_monitoring_poller().snapshot()
+    logger.info("Monitoring poller status requested running=%s poll_count=%s", poller.get("worker_running"), poller.get("poll_count"))
+    return poller
+
+
+@router.get("/machines/{machine_id}/actions")
+def get_machine_actions(
+    machine_id: str,
+    limit: int = 20,
+    action: str | None = None,
+    status: str | None = None,
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, Any]:
+    """Return recent action history for a single machine."""
+    machine = _resolve_machine(store, machine_id)
+    actions = store.list_machine_actions(machine["id"], limit=limit, action=action, status=status)
+    return {"items": actions, "total": len(actions)}
+
+
 @router.get("/status")
-def get_status(ssh: RemoteSSHClient = Depends(get_ssh_client)) -> dict[str, str]:
-    """Return collector running status."""
-    status = resource_collector_status(ssh)
-    logger.info("Monitoring status requested: %s", status)
+def get_status(
+    machine_id: str | None = Query(default=None),
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, str]:
+    """Return collector running status for a given machine."""
+    machine = _resolve_machine(store, machine_id)
+    status = run_machine_operation(machine, store, "status lookup", resource_collector_status)
+    logger.info("Monitoring status requested machine_id=%s status=%s", machine["id"], status)
     return {"status": status}
 
 
@@ -38,17 +97,29 @@ def get_status(ssh: RemoteSSHClient = Depends(get_ssh_client)) -> dict[str, str]
 def get_metrics(
     max_lines: int = 70_000,
     last_seconds: int | None = None,
-    ssh: RemoteSSHClient = Depends(get_ssh_client),
+    machine_id: str | None = Query(default=None),
+    store: SettingsStore = Depends(get_settings_store),
 ) -> dict[str, Any]:
-    """Return resource metric samples from the remote collector."""
-    rows = read_resource_metrics(ssh, max_lines=max_lines)
+    """Return resource metric samples for a given machine."""
+    machine = _resolve_machine(store, machine_id)
+
+    def _read_metrics(client):
+        return read_resource_metrics(client, max_lines=max_lines)
+
+    rows = run_machine_operation(machine, store, "metrics read", _read_metrics)
     if last_seconds is None:
         filtered = rows
         cutoff_ts = 0.0
     else:
         cutoff_ts = time.time() - last_seconds
         filtered = [row for row in rows if float(row.get("ts", 0)) >= cutoff_ts]
-    logger.info("Monitoring metrics requested total=%s filtered=%s last_seconds=%s", len(rows), len(filtered), last_seconds)
+    logger.info(
+        "Monitoring metrics requested machine_id=%s total=%s filtered=%s last_seconds=%s",
+        machine["id"],
+        len(rows),
+        len(filtered),
+        last_seconds,
+    )
     return {
         "samples": filtered,
         "total_samples": len(rows),
@@ -58,41 +129,66 @@ def get_metrics(
 
 
 @router.get("/disk")
-def get_disk_space(ssh: RemoteSSHClient = Depends(get_ssh_client)) -> dict[str, Any]:
-    """Return disk space for the configured media root."""
-    settings = get_settings()
-    path = settings.media_root or "/"
-    logger.info("Monitoring disk requested path=%s", path)
-    return disk_space(ssh, path)
+def get_disk_space(
+    machine_id: str | None = Query(default=None),
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, Any]:
+    """Return disk space for the configured path of a given machine."""
+    machine = _resolve_machine(store, machine_id)
+    app_settings = get_settings()
+    path = str(machine.get("media_root") or app_settings.media_root or "/")
+    logger.info("Monitoring disk requested machine_id=%s path=%s", machine["id"], path)
+    return run_machine_operation(
+        machine,
+        store,
+        f"disk lookup for {path}",
+        lambda client: disk_space(client, path),
+    )
 
 
 @router.post("/start")
-def post_start(ssh: RemoteSSHClient = Depends(get_ssh_client)) -> dict[str, str]:
-    """Start the remote resource collector."""
-    message = start_resource_collector(ssh)
-    logger.info("Monitoring collector start result: %s", message)
+def post_start(
+    machine_id: str | None = Query(default=None),
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, str]:
+    """Start the machine's resource collector."""
+    machine = _resolve_machine(store, machine_id)
+    message = start_collector(machine, store)
+    logger.info("Monitoring collector start result machine_id=%s: %s", machine["id"], message)
     return {"message": message}
 
 
 @router.post("/stop")
-def post_stop(ssh: RemoteSSHClient = Depends(get_ssh_client)) -> dict[str, str]:
-    """Stop the remote resource collector."""
-    message = stop_resource_collector(ssh)
-    logger.info("Monitoring collector stop result: %s", message)
+def post_stop(
+    machine_id: str | None = Query(default=None),
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, str]:
+    """Stop the machine's resource collector."""
+    machine = _resolve_machine(store, machine_id)
+    message = stop_collector(machine, store)
+    logger.info("Monitoring collector stop result machine_id=%s: %s", machine["id"], message)
     return {"message": message}
 
 
 @router.post("/restart")
-def post_restart(ssh: RemoteSSHClient = Depends(get_ssh_client)) -> dict[str, str]:
-    """Restart the remote resource collector."""
-    message = restart_resource_collector(ssh)
-    logger.info("Monitoring collector restart result: %s", message)
+def post_restart(
+    machine_id: str | None = Query(default=None),
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, str]:
+    """Restart the machine's resource collector."""
+    machine = _resolve_machine(store, machine_id)
+    message = restart_collector(machine, store)
+    logger.info("Monitoring collector restart result machine_id=%s: %s", machine["id"], message)
     return {"message": message}
 
 
 @router.get("/diagnostics")
-def get_diagnostics(ssh: RemoteSSHClient = Depends(get_ssh_client)) -> dict[str, str]:
-    """Return collector debug info for troubleshooting."""
-    diagnostics = resource_collector_debug_info(ssh)
-    logger.info("Monitoring diagnostics requested")
+def get_diagnostics(
+    machine_id: str | None = Query(default=None),
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, str]:
+    """Return collector debug info for a given machine."""
+    machine = _resolve_machine(store, machine_id)
+    diagnostics = poll_machine_diagnostics(machine, store)["diagnostics"]
+    logger.info("Monitoring diagnostics requested machine_id=%s", machine["id"])
     return {"diagnostics": diagnostics}

backend/tests/test_api.py

@@ -17,6 +17,7 @@ from media_library_viewer_api.dependencies import (
     get_jellyfin_client,
     get_jellyseerr_client,
     get_mail_queue,
+    get_settings_store,
     get_user_id,
 )
 from media_library_viewer_api.clients.ssh import CommandResult
@@ -159,6 +160,78 @@ class TestDashboard:
         assert data["series"] == 20
         assert data["episodes"] == 500
 
+    def test_monitoring_overview(self, test_client):
+        store = MagicMock()
+        store.list_machines.return_value = [
+            {
+                "id": "local",
+                "name": "This machine",
+                "mode": "local",
+                "enabled": True,
+                "host": "localhost",
+                "port": 22,
+                "username": "",
+                "media_root": "/srv/media",
+                "path_prefix": "",
+                "notes": "",
+            },
+            {
+                "id": "remote1",
+                "name": "Remote",
+                "mode": "ssh",
+                "enabled": True,
+                "host": "server.example.com",
+                "port": 22,
+                "username": "alex",
+                "media_root": "/srv/media",
+                "path_prefix": "",
+                "notes": "",
+            },
+        ]
+        app.dependency_overrides[get_settings_store] = lambda: store
+        try:
+            with (
+                patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=object()),
+                patch("media_library_viewer_api.services.monitoring_actions.resource_collector_status", return_value="running pid=123"),
+                patch(
+                    "media_library_viewer_api.services.monitoring_actions.read_resource_metrics",
+                    return_value=[
+                        {
+                            "ts": 123.0,
+                            "cpu_pct": 10.0,
+                            "iowait_pct": 1.0,
+                            "mem_pct": 20.0,
+                            "net_rx_bytes_per_sec": 100.0,
+                            "net_tx_bytes_per_sec": 50.0,
+                            "disk_read_bps": 1.0,
+                            "disk_write_bps": 2.0,
+                        }
+                    ],
+                ),
+                patch(
+                    "media_library_viewer_api.services.monitoring_actions.disk_space",
+                    return_value={
+                        "filesystem": "/dev/sda1",
+                        "size": 1000,
+                        "used": 200,
+                        "available": 800,
+                        "used_pct": "20.0%",
+                        "mount": "/srv/media",
+                    },
+                ),
+            ):
+                response = test_client.get("/api/dashboard/monitoring")
+        finally:
+            app.dependency_overrides.pop(get_settings_store, None)
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["total"] == 2
+        assert data["enabled"] == 2
+        assert len(data["machines"]) == 2
+        assert data["machines"][0]["latest_sample"]["cpu_pct"] == 10.0
+        assert data["machines"][0]["disk"]["used_pct"] == "20.0%"
+
     def test_libraries(self, test_client):
         response = test_client.get("/api/dashboard/libraries")
         assert response.status_code == 200