feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

.env.example

@@ -1,46 +1,66 @@
-# Optional backend logging level.
+# App
-LOG_LEVEL=INFO
+APP_VERSION=0.1.0
+APP_BUILD_INFO=dev
 
-# Optional SMTP settings for the Users -> message popup.
+# Hosts
+BACKEND_APP_HOST=api.manage.example.com
+FRONTEND_APP_HOST=manage.example.com
+GRAFANA_APP_HOST=grafana.example.com
+BACKEND_APP_PORT=8000
+FRONTEND_APP_PORT=80
+GRAFANA_APP_PORT=3000
+BACKEND_APP_NAME=manage-backend
+FRONTEND_APP_NAME=manage-frontend
+GRAFANA_APP_NAME=grafana
+
+# Traefik / certificates
+CERT_RESOLVER=letsencrypt
+
+# Backend
+LOG_LEVEL=INFO
+LOG_FORMAT=json
+PROMETHEUS_ENABLED=true
+PROMETHEUS_FILE_SD_DIR=/app/backend/.cache/prometheus-file-sd
+ALERTMANAGER_URL=http://alertmanager:9093
+ALERTMANAGER_WEBHOOK_URL=
+BACKEND_CACHE_DIR=./backend-cache
+
+# Auth
+AUTH_ENABLED=true
+OIDC_ISSUER_URL=https://auth.example.com/application/o/manage/
+OIDC_AUDIENCE=manage
+OIDC_JWKS_URL=https://auth.example.com/application/o/manage/jwks/
+OIDC_CLOCK_SKEW_SECONDS=30
+
+# Frontend OIDC
+VITE_API_URL=/api
+VITE_OIDC_ENABLED=true
+VITE_OIDC_ISSUER=https://auth.example.com/application/o/manage/
+VITE_OIDC_CLIENT_ID=manage
+VITE_OIDC_SCOPE=openid profile email
+VITE_OIDC_REDIRECT_URI=https://manage.example.com/oidc/callback
+VITE_OIDC_POST_LOGOUT_REDIRECT_URI=https://manage.example.com/
+VITE_DEV_API_PROXY_TARGET=http://backend:8000
+
+# SMTP
 SMTP_HOST=smtp.example.com
 SMTP_PORT=587
-SMTP_USERNAME=your-smtp-username
+SMTP_USERNAME=
-SMTP_PASSWORD=your-smtp-password
+SMTP_PASSWORD=
 SMTP_FROM_ADDRESS=no-reply@example.com
 SMTP_FROM_NAME=Manage
 SMTP_USE_TLS=true
 SMTP_USE_SSL=false
 SMTP_TIMEOUT=30
 
-# Machine/service configuration now lives in the app's Settings tab.
+# Grafana admin / OAuth
-# The built-in local machine is seeded automatically.
+GRAFANA_ADMIN_USER=admin
-#
+GRAFANA_ADMIN_PASSWORD=change-me
-# Remote SSH machines can store their private key and optional passphrase directly in Settings,
+GF_AUTH_GENERIC_OAUTH_CLIENT_ID=manage
-# so no SSH key mount is required for normal use.
+GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=
+GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.example.com/application/o/manage/authorize/
+GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.example.com/application/o/manage/token/
+GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.example.com/application/o/manage/userinfo/
 
-# For deployment with traefik
+# Alerting
-FRONTEND_APP_NAME=manage
+ALERT_EMAIL_TO=admin@example.com
-FRONTEND_APP_HOST=manage.example.com
-FRONTEND_APP_PORT=5173
-
-BACKEND_APP_NAME=management-api
-BACKEND_APP_HOST=management-api.example.com
-BACKEND_APP_PORT=8000
-
-CERT_RESOLVER=lets-encrypt
-
-# Authentik / OIDC
-# Backend validates every API request with a Bearer JWT.
-AUTH_ENABLED=true
-OIDC_ISSUER_URL=https://authentik.example/application/o/media-library-viewer/
-OIDC_AUDIENCE=media-library-viewer
-OIDC_JWKS_URL=
-OIDC_CLOCK_SKEW_SECONDS=30
-
-# Frontend OIDC settings (Vite build/runtime env)
-VITE_OIDC_ENABLED=true
-VITE_OIDC_ISSUER=https://authentik.example/application/o/media-library-viewer/
-VITE_OIDC_CLIENT_ID=media-library-viewer
-VITE_OIDC_SCOPE=openid profile email
-VITE_OIDC_REDIRECT_URI=http://localhost:8080/
-VITE_OIDC_POST_LOGOUT_REDIRECT_URI=http://localhost:8080/

.gitignore

@@ -15,6 +15,7 @@ dist/
 
 # Virtual environments
 .venv/
+.venv-review/
 venv/
 env/
 
@@ -52,3 +53,7 @@ frontend/dist/
 .fusion/
 .stoneforge/
 .superpowers/
+# Local Pi runtime state
+.atl/
+.pi-map.md
+.pi-map.index.md

backend/pyproject.toml

@@ -13,6 +13,8 @@ dependencies = [
   "pandas>=2.0",
   "PyJWT[crypto]>=2.8",
   "python-multipart>=0.0.9",
+  "prometheus-client>=0.21",
+  "python-json-logger>=2.0",
 ]
 
 [project.optional-dependencies]

backend/src/media_library_viewer_api/clients/jellyfin.py

@@ -53,7 +53,9 @@ class JellyfinClient:
             {
                 "X-Emby-Token": api_key,
                 "Accept": "application/json",
-                "X-Emby-Authorization": 'MediaBrowser Client="MediaLibraryViewer", Device="Streamlit", DeviceId="streamlit", Version="0.1"',
+                "X-Emby-Authorization": (
+                    'MediaBrowser Client="MediaLibraryViewer", Device="Streamlit", DeviceId="streamlit", Version="0.1"'
+                ),
             }
         )
 
@@ -61,9 +63,7 @@ class JellyfinClient:
         """GET a Jellyfin endpoint and include useful response text on errors."""
         clean_params = {k: v for k, v in params.items() if v is not None and v != ""}
         logger.debug("Jellyfin GET %s params=%s", path, sorted(clean_params.keys()))
-        response = self.session.get(
+        response = self.session.get(f"{self.base_url}{path}", params=clean_params, timeout=self.timeout)
-            f"{self.base_url}{path}", params=clean_params, timeout=self.timeout
-        )
         try:
             response.raise_for_status()
         except requests.HTTPError as exc:
@@ -175,14 +175,16 @@ class JellyfinClient:
             series = self.item_count(user_id, "Series", parent_id=lib_id)
             episodes = self.item_count(user_id, "Episode", parent_id=lib_id)
             total = self.item_count(user_id, "Movie,Episode,Video,Audio,Series", parent_id=lib_id)
-            results.append({
+            results.append(
-                "library": lib_name,
+                {
-                "type": lib_type,
+                    "library": lib_name,
-                "movies": movies,
+                    "type": lib_type,
-                "series": series,
+                    "movies": movies,
-                "episodes": episodes,
+                    "series": series,
-                "total": total,
+                    "episodes": episodes,
-            })
+                    "total": total,
+                }
+            )
         return results
 
     def sessions(self, active_within_seconds: int | None = None) -> list[dict[str, Any]]:

backend/src/media_library_viewer_api/clients/jellyseerr.py

@@ -40,16 +40,12 @@ class JellyseerrClient:
         """GET a Jellyseerr endpoint and include useful response text on errors."""
         clean_params = {k: v for k, v in params.items() if v is not None and v != ""}
         logger.debug("Jellyseerr GET %s params=%s", path, sorted(clean_params.keys()))
-        response = self.session.get(
+        response = self.session.get(f"{self.base_url}/api/v1{path}", params=clean_params, timeout=self.timeout)
-            f"{self.base_url}/api/v1{path}", params=clean_params, timeout=self.timeout
-        )
         try:
             response.raise_for_status()
         except requests.HTTPError as exc:
             detail = response.text[:500]
-            logger.warning(
+            logger.warning("Jellyseerr GET %s failed status=%s url=%s", path, response.status_code, response.url)
-                "Jellyseerr GET %s failed status=%s url=%s", path, response.status_code, response.url
-            )
             raise requests.HTTPError(
                 f"{response.status_code} for {response.url}: {detail}",
                 response=response,
@@ -105,7 +101,9 @@ class JellyseerrClient:
                 return results
 
             page_results = payload.get("results") or []
-            page_items = [item for item in page_results if isinstance(item, dict)] if isinstance(page_results, list) else []
+            page_items = (
+                [item for item in page_results if isinstance(item, dict)] if isinstance(page_results, list) else []
+            )
             results.extend(page_items)
 
             page_info = payload.get("pageInfo") or {}

backend/src/media_library_viewer_api/clients/resources.py

@@ -1,340 +0,0 @@
-"""Remote resource collection helpers.
-
-The app does not require Prometheus, Netdata, or sysstat. Instead it can install
-and manage a tiny POSIX-sh collector under /tmp on the remote server. The
-collector samples Linux /proc and /sys counters every 10 seconds and appends JSON
-Lines. This module starts/stops the collector and reads those JSONL samples.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import shlex
-from dataclasses import dataclass
-from typing import Any
-
-from media_library_viewer_api.clients.ssh import RemoteSSHClient
-
-logger = logging.getLogger(__name__)
-
-# POSIX shell script copied to the remote server by start_resource_collector().
-# Keep this script bash-free because many NAS/media servers have minimal shells.
-COLLECTOR_SCRIPT = r'''#!/bin/sh
-set -u
-OUT="${1:-/tmp/media_library_viewer_metrics.jsonl}"
-INTERVAL="${2:-10}"
-RETENTION_SECONDS="${3:-604800}"
-MAX_LINES="${4:-70000}"
-PRUNE_EVERY_SAMPLES="${5:-60}"
-mkdir -p "$(dirname "$OUT")"
-
-echo "collector starting at $(date -Is 2>/dev/null || date), interval=${INTERVAL}s, retention=${RETENTION_SECONDS}s, max_lines=${MAX_LINES}, out=${OUT}"
-
-read_cpu() {
-  awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8+$9+$10, $5+$6, $6}' /proc/stat
-}
-
-read_mem_pct() {
-  awk '
-    /^MemTotal:/ {total=$2}
-    /^MemAvailable:/ {avail=$2}
-    END {if (total > 0) printf "%.2f", (total-avail)*100/total; else printf "0"}
-  ' /proc/meminfo
-}
-
-read_net_bytes() {
-  awk '
-    NR > 2 {
-      split($0, parts, ":")
-      iface = parts[1]
-      stats = parts[2]
-      gsub(/^[ \t]+|[ \t]+$/, "", iface)
-      gsub(/^[ \t]+|[ \t]+$/, "", stats)
-      if (iface == "lo" || iface == "" || stats == "") next
-      split(stats, values, /[ \t]+/)
-      # /proc/net/dev after the colon:
-      # receive bytes are field 1, transmit bytes are field 9.
-      # Trim the stats block before split; otherwise leading whitespace can make
-      # values[1] empty in some awk implementations, resulting in zero rates.
-      rx += values[1] + 0
-      tx += values[9] + 0
-    }
-    END {printf "%.0f %.0f", rx, tx}
-  ' /proc/net/dev
-}
-
-read_disk_bytes() {
-  read_sectors=0
-  written_sectors=0
-  for dev in /sys/block/*; do
-    [ -r "$dev/stat" ] || continue
-    name="$(basename "$dev")"
-    case "$name" in
-      loop*|ram*|fd*|sr*) continue ;;
-    esac
-    # Linux /sys/block/<dev>/stat fields: 3=sectors read, 7=sectors written.
-    # Use POSIX sh parsing instead of bash arrays so this works on minimal systems.
-    set -- $(cat "$dev/stat")
-    sectors_read="${3:-0}"
-    sectors_written="${7:-0}"
-    read_sectors=$((read_sectors + sectors_read))
-    written_sectors=$((written_sectors + sectors_written))
-  done
-  printf "%s %s" "$((read_sectors * 512))" "$((written_sectors * 512))"
-}
-
-set -- $(read_cpu)
-prev_total="${1:-0}"
-prev_idle="${2:-0}"
-prev_iowait="${3:-0}"
-set -- $(read_net_bytes)
-prev_rx="${1:-0}"
-prev_tx="${2:-0}"
-set -- $(read_disk_bytes)
-prev_disk_read="${1:-0}"
-prev_disk_write="${2:-0}"
-prev_ts="$(date +%s)"
-sample_count=0
-
-prune_metrics_file() {
-  [ -f "$OUT" ] || return 0
-  cutoff="$1"
-  tmp="${OUT}.$$.tmp"
-  awk -v cutoff="$cutoff" '
-    match($0, /"ts":[0-9]+/) {
-      ts = substr($0, RSTART + 5, RLENGTH - 5)
-      if (ts >= cutoff) print $0
-    }
-  ' "$OUT" | tail -n "$MAX_LINES" > "$tmp" && mv "$tmp" "$OUT"
-  rm -f "$tmp"
-}
-
-while true; do
-  now_ts="$(date +%s)"
-  dt=$((now_ts - prev_ts))
-  if [ "$dt" -le 0 ]; then dt=1; fi
-
-  set -- $(read_cpu)
-  total="${1:-0}"
-  idle="${2:-0}"
-  iowait="${3:-0}"
-  set -- $(read_net_bytes)
-  rx="${1:-0}"
-  tx="${2:-0}"
-  set -- $(read_disk_bytes)
-  disk_read="${1:-0}"
-  disk_write="${2:-0}"
-  mem_pct="$(read_mem_pct)"
-
-  total_delta=$((total - prev_total))
-  idle_delta=$((idle - prev_idle))
-  iowait_delta=$((iowait - prev_iowait))
-  rx_delta=$((rx - prev_rx))
-  tx_delta=$((tx - prev_tx))
-  disk_read_delta=$((disk_read - prev_disk_read))
-  disk_write_delta=$((disk_write - prev_disk_write))
-
-  cpu_pct="$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN {if (total > 0) printf "%.2f", (total-idle)*100/total; else printf "0"}')"
-  iowait_pct="$(awk -v total="$total_delta" -v iow="$iowait_delta" 'BEGIN {if (total > 0) printf "%.2f", iow*100/total; else printf "0"}')"
-  rx_bytes_per_sec="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
-  tx_bytes_per_sec="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
-  rx_bps="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
-  tx_bps="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
-  disk_read_bps="$(awk -v bytes="$disk_read_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
-  disk_write_bps="$(awk -v bytes="$disk_write_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
-
-  printf '{"ts":%s,"cpu_pct":%s,"iowait_pct":%s,"mem_pct":%s,"net_rx_bytes_per_sec":%s,"net_tx_bytes_per_sec":%s,"net_rx_bps":%s,"net_tx_bps":%s,"disk_read_bps":%s,"disk_write_bps":%s}\n' \
-    "$now_ts" "$cpu_pct" "$iowait_pct" "$mem_pct" "$rx_bytes_per_sec" "$tx_bytes_per_sec" "$rx_bps" "$tx_bps" "$disk_read_bps" "$disk_write_bps" >> "$OUT"
-
-  sample_count=$((sample_count + 1))
-  if [ $((sample_count % PRUNE_EVERY_SAMPLES)) -eq 0 ]; then
-    prune_metrics_file "$((now_ts - RETENTION_SECONDS))"
-  fi
-
-  prev_total="$total"
-  prev_idle="$idle"
-  prev_iowait="$iowait"
-  prev_rx="$rx"
-  prev_tx="$tx"
-  prev_disk_read="$disk_read"
-  prev_disk_write="$disk_write"
-  prev_ts="$now_ts"
-  sleep "$INTERVAL"
-done
-'''
-
-
-@dataclass(frozen=True)
-class ResourceMonitorPaths:
-    """Remote file locations used by the lightweight resource collector."""
-
-    metrics_file: str = "/tmp/media_library_viewer_metrics.jsonl"
-    pid_file: str = "/tmp/media_library_viewer_metrics.pid"
-    script_file: str = "/tmp/media_library_viewer_metrics_collector.sh"
-    log_file: str = "/tmp/media_library_viewer_metrics.log"
-
-
-def start_resource_collector(
-    ssh: RemoteSSHClient,
-    interval_seconds: int = 10,
-    retention_seconds: int = 7 * 24 * 60 * 60,
-    max_lines: int = 70_000,
-    paths: ResourceMonitorPaths = ResourceMonitorPaths(),
-) -> str:
-    """Install and start the remote metrics collector if it is not running.
-
-    Starting a fresh collector removes old metrics/log files because schema
-    changes during development can otherwise leave mixed JSONL records behind.
-    The collector prunes its own metrics file to 7 days / max_lines.
-    """
-    command = f"""
-cat > {shlex.quote(paths.script_file)} <<'MLV_RESOURCE_COLLECTOR'
-{COLLECTOR_SCRIPT}
-MLV_RESOURCE_COLLECTOR
-chmod +x {shlex.quote(paths.script_file)}
-if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
-  echo "already running pid=$(cat {shlex.quote(paths.pid_file)})"
-else
-  rm -f {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)}
-  nohup {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {int(interval_seconds)} {int(retention_seconds)} {int(max_lines)} >> {shlex.quote(paths.log_file)} 2>&1 &
-  echo $! > {shlex.quote(paths.pid_file)}
-  echo "started pid=$(cat {shlex.quote(paths.pid_file)})"
-fi
-"""
-    logger.info(
-        "Starting remote resource collector interval=%ss retention=%ss max_lines=%s", interval_seconds, retention_seconds, max_lines
-    )
-    result = ssh.run(command, timeout=20)
-    if result.exit_status != 0:
-        logger.warning("Failed to start remote resource collector: %s", result.stderr or result.stdout)
-        raise RuntimeError(result.stderr or result.stdout or "failed to start resource collector")
-    logger.info("Remote resource collector start response: %s", result.stdout.strip())
-    return result.stdout.strip()
-
-
-def stop_resource_collector(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
-    """Stop the remote collector process if the pid file points to one."""
-    command = f"""
-if [ -f {shlex.quote(paths.pid_file)} ]; then
-  pid="$(cat {shlex.quote(paths.pid_file)})"
-  if kill -0 "$pid" 2>/dev/null; then
-    kill "$pid"
-    echo "stopped pid=$pid"
-  else
-    echo "not running"
-  fi
-  rm -f {shlex.quote(paths.pid_file)}
-else
-  echo "not running"
-fi
-"""
-    logger.info("Stopping remote resource collector")
-    result = ssh.run(command, timeout=20)
-    if result.exit_status != 0:
-        logger.warning("Failed to stop remote resource collector: %s", result.stderr or result.stdout)
-        raise RuntimeError(result.stderr or result.stdout or "failed to stop resource collector")
-    logger.info("Remote resource collector stop response: %s", result.stdout.strip())
-    return result.stdout.strip()
-
-
-def restart_resource_collector(
-    ssh: RemoteSSHClient,
-    interval_seconds: int = 10,
-    retention_seconds: int = 7 * 24 * 60 * 60,
-    max_lines: int = 70_000,
-    paths: ResourceMonitorPaths = ResourceMonitorPaths(),
-) -> str:
-    logger.info("Restarting remote resource collector")
-    stop_message = stop_resource_collector(ssh, paths)
-    start_message = start_resource_collector(ssh, interval_seconds, retention_seconds, max_lines, paths)
-    return f"{stop_message}\n{start_message}"
-
-
-def resource_collector_status(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
-    """Return a short human-readable status string for the dashboard."""
-    command = f"""
-if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
-  echo "running"
-else
-  echo "not running"
-fi
-"""
-    result = ssh.run(command, timeout=10)
-    if result.exit_status != 0:
-        logger.warning("Failed to read collector status: %s", result.stderr or result.stdout)
-        raise RuntimeError(result.stderr or result.stdout or "failed to check collector status")
-    status = result.stdout.strip()
-    logger.info("Resource collector status: %s", status)
-    return status
-
-
-def resource_collector_debug_info(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
-    """Collect remote diagnostics for troubleshooting missing metrics."""
-    command = f"""
-echo "status:"
-if [ -f {shlex.quote(paths.pid_file)} ]; then
-  pid="$(cat {shlex.quote(paths.pid_file)})"
-  echo "pid_file=$pid"
-  if kill -0 "$pid" 2>/dev/null; then echo "process=running"; else echo "process=not-running"; fi
-else
-  echo "pid_file=missing"
-fi
-
-echo "files:"
-ls -l {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} 2>&1 || true
-
-echo "sample_count:"
-if [ -f {shlex.quote(paths.metrics_file)} ]; then wc -l < {shlex.quote(paths.metrics_file)}; else echo 0; fi
-
-echo "last_samples:"
-if [ -f {shlex.quote(paths.metrics_file)} ]; then tail -n 5 {shlex.quote(paths.metrics_file)}; fi
-
-echo "log_tail:"
-if [ -f {shlex.quote(paths.log_file)} ]; then tail -n 40 {shlex.quote(paths.log_file)}; fi
-
-echo "netdev_snapshot:"
-cat /proc/net/dev 2>&1 || true
-"""
-    logger.info("Collecting resource collector diagnostics")
-    result = ssh.run(command, timeout=20)
-    output = (result.stdout or "") + (result.stderr or "")
-    logger.debug("Resource collector diagnostics length=%s", len(output))
-    return output
-
-
-def read_resource_metrics(ssh: RemoteSSHClient, max_lines: int = 1000, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> list[dict[str, Any]]:
-    """Read recent JSONL metric samples from the remote collector file."""
-    command = f"test -f {shlex.quote(paths.metrics_file)} && tail -n {int(max_lines)} {shlex.quote(paths.metrics_file)} || true"
-    logger.debug("Reading up to %s resource metric lines", max_lines)
-    result = ssh.run(command, timeout=20)
-    if result.exit_status != 0:
-        logger.warning("Failed to read resource metrics: %s", result.stderr or result.stdout)
-        raise RuntimeError(result.stderr or result.stdout or "failed to read resource metrics")
-    rows = []
-    for line in result.stdout.splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        try:
-            rows.append(json.loads(line))
-        except json.JSONDecodeError:
-            continue
-    return rows
-
-
-def disk_space(ssh: RemoteSSHClient, path: str = "/") -> dict[str, Any]:
-    """Return df information for the filesystem containing ``path``."""
-    command = (
-        "df -P -B1 -- "
-        + shlex.quote(path or "/")
-        + " | awk 'NR==2 {printf \"{\\\"filesystem\\\":\\\"%s\\\",\\\"size\\\":%s,\\\"used\\\":%s,\\\"available\\\":%s,\\\"used_pct\\\":\\\"%s\\\",\\\"mount\\\":\\\"%s\\\"}\", $1,$2,$3,$4,$5,$6}'"
-    )
-    logger.debug("Reading disk space for path=%s", path)
-    result = ssh.run(command, timeout=20)
-    if result.exit_status != 0 or not result.stdout.strip():
-        logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout)
-        raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
-    data = json.loads(result.stdout)
-    logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct"))
-    return data

backend/src/media_library_viewer_api/clients/ssh.py

@@ -75,14 +75,10 @@ class RemoteSSHClient:
         client = paramiko.SSHClient()
         client.load_system_host_keys()
         known_hosts_file = Path(self.known_hosts_path) if self.known_hosts_path else None
-        trusted_before = bool(
+        trusted_before = bool(known_hosts_file and has_known_host(self.host, self.port, known_hosts_file))
-            known_hosts_file and has_known_host(self.host, self.port, known_hosts_file)
-        )
         if known_hosts_file and known_hosts_file.is_file():
             client.load_host_keys(str(known_hosts_file))
-        client.set_missing_host_key_policy(
+        client.set_missing_host_key_policy(paramiko.RejectPolicy() if trusted_before else paramiko.AutoAddPolicy())
-            paramiko.RejectPolicy() if trusted_before else paramiko.AutoAddPolicy()
-        )
         connect_kwargs: dict[str, Any] = {
             "hostname": self.host,
             "port": self.port,

backend/src/media_library_viewer_api/config.py

@@ -23,6 +23,7 @@ class Settings(BaseSettings):
 
     # Logging
     log_level: str = "INFO"
+    log_format: str = "text"  # "text" or "json"
 
     # Auth / OIDC (Authentik-compatible JWT validation)
     auth_enabled: bool = False
@@ -48,7 +49,6 @@ class Settings(BaseSettings):
     ssh_port: int = 22
     ssh_key_directory: str = ""
     ssh_key_name: str = ""
-    ssh_key_file: str = "/run/secrets/ssh_private_key"
     ssh_password: str = ""
     ssh_known_hosts_path: str = ""
 
@@ -57,6 +57,12 @@ class Settings(BaseSettings):
     monitoring_poll_initial_delay_seconds: int = 20
     monitoring_action_retention_days: int = 30
 
+    # Observability
+    prometheus_enabled: bool = True
+    prometheus_file_sd_dir: str = "/app/backend/.cache/prometheus-file-sd"
+    alertmanager_url: str = "http://alertmanager:9093"
+    alertmanager_webhook_url: str = ""  # Optional receiver for alertmanager webhook notifications
+
     # Remote paths
     remote_media_root: str = ""
     remote_path_prefix: str = ""
@@ -72,8 +78,6 @@ class Settings(BaseSettings):
 
     @property
     def ssh_key_path(self) -> str:
-        if self.ssh_key_file:
-            return self.ssh_key_file
         if not self.ssh_key_directory or not self.ssh_key_name:
             return ""
         return str(Path(self.ssh_key_directory) / self.ssh_key_name)
@@ -82,7 +86,7 @@ class Settings(BaseSettings):
     def ssh_known_hosts_file(self) -> Path:
         return Path(self.ssh_known_hosts_path or ".cache/media_library_viewer/known_hosts")
 
-    model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"}
+    model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"}  # type: ignore[assignment]
 
 
 def _find_env_file() -> str | None:

backend/src/media_library_viewer_api/dependencies.py

@@ -19,12 +19,16 @@ from media_library_viewer_api.clients.jellyseerr import JellyseerrClient
 from media_library_viewer_api.clients.local import LocalCommandClient
 from media_library_viewer_api.clients.ssh import RemoteSSHClient
 from media_library_viewer_api.config import get_settings
-from media_library_viewer_api.services.mail_queue import MailQueue, get_mail_queue as _get_mail_queue
+from media_library_viewer_api.services.mail_queue import MailQueue
+from media_library_viewer_api.services.mail_queue import get_mail_queue as _get_mail_queue
 from media_library_viewer_api.services.monitoring_poller import (
     MonitoringPoller,
+)
+from media_library_viewer_api.services.monitoring_poller import (
     get_monitoring_poller as _get_monitoring_poller,
 )
-from media_library_viewer_api.services.settings_store import SettingsStore, get_settings_store as _get_settings_store
+from media_library_viewer_api.services.settings_store import SettingsStore
+from media_library_viewer_api.services.settings_store import get_settings_store as _get_settings_store
 
 logger = logging.getLogger(__name__)
 
@@ -39,7 +43,9 @@ def _request_machine_id(request: Request | None) -> str | None:
 @lru_cache(maxsize=32)
 def _jellyfin_client_for(cache_key: tuple[str, str, str]) -> JellyfinClient:
     machine_id, url, api_key = cache_key
-    logger.info("Creating Jellyfin client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>")
+    logger.info(
+        "Creating Jellyfin client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>"
+    )
     return JellyfinClient(url, api_key)
 
 
@@ -52,13 +58,19 @@ def _jellyseerr_client_for(cache_key: tuple[str, str]) -> JellyseerrClient | Non
     api_key = (settings or {}).get("jellyseerr_api_key") if settings else ""
     if not api_key:
         return None
-    logger.info("Creating Jellyseerr client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>")
+    logger.info(
+        "Creating Jellyseerr client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>"
+    )
     return JellyseerrClient(url, api_key)
 
 
 @lru_cache(maxsize=32)
-def _ssh_client_for(cache_key: tuple[str, str, str, int, str, str | None, str | None, str | None, str | None]) -> RemoteSSHClient:
+def _ssh_client_for(
-    machine_id, host, username, port, key_filename, password, private_key, private_key_passphrase, known_hosts_path = cache_key
+    cache_key: tuple[str, str, str, int, str, str | None, str | None, str | None, str | None],
+) -> RemoteSSHClient:
+    machine_id, host, username, port, key_filename, password, private_key, private_key_passphrase, known_hosts_path = (
+        cache_key
+    )
     logger.info(
         "Creating SSH client machine_id=%s host=%s user=%s port=%s key=%s password=%s private_key=%s passphrase=%s",
         machine_id or "<default>",
@@ -141,7 +153,9 @@ def get_jellyfin_client(request: Request = None) -> JellyfinClient:
         cache_key = (machine["id"], machine["jellyfin_url"], machine.get("jellyfin_api_key") or "")
         return _jellyfin_client_for(cache_key)
 
-    raise RuntimeError("No Jellyfin machine is configured. Add a machine with jellyfin_url and jellyfin_api_key in Settings.")
+    raise RuntimeError(
+        "No Jellyfin machine is configured. Add a machine with jellyfin_url and jellyfin_api_key in Settings."
+    )
 
 
 def get_jellyseerr_client(request: Request = None) -> JellyseerrClient | None:
@@ -160,6 +174,37 @@ def get_jellyseerr_client(request: Request = None) -> JellyseerrClient | None:
     return None
 
 
+def _ssh_client_from_machine_config(machine: dict[str, Any], store: SettingsStore | None = None) -> RemoteSSHClient:
+    """Build a RemoteSSHClient from a machine config dict."""
+    store = store or get_settings_store()
+    known_hosts_path = get_settings().ssh_known_hosts_file
+    key_data = None
+    key_passphrase = None
+    ssh_key_id = str(machine.get("ssh_key_id") or "").strip()
+    if ssh_key_id:
+        ssh_key = store.get_ssh_key(ssh_key_id)
+        if ssh_key:
+            key_data = ssh_key.get("private_key") or None
+            key_passphrase = ssh_key.get("passphrase") or None
+    if not key_data and machine.get("ssh_private_key"):
+        key_data = machine.get("ssh_private_key") or None
+        key_passphrase = machine.get("ssh_private_key_passphrase") or None
+    cache_key = (
+        machine["id"],
+        machine["host"],
+        machine["username"],
+        int(machine.get("port") or 22),
+        f"{machine.get('key_directory')}/{machine.get('key_name')}"
+        if machine.get("key_directory") and machine.get("key_name")
+        else "",
+        machine.get("password") or None,
+        key_data,
+        key_passphrase,
+        str(known_hosts_path),
+    )
+    return _ssh_client_for(cache_key)
+
+
 def get_ssh_client(request: Request = None):
     """Return a command client for the selected machine or legacy env fallback."""
     store = get_settings_store()
@@ -172,30 +217,7 @@ def get_ssh_client(request: Request = None):
         logger.info("Creating LocalCommandClient machine_id=%s", machine["id"])
         return LocalCommandClient()
     if machine and machine.get("host") and machine.get("username"):
-        known_hosts_path = get_settings().ssh_known_hosts_file
+        return _ssh_client_from_machine_config(machine, store)
-        key_data = None
-        key_passphrase = None
-        ssh_key_id = str(machine.get("ssh_key_id") or "").strip()
-        if ssh_key_id:
-            ssh_key = store.get_ssh_key(ssh_key_id)
-            if ssh_key:
-                key_data = ssh_key.get("private_key") or None
-                key_passphrase = ssh_key.get("passphrase") or None
-        if not key_data and machine.get("ssh_private_key"):
-            key_data = machine.get("ssh_private_key") or None
-            key_passphrase = machine.get("ssh_private_key_passphrase") or None
-        cache_key = (
-            machine["id"],
-            machine["host"],
-            machine["username"],
-            int(machine.get("port") or 22),
-            f"{machine.get('key_directory')}/{machine.get('key_name')}" if machine.get("key_directory") and machine.get("key_name") else "",
-            machine.get("password") or None,
-            key_data,
-            key_passphrase,
-            str(known_hosts_path),
-        )
-        return _ssh_client_for(cache_key)
 
     settings = get_settings()
     logger.info(
@@ -209,7 +231,19 @@ def get_ssh_client(request: Request = None):
     )
     if not settings.ssh_key_path:
         raise RuntimeError("No SSH machine is configured and SSH key settings must be configured")
-    return _ssh_client_for(("legacy", settings.ssh_host, settings.ssh_username, settings.ssh_port, settings.ssh_key_path, settings.ssh_password or None, None, None, str(settings.ssh_known_hosts_file)))
+    return _ssh_client_for(
+        (
+            "legacy",
+            settings.ssh_host,
+            settings.ssh_username,
+            settings.ssh_port,
+            settings.ssh_key_path,
+            settings.ssh_password or None,
+            None,
+            None,
+            str(settings.ssh_known_hosts_file),
+        )
+    )
 
 
 def get_mail_queue() -> MailQueue:

backend/src/media_library_viewer_api/domain/media.py

@@ -32,7 +32,11 @@ def media_streams(item: dict[str, Any], stream_type: str | None = None) -> list[
         streams.extend(source.get("MediaStreams") or [])
     if stream_type is None:
         return streams
-    return [stream for stream in streams if str(stream.get("Type") or stream.get("codec_type") or "").lower() == stream_type.lower()]
+    return [
+        stream
+        for stream in streams
+        if str(stream.get("Type") or stream.get("codec_type") or "").lower() == stream_type.lower()
+    ]
 
 
 def stream_value(stream: dict[str, Any], *keys: str) -> Any:

backend/src/media_library_viewer_api/jobs.py

@@ -52,6 +52,42 @@ JOB_TEMPLATES: dict[str, JobTemplate] = {
         description="Lists empty directories under the selected path. Does not delete anything.",
         command_template="find {path} -type d -empty -print",
     ),
+    "install_node_exporter": JobTemplate(
+        name="Install Node Exporter",
+        description="Downloads and installs prometheus-node-exporter via package manager (apt/dnf/yum/zypper).",
+        command_template=(
+            "set -e; "
+            "if command -v apt-get >/dev/null 2>&1; then "
+            "sudo apt-get update && sudo apt-get install -y prometheus-node-exporter; "
+            "elif command -v dnf >/dev/null 2>&1; then "
+            "sudo dnf install -y prometheus-node-exporter; "
+            "elif command -v yum >/dev/null 2>&1; then "
+            "sudo yum install -y prometheus-node-exporter; "
+            "elif command -v zypper >/dev/null 2>&1; then "
+            "sudo zypper install -y prometheus-node-exporter; "
+            "else echo 'No supported package manager found' >&2; exit 1; "
+            "fi; "
+            "sudo systemctl enable --now prometheus-node-exporter; "
+            "echo installed at {path}"
+        ),
+    ),
+    "restart_node_exporter": JobTemplate(
+        name="Restart Node Exporter",
+        description="Restarts the prometheus-node-exporter systemd service.",
+        command_template="sudo systemctl restart prometheus-node-exporter; echo restarted at {path}",
+    ),
+    "node_exporter_status": JobTemplate(
+        name="Node Exporter status",
+        description="Checks whether prometheus-node-exporter is installed, enabled, and running.",
+        command_template=(
+            "systemctl status prometheus-node-exporter --no-pager || true; "
+            "echo '---'; "
+            "command -v node_exporter >/dev/null 2>&1 "
+            "&& node_exporter --version 2>&1 | head -1 "
+            "|| echo 'node_exporter binary not found'; "
+            "echo checked {path}"
+        ),
+    ),
 }
 
 

backend/src/media_library_viewer_api/logging_utils.py

@@ -4,19 +4,42 @@ from __future__ import annotations
 
 import logging
 import os
+from typing import Any
 from urllib.parse import urlsplit
 
+from pythonjsonlogger import jsonlogger
 
-def configure_logging(level_name: str | None = None) -> int:
+
+def _json_formatter() -> logging.Formatter:
+    """Return a JSON formatter that preserves timestamp, level, logger, and message."""
+    return jsonlogger.JsonFormatter(
+        fmt="%(timestamp)s %(level)s %(name)s %(message)s",
+        rename_fields={"asctime": "timestamp", "levelname": "level"},
+    )
+
+
+def _text_formatter() -> logging.Formatter:
+    """Return the legacy plain-text formatter."""
+    return logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
+
+
+def configure_logging(level_name: str | None = None, log_format: str | None = None) -> int:
     """Configure root logging once and return the numeric log level."""
     resolved_name = (level_name or os.getenv("LOG_LEVEL", "INFO")).upper()
     level = getattr(logging, resolved_name, logging.INFO)
+    use_json = (log_format or os.getenv("LOG_FORMAT", "text")).lower() == "json"
+
     root = logging.getLogger()
     if not root.handlers:
-        logging.basicConfig(
+        handler = logging.StreamHandler()
-            level=level,
+        handler.setFormatter(_json_formatter() if use_json else _text_formatter())
-            format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        logging.basicConfig(level=level, handlers=[handler])
-        )
+    else:
+        # Reconfigure existing handler formatters if the format changed.
+        formatter = _json_formatter() if use_json else _text_formatter()
+        for handler in root.handlers:
+            handler.setFormatter(formatter)
+
     root.setLevel(level)
     logging.getLogger("media_library_viewer_api").setLevel(level)
     logging.getLogger("uvicorn").setLevel(level)
@@ -40,6 +63,7 @@ def describe_settings(settings: object) -> dict[str, str]:
     """Return a secret-safe summary of the current backend settings."""
     return {
         "log_level": str(getattr(settings, "log_level", "INFO") or "INFO").upper(),
+        "log_format": str(getattr(settings, "log_format", "text") or "text").lower(),
         "auth_enabled": str(bool(getattr(settings, "auth_enabled", True))),
         "oidc_issuer_url": _sanitize_url(getattr(settings, "oidc_issuer_url", "")),
         "oidc_audience": getattr(settings, "oidc_audience", "") or "<unset>",
@@ -60,3 +84,18 @@ def describe_settings(settings: object) -> dict[str, str]:
         "remote_media_root": getattr(settings, "remote_media_root", "") or "<unset>",
         "remote_path_prefix": getattr(settings, "remote_path_prefix", "") or "<unset>",
     }
+
+
+def sanitize_log_extra(extra: dict[str, Any] | None) -> dict[str, Any]:
+    """Remove obvious secret keys from log extra fields before emission."""
+    if not extra:
+        return {}
+    blocked_suffixes = ("_password", "_secret", "_token", "_key", "private_key", "passphrase")
+    sanitized: dict[str, Any] = {}
+    for key, value in extra.items():
+        lower_key = key.lower()
+        if any(lower_key.endswith(suffix) for suffix in blocked_suffixes):
+            sanitized[key] = "<redacted>" if value else ""
+        else:
+            sanitized[key] = value
+    return sanitized

backend/src/media_library_viewer_api/main.py

@@ -7,13 +7,20 @@ import time
 from contextlib import asynccontextmanager
 
 import uvicorn
-from fastapi import FastAPI, Request
+from fastapi import FastAPI, Request, Response
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import Response as FastAPIResponse
 
 from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settings
 from media_library_viewer_api.config import get_settings
-from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller
+from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller, get_settings_store
-from media_library_viewer_api.logging_utils import configure_logging, describe_settings
+from media_library_viewer_api.logging_utils import configure_logging, describe_settings, sanitize_log_extra
+from media_library_viewer_api.observability import (
+    get_request_id,
+    metrics_payload,
+    record_request,
+    set_current_request_id,
+)
 from media_library_viewer_api.routers import backups as backups_router
 from media_library_viewer_api.routers import dashboard, files, jobs, media, monitoring, tasks, users
 from media_library_viewer_api.routers.settings import router as settings_router
@@ -28,10 +35,16 @@ logger = logging.getLogger(__name__)
 async def lifespan(app: FastAPI):
     """Application lifespan — startup/shutdown."""
     settings = get_settings()
-    configure_logging(settings.log_level)
+    configure_logging(settings.log_level, settings.log_format)
     validate_auth_settings(settings)
     logger.info("Backend startup complete: %s", describe_settings(settings))
     logger.info("Managed known_hosts will be populated lazily on first successful SSH connection")
+    try:
+        from media_library_viewer_api.services.targets import write_prometheus_targets
+
+        write_prometheus_targets(get_settings_store())
+    except Exception:
+        logger.exception("Failed to write Prometheus file-SD targets during startup")
     mail_queue = get_mail_queue()
     monitoring_poller = get_monitoring_poller()
     backup_poller = get_backup_poller()
@@ -49,8 +62,7 @@ app = FastAPI(
     title="Manage API",
     version=get_backend_version(),
     description=(
-        "Manage API for Jellyfin media browsing, SSH file inspection, "
+        "Manage API for Jellyfin media browsing, SSH file inspection, server monitoring, and JWT-protected access."
-        "server monitoring, and JWT-protected access."
     ),
     lifespan=lifespan,
 )
@@ -69,30 +81,51 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+
 @app.middleware("http")
 async def enforce_jwt_auth(request: Request, call_next):
-    if request.url.path in {"/api/health", "/api/version"}:
+    if request.url.path in {"/api/health", "/api/version", "/metrics"}:
         return await call_next(request)
     return await require_jwt_auth(request, call_next)
 
+
 @app.middleware("http")
 async def log_requests(request: Request, call_next):
-    """Log every API request with timing and outcome."""
+    """Log every API request with timing, outcome, and request id."""
     start = time.perf_counter()
+    request_id = get_request_id(request)
+    set_current_request_id(request_id)
+    request.state.request_id = request_id
     client_host = request.client.host if request.client else "unknown"
-    logger.info("request start %s %s client=%s", request.method, request.url.path, client_host)
+    extra = sanitize_log_extra({"request_id": request_id, "client_host": client_host})
+    logger.info("request start %s %s", request.method, request.url.path, extra=extra)
     try:
         response = await call_next(request)
     except Exception:
-        logger.exception("request error %s %s client=%s", request.method, request.url.path, client_host)
+        logger.exception(
+            "request error %s %s",
+            request.method,
+            request.url.path,
+            extra=sanitize_log_extra({"request_id": request_id, "client_host": client_host}),
+        )
         raise
     elapsed_ms = (time.perf_counter() - start) * 1000.0
+    response.headers["X-Request-Id"] = request_id
+    record_request(request, response, elapsed_ms / 1000.0)
     logger.info(
         "request end %s %s status=%s elapsed_ms=%.1f",
         request.method,
         request.url.path,
         response.status_code,
         elapsed_ms,
+        extra=sanitize_log_extra(
+            {
+                "request_id": request_id,
+                "client_host": client_host,
+                "status_code": response.status_code,
+                "elapsed_ms": round(elapsed_ms, 1),
+            }
+        ),
     )
     return response
 
@@ -123,5 +156,12 @@ def version_info() -> dict[str, str]:
     return get_version_info()
 
 
+@app.get("/metrics")
+def metrics() -> Response:
+    """Expose Prometheus metrics."""
+    data, content_type = metrics_payload()
+    return FastAPIResponse(content=data, media_type=content_type)
+
+
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=8000)

backend/src/media_library_viewer_api/observability.py

@@ -0,0 +1,154 @@
+"""Application observability: metrics and request correlation.
+
+This module owns Prometheus metrics and request-id generation so that the
+rest of the backend can stay focused on business logic.
+"""
+
+from __future__ import annotations
+
+import uuid
+from contextvars import ContextVar
+from typing import Any
+
+from fastapi import Request, Response
+from prometheus_client import (
+    CONTENT_TYPE_LATEST,
+    Counter,
+    Gauge,
+    Histogram,
+    generate_latest,
+)
+
+# Context-local request id for code paths that cannot receive a Request object.
+_current_request_id: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+# Request metrics
+REQUESTS_TOTAL = Counter(
+    "manage_api_requests_total",
+    "Total API requests",
+    ["method", "path", "status_code"],
+)
+REQUEST_DURATION = Histogram(
+    "manage_api_request_duration_seconds",
+    "API request duration",
+    ["method", "path"],
+    buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0),
+)
+
+# Operation metrics
+SSH_COMMANDS_TOTAL = Counter(
+    "manage_ssh_commands_total",
+    "Total SSH/local commands executed",
+    ["machine_id", "action", "status"],
+)
+SSH_COMMAND_DURATION = Histogram(
+    "manage_ssh_command_duration_seconds",
+    "SSH/local command duration",
+    ["machine_id", "action"],
+    buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0),
+)
+
+MEDIA_INDEX_BUILDS_TOTAL = Counter(
+    "manage_media_index_builds_total",
+    "Total media index build attempts",
+    ["status"],
+)
+MEDIA_INDEX_BUILD_DURATION = Histogram(
+    "manage_media_index_build_duration_seconds",
+    "Media index build duration",
+    buckets=(1.0, 5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0),
+)
+
+BACKUP_RUNS_TOTAL = Counter(
+    "manage_backup_runs_total",
+    "Total backup runs",
+    ["job_name", "status"],
+)
+
+BACKUP_RUNS_LAST_SUCCESS = Gauge(
+    "manage_backup_runs_last_success_timestamp",
+    "Unix timestamp of the last successful backup run per job",
+    ["job_name"],
+)
+
+MAIL_QUEUE_SIZE = Counter(
+    "manage_mail_queue_messages_total",
+    "Total messages enqueued",
+    ["status"],
+)
+
+
+def set_current_request_id(request_id: str | None) -> None:
+    """Set the context-local request id."""
+    _current_request_id.set(request_id)
+
+
+def get_current_request_id() -> str | None:
+    """Return the current context-local request id or None."""
+    return _current_request_id.get()
+
+
+def generate_request_id() -> str:
+    """Return a short unique request id."""
+    return uuid.uuid4().hex[:16]
+
+
+def get_request_id(request: Request | None = None) -> str:
+    """Resolve a request id from the request header, context, or a new value."""
+    if request is not None:
+        header = request.headers.get("x-request-id") or request.headers.get("x-correlation-id")
+        if header:
+            return header.strip()
+    existing = _current_request_id.get()
+    if existing:
+        return existing
+    new_id = generate_request_id()
+    _current_request_id.set(new_id)
+    return new_id
+
+
+def metrics_payload() -> tuple[bytes, str]:
+    """Return the Prometheus metrics payload and content type."""
+    return generate_latest(), CONTENT_TYPE_LATEST
+
+
+def record_request(request: Request, response: Response, duration_seconds: float) -> None:
+    """Record Prometheus metrics for a completed request."""
+    status = str(response.status_code)
+    path = request.url.path
+    method = request.method
+    REQUESTS_TOTAL.labels(method=method, path=path, status_code=status).inc()
+    REQUEST_DURATION.labels(method=method, path=path).observe(duration_seconds)
+
+
+def record_ssh_command(machine_id: str, action: str, status: str, duration_seconds: float) -> None:
+    """Record metrics for an SSH/local command."""
+    SSH_COMMANDS_TOTAL.labels(machine_id=machine_id or "unknown", action=action, status=status).inc()
+    SSH_COMMAND_DURATION.labels(machine_id=machine_id or "unknown", action=action).observe(duration_seconds)
+
+
+def record_media_index_build(status: str, duration_seconds: float | None = None) -> None:
+    """Record metrics for a media index build."""
+    MEDIA_INDEX_BUILDS_TOTAL.labels(status=status).inc()
+    if duration_seconds is not None:
+        MEDIA_INDEX_BUILD_DURATION.observe(duration_seconds)
+
+
+def record_backup_run(job_name: str, status: str, success: bool = False) -> None:
+    """Record metrics for a backup run."""
+    job_name = job_name or "unknown"
+    BACKUP_RUNS_TOTAL.labels(job_name=job_name, status=status).inc()
+    if success:
+        BACKUP_RUNS_LAST_SUCCESS.labels(job_name=job_name).set_to_current_time()
+
+
+def record_mail_queue(status: str) -> None:
+    """Record metrics for a mail queue message outcome."""
+    MAIL_QUEUE_SIZE.labels(status=status).inc()
+
+
+def log_extra(request: Request | None = None, **kwargs: Any) -> dict[str, Any]:
+    """Build a standard extra dict for structured logging."""
+    extra: dict[str, Any] = {"request_id": get_request_id(request)}
+    extra.update(kwargs)
+    return extra

backend/src/media_library_viewer_api/path_utils.py

@@ -57,7 +57,7 @@ def map_path_to_media_root(path: str, media_root: str) -> str:
 
     if root_anchor in raw_parts:
         anchor_index = raw_parts.index(root_anchor)
-        remainder_parts = raw_parts[anchor_index + 1:]
+        remainder_parts = raw_parts[anchor_index + 1 :]
         resolved = posixpath.join(normalized_root, *remainder_parts) if remainder_parts else normalized_root
         logger.debug("Mapped path to media root path=%s media_root=%s resolved=%s", path, normalized_root, resolved)
         return resolved

backend/src/media_library_viewer_api/routers/backups.py

@@ -8,6 +8,7 @@ from ..models.backups import (
     BackupReportRequest,
     BackupRunResponse,
 )
+from ..observability import record_backup_run
 from ..services.backup_alert_engine import generate_alerts_for_run
 from ..services.settings_store import SettingsStore, get_settings_store
 
@@ -17,20 +18,24 @@ router = APIRouter(prefix="/api/backups", tags=["backups"])
 def _get_or_create_job(store: SettingsStore, report: BackupReportRequest) -> dict[str, Any]:
     job = store.get_backup_job_by_name(report.name)
     if not job:
-        job = store.upsert_backup_job({
+        job = store.upsert_backup_job(
-            "name": report.name,
+            {
-            "source": report.source,
+                "name": report.name,
-            "target": report.target,
+                "source": report.source,
-            "schedule_interval_seconds": report.schedule_interval_seconds,
+                "target": report.target,
-        })
+                "schedule_interval_seconds": report.schedule_interval_seconds,
+            }
+        )
     elif report.schedule_interval_seconds:
-        store.upsert_backup_job({
+        store.upsert_backup_job(
-            "id": job["id"],
+            {
-            "name": report.name,
+                "id": job["id"],
-            "source": report.source,
+                "name": report.name,
-            "target": report.target,
+                "source": report.source,
-            "schedule_interval_seconds": report.schedule_interval_seconds,
+                "target": report.target,
-        })
+                "schedule_interval_seconds": report.schedule_interval_seconds,
+            }
+        )
         job = store.get_backup_job(job["id"])
     return job
 
@@ -61,6 +66,7 @@ def post_backup_report(
         "details": report.details,
     }
     run = store.create_backup_run(run_data)
+    record_backup_run(job_name=report.name, status=report.status, success=report.status == "success")
 
     # Generate alerts
     previous_runs = store.list_backup_runs(job_id=job["id"], status="success", limit=20)
@@ -93,6 +99,7 @@ def post_backup_start(
         "status": "in_progress",
     }
     run = store.create_backup_run(run_data)
+    record_backup_run(job_name=report.name, status="in_progress")
     # Map details -> details_json for response model
     run["details_json"] = run.pop("details", None)
     return BackupRunResponse(**run)

backend/src/media_library_viewer_api/routers/dashboard.py

@@ -11,12 +11,10 @@ from fastapi import APIRouter, Depends
 from media_library_viewer_api.clients.jellyfin import JellyfinClient
 from media_library_viewer_api.dependencies import (
     get_jellyfin_client,
-    get_monitoring_poller,
     get_settings_store,
     get_user_id,
 )
 from media_library_viewer_api.models.backups import BackupDashboardSummary
-from media_library_viewer_api.services.monitoring_actions import collect_machine_overview
 from media_library_viewer_api.services.settings_store import SettingsStore
 
 logger = logging.getLogger(__name__)
@@ -46,52 +44,6 @@ def get_library_counts(
     return client.library_item_counts(user_id, libraries)
 
 
-@router.get("/monitoring")
-def get_monitoring_overview(
-    store=Depends(get_settings_store),
-) -> dict[str, Any]:
-    """Return one lightweight monitoring row per configured machine."""
-    machines = [m for m in store.list_machines() if m.get("enabled")]
-    rows: list[dict[str, Any]] = []
-    for machine in machines:
-        try:
-            rows.append(collect_machine_overview(machine, store))
-        except Exception as exc:
-            logger.exception(
-                "Dashboard monitoring overview failed for machine_id=%s",
-                machine.get("id"),
-            )
-            rows.append({
-                "machine": {k: machine.get(k) for k in (
-                    "id", "name", "mode", "enabled", "host", "port", "username",
-                    "media_root", "path_prefix", "notes",
-                )},
-                "status": "",
-                "status_error": str(exc),
-                "metrics_error": str(exc),
-                "disk_error": str(exc),
-                "sample_count": 0,
-                "latest_sample": None,
-                "cpu_summary": None,
-                "iowait_summary": None,
-                "mem_summary": None,
-                "net_rx_summary": None,
-                "net_tx_summary": None,
-                "disk_read_summary": None,
-                "disk_write_summary": None,
-                "disk": None,
-            })
-    poller = get_monitoring_poller().snapshot()
-    enabled_count = sum(1 for machine in machines if machine.get("enabled"))
-    logger.info("Dashboard monitoring machines=%s enabled=%s", len(machines), enabled_count)
-    return {
-        "poller": poller,
-        "machines": rows,
-        "total": len(machines),
-        "enabled": enabled_count,
-    }
-
-
 @router.get("/shortcuts")
 def get_shortcuts(
     store=Depends(get_settings_store),
@@ -144,8 +96,8 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[
         has_item = bool(item)
         series = item.get("SeriesName") or ""
         title = (
-            f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")
+            (f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")) if has_item else "(idle)"
-        ) if has_item else "(idle)"
+        )
 
         if not has_item:
             state_label = "idle"
@@ -162,16 +114,18 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[
             if not transcode_type:
                 transcode_type.append("active")
 
-        results.append({
+        results.append(
-            "user": session.get("UserName") or "Unknown",
+            {
-            "title": title,
+                "user": session.get("UserName") or "Unknown",
-            "type": item.get("Type", "") if has_item else "",
+                "title": title,
-            "state": state_label,
+                "type": item.get("Type", "") if has_item else "",
-            "transcoding": "yes" if is_transcoding else "no",
+                "state": state_label,
-            "transcoding_type": ", ".join(transcode_type),
+                "transcoding": "yes" if is_transcoding else "no",
-            "device": session.get("DeviceName") or session.get("Client") or "",
+                "transcoding_type": ", ".join(transcode_type),
-            "session_id": session.get("Id") or "",
+                "device": session.get("DeviceName") or session.get("Client") or "",
-        })
+                "session_id": session.get("Id") or "",
+            }
+        )
     return results
 
 

backend/src/media_library_viewer_api/routers/media.py

@@ -16,6 +16,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status
 
 from media_library_viewer_api.clients.jellyfin import JellyfinClient
 from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id
+from media_library_viewer_api.observability import record_media_index_build
 from media_library_viewer_api.services.media_index import MediaIndex
 
 logger = logging.getLogger(__name__)
@@ -169,6 +170,7 @@ def post_build_index(
             },
         )
         media_status = index.status()
+    record_media_index_build(status="started")
     logger.info("Media index build started pid=%s", process.pid)
     return {"status": "started", **_serialize_status(media_status)}
 
@@ -256,6 +258,7 @@ def force_stop_build(index: MediaIndex = Depends(get_media_index)) -> dict[str,
             },
         )
         media_status = index.status()
+    record_media_index_build(status="force_stopped")
     return {"status": "force_stopped", **_serialize_status(media_status)}
 
 

backend/src/media_library_viewer_api/routers/monitoring.py

@@ -1,31 +1,70 @@
-"""Monitoring router — metrics, collector controls, and per-machine status."""
+"""Monitoring router — disk checks, action history, and observability stack status."""
 
 from __future__ import annotations
 
 import logging
-import time
 from typing import Any
 
-from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
 
-from media_library_viewer_api.clients.resources import (
-    disk_space,
-    read_resource_metrics,
-    resource_collector_status,
-)
 from media_library_viewer_api.config import get_settings
 from media_library_viewer_api.dependencies import get_settings_store
 from media_library_viewer_api.services.monitoring_actions import (
-    poll_machine_diagnostics,
+    disk_space,
     run_machine_operation,
-    start_collector,
-    stop_collector,
-    restart_collector,
 )
 from media_library_viewer_api.services.settings_store import SettingsStore
+from media_library_viewer_api.services.targets import build_node_exporter_targets
 
 logger = logging.getLogger(__name__)
 
+
+def _alertmanager_client() -> Any:
+    """Return a simple HTTP client for the configured Alertmanager URL."""
+    import requests
+
+    settings = get_settings()
+    return requests.Session(), settings.alertmanager_url
+
+
+def _webhook_client() -> Any:
+    """Return a simple HTTP client for the optional webhook receiver URL."""
+    import requests
+
+    settings = get_settings()
+    return requests.Session(), settings.alertmanager_webhook_url
+
+
+def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
+    """Build a UI-friendly summary from Alertmanager /api/v1/alerts payload."""
+    by_severity: dict[str, int] = {}
+    open_alerts: list[dict[str, Any]] = []
+    for alert in alerts:
+        labels = alert.get("labels") or {}
+        annotations = alert.get("annotations") or {}
+        severity = labels.get("severity", "unknown")
+        by_severity[severity] = by_severity.get(severity, 0) + 1
+        open_alerts.append(
+            {
+                "name": labels.get("alertname", "unknown"),
+                "severity": severity,
+                "category": labels.get("category", ""),
+                "job_name": labels.get("job_name", labels.get("job", "")),
+                "summary": annotations.get("summary", ""),
+                "description": annotations.get("description", ""),
+                "active_since": alert.get("startsAt"),
+                "state": alert.get("status", "firing"),
+                "labels": labels,
+            }
+        )
+    open_alerts.sort(key=lambda a: (a["severity"] not in {"critical", "warning"}, a["severity"], a["name"]))
+    return {
+        "total": len(open_alerts),
+        "by_severity": by_severity,
+        "alerts": open_alerts[:50],
+    }
+
+
 router = APIRouter(prefix="/api/monitoring", tags=["monitoring"])
 
 
@@ -63,7 +102,11 @@ def get_poller_status() -> dict[str, Any]:
     from media_library_viewer_api.dependencies import get_monitoring_poller
 
     poller = get_monitoring_poller().snapshot()
-    logger.info("Monitoring poller status requested running=%s poll_count=%s", poller.get("worker_running"), poller.get("poll_count"))
+    logger.info(
+        "Monitoring poller status requested running=%s poll_count=%s",
+        poller.get("worker_running"),
+        poller.get("poll_count"),
+    )
     return poller
 
 
@@ -81,53 +124,6 @@ def get_machine_actions(
     return {"items": actions, "total": len(actions)}
 
 
-@router.get("/status")
-def get_status(
-    machine_id: str | None = Query(default=None),
-    store: SettingsStore = Depends(get_settings_store),
-) -> dict[str, str]:
-    """Return collector running status for a given machine."""
-    machine = _resolve_machine(store, machine_id)
-    status = run_machine_operation(machine, store, "status lookup", resource_collector_status)
-    logger.info("Monitoring status requested machine_id=%s status=%s", machine["id"], status)
-    return {"status": status}
-
-
-@router.get("/metrics")
-def get_metrics(
-    max_lines: int = 70_000,
-    last_seconds: int | None = None,
-    machine_id: str | None = Query(default=None),
-    store: SettingsStore = Depends(get_settings_store),
-) -> dict[str, Any]:
-    """Return resource metric samples for a given machine."""
-    machine = _resolve_machine(store, machine_id)
-
-    def _read_metrics(client):
-        return read_resource_metrics(client, max_lines=max_lines)
-
-    rows = run_machine_operation(machine, store, "metrics read", _read_metrics)
-    if last_seconds is None:
-        filtered = rows
-        cutoff_ts = 0.0
-    else:
-        cutoff_ts = time.time() - last_seconds
-        filtered = [row for row in rows if float(row.get("ts", 0)) >= cutoff_ts]
-    logger.info(
-        "Monitoring metrics requested machine_id=%s total=%s filtered=%s last_seconds=%s",
-        machine["id"],
-        len(rows),
-        len(filtered),
-        last_seconds,
-    )
-    return {
-        "samples": filtered,
-        "total_samples": len(rows),
-        "filtered_samples": len(filtered),
-        "cutoff_ts": cutoff_ts,
-    }
-
-
 @router.get("/disk")
 def get_disk_space(
     machine_id: str | None = Query(default=None),
@@ -146,49 +142,99 @@ def get_disk_space(
     )
 
 
-@router.post("/start")
+@router.get("/prometheus-targets")
-def post_start(
+def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
-    machine_id: str | None = Query(default=None),
+    """Return Prometheus file-SD targets for remote Node Exporters.
-    store: SettingsStore = Depends(get_settings_store),
+
-) -> dict[str, str]:
+    The backend writes these targets to a JSON file that Prometheus reads via
-    """Start the machine's resource collector."""
+    file_sd_configs. This endpoint returns the same list live from the store so
-    machine = _resolve_machine(store, machine_id)
+    the UI can preview which machines will be scraped.
-    message = start_collector(machine, store)
+    """
-    logger.info("Monitoring collector start result machine_id=%s: %s", machine["id"], message)
+    targets = build_node_exporter_targets(store)
-    return {"message": message}
+    logger.info("Prometheus targets requested count=%s", len(targets))
+    return targets
 
 
-@router.post("/stop")
+@router.get("/alerts")
-def post_stop(
+def get_alertmanager_alerts() -> dict[str, Any]:
-    machine_id: str | None = Query(default=None),
+    """Return a summary of active Alertmanager alerts for the UI.
-    store: SettingsStore = Depends(get_settings_store),
+
-) -> dict[str, str]:
+    Proxies the Alertmanager `/api/v1/alerts` endpoint and reshapes the payload
-    """Stop the machine's resource collector."""
+    into a stable, UI-friendly format. If Alertmanager is unreachable, the
-    machine = _resolve_machine(store, machine_id)
+    endpoint returns an empty summary and logs the failure so the UI can still
-    message = stop_collector(machine, store)
+    render a health card instead of an error page.
-    logger.info("Monitoring collector stop result machine_id=%s: %s", machine["id"], message)
+    """
-    return {"message": message}
+    session, base_url = _alertmanager_client()
+    try:
+        response = session.get(f"{base_url}/api/v1/alerts", timeout=5)
+        response.raise_for_status()
+        data = response.json()
+    except Exception:
+        logger.exception("Failed to fetch Alertmanager alerts from %s", base_url)
+        return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_unreachable"}
+
+    if data.get("status") != "success":
+        return {"total": 0, "by_severity": {}, "alerts": [], "error": data.get("error", "unknown")}
+
+    summary = _summary_from_alerts(data.get("data", []))
+    logger.info("Alertmanager alerts requested total=%s", summary["total"])
+    return summary
 
 
-@router.post("/restart")
+@router.get("/alertmanager-status")
-def post_restart(
+def get_alertmanager_status() -> dict[str, Any]:
-    machine_id: str | None = Query(default=None),
+    """Return Alertmanager cluster/status for the UI health card.
-    store: SettingsStore = Depends(get_settings_store),
+
-) -> dict[str, str]:
+    Uses the Alertmanager `/api/v2/status` endpoint and exposes only the high-
-    """Restart the machine's resource collector."""
+    level fields the UI needs: uptime, version, and whether the cluster is
-    machine = _resolve_machine(store, machine_id)
+    healthy.
-    message = restart_collector(machine, store)
+    """
-    logger.info("Monitoring collector restart result machine_id=%s: %s", machine["id"], message)
+    session, base_url = _alertmanager_client()
-    return {"message": message}
+    try:
+        response = session.get(f"{base_url}/api/v2/status", timeout=5)
+        response.raise_for_status()
+        data = response.json()
+    except Exception:
+        logger.exception("Failed to fetch Alertmanager status from %s", base_url)
+        return {"up": False, "version": "", "uptime": ""}
+
+    cluster = data.get("cluster") or {}
+    status = data.get("clusterStatus") or {}
+    return {
+        "up": True,
+        "version": data.get("versionInfo", {}).get("version", ""),
+        "uptime": status.get("createdAt", ""),
+        "name": "",
+        "peers": [p.get("name", "") for p in cluster.get("peers", [])],
+    }
 
 
-@router.get("/diagnostics")
+@router.post("/alertmanager-webhook")
-def get_diagnostics(
+def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
-    machine_id: str | None = Query(default=None),
+    """Receive alerts from Alertmanager and optionally forward to a webhook URL.
-    store: SettingsStore = Depends(get_settings_store),
+
-) -> dict[str, str]:
+    This endpoint is the receiver referenced by the optional `webhook_configs`
-    """Return collector debug info for a given machine."""
+    block in Alertmanager. It logs the payload for audit/debug purposes and, if
-    machine = _resolve_machine(store, machine_id)
+    `ALERTMANAGER_WEBHOOK_URL` is configured, forwards the alert JSON verbatim.
-    diagnostics = poll_machine_diagnostics(machine, store)["diagnostics"]
+    Forwarding is best-effort: a failure to reach the downstream webhook does
-    logger.info("Monitoring diagnostics requested machine_id=%s", machine["id"])
+    not fail this endpoint, so Alertmanager sees a successful delivery.
-    return {"diagnostics": diagnostics}
+    """
+    alerts = payload.get("alerts", [])
+    logger.info(
+        "Received Alertmanager webhook with %s alert(s), status=%s",
+        len(alerts),
+        payload.get("status", "unknown"),
+    )
+
+    session, webhook_url = _webhook_client()
+    if webhook_url:
+        try:
+            response = session.post(webhook_url, json=payload, timeout=10)
+            response.raise_for_status()
+            logger.info("Forwarded Alertmanager webhook to %s", webhook_url)
+        except Exception:
+            logger.exception("Failed to forward Alertmanager webhook to %s", webhook_url)
+    else:
+        logger.debug("No ALERTMANAGER_WEBHOOK_URL configured; webhook stored in logs only")
+
+    return {"status": "received"}

backend/src/media_library_viewer_api/routers/settings.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import logging
 from io import StringIO
 from typing import Any
 
@@ -12,11 +13,13 @@ from pydantic import BaseModel, Field
 from media_library_viewer_api.clients.ssh import RemoteSSHClient
 from media_library_viewer_api.config import get_settings
 from media_library_viewer_api.dependencies import get_monitoring_poller, get_settings_store
-from media_library_viewer_api.services.monitoring_actions import start_collector
 from media_library_viewer_api.services.db_maintenance import remove_sqlite_database
 from media_library_viewer_api.services.known_hosts import has_known_host
 from media_library_viewer_api.services.media_index import MediaIndex
 from media_library_viewer_api.services.settings_store import SettingsStore
+from media_library_viewer_api.services.targets import write_prometheus_targets
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/settings", tags=["settings"])
 
@@ -95,9 +98,7 @@ def _raise_ssh_validation_error(host: str, port: int, exc: Exception) -> None:
     if "protocol banner" in lowered:
         raise HTTPException(
             status_code=status.HTTP_502_BAD_GATEWAY,
-            detail=(
+            detail=(f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."),
-                f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."
-            ),
         ) from exc
     if "no authentication methods available" in lowered or "authentication failed" in lowered:
         raise HTTPException(
@@ -125,10 +126,12 @@ def _validate_saved_machine_ssh(machine: MonitoringMachineInput, store: Settings
         client.close()
 
 
-def _start_machine_collector(machine: MonitoringMachineInput, store: SettingsStore) -> None:
+def _write_prometheus_targets(store: SettingsStore) -> None:
-    if "monitoring" not in {str(service).strip().lower() for service in machine.services}:
+    """Regenerate Prometheus file-SD targets after machine changes."""
-        return
+    try:
-    start_collector(machine.model_dump(exclude_none=True), store)
+        write_prometheus_targets(store)
+    except Exception:
+        logger.exception("Failed to write Prometheus file-SD targets")
 
 
 @router.post("/machines/test-ssh")
@@ -137,7 +140,9 @@ def test_machine_ssh(
     store: SettingsStore = Depends(get_settings_store),
 ) -> dict[str, Any]:
     if str(machine.mode or "").strip().lower() != "ssh":
-        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines"
+        )
 
     client, host, port = _resolve_ssh_client(machine, store)
     settings = get_settings()
@@ -174,7 +179,8 @@ def test_machine_ssh(
     return {
         "status": "ok",
         "message": (
-            f"SSH connection succeeded for {host}:{port}; host key {'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
+            f"SSH connection succeeded for {host}:{port}; host key "
+            f"{'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
         ),
         "host": host,
         "port": port,
@@ -188,11 +194,11 @@ def post_machine(
     store: SettingsStore = Depends(get_settings_store),
 ) -> dict[str, Any]:
     saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine.id)
+    _write_prometheus_targets(store)
     poller = get_monitoring_poller()
     try:
         saved_machine = MonitoringMachineInput.model_validate(saved)
         _validate_saved_machine_ssh(saved_machine, store)
-        _start_machine_collector(saved_machine, store)
     finally:
         poller.start()
         poller.kick()
@@ -208,11 +214,11 @@ def put_machine(
     if not store.get_machine(machine_id):
         raise HTTPException(status_code=404, detail="Machine not found")
     saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine_id)
+    _write_prometheus_targets(store)
     poller = get_monitoring_poller()
     try:
         saved_machine = MonitoringMachineInput.model_validate(saved)
         _validate_saved_machine_ssh(saved_machine, store)
-        _start_machine_collector(saved_machine, store)
     finally:
         poller.start()
         poller.kick()
@@ -224,6 +230,7 @@ def delete_machine(machine_id: str, store: SettingsStore = Depends(get_settings_
     if not store.get_machine(machine_id):
         raise HTTPException(status_code=404, detail="Machine not found")
     store.delete_machine(machine_id)
+    _write_prometheus_targets(store)
     return {"status": "deleted"}
 
 
@@ -311,8 +318,12 @@ def reset_local_database(
     expected = "RESET LOCAL DATABASE"
     if payload.confirm_phrase.strip().upper() != expected:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Confirmation phrase does not match")
-    if not (payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible):
+    if not (
-        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected")
+        payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected"
+        )
 
     settings_removed = remove_sqlite_database(store.db_path)
     media_index = MediaIndex()

backend/src/media_library_viewer_api/routers/users_impl.py

@@ -212,7 +212,8 @@ def _merge_users(
                 "source_summary": summary,
                 "name_source": name_source,
                 "access_source": access_source,
-                "jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId")) or None,
+                "jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId"))
+                or None,
                 "jellyseerr_username": str(
                     (seerr_user or {}).get("username") or (seerr_user or {}).get("jellyfinUsername") or ""
                 ),
@@ -364,9 +365,10 @@ async def post_user_message(
         text_body=text_body,
         attachments=attachment_payloads,
     )
-    from_address = str(getattr(settings, "smtp_from_address", "") or "").strip() or str(
+    from_address = (
-        getattr(settings, "smtp_username", "") or ""
+        str(getattr(settings, "smtp_from_address", "") or "").strip()
-    ).strip()
+        or str(getattr(settings, "smtp_username", "") or "").strip()
+    )
     logger.info(
         "Users message queued request_id=%s subject=%s recipients=%s attachments=%s skipped=%s",
         request_id,

backend/src/media_library_viewer_api/services/backup_alert_engine.py

@@ -13,20 +13,21 @@ def generate_alerts_for_run(
 
     # 1. Failed status alert
     if run["status"] == "failure":
-        alerts.append({
+        alerts.append(
-            "job_id": job_id,
+            {
-            "run_id": run["id"],
+                "job_id": job_id,
-            "alert_type": "failed_status",
+                "run_id": run["id"],
-            "severity": "critical",
+                "alert_type": "failed_status",
-            "message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
+                "severity": "critical",
-        })
+                "message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
+            }
+        )
 
     # 2. Anomaly size alert
     bytes_transferred = run.get("bytes_transferred")
     if bytes_transferred is not None and previous_runs:
         successful_runs = [
-            r for r in previous_runs
+            r for r in previous_runs if r["status"] == "success" and r.get("bytes_transferred") is not None
-            if r["status"] == "success" and r.get("bytes_transferred") is not None
         ]
         if len(successful_runs) >= 3:
             sizes = [r["bytes_transferred"] for r in successful_runs[-7:]]
@@ -34,24 +35,28 @@ def generate_alerts_for_run(
             if median_size > 0:
                 ratio = bytes_transferred / median_size
                 if bytes_transferred == 0:
-                    alerts.append({
+                    alerts.append(
-                        "job_id": job_id,
+                        {
-                        "run_id": run["id"],
+                            "job_id": job_id,
-                        "alert_type": "anomaly_size",
+                            "run_id": run["id"],
-                        "severity": "warning",
+                            "alert_type": "anomaly_size",
-                        "message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
+                            "severity": "warning",
-                    })
+                            "message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
+                        }
+                    )
                 elif ratio < 0.1 or ratio > 3.0:
-                    alerts.append({
+                    alerts.append(
-                        "job_id": job_id,
+                        {
-                        "run_id": run["id"],
+                            "job_id": job_id,
-                        "alert_type": "anomaly_size",
+                            "run_id": run["id"],
-                        "severity": "warning",
+                            "alert_type": "anomaly_size",
-                        "message": (
+                            "severity": "warning",
-                            f"Backup job '{job_name}' size anomaly: "
+                            "message": (
-                            f"{bytes_transferred} bytes (median: {median_size})"
+                                f"Backup job '{job_name}' size anomaly: "
-                        ),
+                                f"{bytes_transferred} bytes (median: {median_size})"
-                    })
+                            ),
+                        }
+                    )
 
     # 3. Anomaly duration alert
     duration_ms = run.get("duration_ms")
@@ -61,16 +66,17 @@ def generate_alerts_for_run(
             durations = [r["duration_ms"] for r in successful_runs[-7:]]
             median_duration = statistics.median(durations)
             if median_duration > 0 and duration_ms / median_duration > 3.0:
-                alerts.append({
+                alerts.append(
-                    "job_id": job_id,
+                    {
-                    "run_id": run["id"],
+                        "job_id": job_id,
-                    "alert_type": "anomaly_duration",
+                        "run_id": run["id"],
-                    "severity": "warning",
+                        "alert_type": "anomaly_duration",
-                    "message": (
+                        "severity": "warning",
-                        f"Backup job '{job_name}' duration anomaly: "
+                        "message": (
-                        f"{duration_ms}ms (median: {median_duration}ms)"
+                            f"Backup job '{job_name}' duration anomaly: {duration_ms}ms (median: {median_duration}ms)"
-                    ),
+                        ),
-                })
+                    }
+                )
 
     return alerts
 
@@ -82,6 +88,7 @@ def check_missed_schedules(
 ) -> list[dict[str, Any]]:
     alerts = []
     import time
+
     now = int(time.time())
 
     for job in jobs:
@@ -93,31 +100,36 @@ def check_missed_schedules(
         if not latest_run:
             # No runs ever — alert if job is older than interval * 1.5
             if now - job["created_at"] > interval * 1.5:
-                alerts.append({
+                alerts.append(
-                    "job_id": job["id"],
+                    {
-                    "run_id": None,
+                        "job_id": job["id"],
-                    "alert_type": "missed_schedule",
+                        "run_id": None,
-                    "severity": "warning",
+                        "alert_type": "missed_schedule",
-                    "message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
+                        "severity": "warning",
-                })
+                        "message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
+                    }
+                )
         else:
             last_run_time = latest_run["started_at"]
             if now - last_run_time > interval * 1.5:
                 # Check if there's already an unresolved missed_schedule alert
                 has_open_alert = any(
                     a["alert_type"] == "missed_schedule" and a["resolved_at"] is None
-                    for a in existing_alerts if a["job_id"] == job["id"]
+                    for a in existing_alerts
+                    if a["job_id"] == job["id"]
                 )
                 if not has_open_alert:
-                    alerts.append({
+                    alerts.append(
-                        "job_id": job["id"],
+                        {
-                        "run_id": None,
+                            "job_id": job["id"],
-                        "alert_type": "missed_schedule",
+                            "run_id": None,
-                        "severity": "warning",
+                            "alert_type": "missed_schedule",
-                        "message": (
+                            "severity": "warning",
-                        f"Backup job '{job['name']}' missed schedule: "
+                            "message": (
-                        f"last run at {last_run_time} (expected every {interval}s)"
+                                f"Backup job '{job['name']}' missed schedule: "
-                    ),
+                                f"last run at {last_run_time} (expected every {interval}s)"
-                    })
+                            ),
+                        }
+                    )
 
     return alerts

backend/src/media_library_viewer_api/services/mail_queue.py

@@ -15,6 +15,7 @@ import uuid
 from dataclasses import dataclass, field
 from typing import Any
 
+from media_library_viewer_api.observability import record_mail_queue
 from media_library_viewer_api.services.mailer import EmailAttachment, describe_smtp_error, send_email_message
 
 logger = logging.getLogger(__name__)
@@ -206,6 +207,7 @@ class MailQueue:
                     result.get("recipient_count", 0),
                     result.get("attachment_count", 0),
                 )
+                record_mail_queue("sent")
             except Exception as exc:
                 friendly_error = describe_smtp_error(exc)
                 with self._lock:
@@ -220,6 +222,7 @@ class MailQueue:
                     getattr(message, "request_id", "unknown"),
                     friendly_error,
                 )
+                record_mail_queue("failed")
             finally:
                 self._queue.task_done()
 

backend/src/media_library_viewer_api/services/mailer_impl.py

@@ -4,8 +4,8 @@ from __future__ import annotations
 
 import logging
 import mimetypes
-import socket
 import smtplib
+import socket
 import ssl
 from dataclasses import dataclass
 from email.message import EmailMessage
@@ -13,7 +13,6 @@ from email.utils import formataddr
 from html.parser import HTMLParser
 from typing import Any, Iterable
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -164,7 +163,11 @@ def _smtp_sender_not_authorized(error: Exception) -> bool:
     else:
         raw_error_text = str(raw_error)
     text = f"{code} {raw_error_text} {error}".lower()
-    return code in {551, 553} or "not authorised to send from this header address" in text or "not authorized to send from this header address" in text
+    return (
+        code in {551, 553}
+        or "not authorised to send from this header address" in text
+        or "not authorized to send from this header address" in text
+    )
 
 
 def _smtp_attempt_metadata(mode: dict[str, Any]) -> dict[str, Any]:
@@ -203,10 +206,15 @@ def describe_smtp_error(error: Exception) -> str:
             )
         if isinstance(item, (smtplib.SMTPDataError, smtplib.SMTPResponseException)):
             smtp_code = getattr(item, "smtp_code", None)
-            if smtp_code in {551, 553} or "not authorised to send from this header address" in lowered or "not authorized to send from this header address" in lowered:
+            if (
+                smtp_code in {551, 553}
+                or "not authorised to send from this header address" in lowered
+                or "not authorized to send from this header address" in lowered
+            ):
                 return (
-                    "SMTP server rejected the configured From address. Use an authorized alias "
+                    "SMTP server rejected the configured From address. "
-                    "for this account or change SMTP_FROM_ADDRESS to a sender the provider allows."
+                    "Use an authorized alias for this account or change "
+                    "SMTP_FROM_ADDRESS to a sender the provider allows."
                 )
         if isinstance(item, smtplib.SMTPConnectError):
             return "SMTP connection was rejected by the server. Check the host and port."
@@ -247,7 +255,9 @@ def build_email_message(
         msg.set_content(plain_text or "")
 
     for attachment in attachments:
-        content_type = attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream"
+        content_type = (
+            attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream"
+        )
         maintype, subtype = content_type.split("/", 1) if "/" in content_type else ("application", "octet-stream")
         msg.add_attachment(
             attachment.data,
@@ -357,7 +367,9 @@ def send_email_message(
             )
             if _smtp_sender_not_authorized(exc) and fallback_from_address:
                 logger.warning(
-                    "SMTP send sender rejected label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s; retrying with smtp_username",
+                    "SMTP send sender rejected label=%s host=%s "
+                    "port=%s transport=%s auth_user=%s "
+                    "from_address=%s error=%s; retrying with smtp_username",
                     meta["label"],
                     meta["smtp_host"],
                     meta["smtp_port"],
@@ -393,7 +405,9 @@ def send_email_message(
                         }
                     )
                     logger.warning(
-                        "SMTP send fallback failed label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s",
+                        "SMTP send fallback failed label=%s host=%s "
+                        "port=%s transport=%s auth_user=%s "
+                        "from_address=%s error=%s",
                         meta["label"],
                         meta["smtp_host"],
                         meta["smtp_port"],
@@ -416,7 +430,8 @@ def send_email_message(
                     }
                 )
                 logger.info(
-                    "SMTP send succeeded via smtp_username label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s",
+                    "SMTP send succeeded via smtp_username label=%s host=%s "
+                    "port=%s transport=%s auth_user=%s from_address=%s",
                     meta["label"],
                     meta["smtp_host"],
                     meta["smtp_port"],

backend/src/media_library_viewer_api/services/media_index_impl.py

@@ -141,13 +141,20 @@ class MediaIndex:
                     key TEXT PRIMARY KEY,
                     value TEXT
                 );
-                CREATE INDEX IF NOT EXISTS idx_media_type ON media_items(type);
+                CREATE INDEX IF NOT EXISTS idx_media_type
-                CREATE INDEX IF NOT EXISTS idx_media_library ON media_items(library_id);
+                    ON media_items(type);
-                CREATE INDEX IF NOT EXISTS idx_media_title ON media_items(title COLLATE NOCASE);
+                CREATE INDEX IF NOT EXISTS idx_media_library
-                CREATE INDEX IF NOT EXISTS idx_media_series ON media_items(series COLLATE NOCASE);
+                    ON media_items(library_id);
-                CREATE INDEX IF NOT EXISTS idx_media_date_added ON media_items(date_added_ts);
+                CREATE INDEX IF NOT EXISTS idx_media_title
-                CREATE INDEX IF NOT EXISTS idx_media_size ON media_items(size_bytes);
+                    ON media_items(title COLLATE NOCASE);
-                CREATE INDEX IF NOT EXISTS idx_media_bitrate ON media_items(bitrate_bps);
+                CREATE INDEX IF NOT EXISTS idx_media_series
+                    ON media_items(series COLLATE NOCASE);
+                CREATE INDEX IF NOT EXISTS idx_media_date_added
+                    ON media_items(date_added_ts);
+                CREATE INDEX IF NOT EXISTS idx_media_size
+                    ON media_items(size_bytes);
+                CREATE INDEX IF NOT EXISTS idx_media_bitrate
+                    ON media_items(bitrate_bps);
                 """
             )
 
@@ -208,10 +215,7 @@ class MediaIndex:
         try:
             with self.connect() as conn:
                 item_count = int(conn.execute("SELECT COUNT(*) FROM media_items").fetchone()[0])
-                meta = {
+                meta = {row[0]: row[1] for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()}
-                    row[0]: row[1]
-                    for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()
-                }
         except sqlite3.Error:
             return MediaIndexStatus(exists=False)
         updated_at_raw = meta.get("updated_at", "")
@@ -310,7 +314,10 @@ class MediaIndex:
         sort_sql = SORT_COLUMNS.get(sort_key, SORT_COLUMNS["title"])
         direction = "DESC" if sort_order == "Descending" else "ASC"
         # Always add stable tie-breakers.
-        order_sql = f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, season_number ASC, episode ASC, title COLLATE NOCASE ASC"
+        order_sql = (
+            f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, "
+            "season_number ASC, episode ASC, title COLLATE NOCASE ASC"
+        )
 
         with self.connect() as conn:
             total = int(conn.execute("SELECT COUNT(*) FROM media_items" + where_sql, params).fetchone()[0])
@@ -421,10 +428,7 @@ def build_media_index(
                     **row,
                     "path": resolve_remote_media_path(row.get("path", ""), media_root, fallback_prefix),
                 }
-                for row in (
+                for row in (normalize_media_item(item, library_id, current_library_name) for item in items)
-                    normalize_media_item(item, library_id, current_library_name)
-                    for item in items
-                )
             )
             processed_total += len(items)
             current_library_processed += len(items)
@@ -432,7 +436,8 @@ def build_media_index(
             ensure_not_cancelled()
             emit(
                 "building",
-                f"{current_library_name or 'Library'}: {current_library_processed} / {current_library_total or '?'} items",
+                f"{current_library_name or 'Library'}: "
+                f"{current_library_processed} / {current_library_total or '?'} items",
             )
             logger.debug(
                 "Media index progress library=%s processed=%s/%s total_processed=%s",

backend/src/media_library_viewer_api/services/monitoring_actions.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import json
 import logging
+import shlex
 import time
 import uuid
 from typing import Any, Callable
@@ -16,17 +17,9 @@ from typing import Any, Callable
 from fastapi import HTTPException
 
 from media_library_viewer_api.clients.local import LocalCommandClient
-from media_library_viewer_api.clients.resources import (
-    disk_space,
-    read_resource_metrics,
-    resource_collector_debug_info,
-    resource_collector_status,
-    restart_resource_collector,
-    start_resource_collector,
-    stop_resource_collector,
-)
 from media_library_viewer_api.clients.ssh import RemoteSSHClient
 from media_library_viewer_api.config import get_settings
+from media_library_viewer_api.observability import record_ssh_command
 from media_library_viewer_api.services.settings_store import SettingsStore
 
 logger = logging.getLogger(__name__)
@@ -63,6 +56,28 @@ def build_machine_client(machine: dict[str, Any], store: SettingsStore):
     )
 
 
+def disk_space(client: Any, path: str = "/") -> dict[str, Any]:
+    """Return df information for the filesystem containing ``path``.
+
+    Works against any client with a ``run`` method (local shell or SSH).
+    """
+    command = (
+        "df -P -B1 -- " + shlex.quote(path or "/") + " | awk 'NR==2 {printf "
+        '"{\\"filesystem\\":\\"%s\\",\\"size\\":%s,"'
+        '"\\"used\\":%s,\\"available\\":%s,"'
+        '"\\"used_pct\\":\\"%s\\",\\"mount\\":\\"%s\\"}", "'
+        "$1,$2,$3,$4,$5,$6}'"
+    )
+    logger.debug("Reading disk space for path=%s", path)
+    result = client.run(command, timeout=20)
+    if result.exit_status != 0 or not result.stdout.strip():
+        logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout)
+        raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
+    data = json.loads(result.stdout)
+    logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct"))
+    return data
+
+
 def summarize_operation_result(action: str, result: Any) -> str:
     """Turn an operation result into a compact human-readable summary."""
     if result is None:
@@ -73,10 +88,6 @@ def summarize_operation_result(action: str, result: Any) -> str:
     if isinstance(result, list):
         return f"{action}: {len(result)} item(s)"
     if isinstance(result, dict):
-        if action == "metrics read":
-            samples = result.get("samples")
-            if isinstance(samples, list):
-                return f"{action}: {len(samples)} sample(s)"
         if action.startswith("disk lookup"):
             used_pct = result.get("used_pct")
             mount = result.get("mount") or result.get("filesystem")
@@ -104,13 +115,21 @@ def run_machine_operation(
     summarize: Callable[[Any], str] | None = None,
     request_id: str = "",
     raise_http: bool = True,
+    client: Any | None = None,
 ) -> Any:
     """Run a machine operation, record history, and optionally raise on failure."""
     started = time.perf_counter()
-    client = build_machine_client(machine, store)
+    if client is None:
+        client = build_machine_client(machine, store)
     try:
         result = callback(client)
         duration_ms = int((time.perf_counter() - started) * 1000)
+        record_ssh_command(
+            machine_id=machine.get("id") or "unknown",
+            action=action,
+            status="ok",
+            duration_seconds=duration_ms / 1000.0,
+        )
         store.record_machine_action(
             machine,
             action,
@@ -124,6 +143,12 @@ def run_machine_operation(
         raise
     except Exception as exc:  # pragma: no cover - transport/network fallback
         duration_ms = int((time.perf_counter() - started) * 1000)
+        record_ssh_command(
+            machine_id=machine.get("id") or "unknown",
+            action=action,
+            status="error",
+            duration_seconds=duration_ms / 1000.0,
+        )
         logger.exception(
             "Monitoring %s failed machine_id=%s machine_name=%s",
             action,
@@ -157,34 +182,13 @@ def poll_machine_snapshot(
 ) -> dict[str, Any]:
     """Collect a backend-scheduled snapshot for a machine.
 
-    This records the same action-history rows that the UI would otherwise get
+    The legacy POSIX collector has been removed; this now records a lightweight
-    from manual requests, but it runs entirely inside the backend on a schedule.
+    disk-space lookup on the same schedule so action history stays useful.
     """
     request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}"
     results: dict[str, Any] = {"request_id": request_id, "machine_id": machine.get("id"), "actions": []}
 
-    status = run_machine_operation(
+    client = build_machine_client(machine, store)
-        machine,
-        store,
-        "status lookup",
-        resource_collector_status,
-        request_id=request_id,
-        raise_http=False,
-    )
-    results["status"] = status
-    results["actions"].append("status lookup")
-
-    metrics = run_machine_operation(
-        machine,
-        store,
-        "metrics read",
-        lambda client: read_resource_metrics(client, max_lines=metrics_limit),
-        request_id=request_id,
-        raise_http=False,
-    )
-    results["metrics_samples"] = len(metrics or [])
-    results["actions"].append("metrics read")
-
     settings = get_settings()
     path = str(machine.get("media_root") or settings.media_root or "/")
     disk = run_machine_operation(
@@ -194,6 +198,7 @@ def poll_machine_snapshot(
         lambda client: disk_space(client, path),
         request_id=request_id,
         raise_http=False,
+        client=client,
     )
     results["disk_mount"] = (disk or {}).get("mount") if isinstance(disk, dict) else None
     results["actions"].append("disk lookup")
@@ -201,149 +206,3 @@ def poll_machine_snapshot(
     return results
 
 
-def _summarize_metric_samples(samples: list[dict[str, Any]], field: str) -> dict[str, float] | None:
-    values = [float(sample.get(field, 0)) for sample in samples if sample.get(field) is not None]
-    if not values:
-        return None
-    return {
-        "avg": sum(values) / len(values),
-        "min": min(values),
-        "max": max(values),
-        "count": float(len(values)),
-    }
-
-
-def collect_machine_overview(
-    machine: dict[str, Any],
-    store: SettingsStore,
-    *,
-    metrics_window_seconds: int = 600,
-    metrics_limit: int = 70_000,
-) -> dict[str, Any]:
-    """Collect a lightweight machine overview without recording history."""
-    overview: dict[str, Any] = {
-        "machine": {k: machine.get(k) for k in ("id", "name", "mode", "enabled", "host", "port", "username", "media_root", "path_prefix", "notes")},
-        "status": "",
-        "status_error": "",
-        "metrics_error": "",
-        "disk_error": "",
-        "sample_count": 0,
-        "latest_sample": None,
-        "cpu_summary": None,
-        "iowait_summary": None,
-        "mem_summary": None,
-        "net_rx_summary": None,
-        "net_tx_summary": None,
-        "disk_read_summary": None,
-        "disk_write_summary": None,
-        "disk": None,
-    }
-    try:
-        client: Any = build_machine_client(machine, store)
-    except Exception as exc:
-        overview["status_error"] = str(exc)
-        overview["metrics_error"] = str(exc)
-        overview["disk_error"] = str(exc)
-        return overview
-
-    settings = get_settings()
-    path = str(machine.get("media_root") or settings.media_root or "/")
-
-    try:
-        overview["status"] = resource_collector_status(client)
-    except Exception as exc:
-        overview["status_error"] = str(exc)
-
-    try:
-        samples = read_resource_metrics(client, max_lines=metrics_limit)
-        if samples:
-            latest_ts = max(float(sample.get("ts", 0)) for sample in samples)
-            window_start = latest_ts - metrics_window_seconds
-            metrics = [sample for sample in samples if float(sample.get("ts", 0)) >= window_start]
-            if not metrics:
-                metrics = samples
-            overview["sample_count"] = len(metrics)
-            overview["latest_sample"] = metrics[-1]
-            overview["cpu_summary"] = _summarize_metric_samples(metrics, "cpu_pct")
-            overview["iowait_summary"] = _summarize_metric_samples(metrics, "iowait_pct")
-            overview["mem_summary"] = _summarize_metric_samples(metrics, "mem_pct")
-            overview["net_rx_summary"] = _summarize_metric_samples(metrics, "net_rx_bytes_per_sec")
-            overview["net_tx_summary"] = _summarize_metric_samples(metrics, "net_tx_bytes_per_sec")
-            overview["disk_read_summary"] = _summarize_metric_samples(metrics, "disk_read_bps")
-            overview["disk_write_summary"] = _summarize_metric_samples(metrics, "disk_write_bps")
-    except Exception as exc:
-        overview["metrics_error"] = str(exc)
-
-    try:
-        overview["disk"] = disk_space(client, path)
-    except Exception as exc:
-        overview["disk_error"] = str(exc)
-
-    return overview
-
-
-def poll_machine_diagnostics(
-    machine: dict[str, Any],
-    store: SettingsStore,
-    *,
-    request_id: str | None = None,
-) -> dict[str, Any]:
-    """Collect a backend-scheduled diagnostics snapshot for a machine."""
-    request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}"
-    diagnostics = run_machine_operation(
-        machine,
-        store,
-        "collector diagnostics",
-        resource_collector_debug_info,
-        request_id=request_id,
-        raise_http=False,
-    )
-    return {"request_id": request_id, "diagnostics": diagnostics}
-
-
-def start_collector(
-    machine: dict[str, Any],
-    store: SettingsStore,
-    *,
-    request_id: str = "",
-) -> Any:
-    return run_machine_operation(
-        machine,
-        store,
-        "collector start",
-        start_resource_collector,
-        request_id=request_id,
-        raise_http=True,
-    )
-
-
-def stop_collector(
-    machine: dict[str, Any],
-    store: SettingsStore,
-    *,
-    request_id: str = "",
-) -> Any:
-    return run_machine_operation(
-        machine,
-        store,
-        "collector stop",
-        stop_resource_collector,
-        request_id=request_id,
-        raise_http=True,
-    )
-
-
-def restart_collector(
-    machine: dict[str, Any],
-    store: SettingsStore,
-    *,
-    request_id: str = "",
-) -> Any:
-    return run_machine_operation(
-        machine,
-        store,
-        "collector restart",
-        restart_resource_collector,
-        request_id=request_id,
-        raise_http=True,
-    )

backend/src/media_library_viewer_api/services/monitoring_poller.py

@@ -136,16 +136,17 @@ class MonitoringPoller:
                     request_id=f"poll:{machine['id']}:{int(time.time())}",
                 )
                 logger.info(
-                    "Monitoring poll snapshot machine_id=%s request_id=%s status=%s metrics_samples=%s disk_mount=%s",
+                    "Monitoring poll snapshot machine_id=%s request_id=%s disk_mount=%s actions=%s",
                     machine["id"],
                     snapshot.get("request_id"),
-                    snapshot.get("status"),
-                    snapshot.get("metrics_samples"),
                     snapshot.get("disk_mount"),
+                    snapshot.get("actions"),
                 )
             except Exception:
                 cycle_errors += 1
-                logger.exception("Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"])
+                logger.exception(
+                    "Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"]
+                )
         retention_seconds = config.retention_days * 24 * 60 * 60
         cutoff_ts = int(time.time()) - retention_seconds
         removed = store.prune_machine_actions(cutoff_ts)

backend/src/media_library_viewer_api/services/settings_store.py

@@ -48,6 +48,9 @@ def _default_local_machine() -> dict[str, Any]:
         "jellyfin_api_key": "",
         "jellyseerr_url": "",
         "jellyseerr_api_key": "",
+        "node_exporter_enabled": False,
+        "node_exporter_port": 9100,
+        "node_exporter_scrape_host": "",
         "notes": "",
     }
 
@@ -81,9 +84,7 @@ class SettingsStore:
                 )
                 """
             )
-            conn.execute(
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)")
-                "CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)"
-            )
             conn.execute(
                 """
                 CREATE TABLE IF NOT EXISTS monitoring_machine_actions (
@@ -118,9 +119,7 @@ class SettingsStore:
                 )
                 """
             )
-            existing_key_columns = {
+            existing_key_columns = {row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()}
-                row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()
-            }
             if "public_key" not in existing_key_columns:
                 conn.execute("ALTER TABLE ssh_keys ADD COLUMN public_key TEXT NOT NULL DEFAULT ''")
             if "fingerprint" not in existing_key_columns:
@@ -140,9 +139,7 @@ class SettingsStore:
                 )
                 """
             )
-            conn.execute(
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)")
-                "CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)"
-            )
             conn.execute(
                 """
                 CREATE TABLE IF NOT EXISTS saved_task_runs (
@@ -182,10 +179,16 @@ class SettingsStore:
                 "CREATE INDEX IF NOT EXISTS idx_dashboard_shortcuts_type ON dashboard_shortcuts(shortcut_type)"
             )
             conn.execute(
-                "CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time ON monitoring_machine_actions(machine_id, created_at DESC)"
+                """
+                CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time
+                    ON monitoring_machine_actions(machine_id, created_at DESC)
+                """
             )
             conn.execute(
-                "CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status ON monitoring_machine_actions(action, status)"
+                """
+                CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status
+                    ON monitoring_machine_actions(action, status)
+                """
             )
             conn.execute("""
                 CREATE TABLE IF NOT EXISTS backup_jobs (
@@ -251,7 +254,8 @@ class SettingsStore:
 
     def _row_to_machine(self, row: sqlite3.Row) -> dict[str, Any]:
         data = json.loads(row["config_json"])
-        services = self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [])
+        default_services = DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []
+        services = self._normalize_services(data.get("services"), default_services)
         return {
             "id": row["id"],
             "name": row["name"],
@@ -274,6 +278,9 @@ class SettingsStore:
             "jellyfin_api_key_set": bool(data.get("jellyfin_api_key")),
             "jellyseerr_url": data.get("jellyseerr_url", ""),
             "jellyseerr_api_key_set": bool(data.get("jellyseerr_api_key")),
+            "node_exporter_enabled": bool(data.get("node_exporter_enabled", False)),
+            "node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100),
+            "node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""),
             "notes": data.get("notes", ""),
             "created_at": row["created_at"],
             "updated_at": row["updated_at"],
@@ -294,12 +301,18 @@ class SettingsStore:
             "This machine" if mode == "local" else machine_id
         )
         services = self._normalize_services(payload.get("services"), (current or {}).get("services", []))
-        host = str(payload.get("host") if payload.get("host") is not None else (current or {}).get("host", "") or "").strip()
+
+        def _current_str(field: str, default: str = "") -> str:
+            return str(
+                payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default
+            ).strip()
+
+        host = _current_str("host")
         port = int(payload.get("port") or (current or {}).get("port", 22) or 22)
-        username = str(payload.get("username") if payload.get("username") is not None else (current or {}).get("username", "") or "").strip()
+        username = _current_str("username")
-        key_directory = str(payload.get("key_directory") if payload.get("key_directory") is not None else (current or {}).get("key_directory", "") or "").strip()
+        key_directory = _current_str("key_directory")
-        key_name = str(payload.get("key_name") if payload.get("key_name") is not None else (current or {}).get("key_name", "") or "").strip()
+        key_name = _current_str("key_name")
-        ssh_key_id = str(payload.get("ssh_key_id") if payload.get("ssh_key_id") is not None else (current or {}).get("ssh_key_id", "") or "").strip()
+        ssh_key_id = _current_str("ssh_key_id")
         ssh_private_key = payload.get("ssh_private_key")
         if ssh_private_key in (None, ""):
             ssh_private_key = (current or {}).get("ssh_private_key", "")
@@ -312,20 +325,30 @@ class SettingsStore:
         if password in (None, ""):
             password = (current or {}).get("password", "")
         password = str(password or "")
-        media_root = str(payload.get("media_root") if payload.get("media_root") is not None else (current or {}).get("media_root", "") or "").strip()
+        media_root = _current_str("media_root")
-        path_prefix = str(payload.get("path_prefix") if payload.get("path_prefix") is not None else (current or {}).get("path_prefix", "") or "").strip()
+        path_prefix = _current_str("path_prefix")
-        jellyfin_url = str(payload.get("jellyfin_url") if payload.get("jellyfin_url") is not None else (current or {}).get("jellyfin_url", "") or "").strip()
+        jellyfin_url = _current_str("jellyfin_url")
-        jellyfin_user_id = str(payload.get("jellyfin_user_id") if payload.get("jellyfin_user_id") is not None else (current or {}).get("jellyfin_user_id", "") or "").strip()
+        jellyfin_user_id = _current_str("jellyfin_user_id")
         jellyfin_api_key = payload.get("jellyfin_api_key")
         if jellyfin_api_key in (None, ""):
             jellyfin_api_key = (current or {}).get("jellyfin_api_key", "")
         jellyfin_api_key = str(jellyfin_api_key or "")
-        jellyseerr_url = str(payload.get("jellyseerr_url") if payload.get("jellyseerr_url") is not None else (current or {}).get("jellyseerr_url", "") or "").strip()
+        jellyseerr_url = _current_str("jellyseerr_url")
         jellyseerr_api_key = payload.get("jellyseerr_api_key")
         if jellyseerr_api_key in (None, ""):
             jellyseerr_api_key = (current or {}).get("jellyseerr_api_key", "")
         jellyseerr_api_key = str(jellyseerr_api_key or "")
-        notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
+        node_exporter_enabled = bool(
+            payload.get("node_exporter_enabled")
+            if payload.get("node_exporter_enabled") is not None
+            else (current or {}).get("node_exporter_enabled", False)
+        )
+        node_exporter_port_raw = payload.get("node_exporter_port")
+        if node_exporter_port_raw is None:
+            node_exporter_port_raw = (current or {}).get("node_exporter_port", 9100)
+        node_exporter_port = int(node_exporter_port_raw or 9100)
+        node_exporter_scrape_host = _current_str("node_exporter_scrape_host")
+        notes = _current_str("notes")
         if mode == "local":
             host = host or "localhost"
             username = username or ""
@@ -351,6 +374,9 @@ class SettingsStore:
             "jellyfin_api_key": jellyfin_api_key,
             "jellyseerr_url": jellyseerr_url,
             "jellyseerr_api_key": jellyseerr_api_key,
+            "node_exporter_enabled": node_exporter_enabled,
+            "node_exporter_port": node_exporter_port,
+            "node_exporter_scrape_host": node_exporter_scrape_host,
             "notes": notes,
         }
 
@@ -380,6 +406,9 @@ class SettingsStore:
             "jellyfin_api_key": machine["jellyfin_api_key"],
             "jellyseerr_url": machine["jellyseerr_url"],
             "jellyseerr_api_key": machine["jellyseerr_api_key"],
+            "node_exporter_enabled": machine["node_exporter_enabled"],
+            "node_exporter_port": machine["node_exporter_port"],
+            "node_exporter_scrape_host": machine["node_exporter_scrape_host"],
             "notes": machine["notes"],
         }
         with self.connect() as conn:
@@ -431,7 +460,10 @@ class SettingsStore:
             "name": row["name"],
             "mode": row["mode"],
             "enabled": bool(row["enabled"]),
-            "services": self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []),
+            "services": self._normalize_services(
+                data.get("services"),
+                DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [],
+            ),
             "host": data.get("host", ""),
             "port": int(data.get("port", 22) or 22),
             "username": data.get("username", ""),
@@ -448,11 +480,18 @@ class SettingsStore:
             "jellyfin_api_key": data.get("jellyfin_api_key", ""),
             "jellyseerr_url": data.get("jellyseerr_url", ""),
             "jellyseerr_api_key": data.get("jellyseerr_api_key", ""),
+            "node_exporter_enabled": bool(data.get("node_exporter_enabled", False)),
+            "node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100),
+            "node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""),
             "notes": data.get("notes", ""),
         }
 
     def list_machines_for_service(self, service: str) -> list[dict[str, Any]]:
-        return [machine for machine in self.list_machines() if service in machine.get("services", []) and machine.get("enabled")]
+        return [
+            machine
+            for machine in self.list_machines()
+            if service in machine.get("services", []) and machine.get("enabled")
+        ]
 
     def get_machine_for_service(self, service: str, machine_id: str | None = None) -> dict[str, Any] | None:
         if machine_id:
@@ -485,10 +524,16 @@ class SettingsStore:
             "jellyfin_api_key": machine["jellyfin_api_key"],
             "jellyseerr_url": machine["jellyseerr_url"],
             "jellyseerr_api_key": machine["jellyseerr_api_key"],
+            "node_exporter_enabled": machine["node_exporter_enabled"],
+            "node_exporter_port": machine["node_exporter_port"],
+            "node_exporter_scrape_host": machine["node_exporter_scrape_host"],
             "notes": machine["notes"],
         }
         with self.connect() as conn:
-            existing = conn.execute("SELECT created_at FROM monitoring_machines WHERE id = ?", (machine["id"],)).fetchone()
+            existing = conn.execute(
+                "SELECT created_at FROM monitoring_machines WHERE id = ?",
+                (machine["id"],),
+            ).fetchone()
             created_at = int(existing[0]) if existing else now
             conn.execute(
                 """
@@ -538,10 +583,13 @@ class SettingsStore:
             conn.execute(
                 """
                 INSERT INTO monitoring_machine_actions
-                (id, machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail)
+                (
+                    id, machine_id, machine_name, mode, action, status,
+                    created_at, duration_ms, request_id, message, error,
+                    stdout_tail, stderr_tail
+                )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-                """
+                """,
-                ,
                 (
                     uuid.uuid4().hex,
                     str(machine.get("id") or ""),
@@ -577,8 +625,10 @@ class SettingsStore:
             clauses.append("status = ?")
             params.append(status)
         sql = (
-            "SELECT machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail "
+            "SELECT machine_id, machine_name, mode, action, status, "
-            f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} ORDER BY created_at DESC LIMIT ?"
+            "created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail "
+            f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} "
+            "ORDER BY created_at DESC LIMIT ?"
         )
         params.append(max(1, min(int(limit), 200)))
         with self.connect() as conn:
@@ -595,7 +645,6 @@ class SettingsStore:
             )
             return int(cur.rowcount or 0)
 
-
     @staticmethod
     def _private_key_summary(private_key: str) -> dict[str, str]:
         if not private_key:
@@ -639,11 +688,29 @@ class SettingsStore:
         if passphrase in (None, ""):
             passphrase = (current or {}).get("passphrase", "")
         passphrase = str(passphrase or "")
-        notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
+        notes = str(
+            payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or ""
+        ).strip()
         summary = self._private_key_summary(private_key)
-        public_key = str(payload.get("public_key") if payload.get("public_key") is not None else (current or {}).get("public_key", "") or summary["public_key"] or "").strip()
+        public_key = str(
-        fingerprint = str(payload.get("fingerprint") if payload.get("fingerprint") is not None else (current or {}).get("fingerprint", "") or summary["fingerprint"] or "").strip()
+            payload.get("public_key")
-        return {"id": key_id, "name": name, "private_key": private_key, "passphrase": passphrase, "public_key": public_key, "fingerprint": fingerprint, "notes": notes}
+            if payload.get("public_key") is not None
+            else (current or {}).get("public_key", "") or summary["public_key"] or ""
+        ).strip()
+        fingerprint = str(
+            payload.get("fingerprint")
+            if payload.get("fingerprint") is not None
+            else (current or {}).get("fingerprint", "") or summary["fingerprint"] or ""
+        ).strip()
+        return {
+            "id": key_id,
+            "name": name,
+            "private_key": private_key,
+            "passphrase": passphrase,
+            "public_key": public_key,
+            "fingerprint": fingerprint,
+            "notes": notes,
+        }
 
     def list_ssh_keys(self) -> list[dict[str, Any]]:
         self.init_schema()
@@ -666,7 +733,15 @@ class SettingsStore:
         if not row:
             return None
         summary = self._private_key_summary(str(row["private_key"] or ""))
-        return {"id": row["id"], "name": row["name"], "private_key": row["private_key"], "passphrase": row["passphrase"], "notes": row["notes"], "public_key": str(row["public_key"] or summary["public_key"] or ""), "fingerprint": str(row["fingerprint"] or summary["fingerprint"] or "")}
+        return {
+            "id": row["id"],
+            "name": row["name"],
+            "private_key": row["private_key"],
+            "passphrase": row["passphrase"],
+            "notes": row["notes"],
+            "public_key": str(row["public_key"] or summary["public_key"] or ""),
+            "fingerprint": str(row["fingerprint"] or summary["fingerprint"] or ""),
+        }
 
     def upsert_ssh_key(self, payload: dict[str, Any], key_id: str | None = None) -> dict[str, Any]:
         self.init_schema()
@@ -677,7 +752,10 @@ class SettingsStore:
             created_at = int(existing[0]) if existing else now
             conn.execute(
                 """
-                INSERT INTO ssh_keys (id, name, private_key, passphrase, public_key, fingerprint, notes, created_at, updated_at)
+                INSERT INTO ssh_keys (
+                    id, name, private_key, passphrase, public_key, fingerprint,
+                    notes, created_at, updated_at
+                )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                 ON CONFLICT(id) DO UPDATE SET
                     name = excluded.name,
@@ -688,7 +766,17 @@ class SettingsStore:
                     notes = excluded.notes,
                     updated_at = excluded.updated_at
                 """,
-                (key["id"], key["name"], key["private_key"], key["passphrase"], key["public_key"], key["fingerprint"], key["notes"], created_at, now),
+                (
+                    key["id"],
+                    key["name"],
+                    key["private_key"],
+                    key["passphrase"],
+                    key["public_key"],
+                    key["fingerprint"],
+                    key["notes"],
+                    created_at,
+                    now,
+                ),
             )
         return self.get_ssh_key(key["id"]) or key
 
@@ -697,8 +785,6 @@ class SettingsStore:
         with self.connect() as conn:
             conn.execute("DELETE FROM ssh_keys WHERE id = ?", (key_id,))
 
-
-
     def _row_to_task(self, row: sqlite3.Row) -> dict[str, Any]:
         return {
             "id": row["id"],
@@ -719,11 +805,27 @@ class SettingsStore:
         task_type = str(payload.get("task_type") or (current or {}).get("task_type") or "shell").strip().lower()
         if task_type not in {"shell", "python"}:
             task_type = "shell"
-        content = str(payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or "")
+        content = str(
+            payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or ""
+        )
         enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
-        default_machine_id = str(payload.get("default_machine_id") if payload.get("default_machine_id") is not None else (current or {}).get("default_machine_id", "") or "").strip()
+        default_machine_id = str(
-        notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
+            payload.get("default_machine_id")
-        return {"id": task_id, "name": name, "task_type": task_type, "content": content, "enabled": enabled, "default_machine_id": default_machine_id, "notes": notes}
+            if payload.get("default_machine_id") is not None
+            else (current or {}).get("default_machine_id", "") or ""
+        ).strip()
+        notes = str(
+            payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or ""
+        ).strip()
+        return {
+            "id": task_id,
+            "name": name,
+            "task_type": task_type,
+            "content": content,
+            "enabled": enabled,
+            "default_machine_id": default_machine_id,
+            "notes": notes,
+        }
 
     def list_tasks(self) -> list[dict[str, Any]]:
         self.init_schema()
@@ -748,7 +850,10 @@ class SettingsStore:
             created_at = int(existing[0]) if existing else now
             conn.execute(
                 """
-                INSERT INTO saved_tasks (id, name, task_type, content, enabled, default_machine_id, notes, created_at, updated_at)
+                INSERT INTO saved_tasks (
+                    id, name, task_type, content, enabled, default_machine_id,
+                    notes, created_at, updated_at
+                )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                 ON CONFLICT(id) DO UPDATE SET
                     name = excluded.name,
@@ -759,7 +864,17 @@ class SettingsStore:
                     notes = excluded.notes,
                     updated_at = excluded.updated_at
                 """,
-                (task["id"], task["name"], task["task_type"], task["content"], 1 if task["enabled"] else 0, task["default_machine_id"], task["notes"], created_at, now),
+                (
+                    task["id"],
+                    task["name"],
+                    task["task_type"],
+                    task["content"],
+                    1 if task["enabled"] else 0,
+                    task["default_machine_id"],
+                    task["notes"],
+                    created_at,
+                    now,
+                ),
             )
         return self.get_task(task["id"]) or task
 
@@ -796,8 +911,11 @@ class SettingsStore:
         with self.connect() as conn:
             conn.execute(
                 """
-                INSERT INTO saved_task_runs
+                INSERT INTO saved_task_runs (
-                (id, task_id, task_name, machine_id, machine_name, task_type, status, created_at, duration_ms, request_id, stdout_tail, stderr_tail, error)
+                    id, task_id, task_name, machine_id, machine_name, task_type,
+                    status, created_at, duration_ms, request_id, stdout_tail,
+                    stderr_tail, error
+                )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """,
                 (
@@ -837,17 +955,25 @@ class SettingsStore:
     def _normalize_shortcut_payload(self, payload: dict[str, Any], shortcut_id: str | None = None) -> dict[str, Any]:
         current = self.get_shortcut(shortcut_id) if shortcut_id else None
         shortcut_id = str(payload.get("id") or shortcut_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12]
-        shortcut_type = str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower()
+        shortcut_type = (
+            str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower()
+        )
         if shortcut_type not in {"website", "action", "user"}:
             shortcut_type = "website"
         label = str(payload.get("label") or (current or {}).get("label") or "").strip() or shortcut_id
         enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
-        icon = str(payload.get("icon") if payload.get("icon") is not None else (current or {}).get("icon", "") or "").strip()
+
-        url = str(payload.get("url") if payload.get("url") is not None else (current or {}).get("url", "") or "").strip()
+        def _field(field: str, default: str = "") -> str:
-        task_id = str(payload.get("task_id") if payload.get("task_id") is not None else (current or {}).get("task_id", "") or "").strip()
+            return str(
-        machine_id = str(payload.get("machine_id") if payload.get("machine_id") is not None else (current or {}).get("machine_id", "") or "").strip()
+                payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default
-        user_id = str(payload.get("user_id") if payload.get("user_id") is not None else (current or {}).get("user_id", "") or "").strip()
+            ).strip()
-        notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
+
+        icon = _field("icon")
+        url = _field("url")
+        task_id = _field("task_id")
+        machine_id = _field("machine_id")
+        user_id = _field("user_id")
+        notes = _field("notes")
         return {
             "id": shortcut_id,
             "label": label,
@@ -882,7 +1008,10 @@ class SettingsStore:
         shortcut = self._normalize_shortcut_payload(payload, shortcut_id)
         now = int(time.time())
         with self.connect() as conn:
-            existing = conn.execute("SELECT created_at FROM dashboard_shortcuts WHERE id = ?", (shortcut["id"],)).fetchone()
+            existing = conn.execute(
+                "SELECT created_at FROM dashboard_shortcuts WHERE id = ?",
+                (shortcut["id"],),
+            ).fetchone()
             created_at = int(existing[0]) if existing else now
             conn.execute(
                 """
@@ -926,14 +1055,16 @@ class SettingsStore:
             "created_at": row["created_at"],
         }
 
-    def _normalize_backup_job_payload(
+    def _normalize_backup_job_payload(self, payload: dict[str, Any], job_id: str | None = None) -> dict[str, Any]:
-        self, payload: dict[str, Any], job_id: str | None = None
-    ) -> dict[str, Any]:
         current = self.get_backup_job(job_id) if job_id else None
         job_id = str(payload.get("id") or job_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12]
         name = str(payload.get("name") or (current or {}).get("name") or job_id).strip() or job_id
-        source = str(payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or "").strip()
+        source = str(
-        target = str(payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or "").strip()
+            payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or ""
+        ).strip()
+        target = str(
+            payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or ""
+        ).strip()
         schedule_interval_seconds = payload.get("schedule_interval_seconds")
         if schedule_interval_seconds is None:
             schedule_interval_seconds = (current or {}).get("schedule_interval_seconds")
@@ -1040,10 +1171,24 @@ class SettingsStore:
         with self.connect() as conn:
             conn.execute(
                 """
-                INSERT INTO backup_runs (id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, created_at)
+                INSERT INTO backup_runs (
+                    id, job_id, started_at, ended_at, status, bytes_transferred,
+                    duration_ms, error_message, details_json, created_at
+                )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """,
-                (run_id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, now),
+                (
+                    run_id,
+                    job_id,
+                    started_at,
+                    ended_at,
+                    status,
+                    bytes_transferred,
+                    duration_ms,
+                    error_message,
+                    details_json,
+                    now,
+                ),
             )
         return self.get_backup_run(run_id) or {
             "id": run_id,
@@ -1128,7 +1273,10 @@ class SettingsStore:
         with self.connect() as conn:
             conn.execute(
                 """
-                INSERT INTO backup_alerts (id, job_id, run_id, alert_type, severity, message, acknowledged, resolved_at, created_at)
+                INSERT INTO backup_alerts (
+                    id, job_id, run_id, alert_type, severity, message,
+                    acknowledged, resolved_at, created_at
+                )
                 VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?)
                 """,
                 (alert_id, job_id, run_id, alert_type, severity, message, now),
@@ -1238,9 +1386,7 @@ class SettingsStore:
                 )
                 """
             )
-            row = conn.execute(
+            row = conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone()
-                "SELECT value FROM app_settings WHERE key = ?", (key,)
-            ).fetchone()
         return row["value"] if row else default
 
     def update_setting(self, key: str, value: str) -> None:

backend/src/media_library_viewer_api/services/targets.py

@@ -0,0 +1,77 @@
+"""Prometheus file-based service discovery target management.
+
+The backend owns the list of remote Node Exporter targets so that operators can
+enable scraping per machine from the Manage UI.  Prometheus reads the generated
+JSON file via `file_sd_configs`; this keeps Prometheus config static and pushes
+machine-specific changes into a file it can reload.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from media_library_viewer_api.config import get_settings
+from media_library_viewer_api.services.settings_store import SettingsStore
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_NODE_EXPORTER_PORT = 9100
+
+
+def _scrape_address(machine: dict[str, Any]) -> str | None:
+    """Return host:port for the Node Exporter on a machine, or None if disabled."""
+    if not machine.get("node_exporter_enabled"):
+        return None
+    scrape_host = str(machine.get("node_exporter_scrape_host") or "").strip()
+    host = scrape_host or str(machine.get("host") or "").strip()
+    if not host or host == "localhost":
+        return None
+    port = int(machine.get("node_exporter_port") or DEFAULT_NODE_EXPORTER_PORT)
+    return f"{host}:{port}"
+
+
+def build_node_exporter_targets(store: SettingsStore) -> list[dict[str, Any]]:
+    """Build a file-SD target list for all enabled SSH machines.
+
+    Local machines are excluded because the Compose-managed node-exporter
+    service already covers the Docker host.
+    """
+    targets: list[dict[str, Any]] = []
+    for machine in store.list_machines():
+        if not machine.get("enabled"):
+            continue
+        if str(machine.get("mode") or "local").strip().lower() != "ssh":
+            continue
+        address = _scrape_address(machine)
+        if not address:
+            continue
+        targets.append(
+            {
+                "targets": [address],
+                "labels": {
+                    "job": "node-exporter-remote",
+                    "machine_id": str(machine.get("id") or ""),
+                    "machine_name": str(machine.get("name") or ""),
+                    "instance": address,
+                },
+            }
+        )
+    return targets
+
+
+def write_prometheus_targets(store: SettingsStore, file_sd_dir: Path | None = None) -> Path:
+    """Render and persist Prometheus file-SD targets.
+
+    Returns the path written so callers can log or expose it.
+    """
+    settings = get_settings()
+    file_sd_dir = file_sd_dir or Path(settings.prometheus_file_sd_dir)
+    file_sd_dir.mkdir(parents=True, exist_ok=True)
+    file_path = file_sd_dir / "node_exporter_targets.json"
+    targets = build_node_exporter_targets(store)
+    file_path.write_text(json.dumps(targets, indent=2), encoding="utf-8")
+    logger.info("Wrote %s node_exporter targets to %s", len(targets), file_path)
+    return file_path

backend/src/media_library_viewer_api/utils.py

@@ -11,7 +11,6 @@ from datetime import datetime
 from pathlib import PurePosixPath
 from typing import Any
 
-
 VIDEO_FILE_EXTENSIONS = {
     ".3g2",
     ".3gp",

backend/src/media_library_viewer_api/version.py

@@ -3,7 +3,8 @@
 from __future__ import annotations
 
 import os
-from importlib.metadata import PackageNotFoundError, version as package_version
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as package_version
 
 PACKAGE_NAME = "media-library-viewer-backend"
 DEFAULT_VERSION = "0.1.0"

backend/src/media_library_viewer_api/workers/media_index_worker.py

@@ -15,12 +15,12 @@ from typing import Any
 
 from media_library_viewer_api.config import get_settings
 from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id
+from media_library_viewer_api.logging_utils import configure_logging, describe_settings
 from media_library_viewer_api.services.media_index import (
     MediaIndex,
     MediaIndexBuildCancelled,
     build_media_index,
 )
-from media_library_viewer_api.logging_utils import configure_logging, describe_settings
 
 logger = logging.getLogger(__name__)
 

backend/tests/test_api.py

@@ -11,23 +11,23 @@ from unittest.mock import MagicMock, patch
 import pytest
 from fastapi.testclient import TestClient
 
-from media_library_viewer_api.main import app
+from media_library_viewer_api.clients.ssh import CommandResult
 from media_library_viewer_api.dependencies import (
-    get_ssh_client,
     get_jellyfin_client,
     get_jellyseerr_client,
     get_mail_queue,
     get_settings_store,
+    get_ssh_client,
     get_user_id,
 )
-from media_library_viewer_api.clients.ssh import CommandResult
+from media_library_viewer_api.main import app
 from media_library_viewer_api.routers.media import get_media_index
 from media_library_viewer_api.services.media_index import MediaIndex
 from media_library_viewer_api.services.settings_store import SettingsStore
 
-
 # --- Fixtures ---
 
+
 @pytest.fixture
 def mock_jellyfin():
     """Mock Jellyfin client."""
@@ -145,15 +145,29 @@ def test_client(mock_jellyfin, mock_jellyseerr, mock_ssh, tmp_path):
 
 # --- Health ---
 
+
 class TestHealth:
     def test_health(self, test_client):
         response = test_client.get("/api/health")
         assert response.status_code == 200
         assert response.json() == {"status": "ok"}
 
+    def test_metrics_endpoint(self, test_client):
+        response = test_client.get("/metrics")
+        assert response.status_code == 200
+        assert "manage_api_requests_total" in response.text
+        assert response.headers["content-type"].startswith("text/plain")
+
+    def test_request_id_header(self, test_client):
+        response = test_client.get("/api/health")
+        assert response.status_code == 200
+        assert "x-request-id" in response.headers
+        assert len(response.headers["x-request-id"]) > 0
+
 
 # --- Dashboard ---
 
+
 class TestDashboard:
     def test_counts(self, test_client):
         response = test_client.get("/api/dashboard/counts")
@@ -163,78 +177,6 @@ class TestDashboard:
         assert data["series"] == 20
         assert data["episodes"] == 500
 
-    def test_monitoring_overview(self, test_client):
-        store = MagicMock()
-        store.list_machines.return_value = [
-            {
-                "id": "local",
-                "name": "This machine",
-                "mode": "local",
-                "enabled": True,
-                "host": "localhost",
-                "port": 22,
-                "username": "",
-                "media_root": "/srv/media",
-                "path_prefix": "",
-                "notes": "",
-            },
-            {
-                "id": "remote1",
-                "name": "Remote",
-                "mode": "ssh",
-                "enabled": True,
-                "host": "server.example.com",
-                "port": 22,
-                "username": "alex",
-                "media_root": "/srv/media",
-                "path_prefix": "",
-                "notes": "",
-            },
-        ]
-        app.dependency_overrides[get_settings_store] = lambda: store
-        try:
-            with (
-                patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=object()),
-                patch("media_library_viewer_api.services.monitoring_actions.resource_collector_status", return_value="running pid=123"),
-                patch(
-                    "media_library_viewer_api.services.monitoring_actions.read_resource_metrics",
-                    return_value=[
-                        {
-                            "ts": 123.0,
-                            "cpu_pct": 10.0,
-                            "iowait_pct": 1.0,
-                            "mem_pct": 20.0,
-                            "net_rx_bytes_per_sec": 100.0,
-                            "net_tx_bytes_per_sec": 50.0,
-                            "disk_read_bps": 1.0,
-                            "disk_write_bps": 2.0,
-                        }
-                    ],
-                ),
-                patch(
-                    "media_library_viewer_api.services.monitoring_actions.disk_space",
-                    return_value={
-                        "filesystem": "/dev/sda1",
-                        "size": 1000,
-                        "used": 200,
-                        "available": 800,
-                        "used_pct": "20.0%",
-                        "mount": "/srv/media",
-                    },
-                ),
-            ):
-                response = test_client.get("/api/dashboard/monitoring")
-        finally:
-            app.dependency_overrides.pop(get_settings_store, None)
-
-        assert response.status_code == 200
-        data = response.json()
-        assert data["total"] == 2
-        assert data["enabled"] == 2
-        assert len(data["machines"]) == 2
-        assert data["machines"][0]["latest_sample"]["cpu_pct"] == 10.0
-        assert data["machines"][0]["disk"]["used_pct"] == "20.0%"
-
     def test_libraries(self, test_client):
         response = test_client.get("/api/dashboard/libraries")
         assert response.status_code == 200
@@ -268,6 +210,7 @@ class TestDashboard:
 
 # --- Settings reset ---
 
+
 class TestSettingsReset:
     def test_reset_local_database_requires_full_confirmation(self, test_client, tmp_path):
         store = SettingsStore(tmp_path / "settings.sqlite")
@@ -329,8 +272,10 @@ class TestSettingsReset:
         assert store.get_machine("local") is None
         assert len(store.list_machines()) == 0
 
+
 # --- Users ---
 
+
 class TestUsers:
     def test_users_list_enriched(self, test_client):
         response = test_client.get("/api/users")
@@ -418,7 +363,7 @@ class TestUsers:
         )
 
         try:
-            with patch("media_library_viewer_api.routers.users.get_settings", return_value=settings):
+            with patch("media_library_viewer_api.routers.users_impl.get_settings", return_value=settings):
                 response = test_client.post(
                     "/api/users/message",
                     data={
@@ -443,8 +388,10 @@ class TestUsers:
         assert kwargs["subject"] == "Hello team"
         assert kwargs["settings"] is settings
 
+
 # --- Files ---
 
+
 class TestFiles:
     def test_list_directory(self, test_client):
         response = test_client.get("/api/files/list", params={"path": "/media"})
@@ -486,6 +433,7 @@ class TestFiles:
 
 # --- Media index build ---
 
+
 class TestMediaIndexApi:
     def test_status_includes_build_progress(self, test_client, tmp_path):
         index = MediaIndex(tmp_path / "index.sqlite")
@@ -586,9 +534,11 @@ class TestMediaIndexApi:
             alive_calls["count"] += 1
             return alive_calls["count"] <= 2
 
-        with patch("media_library_viewer_api.routers.media._pid_is_alive", side_effect=fake_pid_is_alive), patch(
+        with (
-            "media_library_viewer_api.routers.media.os.killpg"
+            patch("media_library_viewer_api.routers.media._pid_is_alive", side_effect=fake_pid_is_alive),
-        ) as killpg, patch("media_library_viewer_api.routers.media.time.sleep", return_value=None):
+            patch("media_library_viewer_api.routers.media.os.killpg") as killpg,
+            patch("media_library_viewer_api.routers.media.time.sleep", return_value=None),
+        ):
             try:
                 response = test_client.post("/api/media/force-stop")
                 assert response.status_code == 202
@@ -615,6 +565,7 @@ class TestMediaIndexApi:
 
 # --- Jobs ---
 
+
 class TestJobs:
     def test_list_templates(self, test_client):
         response = test_client.get("/api/jobs/templates")
@@ -645,37 +596,21 @@ class TestJobs:
 
 # --- Monitoring ---
 
+
 class TestMonitoring:
     def _ensure_machine(self):
         store = app.dependency_overrides[get_settings_store]()
         if not store.list_machines():
-            store.upsert_machine({
+            store.upsert_machine(
-                "name": "Test Machine",
+                {
-                "mode": "ssh",
+                    "name": "Test Machine",
-                "enabled": True,
+                    "mode": "ssh",
-                "services": ["monitoring", "files", "jellyfin"],
+                    "enabled": True,
-                "host": "test-host",
+                    "services": ["monitoring", "files", "jellyfin"],
-                "username": "test-user",
+                    "host": "test-host",
-            })
+                    "username": "test-user",
-
+                }
-    def test_status(self, test_client, mock_ssh):
+            )
-        self._ensure_machine()
-        mock_ssh.run.return_value = CommandResult(
-            command="...", exit_status=0, stdout="running pid=1234\n", stderr=""
-        )
-        response = test_client.get("/api/monitoring/status")
-        assert response.status_code == 200
-        assert "running" in response.json()["status"]
-
-    def test_metrics_empty(self, test_client, mock_ssh):
-        self._ensure_machine()
-        mock_ssh.run.return_value = CommandResult(
-            command="...", exit_status=0, stdout="", stderr=""
-        )
-        response = test_client.get("/api/monitoring/metrics")
-        assert response.status_code == 200
-        data = response.json()
-        assert data["samples"] == []
 
     def test_disk(self, test_client, mock_ssh):
         self._ensure_machine()
@@ -685,32 +620,162 @@ class TestMonitoring:
             stdout='{"filesystem":"/dev/sda1","size":1000000000,"used":500000000,"available":500000000,"used_pct":"50%","mount":"/"}',
             stderr="",
         )
-        response = test_client.get("/api/monitoring/disk")
+        with patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=mock_ssh):
+            response = test_client.get("/api/monitoring/disk")
         assert response.status_code == 200
         data = response.json()
         assert data["used_pct"] == "50%"
 
-    def test_start(self, test_client, mock_ssh):
+    def test_prometheus_targets_empty(self, test_client):
-        self._ensure_machine()
+        response = test_client.get("/api/monitoring/prometheus-targets")
-        mock_ssh.run.return_value = CommandResult(
-            command="...", exit_status=0, stdout="started pid=5678\n", stderr=""
-        )
-        response = test_client.post("/api/monitoring/start")
         assert response.status_code == 200
-        assert "started" in response.json()["message"]
+        assert response.json() == []
 
-    def test_stop(self, test_client, mock_ssh):
+    def test_prometheus_targets_returns_enabled_ssh_node_exporter(self, test_client):
-        self._ensure_machine()
+        store = app.dependency_overrides[get_settings_store]()
-        mock_ssh.run.return_value = CommandResult(
+        store.upsert_machine(
-            command="...", exit_status=0, stdout="stopped pid=5678\n", stderr=""
+            {
+                "name": "remote1",
+                "mode": "ssh",
+                "enabled": True,
+                "services": ["monitoring"],
+                "host": "10.0.0.5",
+                "username": "u",
+                "node_exporter_enabled": True,
+                "node_exporter_port": 9200,
+                "node_exporter_scrape_host": "1.2.3.4",
+            }
         )
-        response = test_client.post("/api/monitoring/stop")
+        response = test_client.get("/api/monitoring/prometheus-targets")
         assert response.status_code == 200
+        data = response.json()
+        assert len(data) == 1
+        assert data[0]["targets"] == ["1.2.3.4:9200"]
+        assert data[0]["labels"]["job"] == "node-exporter-remote"
 
-    def test_restart(self, test_client, mock_ssh):
+
-        self._ensure_machine()
+class TestSettingsMachines:
-        mock_ssh.run.return_value = CommandResult(
+    def test_post_machine_rewrites_prometheus_targets(self, test_client):
-            command="...", exit_status=0, stdout="stopped pid=5678\nstarted pid=9999\n", stderr=""
+        with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets:
+            with patch("media_library_viewer_api.routers.settings._validate_saved_machine_ssh"):
+                response = test_client.post(
+                    "/api/settings/machines",
+                    json={
+                        "name": "remote1",
+                        "mode": "ssh",
+                        "enabled": True,
+                        "services": ["monitoring"],
+                        "host": "10.0.0.5",
+                        "username": "u",
+                        "node_exporter_enabled": True,
+                        "node_exporter_port": 9200,
+                        "node_exporter_scrape_host": "1.2.3.4",
+                    },
+                )
+        assert response.status_code == 201
+        write_targets.assert_called_once()
+
+    def test_delete_machine_rewrites_prometheus_targets(self, test_client):
+        store = app.dependency_overrides[get_settings_store]()
+        machine = store.upsert_machine(
+            {
+                "name": "remote1",
+                "mode": "ssh",
+                "enabled": True,
+                "services": ["monitoring"],
+                "host": "10.0.0.5",
+                "username": "u",
+            }
         )
-        response = test_client.post("/api/monitoring/restart")
+        with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets:
+            response = test_client.delete(f"/api/settings/machines/{machine['id']}")
         assert response.status_code == 200
+        write_targets.assert_called_once()
+
+
+class TestAlertmanager:
+    def test_alerts_endpoint_when_alertmanager_unreachable(self, test_client):
+        with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
+            session = MagicMock()
+            session.get.side_effect = Exception("connection refused")
+            mock_client.return_value = (session, "http://alertmanager:9093")
+            response = test_client.get("/api/monitoring/alerts")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["total"] == 0
+        assert data["error"] == "alertmanager_unreachable"
+
+    def test_alerts_endpoint_returns_summary(self, test_client):
+        with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
+            session = MagicMock()
+            session.get.return_value.json.return_value = {
+                "status": "success",
+                "data": [
+                    {
+                        "labels": {"alertname": "BackupJobFailed", "severity": "critical", "job_name": "test"},
+                        "annotations": {"summary": "Backup failed", "description": "details"},
+                        "startsAt": "2026-05-11T02:00:00Z",
+                        "status": "firing",
+                    }
+                ],
+            }
+            session.get.return_value.raise_for_status = MagicMock()
+            mock_client.return_value = (session, "http://alertmanager:9093")
+            response = test_client.get("/api/monitoring/alerts")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["total"] == 1
+        assert data["by_severity"]["critical"] == 1
+        assert data["alerts"][0]["name"] == "BackupJobFailed"
+
+    def test_alertmanager_status_endpoint_when_unreachable(self, test_client):
+        with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
+            session = MagicMock()
+            session.get.side_effect = Exception("connection refused")
+            mock_client.return_value = (session, "http://alertmanager:9093")
+            response = test_client.get("/api/monitoring/alertmanager-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["up"] is False
+
+
+class TestAlertmanagerWebhook:
+    """Tests for POST /api/monitoring/alertmanager-webhook."""
+
+    def test_webhook_receives_payload_and_logs(self, test_client, caplog):
+        payload = {
+            "status": "firing",
+            "alerts": [
+                {
+                    "status": "firing",
+                    "labels": {"alertname": "BackupJobFailed", "severity": "critical"},
+                    "annotations": {"summary": "Backup failed"},
+                    "startsAt": "2026-06-13T10:00:00Z",
+                }
+            ],
+        }
+        with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"):
+            response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload)
+        assert response.status_code == 200
+        assert response.json()["status"] == "received"
+        assert "Received Alertmanager webhook with 1 alert(s)" in caplog.text
+
+    def test_webhook_forwards_when_configured(self, test_client, caplog, monkeypatch):
+        payload = {"status": "resolved", "alerts": []}
+        forwarded = {"captured": False}
+
+        class FakeResponse:
+            def raise_for_status(self):
+                pass
+
+        def fake_post(url, json, timeout):
+            forwarded["captured"] = True
+            forwarded["url"] = url
+            forwarded["payload"] = json
+            return FakeResponse()
+
+        monkeypatch.setattr("requests.Session.post", lambda _self, url, json, timeout: fake_post(url, json, timeout))
+        with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"):
+            response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload)
+        assert response.status_code == 200
+        assert forwarded["captured"] is False

backend/tests/test_config.py

@@ -2,6 +2,7 @@
 
 import os
 from unittest.mock import patch
+
 from media_library_viewer_api.config import Settings
 
 

backend/tests/test_domain_media.py

@@ -1,16 +1,15 @@
 """Unit tests for domain/media.py normalization helpers."""
 
-import pytest
 from media_library_viewer_api.domain.media import (
-    first_media_source,
-    media_streams,
-    stream_value,
-    is_hdr_item,
-    format_date_added,
-    timestamp_date_added,
-    format_rate_bits_decimal,
-    normalize_media_item,
     display_media_row,
+    first_media_source,
+    format_date_added,
+    format_rate_bits_decimal,
+    is_hdr_item,
+    media_streams,
+    normalize_media_item,
+    stream_value,
+    timestamp_date_added,
 )
 
 

backend/tests/test_jobs.py

@@ -1,7 +1,6 @@
 """Unit tests for jobs.py template rendering and safety."""
 
-import pytest
+from media_library_viewer_api.jobs import JOB_TEMPLATES, JobTemplate
-from media_library_viewer_api.jobs import JOB_TEMPLATES, JobTemplate, run_job
 
 
 class TestJobTemplate:

backend/tests/test_media_index.py

@@ -1,9 +1,9 @@
 """Unit tests for the SQLite media index service."""
 
-import pytest
-import tempfile
-from pathlib import Path
 from typing import Any
+
+import pytest
+
 from media_library_viewer_api.services.media_index import (
     MediaIndex,
     MediaIndexBuildCancelled,

backend/tests/test_monitoring_actions.py

@@ -3,7 +3,7 @@ from unittest.mock import MagicMock, patch
 from media_library_viewer_api.services.monitoring_actions import poll_machine_snapshot
 
 
-def test_poll_machine_snapshot_records_history_entries():
+def test_poll_machine_snapshot_records_disk_lookup():
     store = MagicMock()
     machine = {
         "id": "local",
@@ -17,14 +17,6 @@ def test_poll_machine_snapshot_records_history_entries():
             "media_library_viewer_api.services.monitoring_actions.build_machine_client",
             return_value=object(),
         ) as build_client,
-        patch(
-            "media_library_viewer_api.services.monitoring_actions.resource_collector_status",
-            return_value="running pid=123",
-        ) as status_fn,
-        patch(
-            "media_library_viewer_api.services.monitoring_actions.read_resource_metrics",
-            return_value=[{"ts": 1.0}, {"ts": 2.0}],
-        ) as metrics_fn,
         patch(
             "media_library_viewer_api.services.monitoring_actions.disk_space",
             return_value={"mount": "/srv/media", "used_pct": "12.5%"},
@@ -33,14 +25,12 @@ def test_poll_machine_snapshot_records_history_entries():
         result = poll_machine_snapshot(machine, store, metrics_limit=123, request_id="poll:test")
 
     assert result["request_id"] == "poll:test"
-    assert result["metrics_samples"] == 2
     assert result["disk_mount"] == "/srv/media"
-    build_client.assert_called_once_with(machine)
+    assert result["actions"] == ["disk lookup"]
-    status_fn.assert_called_once()
+    build_client.assert_called_once_with(machine, store)
-    metrics_fn.assert_called_once_with(build_client.return_value, max_lines=123)
     disk_fn.assert_called_once_with(build_client.return_value, "/srv/media")
-    assert store.record_machine_action.call_count == 3
+    assert store.record_machine_action.call_count == 1
-    recorded_actions = [call.args[1] for call in store.record_machine_action.call_args_list]
+    recorded_action = store.record_machine_action.call_args
-    assert recorded_actions == ["status lookup", "metrics read", "disk lookup for /srv/media"]
+    assert recorded_action.args[1] == "disk lookup for /srv/media"
-    assert all(call.kwargs["request_id"] == "poll:test" for call in store.record_machine_action.call_args_list)
+    assert recorded_action.kwargs["request_id"] == "poll:test"
-    assert all(call.args[2] == "ok" for call in store.record_machine_action.call_args_list)
+    assert recorded_action.args[2] == "ok"

backend/tests/test_observability.py

@@ -0,0 +1,37 @@
+"""Tests for observability metrics helpers."""
+
+import time
+
+from prometheus_client import REGISTRY
+
+from media_library_viewer_api.observability import record_backup_run
+
+
+def _metric_samples(metric_name):
+    return {
+        tuple(s.labels.values()): s.value
+        for family in REGISTRY.collect()
+        for s in family.samples
+        if s.name == metric_name
+    }
+
+
+def test_record_backup_run_increments_counter():
+    record_backup_run("job-a", "success", success=True)
+    record_backup_run("job-a", "failure")
+    record_backup_run("job-b", "success", success=True)
+
+    samples = _metric_samples("manage_backup_runs_total")
+    assert samples[("job-a", "success")] >= 1.0
+    assert samples[("job-a", "failure")] >= 1.0
+    assert samples[("job-b", "success")] >= 1.0
+
+
+def test_record_backup_run_sets_last_success_timestamp():
+    before = time.time()
+    record_backup_run("job-ts", "success", success=True)
+    after = time.time()
+
+    samples = _metric_samples("manage_backup_runs_last_success_timestamp")
+    value = samples[("job-ts",)]
+    assert before <= value <= after

backend/tests/test_path_utils.py

@@ -1,6 +1,5 @@
 """Unit tests for path_utils.py — path resolution logic."""
 
-import pytest
 from media_library_viewer_api.path_utils import (
     apply_remote_path_prefix,
     map_path_to_media_root,
@@ -32,7 +31,9 @@ class TestApplyRemotePathPrefix:
         assert apply_remote_path_prefix("/media/file.mkv", "/srv/") == "/srv/media/file.mkv"
 
     def test_path_with_spaces(self):
-        assert apply_remote_path_prefix("/media/My Movie (2024)/file.mkv", "/srv") == "/srv/media/My Movie (2024)/file.mkv"
+        assert (
+            apply_remote_path_prefix("/media/My Movie (2024)/file.mkv", "/srv") == "/srv/media/My Movie (2024)/file.mkv"
+        )
 
 
 class TestMapPathToMediaRoot:

backend/tests/test_ssh_client.py

@@ -7,7 +7,9 @@ from media_library_viewer_api.clients.ssh import RemoteSSHClient
 
 def test_connect_uses_existing_known_hosts_without_reprobing(tmp_path):
     known_hosts_path = tmp_path / "known_hosts"
-    known_hosts_path.write_text("example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAFAKE\n")
+    known_hosts_path.write_text(
+        "example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID8tdNkXNS1EWbkjb2XxFHQXRrpYzl0MXaVLRmVrbmDZ\n"
+    )
 
     ssh_client = MagicMock()
     ssh_client.connect.return_value = None

backend/tests/test_targets.py

@@ -0,0 +1,109 @@
+"""Tests for Prometheus file-based service discovery target generation."""
+
+from pathlib import Path
+
+import pytest
+
+from media_library_viewer_api.services.settings_store import SettingsStore
+from media_library_viewer_api.services.targets import (
+    build_node_exporter_targets,
+    write_prometheus_targets,
+)
+
+
+@pytest.fixture
+def store(tmp_path: Path) -> SettingsStore:
+    db = SettingsStore(tmp_path / "settings.sqlite")
+    db.init_schema()
+    return db
+
+
+class TestBuildNodeExporterTargets:
+    def test_disabled_machine_excluded(self, store: SettingsStore):
+        store.upsert_machine(
+            {
+                "name": "remote1",
+                "mode": "ssh",
+                "host": "10.0.0.5",
+                "username": "u",
+                "node_exporter_enabled": False,
+                "node_exporter_port": 9200,
+            }
+        )
+        assert build_node_exporter_targets(store) == []
+
+    def test_ssh_enabled_machine_included(self, store: SettingsStore):
+        machine = store.upsert_machine(
+            {
+                "name": "remote1",
+                "mode": "ssh",
+                "host": "10.0.0.5",
+                "username": "u",
+                "node_exporter_enabled": True,
+                "node_exporter_port": 9200,
+                "node_exporter_scrape_host": "1.2.3.4",
+            }
+        )
+        targets = build_node_exporter_targets(store)
+        assert len(targets) == 1
+        assert targets[0]["targets"] == ["1.2.3.4:9200"]
+        assert targets[0]["labels"]["machine_id"] == machine["id"]
+        assert targets[0]["labels"]["machine_name"] == "remote1"
+        assert targets[0]["labels"]["job"] == "node-exporter-remote"
+
+    def test_scrape_host_defaults_to_machine_host(self, store: SettingsStore):
+        store.upsert_machine(
+            {
+                "name": "remote2",
+                "mode": "ssh",
+                "host": "remote2.example.com",
+                "username": "u",
+                "node_exporter_enabled": True,
+                "node_exporter_port": 9100,
+            }
+        )
+        targets = build_node_exporter_targets(store)
+        assert targets[0]["targets"] == ["remote2.example.com:9100"]
+
+    def test_local_machine_excluded(self, store: SettingsStore):
+        store.upsert_machine(
+            {
+                "name": "This machine",
+                "mode": "local",
+                "host": "localhost",
+                "username": "",
+                "node_exporter_enabled": True,
+            }
+        )
+        assert build_node_exporter_targets(store) == []
+
+    def test_missing_host_excluded(self, store: SettingsStore):
+        store.upsert_machine(
+            {
+                "name": "remote3",
+                "mode": "ssh",
+                "host": "",
+                "username": "u",
+                "node_exporter_enabled": True,
+            }
+        )
+        assert build_node_exporter_targets(store) == []
+
+
+class TestWritePrometheusTargets:
+    def test_writes_valid_json(self, store: SettingsStore, tmp_path: Path):
+        store.upsert_machine(
+            {
+                "name": "remote1",
+                "mode": "ssh",
+                "host": "10.0.0.5",
+                "username": "u",
+                "node_exporter_enabled": True,
+                "node_exporter_port": 9200,
+            }
+        )
+        file_path = write_prometheus_targets(store, tmp_path)
+        assert file_path.exists()
+        assert file_path.name == "node_exporter_targets.json"
+        targets = build_node_exporter_targets(store)
+        assert len(targets) == 1

backend/tests/test_utils.py

@@ -1,17 +1,16 @@
 """Unit tests for utils.py formatting helpers."""
 
-import pytest
 from media_library_viewer_api.utils import (
-    ticks_to_minutes,
+    ffprobe_format_summary,
+    format_bitrate,
+    format_duration,
     human_size,
     is_known_video_file,
-    format_duration,
-    format_bitrate,
-    ffprobe_format_summary,
-    summarize_video_streams,
     summarize_audio_streams,
-    summarize_subtitle_streams,
     summarize_streams,
+    summarize_subtitle_streams,
+    summarize_video_streams,
+    ticks_to_minutes,
     timestamp_to_local,
 )
 

context.md

@@ -0,0 +1,207 @@
+# Code Context
+
+## Files Retrieved
+1. `docker-compose.yml` (lines 1–262) – production Compose stack; defines observability services and Traefik routing.
+2. `docker-compose.dev.yml` (lines 1–234) – development Compose stack; same observability services but with host ports exposed and auth disabled.
+3. `.env.example` (lines 1–55) – template with all required environment variables for the stack, including Prometheus/Grafana/Alertmanager/Alloy/Loki and Node Exporter settings.
+4. `monitoring/prometheus/prometheus.yml` (all lines) – Prometheus scrape configuration including the `manage-backend`, `node-exporter`, and file-SD remote targets.
+5. `monitoring/prometheus/rules/backup_alerts.yml` (all lines) – Prometheus alerting rules for backup jobs and observability-stack health.
+6. `monitoring/alertmanager/alertmanager.yml` (all lines) – Alertmanager routing, email/webhook receivers, and inhibition rules.
+7. `monitoring/grafana/grafana.ini` (all lines) – Grafana server, embedding, and generic OAuth (Authentik) configuration.
+8. `monitoring/alloy/config.alloy` (all lines) – Alloy pipeline to discover Docker containers and push logs to Loki.
+9. `monitoring/loki/loki.yml` (all lines) – Single-node Loki configuration with filesystem storage and 30-day retention.
+10. `backend/src/media_library_viewer_api/observability.py` (all lines) – Prometheus metrics definitions and helper functions.
+11. `backend/src/media_library_viewer_api/main.py` (lines 1–128) – FastAPI entrypoint exposing `/metrics` and wiring request/observability middleware.
+12. `backend/src/media_library_viewer_api/services/targets.py` (all lines) – Backend writes file-SD target list for remote Node Exporters.
+13. `backend/src/media_library_viewer_api/config.py` (lines 1–88) – Settings including `prometheus_enabled`, `prometheus_file_sd_dir`, and `alertmanager_url`.
+14. `docs/monitoring-logging-design.md` (all lines) – Architecture/design document describing the observability stack.
+15. `docs/observability-runbooks.md` (all lines) – Operational runbooks for the observability services.
+
+## Key Code
+
+### Backend `/metrics` endpoint
+`backend/src/media_library_viewer_api/main.py`:
+```python
+@app.middleware("http")
+async def enforce_jwt_auth(request: Request, call_next):
+    if request.url.path in {"/api/health", "/api/version", "/metrics"}:
+        return await call_next(request)
+    return await require_jwt_auth(request, call_next)
+
+@app.get("/metrics")
+def metrics() -> Response:
+    """Expose Prometheus metrics."""
+    data, content_type = metrics_payload()
+    return FastAPIResponse(content=data, media_type=content_type)
+```
+
+### Metrics emitted by the backend
+`backend/src/media_library_viewer_api/observability.py`:
+```python
+REQUESTS_TOTAL = Counter("manage_api_requests_total", "Total API requests", ["method", "path", "status_code"])
+REQUEST_DURATION = Histogram("manage_api_request_duration_seconds", "API request duration", ["method", "path"], ...)
+SSH_COMMANDS_TOTAL = Counter("manage_ssh_commands_total", "Total SSH/local commands executed", ["machine_id", "action", "status"])
+MEDIA_INDEX_BUILDS_TOTAL = Counter("manage_media_index_builds_total", "Total media index build attempts", ["status"])
+BACKUP_RUNS_TOTAL = Counter("manage_backup_runs_total", "Total backup runs", ["job_name", "status"])
+BACKUP_RUNS_LAST_SUCCESS = Gauge("manage_backup_runs_last_success_timestamp", "Unix timestamp of the last successful backup run per job", ["job_name"])
+MAIL_QUEUE_SIZE = Counter("manage_mail_queue_messages_total", "Total messages enqueued", ["status"])
+```
+
+### Prometheus scrape configuration
+`monitoring/prometheus/prometheus.yml`:
+```yaml
+scrape_configs:
+  - job_name: manage-backend
+    static_configs:
+      - targets:
+          - backend:8000
+    metrics_path: /metrics
+    scrape_interval: 15s
+
+  - job_name: node-exporter
+    static_configs:
+      - targets:
+          - node-exporter:9100
+
+  - job_name: node-exporter-remote
+    file_sd_configs:
+      - files:
+          - /etc/prometheus/file-sd/node_exporter_targets.json
+        refresh_interval: 30s
+```
+
+### Backend-managed remote Node Exporter targets
+`backend/src/media_library_viewer_api/services/targets.py`:
+```python
+def build_node_exporter_targets(store: SettingsStore) -> list[dict[str, Any]]:
+    ...
+
+def write_prometheus_targets(store: SettingsStore, file_sd_dir: Path | None = None) -> Path:
+    ...
+    file_path = file_sd_dir / "node_exporter_targets.json"
+```
+
+## Architecture
+
+The observability stack is a standard self-hosted Prometheus/Grafana/Loki/Alertmanager deployment running inside the same Docker Compose project as the Manage application:
+
+- **Prometheus** pulls metrics from:
+  - The Manage FastAPI backend via `/metrics` (`job="manage-backend"`).
+  - The local Docker host via `node-exporter` (`job="node-exporter"`).
+  - Remote machines that have Node Exporter enabled through Manage settings (`job="node-exporter-remote"`), discovered through a file-SD JSON file generated by the backend.
+  - The observability services themselves: Loki, Alertmanager, Grafana, and Prometheus self-scrape.
+- **Grafana** visualizes metrics and logs; it is configured for OAuth login through Authentik and is embedded in the Manage React UI via iframes.
+- **Loki** stores logs; retention is 30 days.
+- **Alloy** (Grafana Alloy) collects Docker container logs by mounting the Docker socket and the container log directory, then pushes them to Loki.
+- **Alertmanager** routes alerts by severity (critical vs warning) and delivers email notifications (and optionally a webhook back to the backend).
+
+The backend bridges the stack with the application:
+- It exposes `/metrics` (unauthenticated, along with `/api/health` and `/api/version`).
+- On startup it writes `${PROMETHEUS_FILE_SD_DIR}/node_exporter_targets.json` based on enabled SSH machines in the settings store.
+- It provides proxy endpoints (`/api/monitoring/alerts`, `/api/monitoring/alertmanager-status`, `/api/monitoring/prometheus-targets`) consumed by the frontend.
+
+## Start Here
+
+Open `monitoring/prometheus/prometheus.yml` first to understand what is scraped and how the backend is wired, then read `backend/src/media_library_viewer_api/observability.py` to see the metric names and labels. For environment requirements, read `.env.example`.
+
+## Supervisor coordination
+Not needed — this is a read-only scouting summary.
+
+---
+
+# Monitoring/Observability Setup Summary
+
+## 1. Observability services defined in Compose
+
+Both `docker-compose.yml` and `docker-compose.dev.yml` define the following services:
+
+| Service | Image | Internal endpoint | Purpose |
+|---------|-------|-------------------|---------|
+| `prometheus` | `prom/prometheus:v2.55.1` | `http://prometheus:9090` | Metrics TSDB and alert evaluator |
+| `loki` | `grafana/loki:3.1.1` | `http://loki:3100` | Log aggregation |
+| `alloy` | `grafana/alloy:v1.5.0` | `http://alloy:12345` | Docker log collection agent |
+| `grafana` | `grafana/grafana:11.3.1` | `http://grafana:3000` | Dashboards and visualization |
+| `alertmanager` | `prom/alertmanager:v0.27.0` | `http://alertmanager:9093` | Alert routing/delivery |
+| `node-exporter` | `prom/node-exporter:v1.8.2` | `http://node-exporter:9100` | Host metrics for the Docker host |
+| `backend` | Build from `backend/Dockerfile` | `http://backend:8000` | FastAPI app exposing `/metrics` |
+
+Differences:
+- Production (`docker-compose.yml`): services attach to an external `web` network for Traefik, use `expose` instead of host ports for most services, and require OIDC/auth variables.
+- Development (`docker-compose.dev.yml`): Prometheus/Grafana/Loki/Alertmanager/Node Exporter are published on host ports `9090`, `3000`, `3100`, `9093`, `9100`; auth is disabled (`AUTH_ENABLED=false`).
+
+## 2. Required environment variables
+
+From `.env.example` and the Compose files, the variables relevant to the observability stack are:
+
+### Backend / metrics
+- `PROMETHEUS_ENABLED` – enable metrics endpoint (set to `"true"` in both compose files).
+- `PROMETHEUS_FILE_SD_DIR` – directory where the backend writes `node_exporter_targets.json` (default `/app/backend/.cache/prometheus-file-sd`).
+- `ALERTMANAGER_URL` – backend proxy target (default `http://alertmanager:9093`).
+- `ALERTMANAGER_WEBHOOK_URL` – optional webhook receiver for Alertmanager.
+- `BACKEND_CACHE_DIR` – host directory mounted into backend and Prometheus for file-SD.
+
+### Grafana
+- `GRAFANA_APP_HOST` – public hostname for Grafana (production; required).
+- `GRAFANA_APP_PORT` – defaults to `3000`.
+- `GRAFANA_APP_NAME` – defaults to `grafana`.
+- `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` – local admin credentials.
+- `GF_AUTH_GENERIC_OAUTH_CLIENT_ID`
+- `GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET`
+- `GF_AUTH_GENERIC_OAUTH_AUTH_URL`
+- `GF_AUTH_GENERIC_OAUTH_TOKEN_URL`
+- `GF_AUTH_GENERIC_OAUTH_API_URL`
+
+### Alertmanager
+- `SMTP_HOST` / `SMTP_PORT`
+- `SMTP_USERNAME` / `SMTP_PASSWORD`
+- `SMTP_FROM_ADDRESS`
+- `ALERT_EMAIL_TO`
+
+### Traefik / network (production)
+- `BACKEND_APP_HOST` / `FRONTEND_APP_HOST` / `GRAFANA_APP_HOST`
+- `CERT_RESOLVER` – e.g. `letsencrypt`
+
+### General
+- `LOG_LEVEL` / `LOG_FORMAT` – also passed to Grafana and backend.
+
+## 3. Monitoring config files
+
+| File | Purpose |
+|------|---------|
+| `monitoring/prometheus/prometheus.yml` | Scrape jobs: backend `/metrics`, local node-exporter, remote node-exporter via file-SD, Loki, Alertmanager, Grafana, and self-scrape. |
+| `monitoring/prometheus/rules/backup_alerts.yml` | Alert rules: `BackupJobFailed`, `BackupJobStuck`, `PrometheusTargetMissing`, `AlertmanagerDown`, `GrafanaDown`. |
+| `monitoring/alertmanager/alertmanager.yml` | Routes alerts by severity, sends email to `ALERT_EMAIL_TO`, optional webhook to backend, and inhibits warnings when a critical alert fires. |
+| `monitoring/grafana/grafana.ini` | Enables iframe embedding, OAuth via Authentik, role mapping from groups, and defaults users to `Viewer`. |
+| `monitoring/alloy/config.alloy` | Discovers Docker containers, relabels container/stream labels, and writes logs to `http://loki:3100/loki/api/v1/push`. |
+| `monitoring/loki/loki.yml` | Single-node Loki with filesystem storage, tsdb index, 30-day retention (`720h`). |
+
+## 4. Backend metrics and Prometheus scraping
+
+- The backend exposes Prometheus metrics at `/metrics` on port `8000`.
+- The endpoint is unauthenticated (bypassed in `enforce_jwt_auth`).
+- Prometheus scrapes it as `job="manage-backend"` with `scrape_interval: 15s`.
+- Key application metrics include:
+  - `manage_api_requests_total{method, path, status_code}`
+  - `manage_api_request_duration_seconds{method, path}`
+  - `manage_ssh_commands_total{machine_id, action, status}`
+  - `manage_media_index_builds_total{status}`
+  - `manage_backup_runs_total{job_name, status}`
+  - `manage_backup_runs_last_success_timestamp{job_name}`
+  - `manage_mail_queue_messages_total{status}`
+
+Remote Node Exporter targets are not static: the backend reads machine settings from SQLite and writes `${PROMETHEUS_FILE_SD_DIR}/node_exporter_targets.json`. Prometheus reloads this file every 30 seconds via `file_sd_configs`.
+
+## 5. Setup steps and gotchas
+
+- The observability stack is brought up with the app itself:
+  - Production: `docker compose -f docker-compose.yml up --build`
+  - Development: `docker compose -f docker-compose.dev.yml up --build`
+- Production requires the external `web` network and Traefik already configured; `docker-compose.dev.yml` does not use Traefik and binds ports directly.
+- Export/copy `.env.example` to `.env` and fill required values (`OIDC_ISSUER_URL`, `OIDC_AUDIENCE`, `BACKEND_APP_HOST`, `FRONTEND_APP_HOST`, `GRAFANA_APP_HOST`, `CERT_RESOLVER`, SMTP credentials, Grafana OAuth secrets).
+- `BACKEND_CACHE_DIR` is shared between the backend and Prometheus so file-SD target updates are visible to Prometheus.
+- Alloy must run as `user: root` and mount `/var/run/docker.sock` and `/var/lib/docker/containers`; without these mounts it cannot collect Docker logs.
+- Node Exporter mounts the host root filesystem read-only (`/:/host:ro,rslave`) to report host-level metrics; on production, this exposes the Docker host.
+- Grafana iframe embedding requires `allow_embedding = true` in `grafana.ini` and matching cookie settings; also ensure the reverse proxy/CSP permits embedding.
+- After changing Prometheus rules/config, trigger a reload with `curl -X POST http://localhost:9090/-/reload` (production needs Traefik/network access).
+- Observability data is stored in named volumes: `prometheus_data`, `loki_data`, `grafana_data`, `alertmanager_data`. Back them up as documented in `docs/observability-runbooks.md`.
+- Default retention is 30 days for both Prometheus TSDB and Loki logs.

docker-compose.dev.yml

@@ -11,12 +11,19 @@ services:
       OIDC_AUDIENCE: ${OIDC_AUDIENCE:-}
       OIDC_JWKS_URL: ${OIDC_JWKS_URL:-}
       LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      LOG_FORMAT: ${LOG_FORMAT:-text}
+      PROMETHEUS_ENABLED: "true"
       SSH_KNOWN_HOSTS_PATH: /app/backend/.cache/known_hosts
+      PROMETHEUS_FILE_SD_DIR: /app/backend/.cache/prometheus-file-sd
+      ALERTMANAGER_URL: ${ALERTMANAGER_URL:-http://alertmanager:9093}
     ports:
       - "8000:8000"
     volumes:
       - ./backend:/app/backend
       - ${BACKEND_CACHE_DIR:-./backend-cache}:/app/backend/.cache
+    networks:
+      - web
+      - monitoring
     restart: unless-stopped
 
   frontend:
@@ -38,5 +45,209 @@ services:
       - backend
     restart: unless-stopped
 
+  prometheus:
+    image: prom/prometheus:v2.55.1
+    container_name: prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=30d"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--web.enable-lifecycle"
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
+      - ${BACKEND_CACHE_DIR:-./backend-cache}/prometheus-file-sd:/etc/prometheus/file-sd:ro
+      - prometheus_data:/prometheus
+    ports:
+      - "9090:9090"
+    networks:
+      - monitoring
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "1.00"
+          memory: 1G
+        reservations:
+          cpus: "0.25"
+          memory: 256M
+
+  loki:
+    image: grafana/loki:3.1.1
+    container_name: loki
+    command: -config.file=/etc/loki/loki.yml
+    volumes:
+      - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
+      - loki_data:/loki
+    ports:
+      - "3100:3100"
+    networks:
+      - monitoring
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "1.00"
+          memory: 1G
+        reservations:
+          cpus: "0.25"
+          memory: 256M
+
+  alloy:
+    image: grafana/alloy:v1.5.0
+    container_name: alloy
+    command:
+      - run
+      - /etc/alloy/config.alloy
+      - --storage.path=/var/lib/alloy
+    volumes:
+      - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+    networks:
+      - monitoring
+    depends_on:
+      loki:
+        condition: service_healthy
+    restart: unless-stopped
+    user: root
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:12345/-/healthy"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "0.50"
+          memory: 512M
+        reservations:
+          cpus: "0.10"
+          memory: 128M
+
+  grafana:
+    image: grafana/grafana:11.3.1
+    container_name: grafana
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID:-}
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:-}
+      GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${GF_AUTH_GENERIC_OAUTH_AUTH_URL:-}
+      GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL:-}
+      GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+    volumes:
+      - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - grafana_data:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    networks:
+      - monitoring
+    depends_on:
+      prometheus:
+        condition: service_healthy
+      loki:
+        condition: service_healthy
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "1.00"
+          memory: 512M
+        reservations:
+          cpus: "0.25"
+          memory: 128M
+
+  alertmanager:
+    image: prom/alertmanager:v0.27.0
+    container_name: alertmanager
+    command:
+      - "--config.file=/etc/alertmanager/alertmanager.yml"
+      - "--storage.path=/alertmanager"
+    environment:
+      SMTP_HOST: ${SMTP_HOST:-smtp.example.com}
+      SMTP_PORT: ${SMTP_PORT:-587}
+      SMTP_USERNAME: ${SMTP_USERNAME:-}
+      SMTP_PASSWORD: ${SMTP_PASSWORD:-}
+      SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com}
+      ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
+    volumes:
+      - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - alertmanager_data:/alertmanager
+    ports:
+      - "9093:9093"
+    networks:
+      - monitoring
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "0.50"
+          memory: 256M
+        reservations:
+          cpus: "0.10"
+          memory: 64M
+
+  node-exporter:
+    image: prom/node-exporter:v1.8.2
+    container_name: node-exporter
+    command:
+      - "--path.rootfs=/host"
+    volumes:
+      - /:/host:ro,rslave
+    networks:
+      - monitoring
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    deploy:
+      resources:
+        limits:
+          cpus: "0.25"
+          memory: 128M
+        reservations:
+          cpus: "0.05"
+          memory: 32M
+
+networks:
+  monitoring:
+
 volumes:
   frontend_node_modules:
+  prometheus_data:
+  loki_data:
+  grafana_data:
+  alertmanager_data:

docker-compose.yml

@@ -13,6 +13,8 @@ services:
       OIDC_JWKS_URL: ${OIDC_JWKS_URL:-}
       OIDC_CLOCK_SKEW_SECONDS: ${OIDC_CLOCK_SKEW_SECONDS:-30}
       LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      LOG_FORMAT: ${LOG_FORMAT:-text}
+      PROMETHEUS_ENABLED: "true"
       SMTP_HOST: ${SMTP_HOST:-}
       SMTP_PORT: ${SMTP_PORT:-587}
       SMTP_USERNAME: ${SMTP_USERNAME:-}
@@ -23,11 +25,14 @@ services:
       SMTP_USE_SSL: ${SMTP_USE_SSL:-false}
       SMTP_TIMEOUT: ${SMTP_TIMEOUT:-30}
       SSH_KNOWN_HOSTS_PATH: /app/backend/.cache/known_hosts
+      PROMETHEUS_FILE_SD_DIR: ${PROMETHEUS_FILE_SD_DIR:-/app/backend/.cache/prometheus-file-sd}
+      ALERTMANAGER_URL: ${ALERTMANAGER_URL:-http://alertmanager:9093}
     volumes:
       - ${BACKEND_CACHE_DIR:-./backend-cache}:/app/backend/.cache
     restart: unless-stopped
     networks:
       - web
+      - monitoring
     expose:
       - "8000"
     labels:
@@ -80,6 +85,218 @@ services:
       - "8080:80"
     restart: unless-stopped
 
+  prometheus:
+    image: prom/prometheus:v2.55.1
+    container_name: prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=30d"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--web.enable-lifecycle"
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
+      - ${BACKEND_CACHE_DIR:-./backend-cache}/prometheus-file-sd:/etc/prometheus/file-sd:ro
+      - prometheus_data:/prometheus
+    expose:
+      - "9090"
+    networks:
+      - web
+      - monitoring
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "1.00"
+          memory: 1G
+        reservations:
+          cpus: "0.25"
+          memory: 256M
+
+  loki:
+    image: grafana/loki:3.1.1
+    container_name: loki
+    command: -config.file=/etc/loki/loki.yml
+    volumes:
+      - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
+      - loki_data:/loki
+    expose:
+      - "3100"
+    networks:
+      - monitoring
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "1.00"
+          memory: 1G
+        reservations:
+          cpus: "0.25"
+          memory: 256M
+
+  alloy:
+    image: grafana/alloy:v1.5.0
+    container_name: alloy
+    command:
+      - run
+      - /etc/alloy/config.alloy
+      - --storage.path=/var/lib/alloy
+    volumes:
+      - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+    networks:
+      - monitoring
+    depends_on:
+      loki:
+        condition: service_healthy
+    restart: unless-stopped
+    user: root
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:12345/-/healthy"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "0.50"
+          memory: 512M
+        reservations:
+          cpus: "0.10"
+          memory: 128M
+
+  grafana:
+    image: grafana/grafana:11.3.1
+    container_name: grafana
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID:-}
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:-}
+      GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${GF_AUTH_GENERIC_OAUTH_AUTH_URL:-}
+      GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL:-}
+      GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+    volumes:
+      - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - grafana_data:/var/lib/grafana
+    expose:
+      - "3000"
+    networks:
+      - web
+      - monitoring
+    depends_on:
+      prometheus:
+        condition: service_healthy
+      loki:
+        condition: service_healthy
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.rule=Host(`${GRAFANA_APP_HOST:?set GRAFANA_APP_HOST}`)"
+      - "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.entrypoints=websecure"
+      - "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.tls.certresolver=${CERT_RESOLVER:?set CERT_RESOLVER}"
+      - "traefik.http.services.${GRAFANA_APP_NAME:-grafana}.loadbalancer.server.port=${GRAFANA_APP_PORT:-3000}"
+    deploy:
+      resources:
+        limits:
+          cpus: "1.00"
+          memory: 512M
+        reservations:
+          cpus: "0.25"
+          memory: 128M
+
+  alertmanager:
+    image: prom/alertmanager:v0.27.0
+    container_name: alertmanager
+    command:
+      - "--config.file=/etc/alertmanager/alertmanager.yml"
+      - "--storage.path=/alertmanager"
+    environment:
+      SMTP_HOST: ${SMTP_HOST:-smtp.example.com}
+      SMTP_PORT: ${SMTP_PORT:-587}
+      SMTP_USERNAME: ${SMTP_USERNAME:-}
+      SMTP_PASSWORD: ${SMTP_PASSWORD:-}
+      SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com}
+      ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
+    volumes:
+      - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - alertmanager_data:/alertmanager
+    expose:
+      - "9093"
+    networks:
+      - monitoring
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          cpus: "0.50"
+          memory: 256M
+        reservations:
+          cpus: "0.10"
+          memory: 64M
+
+  node-exporter:
+    image: prom/node-exporter:v1.8.2
+    container_name: node-exporter
+    command:
+      - "--path.rootfs=/host"
+    volumes:
+      - /:/host:ro,rslave
+    networks:
+      - monitoring
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    deploy:
+      resources:
+        limits:
+          cpus: "0.25"
+          memory: 128M
+        reservations:
+          cpus: "0.05"
+          memory: 32M
+
 networks:
   web:
     external: true
+  monitoring:
+
+volumes:
+  prometheus_data:
+  loki_data:
+  grafana_data:
+  alertmanager_data:

docs/REQUIREMENTS.md

@@ -197,6 +197,12 @@ Phase 1: Jellyfin media index, SSH-based remote filesystem inspection, server mo
 
 ## Decision Log
 
+- 2026-06-13: Adopted a dedicated, self-hosted observability subsystem based on Prometheus, Grafana Loki, Grafana, and Alertmanager. Metrics will be pulled from Node Exporter on machines and from application exporters in containers; logs will be structured JSON shipped by Promtail/Grafana Alloy. The existing POSIX remote collector will be removed and backup alerts migrated to Alertmanager rules. See `docs/monitoring-logging-design.md`.
+- 2026-06-13 (Phase 1): Added Prometheus, Loki, Grafana Alloy, Grafana, Alertmanager, and Node Exporter services to `docker-compose.yml` and `docker-compose.dev.yml`. Provisioned Grafana datasources and an initial `Manage Overview` dashboard as code. Configured Alloy to tail Docker logs and ship to Loki. Added Grafana generic OAuth configuration via `monitoring/grafana/grafana.ini` and a dedicated Traefik host rule. Added Alertmanager email routing with env-var interpolation. Added `/grafana` proxy to the Vite dev server for iframe embedding.
+- 2026-06-13 (Phase 2): Extended machine settings with `node_exporter_enabled`, `node_exporter_port`, and `node_exporter_scrape_host`. Added Node Exporter install/restart/status job templates to `jobs.py`. Implemented `media_library_viewer_api.services.targets` to generate Prometheus file-SD target files and wired target regeneration into machine create/update/delete. Added `/api/monitoring/prometheus-targets` for live target previews. Configured Prometheus with a `node-exporter-remote` job reading file SD from the backend cache volume. Added a minimal `Node Exporter Overview` Grafana dashboard. Added unit and integration tests for target generation and the new endpoint.
+- 2026-06-13 (Phase 3): Added `ALERTMANAGER_URL` setting and `GET /api/monitoring/alerts` plus `GET /api/monitoring/alertmanager-status` endpoints that proxy/summarize Alertmanager for the UI. Added `manage_backup_runs_last_success_timestamp` Prometheus gauge and wired `record_backup_run` to update it on success so the existing `BackupJobStuck` Alertmanager rule works. Updated `docker-compose.yml`, `docker-compose.dev.yml`, and `.env.example` with `ALERTMANAGER_URL`. Added tests for the new endpoints and metric helpers.
+- 2026-06-13 (Phase 4): Added `/observability` React page (`frontend/src/components/ObservabilityPage.tsx`) with health cards for Alertmanager, active alerts, Prometheus targets, and machines; a recent-alerts list; machine selector; and Grafana iframe panels for Node Exporter metrics and Loki logs. Added `useObservability` hook and API client wrappers for `/api/monitoring/alerts`, `/api/monitoring/alertmanager-status`, and `/api/monitoring/prometheus-targets`. Added TypeScript types for Alertmanager summary/status and Prometheus targets. Wired the new route into `App.tsx` and the sidebar. Added shadcn/ui `card`, `badge`, `alert`, `skeleton`, and `select` components. Frontend build (`npm run build`) passes; lint has only pre-existing warnings.
+- 2026-06-13 (Phase 5): Hardened observability containers with health checks and resource limits in both compose files; added service-healthy `depends_on` conditions. Added Prometheus scrape jobs for Loki, Alertmanager, and Grafana, plus new `observability_health` alert rules. Added `ALERTMANAGER_WEBHOOK_URL` backend setting, `POST /api/monitoring/alertmanager-webhook` receiver, and Alertmanager `webhook` receiver config. Added `docs/observability-runbooks.md` with operational playbooks. Updated `.env.example` to include all observability variables.
 - 2026-05-03: Reaffirmed that the Monitoring tab charts should be rendered directly with D3 and expose brush-based time-range selection plus moving averages.
 - 2026-05-03: Added hover tooltips, summary chips, a moving vertical cursor, snapped point markers, and a selected-range label to the D3 Monitoring charts for faster visual inspection.
 - 2026-05-03: Combined network download/upload into one traffic chart and disk read/write into one I/O chart for clearer Monitoring layout.

docs/monitoring-logging-design.md

@@ -0,0 +1,438 @@
+# Monitoring and Logging Design — Manage
+
+## Executive Summary
+
+Manage currently uses ad-hoc observability: plain-text Python logs, a custom POSIX shell metrics collector on remote machines, and a background poller that stores snapshots in SQLite. This works for a single-instance homelab but becomes painful as the fleet grows and as users need faster incident response.
+
+This document proposes a dedicated, self-hosted observability subsystem built on the standard Prometheus/Grafana stack:
+
+- **Metrics**: Prometheus pulling from Node Exporter on machines and from application exporters in containers.
+- **Logs**: Structured JSON logs shipped to **Grafana Loki** by **Promtail/Grafana Alloy**.
+- **Dashboards**: Grafana for deep-dive dashboards, embedded in the Manage React UI via iframes.
+- **Alerting**: Prometheus Alertmanager for routing and notifications (email first, webhooks later).
+- **Auth**: Grafana authenticates through the existing OIDC/Authentik provider.
+
+The existing POSIX remote collector will be removed, and the Python backup alert engine will be migrated to Alertmanager rules.
+
+---
+
+## Goals
+
+1. **Fast query and alerting**: move from SQLite scan-based history to a real time-series database and indexed log store.
+2. **Unified view**: monitor both local containers/apps and remote Linux machines from one place.
+3. **Standard tooling**: use de-facto open-source tools so dashboards, exporters, and runbooks are reusable.
+4. **Room to grow**: design supports adding traces, more notification channels, and longer retention later without re-architecture.
+
+## Non-Goals
+
+1. **Traces**: deferred to a later phase; the data flow and collector choice (Promtail/Alloy) will be trace-ready.
+2. **Multi-tenant RBAC**: Manage is single-instance/homelab; Grafana teams are sufficient for now.
+3. **SLA/SLO framework**: out of scope; we focus on metrics, logs, and alerts, not SLO budgeting.
+4. **Cloud-hosted observability vendors**: all components run self-hosted in Docker Compose.
+
+---
+
+## Decisions
+
+| Area | Decision | Rationale |
+|------|----------|-----------|
+| Coupling | Dedicated observability subsystem consumed by Manage | Keeps Manage fast and lets the observability stack evolve independently. |
+| Metrics backend | Prometheus | Pull model, huge ecosystem, standard exporters, easy Grafana integration. |
+| Machine metrics | Node Exporter | Rich OS metrics, reusable dashboards, no custom shell to maintain. |
+| Log backend | Grafana Loki | Prometheus-style labels, low resource use, tight Grafana integration. |
+| Log collection | Promtail / Grafana Alloy | Tails Docker logs and journald; no per-app network calls. |
+| App logs | Structured JSON to stdout | Standard 12-factor pattern; collector handles routing. |
+| Dashboards | Grafana + iframe embeds | Fast to implement, rich dashboards, Manage UI stays focused on summary. |
+| Alerting | Prometheus Alertmanager | Mature routing, silencing, inhibition; single source of truth for infra alerts. |
+| Auth | Grafana OAuth via Authentik | Reuses existing identity provider; consistent UX. |
+| Retention | 30 days metrics, 30 days logs | Matches current retention policy; disk usage stays predictable. |
+| Migration | Remove POSIX collector, migrate backup alerts | Eliminates duplicate alerting paths and custom remote code. |
+
+---
+
+## Current State
+
+### Logging
+
+- `backend/src/media_library_viewer_api/logging_utils.py` configures stdlib `logging` with a plain-text format.
+- `main.py` has a `log_requests` middleware that emits method, path, client IP, status code, and elapsed time.
+- Frontend uses standard `console.log` / browser dev tools; no server-side log aggregation.
+
+### Metrics
+
+- `backend/src/media_library_viewer_api/clients/resources.py` deploys a POSIX shell collector to `/tmp` on each remote machine.
+- The collector samples `/proc/stat`, `/proc/meminfo`, `/proc/net/dev`, and `/sys/block/*/stat` every 10s and writes JSONL to `/tmp/media_library_viewer_metrics.jsonl`.
+- `MonitoringPoller` (`monitoring_poller.py`) runs every 5 minutes, reads the remote JSONL, and stores snapshots in SQLite (`monitoring_machine_actions`).
+- Retention defaults to 30 days with periodic pruning.
+
+### Alerting
+
+- `backup_alert_engine.py` / `backup_poller.py` generate backup-related alerts (failure, anomaly, missed schedule) and store them in SQLite.
+- No general infrastructure alerting (disk full, machine down, high CPU, etc.).
+
+---
+
+## Target Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                              Docker Compose Network                           │
+│                                                                              │
+│   ┌─────────────┐     scrape      ┌──────────────┐                         │
+│   │  Prometheus │◄────────────────│ Node Exporter│◄── host / remote hosts  │
+│   │   (TSDB)    │                 └──────────────┘                         │
+│   └──────┬──────┘                                                            │
+│          │ query                                                             │
+│          ▼                                                                   │
+│   ┌─────────────┐     alert     ┌─────────────┐     email     ┌──────────┐  │
+│   │   Grafana   │──────────────►│ Alertmanager│──────────────►│  SMTP    │  │
+│   │  (OAuth)    │               └─────────────┘             └──────────┘  │
+│   └──────┬──────┘                                                            │
+│          │ embed (iframe)                                                   │
+│          ▼                                                                   │
+│   ┌─────────────────┐                                                        │
+│   │  Manage React   │                                                        │
+│   │     (summary)   │                                                        │
+│   └─────────────────┘                                                        │
+│                                                                              │
+│   Logs:                                                                      │
+│   Manage / containers ──stdout──► Promtail/Alloy ──push──► Loki ◄──────┐    │
+│   host / remote journald ───────► Promtail/Alloy ──push──► Loki        │    │
+│                                                                        │    │
+│   Grafana queries Loki for logs ◄──────────────────────────────────────┘    │
+└─────────────────────────────────────────────────────────────────────────────┘
+```
+
+### Component Responsibilities
+
+| Component | Responsibility |
+|-----------|----------------|
+| **Prometheus** | Scrape and store metrics; evaluate alert rules. |
+| **Node Exporter** | Expose host-level metrics (CPU, memory, disk, network, filesystem). |
+| **Loki** | Store and index log streams by labels. |
+| **Promtail / Alloy** | Discover log sources, parse labels, and push to Loki. |
+| **Grafana** | Visualize metrics and logs; serve as the alert UI. |
+| **Alertmanager** | Deduplicate, group, route, and deliver alerts. |
+| **Manage backend** | Emit structured logs; expose `/metrics` for Prometheus; forward health/status to summary endpoints. |
+| **Manage frontend** | Embed Grafana panels; show high-level status cards. |
+
+---
+
+## Instrumentation Changes
+
+### Backend Logging
+
+1. Switch to structured JSON logging via `python-json-logger` or `structlog`.
+2. Include fields:
+   - `timestamp`, `level`, `logger`, `message`
+   - `request_id` (correlation ID propagated from frontend or generated)
+   - `method`, `path`, `status_code`, `elapsed_ms`
+   - `user_id`, `machine_id` where relevant
+   - `error` / `error_type` / `traceback` for exceptions
+3. Keep emitting to stdout; Promtail/Alloy will parse JSON.
+
+**Status**: implemented in `backend/src/media_library_viewer_api/logging_utils.py` with `LOG_FORMAT=json|text`, secret-safe `sanitize_log_extra`, and request logging in `main.py`.
+
+### Request Middleware
+
+- Extend `log_requests` to attach `request_id` to `request.state`.
+- Include `request_id` in response headers (`X-Request-Id`) so the frontend can correlate.
+- Log all outbound SSH commands with `machine_id`, `action`, `duration_ms`, and `request_id`.
+
+**Status**: implemented in `main.py` and `observability.py`; `record_ssh_command` is called from `monitoring_actions.py` for every machine operation.
+
+### Application Metrics Endpoint
+
+- Add a `/metrics` endpoint using `prometheus-client`.
+- Initial counters/gauges:
+  - `manage_api_requests_total` (method, path, status)
+  - `manage_api_request_duration_seconds` histogram
+  - `manage_ssh_commands_total` (machine_id, action, status)
+  - `manage_ssh_command_duration_seconds` histogram
+  - `manage_media_index_build_duration_seconds`
+  - `manage_backup_runs_total` (job_name, status)
+  - `manage_mail_queue_size`, `manage_mail_queue_failures_total`
+
+**Status**: implemented in `backend/src/media_library_viewer_api/observability.py` and wired into `main.py`, `monitoring_actions.py`, `backups.py`, `media.py`, and `mail_queue.py`.
+
+### Frontend Observability
+
+- Keep first phase minimal: capture JS errors and send them to the backend as structured log events.
+- Optional later: expose RUM-style metrics (page loads, API call latencies) via Prometheus client library or manual instrumentation.
+
+---
+
+## Node Exporter Deployment
+
+### Local / Docker Host
+
+- Add a `node-exporter` service to `docker-compose.yml` with host PID/network mounts.
+- Prometheus scrapes it as `job="node"`.
+
+### Remote Machines
+
+- Add a managed task/template in `jobs.py` to install/upgrade Node Exporter via the package manager or a static binary.
+- Manage exposes a settings flag per machine: `node_exporter_enabled`.
+- For machines behind NAT, use one of:
+  1. Reverse SSH tunnel from machine to Manage host.
+  2. VPN/Wireguard already in place.
+  3. Prometheus federation or pushgateway for unreachable targets (later phase).
+- If Node Exporter cannot be installed, temporarily keep the POSIX collector as a fallback until migration is complete.
+
+---
+
+## Log Shipping
+
+### Docker Compose Services
+
+- Add `logging` driver config or Promtail sidecar to each service.
+- Preferred: run **Grafana Alloy** as a single daemon container with `docker_sd_config` to discover all Compose services automatically.
+
+### Host Logs
+
+- Alloy mounts `/var/log` and `/var/lib/docker/containers` (read-only).
+- Alloy also tails journald where available.
+
+### Remote Machines
+
+- Option A: install Alloy on remote hosts and have it push logs to Loki.
+- Option B: keep logs on remote hosts and use Node Exporter logs only; defer centralized remote logs.
+- Recommendation: **Option A** for important machines, **Option B** for constrained ones.
+
+---
+
+## Dashboards
+
+### Grafana
+
+- Provision dashboards from YAML/JSON in version control:
+  - Node Exporter Full dashboard (import from Grafana.com).
+  - Manage API overview (request rate, latency, errors).
+  - Manage operations (SSH commands, media index builds, mail queue).
+  - Backup runs and alert history.
+- Manage iframe embeds point to specific dashboard panels using Grafana's `panelId` and ` kiosk` mode.
+
+### Manage React UI
+
+- Add an "Observability" page with:
+  - System health cards (Prometheus up, Loki up, Alertmanager up).
+  - Recent alerts summary from Alertmanager API.
+  - Iframe panels for key metrics (CPU/memory of selected machine, recent logs).
+- Drill-down links open the full Grafana dashboard.
+
+---
+
+## Alerting
+
+### Alertmanager Configuration
+
+- Reuse existing SMTP settings for email notifications.
+- Initial routing:
+  - `severity=critical` → email immediately.
+  - `severity=warning` → email with 5-minute group wait.
+  - `job=backup` → grouped by job name.
+
+### Initial Alert Rules
+
+- Infrastructure:
+  - Node down for > 5 minutes.
+  - Disk usage > 85% (warning), > 95% (critical).
+  - Memory usage > 90% for > 10 minutes.
+  - CPU iowait > 30% for > 10 minutes.
+- Application:
+  - Manage API 5xx rate > 1% over 5 minutes.
+  - SSH command failure rate > 10% over 5 minutes.
+  - Mail queue growing or failures increasing.
+- Backup:
+  - Backup job failed (`manage_backup_runs_total{status="failure"}`).
+  - Backup job missing for > 1.5× schedule interval.
+  - Backup run duration or size anomaly compared to rolling median.
+
+### Backup Alert Migration
+
+- Re-implement rules as Prometheus recording/alerting rules where possible.
+- Keep historical comparison logic (median duration/size) as a small scheduled task that writes anomaly metrics to a Pushgateway or custom exporter, then Alertmanager consumes them.
+- Preserve acknowledge/resolve workflow by storing Alertmanager webhook events in SQLite if needed, or by using Grafana alert annotations.
+
+---
+
+## Authentication
+
+- Grafana configured with generic OAuth pointing at Authentik (same issuer as Manage).
+- Grafana role mapping: default `Viewer`; admin group mapped to `Admin`.
+- Traefik routes `grafana.${BACKEND_APP_HOST}` or a sub-path.
+- Iframe embedding requires Grafana `allow_embedding = true` and matching cookie domain/samesite settings.
+
+---
+
+## Retention and Storage
+
+| Store | Retention | Notes |
+|-------|-----------|-------|
+| Prometheus | 30 days | Default TSDB block compaction. |
+| Loki | 30 days | Single-store boltdb-shipper or filesystem target. |
+| Grafana | persistent SQLite/Postgres later | Dashboards and users are config, not runtime data. |
+
+- Volumes: `prometheus-data`, `loki-data`, `grafana-data`.
+- Backups: snapshot these volumes alongside existing `backend_cache`.
+
+---
+
+## Security
+
+1. Network: all observability services on an internal Docker network; exposed only through Traefik where needed.
+2. Node Exporter: bind to localhost on remote hosts and use a reverse tunnel, or firewall to Manage IP only.
+3. Secrets: SMTP password, OIDC client secret, and any remote scrape credentials in environment variables or Docker secrets; never commit them.
+4. Logs: sanitize tokens, passwords, and private keys before JSON serialization.
+5. Alertmanager: disable unauthenticated UI if exposed publicly; rely on OIDC/Traefik.
+
+---
+
+## Implementation Plan
+
+### Phase 0 — Foundation and Cleanup
+
+- [x] Add `prometheus-client` and `python-json-logger` to `backend/pyproject.toml`.
+- [x] Refactor `logging_utils.py` to emit JSON when `LOG_FORMAT=json`.
+- [x] Add `request_id` propagation in `log_requests` middleware.
+- [x] Add `/metrics` endpoint with initial counters/gauges.
+- [x] Remove the POSIX remote collector code in `resources.py`; keep `disk_space` as a lightweight SSH/local helper in `monitoring_actions.py`.
+- [x] Add `X-Request-Id` response header.
+
+### Phase 1 — Local Observability Stack
+
+- [x] Add services to `docker-compose.yml`: Prometheus, Loki, Grafana, Alertmanager, Grafana Alloy.
+- [x] Add `node-exporter` service for the Docker host.
+- [x] Configure Alloy to scrape all Docker container logs and ship to Loki.
+- [x] Configure Prometheus to scrape `node-exporter` and Manage `/metrics`.
+- [x] Provision Grafana datasources and a basic Manage API dashboard.
+- [x] Wire Grafana OAuth to Authentik.
+
+**Phase 1 files**:
+- `monitoring/prometheus/prometheus.yml`
+- `monitoring/prometheus/rules/backup_alerts.yml`
+- `monitoring/loki/loki.yml`
+- `monitoring/alloy/config.alloy`
+- `monitoring/alertmanager/alertmanager.yml`
+- `monitoring/grafana/grafana.ini`
+- `monitoring/grafana/provisioning/datasources/datasources.yml`
+- `monitoring/grafana/provisioning/dashboards-json/dashboards.yml`
+- `monitoring/grafana/provisioning/dashboards-json/dashboards/manage-overview.json`
+- `docker-compose.yml` and `docker-compose.dev.yml` updated with observability services.
+- `frontend/vite.config.ts` updated with `/grafana` dev proxy.
+- `.env.example` updated with Grafana OAuth and alerting variables.
+
+### Phase 2 — Remote Machine Metrics
+
+- [x] Add Node Exporter install/restart/status job templates in `jobs.py` (`install_node_exporter`, `restart_node_exporter`, `node_exporter_status`).
+- [x] Add `node_exporter_enabled`, `node_exporter_port`, and `node_exporter_scrape_host` fields to `MonitoringMachineInput` and `SettingsStore`.
+- [x] Implement `media_library_viewer_api.services.targets` to build Prometheus file-SD target lists for enabled SSH machines and write them to `PROMETHEUS_FILE_SD_DIR/node_exporter_targets.json`.
+- [x] Regenerate file-SD targets on machine create/update/delete in `routers/settings.py`.
+- [x] Add `/api/monitoring/prometheus-targets` endpoint returning live targets from the store.
+- [x] Configure Prometheus `node-exporter-remote` job with `file_sd_configs` reading `/etc/prometheus/file-sd/node_exporter_targets.json`.
+- [x] Mount the backend cache `prometheus-file-sd` directory into the Prometheus container as a read-only file-SD source.
+- [x] Add `PROMETHEUS_FILE_SD_DIR` setting and `.env.example` entry.
+- [x] Provision a minimal `Node Exporter Overview` Grafana dashboard (`monitoring/grafana/provisioning/dashboards-json/dashboards/node-exporter-overview.json`) covering CPU, memory, root disk, and network traffic.
+- [x] Remove POSIX collector fallback. The legacy collector code in `backend/src/media_library_viewer_api/clients/resources.py` has been deleted, the collector control endpoints were removed from `routers/monitoring.py`, and `disk_space` was relocated to `services/monitoring_actions.py` as a lightweight SSH/local helper. Metrics are now sourced exclusively from Prometheus/Node Exporter.
+
+**Phase 2 files**:
+- `backend/src/media_library_viewer_api/jobs.py` (Node Exporter job templates).
+- `backend/src/media_library_viewer_api/routers/settings.py` (machine input fields + target regeneration).
+- `backend/src/media_library_viewer_api/services/settings_store.py` (machine persistence fields).
+- `backend/src/media_library_viewer_api/services/targets.py` (file-SD target builder/writer).
+- `backend/src/media_library_viewer_api/routers/monitoring.py` (`/prometheus-targets` endpoint).
+- `backend/src/media_library_viewer_api/config.py` (`prometheus_file_sd_dir` setting).
+- `backend/tests/test_targets.py` and `backend/tests/test_api.py` (target + endpoint tests).
+- `monitoring/prometheus/prometheus.yml` (`node-exporter-remote` file SD job).
+- `monitoring/grafana/provisioning/dashboards-json/dashboards/node-exporter-overview.json`.
+- `docker-compose.yml` and `docker-compose.dev.yml` (file-SD volume mount + backend env var).
+- `.env.example` (`PROMETHEUS_FILE_SD_DIR`).
+
+### Phase 3 — Alerting
+
+- [x] Define initial Prometheus alert rules for backup failures (infrastructure rules deferred to Phase 2/3).
+- [x] Configure Alertmanager with email routing using existing SMTP settings; `monitoring/alertmanager/alertmanager.yml` uses env vars for SMTP and routing.
+- [x] Migrate backup alert rules to Alertmanager:
+  - `BackupJobFailed` triggers on `increase(manage_backup_runs_total{status="failed"}[1h]) > 0`.
+  - `BackupJobStuck` triggers on `time() - manage_backup_runs_last_success_timestamp > 86400`.
+  - Added `manage_backup_runs_last_success_timestamp` gauge in `observability.py` and updated `routers/backups.py` to set it on successful runs.
+  - SQLite backup alerts (`backup_alert_engine.py` and `backup_poller.py`) are preserved for now alongside Alertmanager rules; the UI can consume either source during transition.
+- [x] Add Alertmanager status summary endpoints in Manage backend:
+  - `GET /api/monitoring/alerts` proxies `/api/v1/alerts` and returns a UI-friendly summary (total, by_severity, alerts list).
+  - `GET /api/monitoring/alertmanager-status` proxies `/api/v2/status` and returns `up`, `version`, `uptime`, `peers`.
+- [x] Added `alertmanager_url` setting to `config.py` (default `http://alertmanager:9093`) and `ALERTMANAGER_URL` env var in both compose files and `.env.example`.
+- [x] Added tests for the Alertmanager endpoints and the backup success gauge.
+
+**Phase 3 files**:
+- `backend/src/media_library_viewer_api/routers/monitoring.py` (`/alerts` and `/alertmanager-status` endpoints).
+- `backend/src/media_library_viewer_api/observability.py` (`BACKUP_RUNS_LAST_SUCCESS` gauge + updated `record_backup_run`).
+- `backend/src/media_library_viewer_api/routers/backups.py` (pass `success=True` to `record_backup_run` on successful reports).
+- `backend/src/media_library_viewer_api/config.py` (`alertmanager_url` setting).
+- `monitoring/alertmanager/alertmanager.yml` (SMTP + routing config).
+- `monitoring/prometheus/rules/backup_alerts.yml` (backup alert rules).
+- `docker-compose.yml` / `docker-compose.dev.yml` (`ALERTMANAGER_URL` env var).
+- `.env.example` (`ALERTMANAGER_URL`).
+- `backend/tests/test_api.py` (`TestAlertmanager` tests).
+- `backend/tests/test_observability.py` (backup metric tests).
+
+### Phase 4 — Manage UI Integration
+
+- [x] Add "Observability" page in React with summary cards (Alertmanager health, active alerts, Prometheus targets, machines) and Grafana iframe panels.
+- [x] Add recent alerts list from Alertmanager API via `GET /api/monitoring/alerts`.
+- [x] Add drill-down links to full Grafana dashboards for Node Exporter metrics and Loki logs.
+- [x] Handle iframe sandbox attributes (`allow-scripts allow-same-origin allow-popups allow-forms`); CSP is delegated to the reverse proxy / Grafana `allow_embedding` configuration.
+- [x] Add `useObservability` hook and API client wrappers for alerts, Alertmanager status, and Prometheus targets.
+- [x] Add TypeScript types for Alertmanager summary/status and Prometheus targets.
+- [x] Wire the new `/observability` route into `App.tsx` and the sidebar navigation.
+
+**Phase 4 files**:
+- `frontend/src/components/ObservabilityPage.tsx` (page component).
+- `frontend/src/hooks/useObservability.ts` (React Query hooks).
+- `frontend/src/api/client.ts` (API client functions).
+- `frontend/src/types/index.ts` (new interfaces).
+- `frontend/src/App.tsx` (route + nav item).
+- `frontend/src/components/ui/{card,badge,alert,skeleton,select}.tsx` (shadcn/ui components).
+- `frontend/vite.config.ts` already has `/grafana` dev proxy for iframe source.
+
+### Phase 5 — Hardening and Future-Proofing
+
+- [x] Add health checks and `deploy.resources` limits for Prometheus, Loki, Alloy, Grafana, Alertmanager, and Node Exporter in both compose files.
+- [x] Use `depends_on` with `condition: service_healthy` for Alloy → Loki and Grafana → Prometheus/Loki.
+- [x] Add Prometheus scrape jobs for Loki, Alertmanager, and Grafana so their `up` metrics are available for health alerts.
+- [x] Add observability health alerting rules (`PrometheusTargetMissing`, `AlertmanagerDown`, `GrafanaDown`).
+- [x] Add `ALERTMANAGER_WEBHOOK_URL` backend setting, `POST /api/monitoring/alertmanager-webhook` receiver, and Alertmanager `webhook` receiver config.
+- [x] Document runbooks for common alerts.
+- [x] Add volume backups for Prometheus/Loki/Grafana data. Backup/restore procedures for `prometheus_data`, `loki_data`, `grafana_data`, and `alertmanager_data` are documented in `docs/observability-runbooks.md`.
+- [ ] Optional: add OpenTelemetry Collector as a translation layer for traces later.
+
+**Phase 5 files**:
+- `docker-compose.yml` and `docker-compose.dev.yml` (health checks, resource limits, `depends_on` conditions).
+- `monitoring/prometheus/prometheus.yml` (additional scrape jobs for observability services).
+- `monitoring/prometheus/rules/backup_alerts.yml` (renamed scope to include observability health alerts).
+- `backend/src/media_library_viewer_api/config.py` (`alertmanager_webhook_url` setting).
+- `backend/src/media_library_viewer_api/routers/monitoring.py` (`POST /api/monitoring/alertmanager-webhook`).
+- `monitoring/alertmanager/alertmanager.yml` (`webhook` receiver).
+- `backend/tests/test_api.py` (`TestAlertmanagerWebhook`).
+- `docs/observability-runbooks.md` (new runbook documentation).
+
+---
+
+## Risks and Mitigations
+
+| Risk | Mitigation |
+|------|------------|
+| Node Exporter hard to install on NAS/minimal hosts | Keep POSIX collector as opt-in fallback; document manual install steps. |
+| Grafana iframe embedding blocked by CSP or cookies | Test early in Phase 4; use matching domains and `allow_embedding`. |
+| Prometheus storage grows faster than expected | Start with 30-day retention; add compaction alerts. |
+| Backup alert semantics lost in migration | Write tests comparing old Python alerts vs new Alertmanager rules. |
+| OIDC configuration drift between Manage and Grafana | Use same env vars/Authentik application for both. |
+| Remote scrape requires network path | Use reverse SSH tunnels or defer remote scraping until VPN is ready. |
+
+---
+
+## Open Questions
+
+1. ~~What sub-domain or sub-path should Grafana use? (`grafana.${BACKEND_APP_HOST}` vs `${BACKEND_APP_HOST}/grafana`)~~ Decided: dedicated `GRAFANA_APP_HOST` subdomain in production; dev uses port 3000 and a `/grafana` proxy in Vite.
+2. Should remote machines run Node Exporter as a systemd service or a container?
+3. Do we need remote log aggregation immediately, or can it wait until after metrics alerting is stable?
+4. Should the existing backup alert acknowledgement/resolve UI be rebuilt on top of Alertmanager, or replaced by Grafana alert annotations?

docs/observability-runbooks.md

@@ -0,0 +1,190 @@
+# Observability Runbooks
+
+Operational playbooks for the Manage self-hosted observability stack (Prometheus, Grafana, Loki, Alertmanager).
+
+## Service Overview
+
+| Service | Compose name | Internal URL | Health check |
+|---------|--------------|--------------|--------------|
+| Prometheus | `prometheus` | `http://prometheus:9090` | `/-/healthy` |
+| Grafana | `grafana` | `http://grafana:3000` | `/api/health` |
+| Loki | `loki` | `http://loki:3100` | `/ready` |
+| Alloy | `alloy` | `http://alloy:12345` | `/-/healthy` |
+| Alertmanager | `alertmanager` | `http://alertmanager:9093` | `/-/healthy` |
+| Node Exporter | `node-exporter` | `http://node-exporter:9100` | `/` |
+| Manage backend | `backend` | `http://backend:8000` | `/api/health` |
+
+---
+
+## Alert: `BackupJobFailed`
+
+**Severity**: critical  
+**Meaning**: A backup job reported `status=failed` within the last hour.
+
+### Steps
+
+1. Open **Manage → Backups** and identify the failed job/run.
+2. Check the run output / logs for the failure reason.
+3. Search Loki for `{container="backend"} | json | message=~"(?i)backup"` around the failure time.
+4. If transient (network, lock file), retry the job.
+5. If persistent, open a task to fix the backup script or credentials.
+
+---
+
+## Alert: `BackupJobStuck`
+
+**Severity**: warning  
+**Meaning**: No successful backup run has been recorded for a job in the last 24 hours.
+
+### Steps
+
+1. Confirm the job is still scheduled and expected to run.
+2. Check whether the backup scheduler/host is running.
+3. Verify the job can still report success to `POST /api/backups/reports`.
+4. Inspect Prometheus graph for `manage_backup_runs_last_success_timestamp` by `job_name`.
+5. If the job was intentionally retired, remove or disable its reporting.
+
+---
+
+## Alert: `PrometheusTargetMissing`
+
+**Severity**: warning  
+**Meaning**: A Prometheus scrape target is down (`up == 0`) for more than 2 minutes.
+
+### Steps
+
+1. Identify `job` and `instance` from the alert labels.
+2. Check the container/process status:
+   - `docker compose ps <service>`
+   - `docker compose logs --tail 100 <service>`
+3. Verify network reachability from the Prometheus container:
+   - `docker compose exec prometheus wget -qO- http://<instance>/`
+4. If the target is a remote Node Exporter:
+   - Check the machine is reachable over SSH.
+   - Verify Node Exporter is installed and running (`systemctl status node_exporter`).
+   - Confirm the scrape host/port in Manage → Settings for that machine.
+5. Restart if needed: `docker compose restart <service>`.
+
+---
+
+## Alert: `AlertmanagerDown`
+
+**Severity**: critical  
+**Meaning**: Prometheus cannot scrape Alertmanager; new alerts may not be delivered.
+
+### Steps
+
+1. Check container status: `docker compose ps alertmanager`
+2. Review logs: `docker compose logs --tail 200 alertmanager`
+3. Validate config syntax:
+   - `docker compose exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml`
+4. Verify SMTP environment variables are present if using email receivers.
+5. Restart: `docker compose restart alertmanager`
+
+---
+
+## Alert: `GrafanaDown`
+
+**Severity**: warning  
+**Meaning**: Grafana is unreachable; dashboards and iframe panels in Manage are unavailable.
+
+### Steps
+
+1. Check container status and logs.
+2. Verify the OAuth client configuration is correct (`GF_AUTH_GENERIC_OAUTH_*`).
+3. If embedded panels are blank, confirm Grafana `allow_embedding = true` and cookie settings.
+4. Restart: `docker compose restart grafana`
+
+---
+
+## Routine Maintenance
+
+### Check overall health
+
+```bash
+cd /path/to/manage
+docker compose ps
+docker compose exec prometheus wget -qO- http://127.0.0.1:9090/-/healthy
+docker compose exec grafana wget -qO- http://127.0.0.1:3000/api/health
+docker compose exec loki wget -qO- http://127.0.0.1:3100/ready
+docker compose exec alertmanager wget -qO- http://127.0.0.1:9093/-/healthy
+```
+
+### Reload Prometheus after rule/config changes
+
+Prometheus is started with `--web.enable-lifecycle`, so a SIGHUP or HTTP call reloads config:
+
+```bash
+curl -X POST http://localhost:9090/-/reload
+```
+
+### Inspect logs
+
+```bash
+# All backend logs in Loki via Grafana Explore, or locally:
+docker compose logs --tail 500 backend
+
+# Specific service:
+docker compose logs -f prometheus
+```
+
+### Storage usage
+
+```bash
+docker system df -v
+docker compose exec prometheus du -sh /prometheus
+docker compose exec loki du -sh /loki
+docker compose exec grafana du -sh /var/lib/grafana
+```
+
+---
+
+## Backup and Disaster Recovery
+
+The observability data lives in named volumes:
+
+- `prometheus_data`
+- `loki_data`
+- `grafana_data`
+- `alertmanager_data`
+
+### Backup volumes
+
+```bash
+# Stop the stack to ensure consistency
+docker compose down
+
+# Back up each volume to a tarball
+docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \
+  tar czf /backups/prometheus-$(date +%F).tar.gz -C /data .
+docker run --rm -v manage_loki_data:/data -v $(pwd)/backups:/backups alpine \
+  tar czf /backups/loki-$(date +%F).tar.gz -C /data .
+docker run --rm -v manage_grafana_data:/data -v $(pwd)/backups:/backups alpine \
+  tar czf /backups/grafana-$(date +%F).tar.gz -C /data .
+docker run --rm -v manage_alertmanager_data:/data -v $(pwd)/backups:/backups alpine \
+  tar czf /backups/alertmanager-$(date +%F).tar.gz -C /data .
+
+# Start the stack again
+docker compose up -d
+```
+
+> Replace `manage_` with your actual Docker Compose project name if different.
+
+### Restore a volume
+
+```bash
+docker compose down
+docker volume rm manage_prometheus_data
+docker volume create manage_prometheus_data
+docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \
+  tar xzf /backups/prometheus-YYYY-MM-DD.tar.gz -C /data
+docker compose up -d
+```
+
+---
+
+## Scaling Notes
+
+- The current `deploy.resources` blocks are tuned for a small homelab. Raise memory limits if you monitor many machines or retain logs longer than 30 days.
+- Loki is configured for single-node filesystem storage. For larger deployments, migrate to object storage (S3/GCS/MinIO) and a shared index.
+- Prometheus remote-write or Thanos/Cortex can be added later for long-term metrics without changing application instrumentation.

frontend/package-lock.json

@@ -35,7 +35,7 @@
         "@tailwindcss/postcss": "^4.3.0",
         "@tailwindcss/vite": "^4.3.0",
         "@types/d3": "^7.4.3",
-        "@types/node": "^24.12.2",
+        "@types/node": "^24.13.2",
         "@types/react": "^19.2.14",
         "@types/react-dom": "^19.2.3",
         "@vitejs/plugin-react": "^6.0.1",
@@ -3375,9 +3375,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3395,9 +3392,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3415,9 +3409,6 @@
         "ppc64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3435,9 +3426,6 @@
         "s390x"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3455,9 +3443,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3475,9 +3460,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3737,9 +3719,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3757,9 +3736,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3777,9 +3753,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -3797,9 +3770,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -4254,12 +4224,12 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "24.12.2",
+      "version": "24.13.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.2.tgz",
-      "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==",
+      "integrity": "sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==",
       "license": "MIT",
       "dependencies": {
-        "undici-types": "~7.16.0"
+        "undici-types": "~7.18.0"
       }
     },
     "node_modules/@types/parse-json": {
@@ -7532,9 +7502,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -7556,9 +7523,6 @@
         "arm64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -7580,9 +7544,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "glibc"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -7604,9 +7565,6 @@
         "x64"
       ],
       "dev": true,
-      "libc": [
-        "musl"
-      ],
       "license": "MPL-2.0",
       "optional": true,
       "os": [
@@ -9774,9 +9732,9 @@
       }
     },
     "node_modules/undici-types": {
-      "version": "7.16.0",
+      "version": "7.18.2",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz",
-      "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
+      "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
       "license": "MIT"
     },
     "node_modules/unicorn-magic": {

frontend/package.json

@@ -37,7 +37,7 @@
     "@tailwindcss/postcss": "^4.3.0",
     "@tailwindcss/vite": "^4.3.0",
     "@types/d3": "^7.4.3",
-    "@types/node": "^24.12.2",
+    "@types/node": "^24.13.2",
     "@types/react": "^19.2.14",
     "@types/react-dom": "^19.2.3",
     "@vitejs/plugin-react": "^6.0.1",

frontend/src/App.tsx

@@ -5,6 +5,7 @@ import {
 	NavLink,
 	useLocation,
 	Outlet,
+	Navigate,
 } from "react-router-dom";
 import {
 	QueryClient,
@@ -14,13 +15,13 @@ import {
 import { useEffect, useMemo, useState } from "react";
 import { AuthProvider, useAuth } from "react-oidc-context";
 import { Dashboard } from "./pages/Dashboard";
-import { Monitoring } from "./pages/Monitoring";
 import { Applications } from "./pages/Applications";
 import { Settings } from "./pages/Settings";
 import { UsersPage } from "./pages/Users";
 import { FileBrowser } from "./pages/FileBrowser";
 import { Actions } from "./pages/Actions";
 import BackupsPage from "./components/BackupsPage";
+import { ObservabilityPage } from "./components/ObservabilityPage";
 import { getOidcConfig, isOidcConfigured, setAccessToken } from "./auth";
 import { fetchAppVersion } from "./api/client";
 import { FRONTEND_VERSION_LABEL } from "./version";
@@ -80,7 +81,7 @@ function useDarkMode() {
 // Navigation items for sidebar
 const navItems = [
 	{ path: "/", label: "Dashboard", icon: LayoutDashboard },
-	{ path: "/monitoring", label: "Monitoring", icon: Activity },
+	{ path: "/observability", label: "Observability", icon: Activity },
 	{ path: "/applications", label: "Media", icon: Monitor },
 	{ path: "/files", label: "Files", icon: FolderOpen },
 	{ path: "/users", label: "Users", icon: Users },
@@ -114,9 +115,7 @@ function Sidebar({
 						<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-primary font-bold text-primary-foreground">
 							M
 						</div>
-						{!collapsed && (
+						{!collapsed && <span className="font-semibold">Manage</span>}
-							<span className="font-semibold">Manage</span>
-						)}
 					</div>
 					<Button
 						variant="ghost"
@@ -155,9 +154,7 @@ function Sidebar({
 											<Icon className="h-5 w-5" />
 										</NavLink>
 									</TooltipTrigger>
-									<TooltipContent side="right">
+									<TooltipContent side="right">{item.label}</TooltipContent>
-										{item.label}
-									</TooltipContent>
 								</Tooltip>
 							) : (
 								<NavLink
@@ -313,8 +310,8 @@ function ShellLayout({
 	onToggleDarkMode: () => void;
 }) {
 	const [sidebarCollapsed, setSidebarCollapsed] = useState(false);
-	const [isMobile, setIsMobile] = useState(() =>
+	const [isMobile, setIsMobile] = useState(
-		window.matchMedia("(max-width: 768px)").matches,
+		() => window.matchMedia("(max-width: 768px)").matches,
 	);
 
 	useEffect(() => {
@@ -435,13 +432,14 @@ function AppInner() {
 						<Routes>
 							<Route element={<AuthenticatedApp />}>
 								<Route path="/" element={<Dashboard />} />
-								<Route path="/monitoring" element={<Monitoring />} />
+								<Route path="/monitoring" element={<Navigate to="/observability" replace />} />
 								<Route path="/applications" element={<Applications />} />
 								<Route path="/media" element={<Applications />} />
 								<Route path="/users" element={<UsersPage />} />
 								<Route path="/actions" element={<Actions />} />
 								<Route path="/files" element={<FileBrowser />} />
 								<Route path="/backups" element={<BackupsPage />} />
+								<Route path="/observability" element={<ObservabilityPage />} />
 								<Route path="/settings" element={<Settings />} />
 							</Route>
 						</Routes>
@@ -459,13 +457,14 @@ function AppInner() {
 							}
 						>
 							<Route path="/" element={<Dashboard />} />
-							<Route path="/monitoring" element={<Monitoring />} />
+							<Route path="/monitoring" element={<Navigate to="/observability" replace />} />
 							<Route path="/applications" element={<Applications />} />
 							<Route path="/media" element={<Applications />} />
 							<Route path="/users" element={<UsersPage />} />
 							<Route path="/actions" element={<Actions />} />
 							<Route path="/files" element={<FileBrowser />} />
 							<Route path="/backups" element={<BackupsPage />} />
+							<Route path="/observability" element={<ObservabilityPage />} />
 							<Route path="/settings" element={<Settings />} />
 						</Route>
 					</Routes>

frontend/src/api/client.ts

@@ -10,12 +10,7 @@ import type {
 	UserMessageResponse,
 	UserMessageQueueStatus,
 	NowPlayingSession,
-	MonitoringPollerStatus,
-	MonitoringOverviewResponse,
 	AppVersionInfo,
-	MonitoringStatus,
-	MonitoringMetrics,
-	DiskSpace,
 	SSHKey,
 	SSHKeyInput,
 	SSHKeyGenerated,
@@ -24,7 +19,6 @@ import type {
 	SavedTaskRun,
 	MonitoringMachine,
 	MonitoringMachineInput,
-	MonitoringMachineAction,
 	MediaIndexStatus,
 	MediaIndexActionResponse,
 	MediaQueryResponse,
@@ -37,6 +31,9 @@ import type {
 	SSHValidationResult,
 	DashboardShortcut,
 	DashboardShortcutInput,
+	AlertmanagerAlertSummary,
+	AlertmanagerStatus,
+	PrometheusTarget,
 } from "../types";
 
 const BASE_URL = import.meta.env.VITE_API_URL || "/api";
@@ -165,11 +162,7 @@ export const fetchNowPlaying = fetchActivity;
 // Monitoring
 export const fetchMonitoringMachines = () =>
 	get<MonitoringMachine[]>("/api/monitoring/machines");
-export const fetchMonitoringPoller = () =>
-	get<MonitoringPollerStatus>("/api/monitoring/poller");
 export const fetchAppVersion = () => get<AppVersionInfo>("/api/version");
-export const fetchMonitoringOverview = () =>
-	get<MonitoringOverviewResponse>("/api/dashboard/monitoring");
 export const fetchDashboardShortcuts = () =>
 	get<DashboardShortcut[]>("/api/dashboard/shortcuts");
 export const saveDashboardShortcut = (shortcut: DashboardShortcutInput) =>
@@ -194,45 +187,6 @@ export const deleteDashboardShortcut = (shortcutId: string) =>
 	del<{ status: string }>(
 		`/api/dashboard/shortcuts/${encodeURIComponent(shortcutId)}`,
 	);
-export const fetchMonitoringStatus = (machineId?: string) =>
-	get<MonitoringStatus>(
-		"/api/monitoring/status",
-		machineId ? { machine_id: machineId } : undefined,
-	);
-export const fetchMonitoringMetrics = (
-	lastSeconds?: number | null,
-	maxLines = 70_000,
-	machineId?: string,
-) =>
-	get<MonitoringMetrics>("/api/monitoring/metrics", {
-		...(lastSeconds == null ? {} : { last_seconds: String(lastSeconds) }),
-		max_lines: String(maxLines),
-		...(machineId ? { machine_id: machineId } : {}),
-	});
-export const fetchDiskSpace = (machineId?: string) =>
-	get<DiskSpace>(
-		"/api/monitoring/disk",
-		machineId ? { machine_id: machineId } : undefined,
-	);
-export const startCollector = (machineId?: string) =>
-	post<{ message: string }>(
-		machineId
-			? `/api/monitoring/start?machine_id=${encodeURIComponent(machineId)}`
-			: "/api/monitoring/start",
-	);
-export const stopCollector = (machineId?: string) =>
-	post<{ message: string }>(
-		machineId
-			? `/api/monitoring/stop?machine_id=${encodeURIComponent(machineId)}`
-			: "/api/monitoring/stop",
-	);
-export const restartCollector = (machineId?: string) =>
-	post<{ message: string }>(
-		machineId
-			? `/api/monitoring/restart?machine_id=${encodeURIComponent(machineId)}`
-			: "/api/monitoring/restart",
-	);
-
 export const fetchMonitoringSettings = () =>
 	get<MonitoringMachine[]>("/api/settings/machines");
 export const fetchSSHKeys = () => get<SSHKey[]>("/api/settings/ssh-keys");
@@ -242,11 +196,6 @@ export const generateSSHKey = (payload: {
 	notes: string;
 	bits?: number;
 }) => post<SSHKeyGenerated>("/api/settings/ssh-keys/generate", payload);
-export const fetchMonitoringMachineActions = (machineId: string, limit = 10) =>
-	get<{ items: MonitoringMachineAction[]; total: number }>(
-		`/api/monitoring/machines/${encodeURIComponent(machineId)}/actions`,
-		{ limit: String(limit) },
-	);
 export const saveSSHKey = (key: SSHKeyInput) =>
 	fetch(
 		buildUrl(
@@ -428,3 +377,13 @@ export const fetchUserMessageQueueStatus = () =>
 
 export const sendUserMessage = (formData: FormData) =>
 	postForm<UserMessageResponse>("/api/users/message", formData);
+
+// Observability summary endpoints
+export const fetchAlertmanagerAlerts = () =>
+	get<AlertmanagerAlertSummary>("/api/monitoring/alerts");
+
+export const fetchAlertmanagerStatus = () =>
+	get<AlertmanagerStatus>("/api/monitoring/alertmanager-status");
+
+export const fetchPrometheusTargets = () =>
+	get<PrometheusTarget[]>("/api/monitoring/prometheus-targets");

frontend/src/components/MachineMonitoringSection.tsx

@@ -1,354 +0,0 @@
-import { useMemo, useState } from "react";
-import {
-	Alert,
-	Box,
-	Button,
-	Chip,
-	FormControl,
-	Grid,
-	InputLabel,
-	MenuItem,
-	Paper,
-	Select,
-	Stack,
-	Table,
-	TableBody,
-	TableCell,
-	TableContainer,
-	TableHead,
-	TableRow,
-	Typography,
-} from "@mui/material";
-import { MetricCard } from "./MetricCard";
-import { MonitoringCharts } from "./MonitoringCharts";
-import type { MonitoringMachine } from "../types";
-import {
-	useCollectorControls,
-	useDiskSpace,
-	useMachineActions,
-	useMonitoringMetrics,
-	useMonitoringStatus,
-} from "../hooks/useMonitoring";
-
-function formatBytes(bytes: number): string {
-	if (!bytes || bytes === 0) return "0 B";
-	const units = ["B", "KB", "MB", "GB", "TB"];
-	let value = bytes;
-	let unitIdx = 0;
-	while (value >= 1000 && unitIdx < units.length - 1) {
-		value /= 1000;
-		unitIdx++;
-	}
-	return `${value.toFixed(1)} ${units[unitIdx]}`;
-}
-
-function formatRate(bytes: number): string {
-	return `${formatBytes(bytes)}/s`;
-}
-
-function avg(arr: number[]) {
-	return arr.length ? arr.reduce((a, b) => a + b, 0) / arr.length : 0;
-}
-function max(arr: number[]) {
-	return arr.length ? Math.max(...arr) : 0;
-}
-
-function formatActionTime(epochSeconds: number): string {
-	return new Date(epochSeconds * 1000).toLocaleString();
-}
-
-function ErrorBanner({ label, error }: { label: string; error: unknown }) {
-	if (!error) return null;
-	return (
-		<Alert severity="error">
-			{label}: {String(error)}
-		</Alert>
-	);
-}
-
-export function MachineMonitoringSection({
-	machine,
-}: {
-	machine: MonitoringMachine;
-}) {
-	const statusQuery = useMonitoringStatus(machine.id, machine.enabled);
-	const metricsQuery = useMonitoringMetrics(machine.id, machine.enabled);
-	const diskQuery = useDiskSpace(machine.id, machine.enabled);
-	const { start, stop, restart } = useCollectorControls(machine.id);
-	const actionsQuery = useMachineActions(machine.id, machine.enabled);
-	const [actionFilter, setActionFilter] = useState("all");
-	const [resultFilter, setResultFilter] = useState("all");
-
-	const status = statusQuery.data;
-	const metrics = metricsQuery.data;
-	const disk = diskQuery.data;
-	const samples = metrics?.samples ?? [];
-	const latest = samples.at(-1);
-	const cpuArr = samples.map((s) => s.cpu_pct);
-	const iowArr = samples.map((s) => s.iowait_pct ?? 0);
-	const memArr = samples.map((s) => s.mem_pct);
-	const netDownArr = samples.map((s) => s.net_rx_bytes_per_sec);
-	const netUpArr = samples.map((s) => s.net_tx_bytes_per_sec);
-	const diskReadArr = samples.map((s) => s.disk_read_bps);
-	const diskWriteArr = samples.map((s) => s.disk_write_bps);
-	const actions = actionsQuery.data?.items ?? [];
-	const visibleActions = useMemo(
-		() =>
-			actions.filter((action) => {
-				const actionMatches =
-					actionFilter === "all" || action.action === actionFilter;
-				const resultMatches =
-					resultFilter === "all" || action.status === resultFilter;
-				return actionMatches && resultMatches;
-			}),
-		[actions, actionFilter, resultFilter],
-	);
-	const hasQueryError =
-		statusQuery.error ||
-		metricsQuery.error ||
-		diskQuery.error ||
-		actionsQuery.error;
-
-	return (
-		<Stack spacing={2}>
-			<Stack
-				direction="row"
-				spacing={1.5}
-				sx={{ alignItems: "center", flexWrap: "wrap" }}
-			>
-				<Box>
-					<Typography variant="h6" sx={{ fontWeight: 700 }}>
-						{machine.name}
-					</Typography>
-					<Typography variant="caption" color="text.secondary">
-						{machine.mode === "local"
-							? "Local API host"
-							: `${machine.username || "user"}@${machine.host || "host"}:${machine.port}`}
-						{machine.media_root ? ` · media root ${machine.media_root}` : ""}
-					</Typography>
-				</Box>
-				<Chip label={machine.mode} variant="outlined" />
-				<Chip
-					label={machine.enabled ? "Enabled" : "Disabled"}
-					color={machine.enabled ? "success" : "default"}
-					variant="outlined"
-				/>
-				<Chip
-					label={status?.status ?? "unknown"}
-					color={status?.status?.includes("running") ? "success" : "primary"}
-					variant="outlined"
-				/>
-				<Button
-					size="small"
-					variant="outlined"
-					onClick={() => start.mutate()}
-					disabled={start.isPending || !machine.enabled}
-				>
-					Start
-				</Button>
-				<Button
-					size="small"
-					variant="outlined"
-					onClick={() => restart.mutate()}
-					disabled={restart.isPending || !machine.enabled}
-				>
-					Restart
-				</Button>
-				<Button
-					size="small"
-					variant="outlined"
-					onClick={() => stop.mutate()}
-					disabled={stop.isPending || !machine.enabled}
-				>
-					Stop
-				</Button>
-			</Stack>
-
-			{hasQueryError && (
-				<Stack spacing={1}>
-					<ErrorBanner label="Status" error={statusQuery.error} />
-					<ErrorBanner label="Metrics" error={metricsQuery.error} />
-					<ErrorBanner label="Disk" error={diskQuery.error} />
-					<ErrorBanner label="Recent activity" error={actionsQuery.error} />
-				</Stack>
-			)}
-
-			<Grid container spacing={1.5}>
-				<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
-					<MetricCard
-						label="CPU now"
-						value={latest ? `${latest.cpu_pct.toFixed(1)}%` : "-"}
-						subtext={`avg ${avg(cpuArr).toFixed(1)}%\npeak ${max(cpuArr).toFixed(1)}%`}
-					/>
-				</Grid>
-				<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
-					<MetricCard
-						label="IO Wait"
-						value={latest ? `${(latest.iowait_pct ?? 0).toFixed(1)}%` : "-"}
-						subtext={`avg ${avg(iowArr).toFixed(1)}%\npeak ${max(iowArr).toFixed(1)}%`}
-					/>
-				</Grid>
-				<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
-					<MetricCard
-						label="RAM now"
-						value={latest ? `${latest.mem_pct.toFixed(1)}%` : "-"}
-						subtext={`avg ${avg(memArr).toFixed(1)}%\npeak ${max(memArr).toFixed(1)}%`}
-					/>
-				</Grid>
-				<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
-					<MetricCard
-						label="Net down"
-						value={latest ? formatRate(latest.net_rx_bytes_per_sec) : "-"}
-						subtext={`avg ${formatRate(avg(netDownArr))}\npeak ${formatRate(max(netDownArr))}`}
-					/>
-				</Grid>
-				<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
-					<MetricCard
-						label="Net up"
-						value={latest ? formatRate(latest.net_tx_bytes_per_sec) : "-"}
-						subtext={`avg ${formatRate(avg(netUpArr))}\npeak ${formatRate(max(netUpArr))}`}
-					/>
-				</Grid>
-				<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
-					<MetricCard
-						label="Disk read"
-						value={latest ? formatRate(latest.disk_read_bps) : "-"}
-						subtext={`avg ${formatRate(avg(diskReadArr))}\npeak ${formatRate(max(diskReadArr))}`}
-					/>
-				</Grid>
-				<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
-					<MetricCard
-						label="Disk write"
-						value={latest ? formatRate(latest.disk_write_bps) : "-"}
-						subtext={`avg ${formatRate(avg(diskWriteArr))}\npeak ${formatRate(max(diskWriteArr))}`}
-					/>
-				</Grid>
-			</Grid>
-
-			{disk && (
-				<Grid container spacing={1.5}>
-					<Grid size={{ xs: 6, md: 3 }}>
-						<MetricCard label="Disk used" value={formatBytes(disk.used)} />
-					</Grid>
-					<Grid size={{ xs: 6, md: 3 }}>
-						<MetricCard
-							label="Disk available"
-							value={formatBytes(disk.available)}
-						/>
-					</Grid>
-					<Grid size={{ xs: 6, md: 3 }}>
-						<MetricCard label="Disk total" value={formatBytes(disk.size)} />
-					</Grid>
-					<Grid size={{ xs: 6, md: 3 }}>
-						<MetricCard label="Used %" value={disk.used_pct} />
-					</Grid>
-				</Grid>
-			)}
-
-			<Box>
-				<MonitoringCharts samples={samples} />
-			</Box>
-
-			<Stack spacing={1.5}>
-				<Stack
-					direction="row"
-					spacing={1}
-					sx={{ alignItems: "center", flexWrap: "wrap" }}
-				>
-					<Box>
-						<Typography variant="subtitle1" sx={{ fontWeight: 700 }}>
-							Recent activity
-						</Typography>
-						<Typography variant="caption" color="text.secondary">
-							Collected automatically by the backend poller.
-						</Typography>
-					</Box>
-					<Chip
-						label={`${visibleActions.length}/${actions.length || 0}`}
-						size="small"
-						variant="outlined"
-					/>
-					<FormControl size="small" sx={{ minWidth: 150 }}>
-						<InputLabel>Action</InputLabel>
-						<Select
-							label="Action"
-							value={actionFilter}
-							onChange={(e) => setActionFilter(e.target.value as string)}
-						>
-							<MenuItem value="all">All actions</MenuItem>
-							<MenuItem value="status lookup">Status</MenuItem>
-							<MenuItem value="metrics read">Metrics</MenuItem>
-							<MenuItem value="disk lookup">Disk</MenuItem>
-							<MenuItem value="collector start">Start</MenuItem>
-							<MenuItem value="collector stop">Stop</MenuItem>
-							<MenuItem value="collector restart">Restart</MenuItem>
-							<MenuItem value="collector diagnostics">Diagnostics</MenuItem>
-						</Select>
-					</FormControl>
-					<FormControl size="small" sx={{ minWidth: 120 }}>
-						<InputLabel>Result</InputLabel>
-						<Select
-							label="Result"
-							value={resultFilter}
-							onChange={(e) => setResultFilter(e.target.value as string)}
-						>
-							<MenuItem value="all">All results</MenuItem>
-							<MenuItem value="ok">OK</MenuItem>
-							<MenuItem value="error">Error</MenuItem>
-						</Select>
-					</FormControl>
-				</Stack>
-				{actionsQuery.error && (
-					<Alert severity="error">
-						Recent activity: {String(actionsQuery.error)}
-					</Alert>
-				)}
-				<TableContainer component={Paper} variant="outlined">
-					<Table size="small">
-						<TableHead>
-							<TableRow>
-								<TableCell>Time</TableCell>
-								<TableCell>Action</TableCell>
-								<TableCell>Status</TableCell>
-								<TableCell align="right">Duration</TableCell>
-								<TableCell>Message</TableCell>
-							</TableRow>
-						</TableHead>
-						<TableBody>
-							{visibleActions.length === 0 ? (
-								<TableRow>
-									<TableCell colSpan={5}>
-										<Typography variant="body2" color="text.secondary">
-											No activity matches the current filters.
-										</Typography>
-									</TableCell>
-								</TableRow>
-							) : (
-								visibleActions.map((action) => (
-									<TableRow
-										key={`${action.machine_id}-${action.created_at}-${action.action}-${action.status}`}
-									>
-										<TableCell>{formatActionTime(action.created_at)}</TableCell>
-										<TableCell>{action.action}</TableCell>
-										<TableCell>
-											<Chip
-												size="small"
-												label={action.status}
-												color={action.status === "ok" ? "success" : "error"}
-												variant="outlined"
-											/>
-										</TableCell>
-										<TableCell align="right">{action.duration_ms} ms</TableCell>
-										<TableCell>
-											{action.message || action.error || "-"}
-										</TableCell>
-									</TableRow>
-								))
-							)}
-						</TableBody>
-					</Table>
-				</TableContainer>
-			</Stack>
-		</Stack>
-	);
-}

frontend/src/components/MonitoringCharts.tsx

@@ -1 +0,0 @@
-export { MonitoringCharts } from "./MonitoringCharts.impl";

frontend/src/components/MonitoringOverviewTable.tsx

@@ -1,716 +0,0 @@
-import { useMemo, useState } from "react";
-import {
-	Box,
-	Chip,
-	Paper,
-	Stack,
-	Table,
-	TableBody,
-	TableCell,
-	TableContainer,
-	TableHead,
-	TableRow,
-	TableSortLabel,
-	Tooltip,
-	Typography,
-} from "@mui/material";
-import type {
-	MonitoringMachineOverview,
-	MonitoringOverviewResponse,
-} from "../types";
-
-type SortKey =
-	| "machine"
-	| "mode"
-	| "status"
-	| "cpu"
-	| "iowait"
-	| "mem"
-	| "net_rx"
-	| "net_tx"
-	| "disk_read"
-	| "disk_write"
-	| "disk_used"
-	| "updated";
-
-function formatBytes(bytes: number): string {
-	if (!bytes || bytes === 0) return "0 B";
-	const units = ["B", "KB", "MB", "GB", "TB"];
-	let value = bytes;
-	let unitIdx = 0;
-	while (value >= 1000 && unitIdx < units.length - 1) {
-		value /= 1000;
-		unitIdx++;
-	}
-	return `${value.toFixed(1)} ${units[unitIdx]}`;
-}
-
-function formatRate(bytes: number): string {
-	return `${formatBytes(bytes)}/s`;
-}
-
-function formatAge(epochSeconds: number | null): string {
-	if (!epochSeconds) return "-";
-	const diff = Date.now() / 1000 - epochSeconds;
-	if (diff < 60) return `${Math.max(0, Math.round(diff))}s ago`;
-	if (diff < 3600) return `${Math.round(diff / 60)}m ago`;
-	return `${Math.round(diff / 3600)}h ago`;
-}
-
-function formatReadableTime(epochSeconds: number | null): string {
-	if (!epochSeconds) return "-";
-	const date = new Date(epochSeconds * 1000);
-	return date.toLocaleString("en-US", {
-		month: "short",
-		day: "numeric",
-		hour: "2-digit",
-		minute: "2-digit",
-		hour12: false,
-	});
-}
-
-function formatClockTime(epochSeconds: number | null): string {
-	if (!epochSeconds) return "-";
-	const date = new Date(epochSeconds * 1000);
-	return date.toLocaleTimeString("en-US", {
-		hour: "2-digit",
-		minute: "2-digit",
-		hour12: false,
-	});
-}
-
-function formatUpdatedDetails(epochSeconds: number | null): [string, string] {
-	if (!epochSeconds) return ["-", "-"];
-	return [
-		formatReadableTime(epochSeconds),
-		`${formatClockTime(epochSeconds)} · ${formatAge(epochSeconds)}`,
-	];
-}
-
-function formatSummary(
-	summary: { avg: number; min: number; max: number } | null,
-	formatter: (value: number) => string,
-) {
-	if (!summary) return { value: "-", min: "", max: "" };
-	return {
-		value: formatter(summary.avg),
-		min: formatter(summary.min),
-		max: formatter(summary.max),
-	};
-}
-
-function sortableText(value: string) {
-	return value.toLowerCase();
-}
-
-function getStatusChipProps(row: MonitoringMachineOverview) {
-	if (row.status_error) {
-		return { color: "error" as const, label: row.status_error };
-	}
-	if (row.status === "running") {
-		return { color: "success" as const, label: "running" };
-	}
-	if (row.status === "not running") {
-		return { color: "warning" as const, label: "not running" };
-	}
-	return { color: "default" as const, label: row.status || "-" };
-}
-
-function metricSortValue(
-	row: MonitoringMachineOverview,
-	key: SortKey,
-): number | string {
-	switch (key) {
-		case "machine":
-			return sortableText(row.machine.name);
-		case "mode":
-			return row.machine.mode;
-		case "status":
-			return sortableText(row.status || row.status_error || "");
-		case "cpu":
-			return row.cpu_summary?.avg ?? -1;
-		case "iowait":
-			return row.iowait_summary?.avg ?? -1;
-		case "mem":
-			return row.mem_summary?.avg ?? -1;
-		case "net_rx":
-			return row.net_rx_summary?.avg ?? -1;
-		case "net_tx":
-			return row.net_tx_summary?.avg ?? -1;
-		case "disk_read":
-			return row.disk_read_summary?.avg ?? -1;
-		case "disk_write":
-			return row.disk_write_summary?.avg ?? -1;
-		case "disk_used":
-			return parseFloat((row.disk?.used_pct || "0").replace("%", "")) || -1;
-		case "updated":
-			return row.latest_sample?.ts ?? -1;
-		default:
-			return 0;
-	}
-}
-
-function NoDataCell() {
-	return (
-		<Stack
-			sx={{
-				width: "100%",
-				minWidth: 0,
-				height: "100%",
-				minHeight: 118,
-				textAlign: "center",
-				justifyContent: "center",
-				py: 0.5,
-			}}
-		>
-			<Typography
-				variant="caption"
-				color="text.secondary"
-				sx={{
-					fontSize: "0.7rem",
-					fontStyle: "italic",
-					lineHeight: 1.2,
-				}}
-			>
-				No data
-			</Typography>
-		</Stack>
-	);
-}
-
-function MetricCell({
-	value,
-	min,
-	max,
-}: {
-	value: string;
-	min?: string;
-	max?: string;
-}) {
-	return (
-		<Stack
-			sx={{
-				width: "100%",
-				minWidth: 0,
-				height: "100%",
-				minHeight: 118,
-				textAlign: "center",
-				justifyContent: "space-between",
-				py: 0.5,
-			}}
-		>
-			<Box
-				sx={{
-					flex: 1,
-					display: "flex",
-					alignItems: "center",
-					justifyContent: "center",
-					px: 1.5,
-					py: 1.25,
-					minHeight: 68,
-					borderRadius: 1,
-				}}
-			>
-				<Stack spacing={0.25} sx={{ alignItems: "center" }}>
-					<Typography
-						variant="caption"
-						color="text.secondary"
-						sx={{
-							fontSize: "0.6rem",
-							lineHeight: 1,
-							textTransform: "uppercase",
-							letterSpacing: "0.04em",
-						}}
-					>
-						10m avg
-					</Typography>
-					<Typography
-						variant="body1"
-						sx={{
-							fontWeight: 900,
-							fontSize: "1.12rem",
-							lineHeight: 1,
-							textAlign: "center",
-							whiteSpace: "nowrap",
-							fontVariantNumeric: "tabular-nums",
-						}}
-					>
-						{value}
-					</Typography>
-				</Stack>
-			</Box>
-			<Stack spacing={0.25} sx={{ width: "100%" }}>
-				{min ? (
-					<Box
-						sx={{
-							width: "100%",
-							border: 1,
-							borderColor: "divider",
-							borderRadius: 999,
-							px: 0.75,
-							py: 0.15,
-							fontSize: "0.6rem",
-							lineHeight: 1.2,
-							color: "text.secondary",
-							textAlign: "center",
-							whiteSpace: "nowrap",
-							overflow: "hidden",
-							textOverflow: "ellipsis",
-						}}
-					>
-						Min {min}
-					</Box>
-				) : null}
-				{max ? (
-					<Box
-						sx={{
-							width: "100%",
-							border: 1,
-							borderColor: "divider",
-							borderRadius: 999,
-							px: 0.75,
-							py: 0.15,
-							fontSize: "0.6rem",
-							lineHeight: 1.2,
-							color: "text.secondary",
-							textAlign: "center",
-							whiteSpace: "nowrap",
-							overflow: "hidden",
-							textOverflow: "ellipsis",
-						}}
-					>
-						Max {max}
-					</Box>
-				) : null}
-			</Stack>
-		</Stack>
-	);
-}
-
-export function MonitoringOverviewTable({
-	overview,
-	embedded = false,
-}: {
-	overview?: MonitoringOverviewResponse;
-	embedded?: boolean;
-}) {
-	const poller = overview?.poller;
-	const rows = overview?.machines ?? [];
-	const [sortKey, setSortKey] = useState<SortKey>("machine");
-	const [sortDirection, setSortDirection] = useState<"asc" | "desc">("asc");
-
-	const sortedRows = useMemo(() => {
-		const factor = sortDirection === "asc" ? 1 : -1;
-		return [...rows].sort((a, b) => {
-			const av = metricSortValue(a, sortKey);
-			const bv = metricSortValue(b, sortKey);
-			if (typeof av === "number" && typeof bv === "number") {
-				return (av - bv) * factor;
-			}
-			return String(av).localeCompare(String(bv)) * factor;
-		});
-	}, [rows, sortDirection, sortKey]);
-
-	const setSort = (key: SortKey) => {
-		if (sortKey === key) {
-			setSortDirection((current) => (current === "asc" ? "desc" : "asc"));
-			return;
-		}
-		setSortKey(key);
-		setSortDirection("asc");
-	};
-
-	return (
-		<Stack spacing={1.25}>
-			{!embedded ? (
-				<Stack
-					direction="row"
-					spacing={1}
-					sx={{ alignItems: "center", flexWrap: "wrap" }}
-				>
-					<Typography variant="subtitle1" sx={{ fontWeight: 700 }}>
-						Machine monitoring
-					</Typography>
-					<Chip
-						size="small"
-						variant="outlined"
-						color={poller?.worker_running ? "success" : "default"}
-						label={poller?.worker_running ? "running" : "stopped"}
-					/>
-					<Chip
-						size="small"
-						variant="outlined"
-						label={`Machines: ${overview?.enabled ?? 0}/${overview?.total ?? 0}`}
-					/>
-					<Typography variant="caption" color="text.secondary">
-						Last success:{" "}
-						{poller?.last_success_at
-							? new Date(poller.last_success_at * 1000).toLocaleString()
-							: "-"}{" "}
-						· last run: {formatAge(poller?.last_run_at ?? null)}
-					</Typography>
-				</Stack>
-			) : null}
-			{!embedded && poller?.last_error ? (
-				<Box>
-					<Typography variant="caption" color="error.main">
-						Poller error: {poller.last_error}
-					</Typography>
-				</Box>
-			) : null}
-			<TableContainer
-				component={embedded ? Box : Paper}
-				variant={embedded ? undefined : "outlined"}
-				sx={
-					embedded
-						? {
-								border: 1,
-								borderColor: "divider",
-								borderRadius: 1,
-								overflowX: "auto",
-								overflowY: "hidden",
-							}
-						: {
-								maxWidth: "100%",
-								overflowX: "auto",
-							}
-				}
-			>
-				<Table
-					size="small"
-					sx={{
-						minWidth: 1500,
-						tableLayout: "fixed",
-						"& .MuiTableCell-root": {
-							px: 1.1,
-							py: 0.9,
-							verticalAlign: "middle",
-						},
-						"& .MuiTableHead .MuiTableCell-root": {
-							fontSize: "0.7rem",
-							fontWeight: 700,
-							lineHeight: 1.15,
-							whiteSpace: "nowrap",
-						},
-					}}
-				>
-					<TableHead>
-						<TableRow>
-							<TableCell
-								sx={{ width: 260 }}
-								sortDirection={sortKey === "machine" ? sortDirection : false}
-							>
-								<TableSortLabel
-									active={sortKey === "machine"}
-									direction={sortKey === "machine" ? sortDirection : "asc"}
-									onClick={() => setSort("machine")}
-								>
-									Machine
-								</TableSortLabel>
-							</TableCell>
-							<TableCell
-								sx={{ width: 90 }}
-								sortDirection={sortKey === "mode" ? sortDirection : false}
-							>
-								<TableSortLabel
-									active={sortKey === "mode"}
-									direction={sortKey === "mode" ? sortDirection : "asc"}
-									onClick={() => setSort("mode")}
-								>
-									Mode
-								</TableSortLabel>
-							</TableCell>
-							<TableCell
-								sx={{ width: 120 }}
-								sortDirection={sortKey === "status" ? sortDirection : false}
-							>
-								<TableSortLabel
-									active={sortKey === "status"}
-									direction={sortKey === "status" ? sortDirection : "asc"}
-									onClick={() => setSort("status")}
-								>
-									Status
-								</TableSortLabel>
-							</TableCell>
-							{[
-								["cpu", "CPU"],
-								["iowait", "IO wait"],
-								["mem", "RAM"],
-								["net_rx", "Net down"],
-								["net_tx", "Net up"],
-								["disk_read", "Disk read"],
-								["disk_write", "Disk write"],
-								["disk_used", "Disk used"],
-							].map(([key, label]) => (
-								<TableCell
-									key={key}
-									align="center"
-									sx={{ width: 150 }}
-									sortDirection={sortKey === key ? sortDirection : false}
-								>
-									<TableSortLabel
-										active={sortKey === key}
-										direction={sortKey === key ? sortDirection : "asc"}
-										onClick={() => setSort(key as SortKey)}
-									>
-										{label}
-									</TableSortLabel>
-								</TableCell>
-							))}
-							<TableCell
-								sx={{ width: 180 }}
-								sortDirection={sortKey === "updated" ? sortDirection : false}
-							>
-								<TableSortLabel
-									active={sortKey === "updated"}
-									direction={sortKey === "updated" ? sortDirection : "asc"}
-									onClick={() => setSort("updated")}
-								>
-									Updated
-								</TableSortLabel>
-							</TableCell>
-						</TableRow>
-					</TableHead>
-					<TableBody>
-						{sortedRows.length === 0 ? (
-							<TableRow>
-								<TableCell colSpan={12}>
-									<Typography variant="body2" color="text.secondary">
-										No monitoring machines are configured.
-									</Typography>
-								</TableCell>
-							</TableRow>
-						) : (
-							sortedRows.map((row) => {
-								const machine = row.machine;
-								const hasMetrics = row.sample_count > 0;
-								const cpu = formatSummary(
-									row.cpu_summary,
-									(value) => `${value.toFixed(1)}%`,
-								);
-								const iowait = formatSummary(
-									row.iowait_summary,
-									(value) => `${value.toFixed(1)}%`,
-								);
-								const mem = formatSummary(
-									row.mem_summary,
-									(value) => `${value.toFixed(1)}%`,
-								);
-								const netDown = formatSummary(row.net_rx_summary, formatRate);
-								const netUp = formatSummary(row.net_tx_summary, formatRate);
-								const diskRead = formatSummary(
-									row.disk_read_summary,
-									formatRate,
-								);
-								const diskWrite = formatSummary(
-									row.disk_write_summary,
-									formatRate,
-								);
-								const disk = row.disk
-									? {
-											value: row.disk.used_pct,
-											min: `Used ${formatBytes(row.disk.used)}`,
-											max: `Avail ${formatBytes(row.disk.available)}`,
-										}
-									: null;
-								const updated = formatUpdatedDetails(
-									row.latest_sample?.ts ?? null,
-								);
-								return (
-									<TableRow key={machine.id}>
-										<TableCell sx={{ width: 260 }}>
-											<Stack spacing={0.25} sx={{ minWidth: 0 }}>
-												<Stack
-													direction="row"
-													spacing={0.75}
-													sx={{
-														alignItems: "center",
-														flexWrap: "nowrap",
-														minWidth: 0,
-													}}
-												>
-													<Typography
-														variant="body2"
-														sx={{
-															fontWeight: 700,
-															minWidth: 0,
-															overflow: "hidden",
-															textOverflow: "ellipsis",
-															whiteSpace: "nowrap",
-														}}
-													>
-														{machine.name}
-													</Typography>
-													{!machine.enabled && (
-														<Chip
-															size="small"
-															label="disabled"
-															variant="outlined"
-														/>
-													)}
-												</Stack>
-												<Typography
-													variant="caption"
-													color="text.secondary"
-													sx={{
-														minWidth: 0,
-														overflow: "hidden",
-														textOverflow: "ellipsis",
-														whiteSpace: "nowrap",
-													}}
-												>
-													{machine.mode === "local"
-														? "Local API host"
-														: `${machine.username || "user"}@${machine.host || "host"}:${machine.port}`}
-												</Typography>
-											</Stack>
-										</TableCell>
-										<TableCell sx={{ width: 90 }}>{machine.mode}</TableCell>
-							<TableCell sx={{ width: 120 }}>
-								{(() => {
-									const chip = getStatusChipProps(row);
-									return (
-										<Tooltip
-											title={
-												row.metrics_error
-													? `Metrics: ${row.metrics_error}`
-													: row.sample_count === 0
-														? "No metrics collected yet. Start the collector to begin monitoring."
-														: ""
-											}
-											arrow
-										>
-											<Chip
-												size="small"
-												variant="outlined"
-												color={chip.color}
-												label={chip.label}
-											/>
-										</Tooltip>
-									);
-								})()}
-							</TableCell>
-										<TableCell align="center">
-											{hasMetrics ? (
-												<MetricCell
-													value={cpu.value}
-													min={cpu.min}
-													max={cpu.max}
-												/>
-											) : (
-												<NoDataCell />
-											)}
-										</TableCell>
-										<TableCell align="center">
-											{hasMetrics ? (
-												<MetricCell
-													value={iowait.value}
-													min={iowait.min}
-													max={iowait.max}
-												/>
-											) : (
-												<NoDataCell />
-											)}
-										</TableCell>
-										<TableCell align="center">
-											{hasMetrics ? (
-												<MetricCell
-													value={mem.value}
-													min={mem.min}
-													max={mem.max}
-												/>
-											) : (
-												<NoDataCell />
-											)}
-										</TableCell>
-										<TableCell align="center">
-											{hasMetrics ? (
-												<MetricCell
-													value={netDown.value}
-													min={netDown.min}
-													max={netDown.max}
-												/>
-											) : (
-												<NoDataCell />
-											)}
-										</TableCell>
-										<TableCell align="center">
-											{hasMetrics ? (
-												<MetricCell
-													value={netUp.value}
-													min={netUp.min}
-													max={netUp.max}
-												/>
-											) : (
-												<NoDataCell />
-											)}
-										</TableCell>
-										<TableCell align="center">
-											{hasMetrics ? (
-												<MetricCell
-													value={diskRead.value}
-													min={diskRead.min}
-													max={diskRead.max}
-												/>
-											) : (
-												<NoDataCell />
-											)}
-										</TableCell>
-										<TableCell align="center">
-											{hasMetrics ? (
-												<MetricCell
-													value={diskWrite.value}
-													min={diskWrite.min}
-													max={diskWrite.max}
-												/>
-											) : (
-												<NoDataCell />
-											)}
-										</TableCell>
-										<TableCell align="center">
-											{disk ? (
-												<MetricCell
-													value={disk.value}
-													min={disk.min}
-													max={disk.max}
-												/>
-											) : hasMetrics ? (
-												"-"
-											) : (
-												<NoDataCell />
-											)}
-										</TableCell>
-										<TableCell sx={{ width: 180 }}>
-											<Stack spacing={0.1} sx={{ alignItems: "center" }}>
-												{hasMetrics ? (
-													updated.map((line) => (
-														<Typography
-															key={line}
-															variant="caption"
-															color="text.secondary"
-															sx={{ lineHeight: 1.1 }}
-														>
-															{line}
-														</Typography>
-													))
-												) : (
-													<Typography
-														variant="caption"
-														color="text.secondary"
-														sx={{ lineHeight: 1.1, fontStyle: "italic" }}
-													>
-														No data yet
-													</Typography>
-												)}
-											</Stack>
-										</TableCell>
-									</TableRow>
-								);
-							})
-						)}
-					</TableBody>
-				</Table>
-			</TableContainer>
-		</Stack>
-	);
-}

frontend/src/components/ObservabilityPage.tsx

@@ -0,0 +1,698 @@
+import {
+	Component,
+	useEffect,
+	useMemo,
+	useRef,
+	useState,
+	type ElementType,
+	type ErrorInfo,
+	type ReactNode,
+} from "react";
+import { Link } from "react-router-dom";
+import {
+	Activity,
+	AlertTriangle,
+	Bell,
+	CheckCircle2,
+	ChevronDown,
+	ExternalLink,
+	Inbox,
+	PanelTop,
+	Radio,
+	RefreshCw,
+	Server,
+	ServerOff,
+	XCircle,
+} from "lucide-react";
+import {
+	useAlertmanagerAlerts,
+	useAlertmanagerStatus,
+	usePrometheusTargets,
+	useMonitoringMachines,
+} from "../hooks/useObservability";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Badge } from "@/components/ui/badge";
+import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
+import { Button } from "@/components/ui/button";
+import {
+	Select,
+	SelectContent,
+	SelectItem,
+	SelectTrigger,
+	SelectValue,
+} from "@/components/ui/select";
+import { Skeleton } from "@/components/ui/skeleton";
+import {
+	Collapsible,
+	CollapsibleContent,
+	CollapsibleTrigger,
+} from "@/components/ui/collapsible";
+import type {
+	AlertmanagerAlert,
+	MonitoringMachine,
+	PrometheusTarget,
+} from "../types";
+
+function severityVariant(
+	severity: string,
+): "default" | "secondary" | "destructive" | "outline" {
+	switch (severity.toLowerCase()) {
+		case "critical":
+			return "destructive";
+		case "warning":
+			return "default";
+		case "info":
+			return "secondary";
+		default:
+			return "outline";
+	}
+}
+
+function HealthCard({
+	title,
+	status,
+	detail,
+	icon: Icon,
+	isLoading,
+}: {
+	title: string;
+	status: "ok" | "warning" | "error" | "unknown";
+	detail: string;
+	icon: ElementType;
+	isLoading?: boolean;
+}) {
+	const statusIcon =
+		status === "ok" ? (
+			<CheckCircle2 className="h-5 w-5 text-green-500" />
+		) : status === "warning" ? (
+			<AlertTriangle className="h-5 w-5 text-amber-500" />
+		) : status === "error" ? (
+			<XCircle className="h-5 w-5 text-red-500" />
+		) : (
+			<Radio className="h-5 w-5 text-muted-foreground" />
+		);
+
+	return (
+		<Card>
+			<CardHeader className="flex flex-row items-center justify-between space-y-0 pb-2">
+				<CardTitle className="text-sm font-medium">{title}</CardTitle>
+				<Icon className="h-4 w-4 text-muted-foreground" />
+			</CardHeader>
+			<CardContent>
+				<div className="flex items-center gap-2">
+					{isLoading ? <Skeleton className="h-5 w-5" /> : statusIcon}
+					<span className="text-2xl font-bold capitalize">{status}</span>
+				</div>
+				<p className="mt-1 text-xs text-muted-foreground">{detail}</p>
+			</CardContent>
+		</Card>
+	);
+}
+
+function EmptyState({
+	icon: Icon,
+	title,
+	description,
+	action,
+}: {
+	icon: ElementType;
+	title: string;
+	description: string;
+	action?: ReactNode;
+}) {
+	return (
+		<div className="flex h-full min-h-[160px] flex-col items-center justify-center gap-2 rounded-md border p-6 text-center">
+			<Icon className="h-8 w-8 text-muted-foreground" />
+			<div className="font-medium">{title}</div>
+			<div className="max-w-md text-sm text-muted-foreground">
+				{description}
+			</div>
+			{action ? <div className="mt-2">{action}</div> : null}
+		</div>
+	);
+}
+
+function QueryError({
+	label,
+	error,
+	refetch,
+}: {
+	label: string;
+	error: Error | null;
+	refetch: () => void;
+}) {
+	if (!error) return null;
+	return (
+		<Alert variant="destructive">
+			<AlertTitle>{label} failed</AlertTitle>
+			<AlertDescription className="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
+				<span className="break-words">{error.message}</span>
+				<Button variant="outline" size="sm" onClick={() => refetch()}>
+					<RefreshCw className="mr-1 h-3 w-3" />
+					Retry
+				</Button>
+			</AlertDescription>
+		</Alert>
+	);
+}
+
+function AlertItem({ alert }: { alert: AlertmanagerAlert }) {
+	return (
+		<Collapsible>
+			<CollapsibleTrigger asChild>
+				<div className="group cursor-pointer rounded-lg border p-3 transition-colors hover:bg-muted/50">
+					<div className="flex items-start justify-between gap-2">
+						<div className="font-medium text-sm">{alert.name}</div>
+						<div className="flex items-center gap-1">
+							<Badge variant={severityVariant(alert.severity)}>
+								{alert.severity}
+							</Badge>
+							<ChevronDown className="h-4 w-4 text-muted-foreground transition-transform group-data-[state=open]:rotate-180" />
+						</div>
+					</div>
+					<div className="mt-1 text-xs text-muted-foreground">
+						{alert.summary || alert.description}
+					</div>
+					{alert.active_since && (
+						<div className="mt-1 text-[10px] text-muted-foreground">
+							Since {new Date(alert.active_since).toLocaleString()}
+						</div>
+					)}
+				</div>
+			</CollapsibleTrigger>
+			<CollapsibleContent className="overflow-hidden">
+				<div className="space-y-2 rounded-b-lg border-x border-b p-3 text-sm">
+					{alert.description && (
+						<div>
+							<span className="font-medium">Description:</span>{" "}
+							{alert.description}
+						</div>
+					)}
+					<div className="grid grid-cols-2 gap-2 text-xs">
+						{alert.job_name && (
+							<div>
+								<span className="font-medium">Job:</span> {alert.job_name}
+							</div>
+						)}
+						{alert.category && (
+							<div>
+								<span className="font-medium">Category:</span> {alert.category}
+							</div>
+						)}
+						<div>
+							<span className="font-medium">State:</span> {alert.state}
+						</div>
+						<div>
+							<span className="font-medium">Since:</span>{" "}
+							{alert.active_since
+								? new Date(alert.active_since).toLocaleString()
+								: "unknown"}
+						</div>
+					</div>
+					{alert.labels && Object.keys(alert.labels).length > 0 && (
+						<div className="flex flex-wrap gap-1 pt-1">
+							{Object.entries(alert.labels).map(([key, value]) => (
+								<Badge key={key} variant="secondary" className="text-[10px]">
+									{key}={value}
+								</Badge>
+							))}
+						</div>
+					)}
+				</div>
+			</CollapsibleContent>
+		</Collapsible>
+	);
+}
+
+function TargetsTable({ targets }: { targets: PrometheusTarget[] }) {
+	return (
+		<div className="space-y-3">
+			{targets.map((target, idx) => (
+				<div key={idx} className="rounded-lg border p-3">
+					<div className="font-mono text-sm">{target.targets.join(", ")}</div>
+					{target.labels && Object.keys(target.labels).length > 0 && (
+						<div className="mt-2 flex flex-wrap gap-1">
+							{Object.entries(target.labels).map(([key, value]) => (
+								<Badge key={key} variant="outline" className="text-[10px]">
+									{key}: {value}
+								</Badge>
+							))}
+						</div>
+					)}
+				</div>
+			))}
+		</div>
+	);
+}
+
+class GrafanaErrorBoundary extends Component<
+	{ children: ReactNode; fallback: ReactNode },
+	{ hasError: boolean }
+> {
+	constructor(props: { children: ReactNode; fallback: ReactNode }) {
+		super(props);
+		this.state = { hasError: false };
+	}
+
+	static getDerivedStateFromError() {
+		return { hasError: true };
+	}
+
+	componentDidCatch(error: Error, errorInfo: ErrorInfo) {
+		console.error("Grafana panel error:", error, errorInfo);
+	}
+
+	render() {
+		if (this.state.hasError) {
+			return this.props.fallback;
+		}
+		return this.props.children;
+	}
+}
+
+function GrafanaPanel({ src, title }: { src: string; title: string }) {
+	const [loaded, setLoaded] = useState(false);
+	const [failed, setFailed] = useState(false);
+	const [iframeKey, setIframeKey] = useState(0);
+	const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+	useEffect(() => {
+		timerRef.current = setTimeout(() => setFailed(true), 10_000);
+		return () => {
+			if (timerRef.current) clearTimeout(timerRef.current);
+		};
+	}, [iframeKey]);
+
+	const handleLoad = () => {
+		if (timerRef.current) clearTimeout(timerRef.current);
+		setLoaded(true);
+		setFailed(false);
+	};
+
+	const handleError = () => {
+		if (timerRef.current) clearTimeout(timerRef.current);
+		setFailed(true);
+	};
+
+	const reload = () => {
+		setLoaded(false);
+		setFailed(false);
+		setIframeKey((k) => k + 1);
+	};
+
+	if (!src) {
+		return (
+			<EmptyState
+				icon={PanelTop}
+				title="No Grafana URL"
+				description="Select a machine to load a Grafana panel."
+			/>
+		);
+	}
+
+	const fallback = (
+		<EmptyState
+			icon={PanelTop}
+			title="Grafana panel unavailable"
+			description="The panel did not load in time or Grafana is unreachable."
+			action={
+				<div className="flex flex-wrap justify-center gap-2">
+					<Button variant="outline" size="sm" onClick={reload}>
+						<RefreshCw className="mr-1 h-3 w-3" />
+						Reload
+					</Button>
+					<Button variant="outline" size="sm" asChild>
+						<a href={src} target="_blank" rel="noopener noreferrer">
+							<ExternalLink className="mr-1 h-3 w-3" />
+							Open in Grafana
+						</a>
+					</Button>
+				</div>
+			}
+		/>
+	);
+
+	return (
+		<GrafanaErrorBoundary key={iframeKey} fallback={fallback}>
+			<div className="relative h-full min-h-[320px] w-full overflow-hidden rounded-md border">
+				{!loaded && !failed && (
+					<div className="absolute inset-0 z-10 p-4">
+						<Skeleton className="h-full w-full" />
+					</div>
+				)}
+				{failed ? (
+					<div className="absolute inset-0 z-10 bg-background p-2">
+						{fallback}
+					</div>
+				) : (
+					<iframe
+						key={iframeKey}
+						title={title}
+						src={src}
+						className="h-full min-h-[320px] w-full"
+						allow="fullscreen"
+						sandbox="allow-scripts allow-same-origin allow-popups allow-forms"
+						onLoad={handleLoad}
+						onError={handleError}
+					/>
+				)}
+			</div>
+		</GrafanaErrorBoundary>
+	);
+}
+
+export function ObservabilityPage() {
+	const {
+		data: alertsSummary,
+		isLoading: alertsLoading,
+		error: alertsError,
+		refetch: refetchAlerts,
+	} = useAlertmanagerAlerts();
+	const {
+		data: alertmanagerStatus,
+		isLoading: statusLoading,
+		error: statusError,
+		refetch: refetchStatus,
+	} = useAlertmanagerStatus();
+	const {
+		data: prometheusTargets,
+		isLoading: targetsLoading,
+		error: targetsError,
+		refetch: refetchTargets,
+	} = usePrometheusTargets();
+	const {
+		data: machines = [],
+		isLoading: machinesLoading,
+		error: machinesError,
+		refetch: refetchMachines,
+	} = useMonitoringMachines();
+	const [selectedMachineId, setSelectedMachineId] = useState<string>("");
+
+	const selectedMachine = useMemo<MonitoringMachine | null>(
+		() =>
+			machines.find((m) => m.id === selectedMachineId) ?? machines[0] ?? null,
+		[machines, selectedMachineId],
+	);
+
+	const grafanaBase = "/grafana";
+
+	const nodeExporterDashboardUrl = useMemo(() => {
+		if (!selectedMachine) return "";
+		const instance = `${selectedMachine.host || "localhost"}:9100`;
+		return `${grafanaBase}/d/node-exporter-overview/node-exporter-overview?kiosk&var-instance=${encodeURIComponent(instance)}`;
+	}, [selectedMachine]);
+
+	const logsUrl = useMemo(() => {
+		if (!selectedMachine) return "";
+		const container =
+			selectedMachine.mode === "local" ? "backend" : selectedMachine.name;
+		return `${grafanaBase}/explore?orgId=1&left=${encodeURIComponent(
+			JSON.stringify({
+				datasource: "Loki",
+				queries: [{ refId: "A", expr: `{container="${container}"}` }],
+				range: { from: "now-1h", to: "now" },
+			}),
+		)}`;
+	}, [selectedMachine]);
+
+	const alertmanagerStatusDetail = alertmanagerStatus?.up
+		? alertmanagerStatus.version
+			? `version ${alertmanagerStatus.version}`
+			: "reachable"
+		: "unreachable";
+
+	const targetsCount = prometheusTargets?.length ?? 0;
+	const targetsStatus: "ok" | "warning" | "error" | "unknown" = targetsLoading
+		? "unknown"
+		: targetsError
+			? "error"
+			: targetsCount > 0
+				? "ok"
+				: "warning";
+
+	const alertStatus: "ok" | "warning" | "error" | "unknown" = alertsLoading
+		? "unknown"
+		: alertsError
+			? "error"
+			: (alertsSummary?.total ?? 0) > 0
+				? alertsSummary?.alerts.some((a) => a.severity === "critical")
+					? "error"
+					: "warning"
+				: "ok";
+
+	const machinesStatus: "ok" | "warning" | "error" | "unknown" = machinesLoading
+		? "unknown"
+		: machinesError
+			? "error"
+			: machines.length > 0
+				? "ok"
+				: "warning";
+
+	return (
+		<div className="space-y-6">
+			<div className="space-y-1">
+				<h1 className="text-2xl font-bold tracking-tight">Observability</h1>
+				<p className="text-sm text-muted-foreground">
+					Unified view of metrics, logs, and alerts from Prometheus, Grafana,
+					Loki, and Alertmanager.
+				</p>
+			</div>
+
+			<div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-4">
+				<HealthCard
+					title="Alertmanager"
+					status={
+						statusError
+							? "error"
+							: alertmanagerStatus?.up
+								? "ok"
+								: statusLoading
+									? "unknown"
+									: "error"
+					}
+					detail={alertmanagerStatusDetail}
+					icon={Bell}
+					isLoading={statusLoading}
+				/>
+				<HealthCard
+					title="Active Alerts"
+					status={alertStatus}
+					detail={`${alertsSummary?.total ?? 0} firing alert${(alertsSummary?.total ?? 0) === 1 ? "" : "s"}`}
+					icon={AlertTriangle}
+					isLoading={alertsLoading}
+				/>
+				<HealthCard
+					title="Prometheus Targets"
+					status={targetsStatus}
+					detail={`${targetsCount} remote Node Exporter target${targetsCount === 1 ? "" : "s"}`}
+					icon={Radio}
+					isLoading={targetsLoading}
+				/>
+				<HealthCard
+					title="Machines"
+					status={machinesStatus}
+					detail={`${machines.length} monitoring machine${machines.length === 1 ? "" : "s"}`}
+					icon={Server}
+					isLoading={machinesLoading}
+				/>
+			</div>
+
+			<div className="space-y-3">
+				{statusError && (
+					<QueryError
+						label="Alertmanager status"
+						error={statusError}
+						refetch={refetchStatus}
+					/>
+				)}
+				{alertsError && (
+					<QueryError
+						label="Active alerts"
+						error={alertsError}
+						refetch={refetchAlerts}
+					/>
+				)}
+				{targetsError && (
+					<QueryError
+						label="Prometheus targets"
+						error={targetsError}
+						refetch={refetchTargets}
+					/>
+				)}
+				{machinesError && (
+					<QueryError
+						label="Monitoring machines"
+						error={machinesError}
+						refetch={refetchMachines}
+					/>
+				)}
+			</div>
+
+			{alertsSummary?.error && (
+				<Alert variant="destructive">
+					<AlertTitle>Alertmanager unreachable</AlertTitle>
+					<AlertDescription>
+						The UI cannot reach Alertmanager right now. Alerts shown here may be
+						stale.
+					</AlertDescription>
+				</Alert>
+			)}
+
+			<div className="grid gap-6 lg:grid-cols-2">
+				<div className="space-y-6">
+					<Card>
+						<CardHeader>
+							<CardTitle className="flex items-center gap-2">
+								<Bell className="h-4 w-4" />
+								Recent Alerts
+							</CardTitle>
+						</CardHeader>
+						<CardContent className="space-y-3">
+							{alertsLoading ? (
+								<div className="space-y-2">
+									<Skeleton className="h-16 w-full" />
+									<Skeleton className="h-16 w-full" />
+									<Skeleton className="h-16 w-full" />
+								</div>
+							) : !alertsSummary || alertsSummary.total === 0 ? (
+								<EmptyState
+									icon={Inbox}
+									title="No active alerts"
+									description="Everything looks quiet. Alertmanager will list firing alerts here when they occur."
+								/>
+							) : (
+								<>
+									{alertsSummary.alerts.map((alert, idx) => (
+										<AlertItem key={`${alert.name}-${idx}`} alert={alert} />
+									))}
+									{alertsSummary.total > alertsSummary.alerts.length && (
+										<div className="text-center text-xs text-muted-foreground">
+											{alertsSummary.total - alertsSummary.alerts.length} more
+											alert
+											{alertsSummary.total - alertsSummary.alerts.length === 1
+												? ""
+												: "s"}{" "}
+											in Alertmanager
+										</div>
+									)}
+								</>
+							)}
+						</CardContent>
+					</Card>
+
+					<Card>
+						<CardHeader>
+							<CardTitle className="flex items-center gap-2">
+								<Radio className="h-4 w-4" />
+								Prometheus Targets
+							</CardTitle>
+						</CardHeader>
+						<CardContent>
+							{targetsLoading ? (
+								<div className="space-y-2">
+									<Skeleton className="h-16 w-full" />
+									<Skeleton className="h-16 w-full" />
+								</div>
+							) : !prometheusTargets || prometheusTargets.length === 0 ? (
+								<EmptyState
+									icon={Radio}
+									title="No Node Exporter targets"
+									description="Enable Node Exporter on an SSH machine in Settings to populate Prometheus scrape targets."
+									action={
+										<Button variant="outline" size="sm" asChild>
+											<Link to="/settings">Open Settings</Link>
+										</Button>
+									}
+								/>
+							) : (
+								<TargetsTable targets={prometheusTargets} />
+							)}
+						</CardContent>
+					</Card>
+				</div>
+
+				<Card>
+					<CardHeader className="flex flex-col gap-4 sm:flex-row sm:items-center sm:justify-between">
+						<CardTitle className="flex items-center gap-2">
+							<Activity className="h-4 w-4" />
+							Machine Dashboard
+						</CardTitle>
+						<Select
+							value={selectedMachine?.id ?? ""}
+							onValueChange={setSelectedMachineId}
+							disabled={machines.length === 0}
+						>
+							<SelectTrigger className="w-full sm:w-[240px]">
+								<SelectValue placeholder="Select machine" />
+							</SelectTrigger>
+							<SelectContent>
+								{machines.map((machine) => (
+									<SelectItem key={machine.id} value={machine.id}>
+										{machine.name}
+									</SelectItem>
+								))}
+							</SelectContent>
+						</Select>
+					</CardHeader>
+					<CardContent className="space-y-4">
+						{selectedMachine ? (
+							<>
+								<div className="flex items-center justify-between">
+									<div className="text-sm font-medium">
+										{selectedMachine.name} metrics
+									</div>
+									<Button variant="outline" size="sm" asChild>
+										<a
+											href={nodeExporterDashboardUrl}
+											target="_blank"
+											rel="noopener noreferrer"
+											className="gap-1"
+										>
+											Open in Grafana
+											<ExternalLink className="h-3 w-3" />
+										</a>
+									</Button>
+								</div>
+								<GrafanaPanel
+									key={nodeExporterDashboardUrl}
+									src={nodeExporterDashboardUrl}
+									title={`${selectedMachine.name} metrics`}
+								/>
+								<div className="flex items-center justify-between pt-2">
+									<div className="text-sm font-medium">Recent logs</div>
+									<Button variant="outline" size="sm" asChild>
+										<a
+											href={logsUrl}
+											target="_blank"
+											rel="noopener noreferrer"
+											className="gap-1"
+										>
+											Explore in Grafana
+											<ExternalLink className="h-3 w-3" />
+										</a>
+									</Button>
+								</div>
+								<GrafanaPanel
+									key={logsUrl}
+									src={logsUrl}
+									title={`${selectedMachine.name} logs`}
+								/>
+							</>
+						) : (
+							<EmptyState
+								icon={ServerOff}
+								title="No machine selected"
+								description="Add monitoring machines in Settings to embed Grafana dashboards."
+								action={
+									<Button variant="outline" size="sm" asChild>
+										<Link to="/settings">Open Settings</Link>
+									</Button>
+								}
+							/>
+						)}
+					</CardContent>
+				</Card>
+			</div>
+		</div>
+	);
+}

frontend/src/components/ui/alert.tsx

@@ -0,0 +1,76 @@
+import * as React from "react"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/lib/utils"
+
+const alertVariants = cva(
+  "group/alert relative grid w-full gap-0.5 rounded-lg border px-2.5 py-2 text-left text-sm has-data-[slot=alert-action]:relative has-data-[slot=alert-action]:pr-18 has-[>svg]:grid-cols-[auto_1fr] has-[>svg]:gap-x-2 *:[svg]:row-span-2 *:[svg]:translate-y-0.5 *:[svg]:text-current *:[svg:not([class*='size-'])]:size-4",
+  {
+    variants: {
+      variant: {
+        default: "bg-card text-card-foreground",
+        destructive:
+          "bg-card text-destructive *:data-[slot=alert-description]:text-destructive/90 *:[svg]:text-current",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+)
+
+function Alert({
+  className,
+  variant,
+  ...props
+}: React.ComponentProps<"div"> & VariantProps<typeof alertVariants>) {
+  return (
+    <div
+      data-slot="alert"
+      role="alert"
+      className={cn(alertVariants({ variant }), className)}
+      {...props}
+    />
+  )
+}
+
+function AlertTitle({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="alert-title"
+      className={cn(
+        "font-medium group-has-[>svg]/alert:col-start-2 [&_a]:underline [&_a]:underline-offset-3 [&_a]:hover:text-foreground",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function AlertDescription({
+  className,
+  ...props
+}: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="alert-description"
+      className={cn(
+        "text-sm text-balance text-muted-foreground md:text-pretty [&_a]:underline [&_a]:underline-offset-3 [&_a]:hover:text-foreground [&_p:not(:last-child)]:mb-4",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function AlertAction({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="alert-action"
+      className={cn("absolute top-2 right-2", className)}
+      {...props}
+    />
+  )
+}
+
+export { Alert, AlertTitle, AlertDescription, AlertAction }

frontend/src/components/ui/badge.tsx

@@ -0,0 +1,49 @@
+import * as React from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+import { Slot } from "radix-ui";
+
+import { cn } from "@/lib/utils";
+
+const badgeVariants = cva(
+	"group/badge inline-flex h-5 w-fit shrink-0 items-center justify-center gap-1 overflow-hidden rounded-4xl border border-transparent px-2 py-0.5 text-xs font-medium whitespace-nowrap transition-all focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 aria-invalid:border-destructive aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 [&>svg]:pointer-events-none [&>svg]:size-3!",
+	{
+		variants: {
+			variant: {
+				default: "bg-primary text-primary-foreground [a]:hover:bg-primary/80",
+				secondary:
+					"bg-secondary text-secondary-foreground [a]:hover:bg-secondary/80",
+				destructive:
+					"bg-destructive/10 text-destructive focus-visible:ring-destructive/20 dark:bg-destructive/20 dark:focus-visible:ring-destructive/40 [a]:hover:bg-destructive/20",
+				outline:
+					"border-border text-foreground [a]:hover:bg-muted [a]:hover:text-muted-foreground",
+				ghost:
+					"hover:bg-muted hover:text-muted-foreground dark:hover:bg-muted/50",
+				link: "text-primary underline-offset-4 hover:underline",
+			},
+		},
+		defaultVariants: {
+			variant: "default",
+		},
+	},
+);
+
+export function Badge({
+	className,
+	variant = "default",
+	asChild = false,
+	...props
+}: React.ComponentProps<"span"> &
+	VariantProps<typeof badgeVariants> & { asChild?: boolean }) {
+	const Comp = asChild ? Slot.Root : "span";
+
+	return (
+		<Comp
+			data-slot="badge"
+			data-variant={variant}
+			className={cn(badgeVariants({ variant }), className)}
+			{...props}
+		/>
+	);
+}
+
+Badge.displayName = "Badge";

frontend/src/components/ui/card.tsx

@@ -0,0 +1,103 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Card({
+  className,
+  size = "default",
+  ...props
+}: React.ComponentProps<"div"> & { size?: "default" | "sm" }) {
+  return (
+    <div
+      data-slot="card"
+      data-size={size}
+      className={cn(
+        "group/card flex flex-col gap-(--card-spacing) overflow-hidden rounded-xl bg-card py-(--card-spacing) text-sm text-card-foreground ring-1 ring-foreground/10 [--card-spacing:--spacing(4)] has-data-[slot=card-footer]:pb-0 has-[>img:first-child]:pt-0 data-[size=sm]:[--card-spacing:--spacing(3)] data-[size=sm]:has-data-[slot=card-footer]:pb-0 *:[img:first-child]:rounded-t-xl *:[img:last-child]:rounded-b-xl",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function CardHeader({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="card-header"
+      className={cn(
+        "group/card-header @container/card-header grid auto-rows-min items-start gap-1 rounded-t-xl px-(--card-spacing) has-data-[slot=card-action]:grid-cols-[1fr_auto] has-data-[slot=card-description]:grid-rows-[auto_auto] [.border-b]:pb-(--card-spacing)",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function CardTitle({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="card-title"
+      className={cn(
+        "text-base leading-snug font-medium group-data-[size=sm]/card:text-sm",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function CardDescription({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="card-description"
+      className={cn("text-sm text-muted-foreground", className)}
+      {...props}
+    />
+  )
+}
+
+function CardAction({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="card-action"
+      className={cn(
+        "col-start-2 row-span-2 row-start-1 self-start justify-self-end",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function CardContent({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="card-content"
+      className={cn("px-(--card-spacing)", className)}
+      {...props}
+    />
+  )
+}
+
+function CardFooter({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="card-footer"
+      className={cn(
+        "flex items-center rounded-b-xl border-t bg-muted/50 p-(--card-spacing)",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+export {
+  Card,
+  CardHeader,
+  CardFooter,
+  CardTitle,
+  CardAction,
+  CardDescription,
+  CardContent,
+}

frontend/src/components/ui/select.tsx

@@ -0,0 +1,190 @@
+import * as React from "react"
+import { Select as SelectPrimitive } from "radix-ui"
+
+import { cn } from "@/lib/utils"
+import { ChevronDownIcon, CheckIcon, ChevronUpIcon } from "lucide-react"
+
+function Select({
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.Root>) {
+  return <SelectPrimitive.Root data-slot="select" {...props} />
+}
+
+function SelectGroup({
+  className,
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.Group>) {
+  return (
+    <SelectPrimitive.Group
+      data-slot="select-group"
+      className={cn("scroll-my-1 p-1", className)}
+      {...props}
+    />
+  )
+}
+
+function SelectValue({
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.Value>) {
+  return <SelectPrimitive.Value data-slot="select-value" {...props} />
+}
+
+function SelectTrigger({
+  className,
+  size = "default",
+  children,
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.Trigger> & {
+  size?: "sm" | "default"
+}) {
+  return (
+    <SelectPrimitive.Trigger
+      data-slot="select-trigger"
+      data-size={size}
+      className={cn(
+        "flex w-fit items-center justify-between gap-1.5 rounded-lg border border-input bg-transparent py-2 pr-2 pl-2.5 text-sm whitespace-nowrap transition-colors outline-none select-none focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 disabled:cursor-not-allowed disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 data-placeholder:text-muted-foreground data-[size=default]:h-8 data-[size=sm]:h-7 data-[size=sm]:rounded-[min(var(--radius-md),10px)] *:data-[slot=select-value]:line-clamp-1 *:data-[slot=select-value]:flex *:data-[slot=select-value]:items-center *:data-[slot=select-value]:gap-1.5 dark:bg-input/30 dark:hover:bg-input/50 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
+        className
+      )}
+      {...props}
+    >
+      {children}
+      <SelectPrimitive.Icon asChild>
+        <ChevronDownIcon className="pointer-events-none size-4 text-muted-foreground" />
+      </SelectPrimitive.Icon>
+    </SelectPrimitive.Trigger>
+  )
+}
+
+function SelectContent({
+  className,
+  children,
+  position = "item-aligned",
+  align = "center",
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.Content>) {
+  return (
+    <SelectPrimitive.Portal>
+      <SelectPrimitive.Content
+        data-slot="select-content"
+        data-align-trigger={position === "item-aligned"}
+        className={cn("relative z-50 max-h-(--radix-select-content-available-height) min-w-36 origin-(--radix-select-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-lg bg-popover text-popover-foreground shadow-md ring-1 ring-foreground/10 duration-100 data-[align-trigger=true]:animate-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-open:animate-in data-open:fade-in-0 data-open:zoom-in-95 data-closed:animate-out data-closed:fade-out-0 data-closed:zoom-out-95", position ==="popper"&&"data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1", className )}
+        position={position}
+        align={align}
+        {...props}
+      >
+        <SelectScrollUpButton />
+        <SelectPrimitive.Viewport
+          data-position={position}
+          className={cn(
+            "data-[position=popper]:h-(--radix-select-trigger-height) data-[position=popper]:w-full data-[position=popper]:min-w-(--radix-select-trigger-width)",
+            position === "popper" && ""
+          )}
+        >
+          {children}
+        </SelectPrimitive.Viewport>
+        <SelectScrollDownButton />
+      </SelectPrimitive.Content>
+    </SelectPrimitive.Portal>
+  )
+}
+
+function SelectLabel({
+  className,
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.Label>) {
+  return (
+    <SelectPrimitive.Label
+      data-slot="select-label"
+      className={cn("px-1.5 py-1 text-xs text-muted-foreground", className)}
+      {...props}
+    />
+  )
+}
+
+function SelectItem({
+  className,
+  children,
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.Item>) {
+  return (
+    <SelectPrimitive.Item
+      data-slot="select-item"
+      className={cn(
+        "relative flex w-full cursor-default items-center gap-1.5 rounded-md py-1 pr-8 pl-1.5 text-sm outline-hidden select-none focus:bg-accent focus:text-accent-foreground not-data-[variant=destructive]:focus:**:text-accent-foreground data-disabled:pointer-events-none data-disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 *:[span]:last:flex *:[span]:last:items-center *:[span]:last:gap-2",
+        className
+      )}
+      {...props}
+    >
+      <span className="pointer-events-none absolute right-2 flex size-4 items-center justify-center">
+        <SelectPrimitive.ItemIndicator>
+          <CheckIcon className="pointer-events-none" />
+        </SelectPrimitive.ItemIndicator>
+      </span>
+      <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+    </SelectPrimitive.Item>
+  )
+}
+
+function SelectSeparator({
+  className,
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.Separator>) {
+  return (
+    <SelectPrimitive.Separator
+      data-slot="select-separator"
+      className={cn("pointer-events-none -mx-1 my-1 h-px bg-border", className)}
+      {...props}
+    />
+  )
+}
+
+function SelectScrollUpButton({
+  className,
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.ScrollUpButton>) {
+  return (
+    <SelectPrimitive.ScrollUpButton
+      data-slot="select-scroll-up-button"
+      className={cn(
+        "z-10 flex cursor-default items-center justify-center bg-popover py-1 [&_svg:not([class*='size-'])]:size-4",
+        className
+      )}
+      {...props}
+    >
+      <ChevronUpIcon
+      />
+    </SelectPrimitive.ScrollUpButton>
+  )
+}
+
+function SelectScrollDownButton({
+  className,
+  ...props
+}: React.ComponentProps<typeof SelectPrimitive.ScrollDownButton>) {
+  return (
+    <SelectPrimitive.ScrollDownButton
+      data-slot="select-scroll-down-button"
+      className={cn(
+        "z-10 flex cursor-default items-center justify-center bg-popover py-1 [&_svg:not([class*='size-'])]:size-4",
+        className
+      )}
+      {...props}
+    >
+      <ChevronDownIcon
+      />
+    </SelectPrimitive.ScrollDownButton>
+  )
+}
+
+export {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectLabel,
+  SelectScrollDownButton,
+  SelectScrollUpButton,
+  SelectSeparator,
+  SelectTrigger,
+  SelectValue,
+}

frontend/src/components/ui/skeleton.tsx

@@ -0,0 +1,13 @@
+import { cn } from "@/lib/utils"
+
+function Skeleton({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="skeleton"
+      className={cn("animate-pulse rounded-md bg-muted", className)}
+      {...props}
+    />
+  )
+}
+
+export { Skeleton }

frontend/src/hooks/useDashboard.ts

@@ -5,7 +5,6 @@ import {
 	fetchCounts,
 	fetchDashboardShortcuts,
 	fetchLibraries,
-	fetchMonitoringOverview,
 	saveDashboardShortcut,
 } from "../api/client";
 import type { DashboardShortcutInput } from "../types";
@@ -34,14 +33,6 @@ export function useActivity(machineId?: string) {
 	});
 }
 
-export function useMonitoringOverview() {
-	return useQuery({
-		queryKey: ["dashboard", "monitoring"],
-		queryFn: fetchMonitoringOverview,
-		refetchInterval: 30_000,
-	});
-}
-
 // Backward-compatible alias used by older code.
 export const useNowPlaying = useActivity;
 

frontend/src/hooks/useMonitoring.ts

@@ -1,85 +0,0 @@
-import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
-import {
-	fetchMonitoringStatus,
-	fetchMonitoringMetrics,
-	fetchDiskSpace,
-	fetchMonitoringMachines,
-	fetchMonitoringMachineActions,
-	fetchMonitoringPoller,
-	startCollector,
-	stopCollector,
-	restartCollector,
-} from "../api/client";
-
-export function useMonitoringMachines() {
-	return useQuery({
-		queryKey: ["monitoring", "machines"],
-		queryFn: fetchMonitoringMachines,
-		refetchInterval: 30_000,
-	});
-}
-
-export function useMonitoringPoller() {
-	return useQuery({
-		queryKey: ["monitoring", "poller"],
-		queryFn: fetchMonitoringPoller,
-		refetchInterval: 30_000,
-	});
-}
-
-export function useMonitoringStatus(machineId?: string, enabled = true) {
-	return useQuery({
-		queryKey: ["monitoring", "status", machineId ?? "default"],
-		queryFn: () => fetchMonitoringStatus(machineId),
-		refetchInterval: 30_000,
-		enabled,
-	});
-}
-
-export function useMonitoringMetrics(machineId?: string, enabled = true) {
-	return useQuery({
-		queryKey: ["monitoring", "metrics", machineId ?? "default"],
-		queryFn: () => fetchMonitoringMetrics(undefined, 70_000, machineId),
-		refetchInterval: 15_000,
-		enabled,
-	});
-}
-
-export function useDiskSpace(machineId?: string, enabled = true) {
-	return useQuery({
-		queryKey: ["monitoring", "disk", machineId ?? "default"],
-		queryFn: () => fetchDiskSpace(machineId),
-		staleTime: 60_000,
-		enabled,
-	});
-}
-
-export function useMachineActions(machineId: string, enabled = true) {
-	return useQuery({
-		queryKey: ["monitoring", "actions", machineId],
-		queryFn: () => fetchMonitoringMachineActions(machineId),
-		refetchInterval: 15_000,
-		enabled,
-	});
-}
-
-export function useCollectorControls(machineId?: string) {
-	const queryClient = useQueryClient();
-	const invalidate = () =>
-		queryClient.invalidateQueries({ queryKey: ["monitoring"] });
-
-	const start = useMutation({
-		mutationFn: () => startCollector(machineId),
-		onSuccess: invalidate,
-	});
-	const stop = useMutation({
-		mutationFn: () => stopCollector(machineId),
-		onSuccess: invalidate,
-	});
-	const restart = useMutation({
-		mutationFn: () => restartCollector(machineId),
-		onSuccess: invalidate,
-	});
-
-	return { start, stop, restart };
-}

frontend/src/hooks/useObservability.ts

@@ -0,0 +1,47 @@
+import { useQuery } from "@tanstack/react-query";
+import {
+	fetchAlertmanagerAlerts,
+	fetchAlertmanagerStatus,
+	fetchPrometheusTargets,
+	fetchMonitoringMachines,
+} from "../api/client";
+
+export function useAlertmanagerAlerts() {
+	return useQuery({
+		queryKey: ["observability", "alerts"],
+		queryFn: fetchAlertmanagerAlerts,
+		retry: 2,
+		staleTime: 10_000,
+		refetchInterval: 15_000,
+	});
+}
+
+export function useAlertmanagerStatus() {
+	return useQuery({
+		queryKey: ["observability", "alertmanager-status"],
+		queryFn: fetchAlertmanagerStatus,
+		retry: 2,
+		staleTime: 10_000,
+		refetchInterval: 30_000,
+	});
+}
+
+export function usePrometheusTargets() {
+	return useQuery({
+		queryKey: ["observability", "prometheus-targets"],
+		queryFn: fetchPrometheusTargets,
+		retry: 2,
+		staleTime: 10_000,
+		refetchInterval: 30_000,
+	});
+}
+
+export function useMonitoringMachines() {
+	return useQuery({
+		queryKey: ["monitoring", "machines"],
+		queryFn: fetchMonitoringMachines,
+		retry: 2,
+		staleTime: 10_000,
+		refetchInterval: 30_000,
+	});
+}

frontend/src/pages/Dashboard.tsx

@@ -26,13 +26,11 @@ import {
 	useActivity,
 	useDashboardShortcuts,
 	useDeleteDashboardShortcut,
-	useMonitoringOverview,
 	useSaveDashboardShortcut,
 } from "../hooks/useDashboard";
 import { useMonitoringSettings } from "../hooks/useSettings";
 import type { DashboardShortcut, DashboardShortcutInput } from "../types";
 import { NowPlaying } from "../components/NowPlaying";
-import { MonitoringOverviewTable } from "../components/MonitoringOverviewTable";
 import { SectionCard } from "../components/SectionCard";
 import { DialogFooter } from "../components/DialogFooter";
 import BackupDashboardWidget from "../components/BackupDashboardWidget";
@@ -325,7 +323,6 @@ export function Dashboard() {
 	const selectedJellyfinId =
 		activeJellyfinMachineId || jellyfinMachines[0]?.id || "";
 	const { data: activity } = useActivity(selectedJellyfinId || undefined);
-	const { data: monitoringOverview } = useMonitoringOverview();
 	const { data: shortcuts = [] } = useDashboardShortcuts();
 	const saveShortcut = useSaveDashboardShortcut();
 	const deleteShortcut = useDeleteDashboardShortcut();
@@ -438,13 +435,6 @@ export function Dashboard() {
 				) : null}
 			</SectionCard>
 
-			<SectionCard
-				title="Monitoring overview"
-				description="10-minute averages and fleet status across all configured machines."
-			>
-				<MonitoringOverviewTable overview={monitoringOverview} embedded />
-			</SectionCard>
-
 			<BackupDashboardWidget />
 
 			<ShortcutDialog

frontend/src/pages/Monitoring.tsx

@@ -1,106 +0,0 @@
-import { useMemo, useState } from "react";
-import { Alert, Button, Stack, Tab, Typography } from "@mui/material";
-import { useNavigate } from "react-router-dom";
-import {
-	useMonitoringMachines,
-	useMonitoringPoller,
-} from "../hooks/useMonitoring";
-import { useMonitoringOverview } from "../hooks/useDashboard";
-import { MachineMonitoringSection } from "../components/MachineMonitoringSection";
-import { MonitoringOverviewTable } from "../components/MonitoringOverviewTable";
-import { SectionCard } from "../components/SectionCard";
-import { TabbedCard } from "../components/TabbedCard";
-
-type MonitoringTab = "overview" | string;
-
-export function Monitoring() {
-	const { data: machines = [], isLoading, error } = useMonitoringMachines();
-	const { data: poller } = useMonitoringPoller();
-	const { data: overview } = useMonitoringOverview();
-	const navigate = useNavigate();
-	const [tab, setTab] = useState<MonitoringTab>("overview");
-
-	const selectedMachine = useMemo(
-		() => machines.find((machine) => machine.id === tab) ?? null,
-		[machines, tab],
-	);
-
-	return (
-		<Stack spacing={2.25}>
-			<Stack spacing={0.5}>
-				<Typography variant="h5" sx={{ fontWeight: 800 }}>
-					Monitoring
-				</Typography>
-				<Typography variant="body2" color="text.secondary">
-					Review fleet health first, then switch to a machine-specific tab.
-				</Typography>
-			</Stack>
-
-			{error ? <Alert severity="error">{String(error)}</Alert> : null}
-			{isLoading ? (
-				<Alert severity="info">Loading monitoring machines...</Alert>
-			) : null}
-
-			<TabbedCard
-				value={tab}
-				onChange={setTab}
-				tabs={[
-					<Tab key="overview" value="overview" label="Overview" />,
-					...machines.map((machine) => (
-						<Tab key={machine.id} value={machine.id} label={machine.name} />
-					)),
-				]}
-				contentSx={{ p: 1.5 }}
-			>
-				{tab === "overview" ? (
-					<Stack spacing={1.5}>
-						<Stack
-							direction="row"
-							spacing={1}
-							sx={{ alignItems: "center", flexWrap: "wrap" }}
-						>
-							<Typography variant="subtitle1" sx={{ fontWeight: 700 }}>
-								Fleet overview
-							</Typography>
-							<Typography variant="caption" color="text.secondary">
-								{poller?.worker_running ? "running" : "stopped"}
-							</Typography>
-						</Stack>
-
-						{!isLoading && machines.length === 0 ? (
-							<Alert
-								severity="warning"
-								action={
-									<Button
-										color="inherit"
-										size="small"
-										onClick={() => navigate("/settings")}
-									>
-										Open Settings
-									</Button>
-								}
-							>
-								No monitoring machines are configured yet.
-							</Alert>
-						) : overview ? (
-							<MonitoringOverviewTable overview={overview} embedded />
-						) : (
-							<Alert severity="info">Loading fleet overview...</Alert>
-						)}
-					</Stack>
-				) : selectedMachine ? (
-					<SectionCard
-						title={selectedMachine.name}
-						description={
-							selectedMachine.mode === "local"
-								? "Local API host"
-								: `${selectedMachine.username || "user"}@${selectedMachine.host || "host"}:${selectedMachine.port}`
-						}
-					>
-						<MachineMonitoringSection machine={selectedMachine} />
-					</SectionCard>
-				) : null}
-			</TabbedCard>
-		</Stack>
-	);
-}

frontend/src/types/index.ts

@@ -246,28 +246,6 @@ export interface SavedTaskRun {
 	error: string;
 }
 
-export interface MonitoringMachineAction {
-	machine_id: string;
-	machine_name: string;
-	mode: "local" | "ssh";
-	action: string;
-	status: string;
-	created_at: number;
-	duration_ms: number;
-	request_id: string;
-	message: string;
-	error: string;
-	stdout_tail: string;
-	stderr_tail: string;
-}
-
-export interface MetricSummary {
-	avg: number;
-	min: number;
-	max: number;
-	count: number;
-}
-
 export interface ResetLocalDatabaseInput {
 	confirm_phrase: string;
 	acknowledge_settings_loss: boolean;
@@ -291,20 +269,6 @@ export interface SSHValidationResult {
 	known_hosts_updated: boolean;
 }
 
-export interface MonitoringPollerStatus {
-	worker_running: boolean;
-	stop_requested: boolean;
-	last_run_at: number | null;
-	last_success_at: number | null;
-	last_error: string;
-	last_cycle_ms: number | null;
-	poll_count: number;
-	error_count: number;
-	interval_seconds: number;
-	initial_delay_seconds: number;
-	retention_days: number;
-}
-
 export interface AppVersionInfo {
 	app: string;
 	backend_version: string;
@@ -312,53 +276,6 @@ export interface AppVersionInfo {
 	backend_label: string;
 }
 
-export interface MonitoringMachineOverview {
-	machine: MonitoringMachine;
-	status: string;
-	status_error: string;
-	metrics_error: string;
-	disk_error: string;
-	latest_sample: MonitoringSample | null;
-	sample_count: number;
-	cpu_summary: MetricSummary | null;
-	iowait_summary: MetricSummary | null;
-	mem_summary: MetricSummary | null;
-	net_rx_summary: MetricSummary | null;
-	net_tx_summary: MetricSummary | null;
-	disk_read_summary: MetricSummary | null;
-	disk_write_summary: MetricSummary | null;
-	disk: DiskSpace | null;
-}
-
-export interface MonitoringOverviewResponse {
-	poller: MonitoringPollerStatus;
-	machines: MonitoringMachineOverview[];
-	total: number;
-	enabled: number;
-}
-
-export interface MonitoringStatus {
-	status: string;
-}
-
-export interface MonitoringSample {
-	ts: number;
-	cpu_pct: number;
-	iowait_pct?: number;
-	mem_pct: number;
-	net_rx_bytes_per_sec: number;
-	net_tx_bytes_per_sec: number;
-	disk_read_bps: number;
-	disk_write_bps: number;
-}
-
-export interface MonitoringMetrics {
-	samples: MonitoringSample[];
-	total_samples: number;
-	filtered_samples: number;
-	cutoff_ts: number;
-}
-
 export interface DiskSpace {
 	filesystem: string;
 	size: number;
@@ -503,3 +420,35 @@ export interface DashboardShortcutInput {
 	user_id: string;
 	notes: string;
 }
+
+export interface AlertmanagerAlert {
+	name: string;
+	severity: string;
+	category: string;
+	job_name: string;
+	summary: string;
+	description: string;
+	active_since: string;
+	state: string;
+	labels: Record<string, string>;
+}
+
+export interface AlertmanagerAlertSummary {
+	total: number;
+	by_severity: Record<string, number>;
+	alerts: AlertmanagerAlert[];
+	error?: string;
+}
+
+export interface AlertmanagerStatus {
+	up: boolean;
+	version: string;
+	uptime: string;
+	name: string;
+	peers: string[];
+}
+
+export interface PrometheusTarget {
+	labels: Record<string, string>;
+	targets: string[];
+}

frontend/vite.config.ts

@@ -30,6 +30,11 @@ export default defineConfig(({ mode }) => {
 					changeOrigin: true,
 					secure: false,
 				},
+				"/grafana": {
+					target: "http://localhost:3000",
+					changeOrigin: true,
+					rewrite: (path: string) => path.replace(/^\/grafana/, ""),
+				},
 			},
 		},
 		resolve: {

monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,44 @@
+global:
+  smtp_smarthost: '${SMTP_HOST:-smtp.example.com}:${SMTP_PORT:-587}'
+  smtp_from: '${SMTP_FROM_ADDRESS:-alerts@example.com}'
+  smtp_auth_username: '${SMTP_USERNAME:-}'
+  smtp_auth_password: '${SMTP_PASSWORD}'
+
+route:
+  receiver: 'default'
+  group_by: ['alertname', 'severity', 'category']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+  routes:
+    - match:
+        severity: critical
+      receiver: 'critical'
+      continue: true
+
+receivers:
+  - name: 'default'
+    email_configs:
+      - to: '${ALERT_EMAIL_TO}'
+        send_resolved: true
+        headers:
+          Subject: '[Manage] {{ .GroupLabels.alertname }}'
+
+  - name: 'critical'
+    email_configs:
+      - to: '${ALERT_EMAIL_TO}'
+        send_resolved: true
+        headers:
+          Subject: '[Manage CRITICAL] {{ .GroupLabels.alertname }}'
+
+  - name: 'webhook'
+    webhook_configs:
+      - url: '${ALERTMANAGER_WEBHOOK_URL:-http://backend:8000/api/monitoring/alertmanager-webhook}'
+        send_resolved: true
+
+inhibit_rules:
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'
+    equal: ['alertname', 'instance']

monitoring/alloy/config.alloy

@@ -0,0 +1,39 @@
+logging {
+  level = "info"
+}
+
+// Promtail-compatible Docker log scraping via the local container engine.
+// Requires mounting /var/run/docker.sock and /var/lib/docker/containers.
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+discovery.relabel "container_labels" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    target_label  = "container"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+}
+
+loki.source.docker "default" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.relabel.container_labels.output
+  forward_to = [loki.write.local.receiver]
+  labels = {
+    job = "docker",
+  }
+}
+
+loki.write "local" {
+  endpoint {
+    name = "loki"
+    url  = "http://loki:3100/loki/api/v1/push"
+  }
+}

monitoring/grafana/grafana.ini

@@ -0,0 +1,32 @@
+[server]
+# Allow embedding Manage in iframes
+root_url = %(protocol)s://%(domain)s:%(http_port)s/
+serve_from_sub_path = false
+
+[security]
+allow_embedding = true
+cookie_samesite = lax
+
+[auth]
+disable_login_form = false
+disable_signout_menu = false
+
+[auth.generic_oauth]
+enabled = true
+name = Authentik
+allow_sign_up = true
+client_id = ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID}
+client_secret = ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET}
+scopes = openid profile email
+auth_url = ${GF_AUTH_GENERIC_OAUTH_AUTH_URL}
+token_url = ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL}
+api_url = ${GF_AUTH_GENERIC_OAUTH_API_URL}
+role_attribute_path = "contains(groups[*], 'grafana-admins') && 'Admin' || contains(groups[*], 'grafana-editors') && 'Editor' || 'Viewer'"
+
+[users]
+auto_assign_org = true
+auto_assign_org_role = Viewer
+
+[dataproxy]
+# Keep dashboard queries responsive in the iframe
+timeout = 30

monitoring/grafana/provisioning/dashboards-json/dashboards.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: "Manage dashboards"
+    orgId: 1
+    folder: "Manage"
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 60
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/provisioning/dashboards-json/dashboards

monitoring/grafana/provisioning/dashboards-json/dashboards/manage-overview.json

@@ -0,0 +1,312 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "sum(rate(manage_api_requests_total[5m])) by (method)",
+          "refId": "A"
+        }
+      ],
+      "title": "API Request Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 500
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "sum(increase(manage_backup_runs_total[1h]))",
+          "refId": "A"
+        }
+      ],
+      "title": "Backup Runs (1h)",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "pieType": "donut",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "sum by (status) (manage_backup_runs_total)",
+          "legendFormat": "{{ status }}",
+          "refId": "A"
+        }
+      ],
+      "title": "Backup Runs by Status",
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki_datasource}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "dedupStrategy": "none"
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "expr": "{container=\"/backend\"} |= \"\"",
+          "refId": "A"
+        }
+      ],
+      "title": "Backend Logs",
+      "type": "logs"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "tags": ["manage"],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Metrics",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "Loki",
+          "value": "Loki"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Logs",
+        "multi": false,
+        "name": "loki_datasource",
+        "options": [],
+        "query": "loki",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Manage Overview",
+  "uid": "manage-overview",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/grafana/provisioning/dashboards-json/dashboards/node-exporter-overview.json

@@ -0,0 +1,400 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "legendFormat": "{{ instance }}",
+          "refId": "A"
+        }
+      ],
+      "title": "CPU Utilization",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
+          "legendFormat": "{{ instance }}",
+          "refId": "A"
+        }
+      ],
+      "title": "Memory Utilization",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 85
+              },
+              {
+                "color": "red",
+                "value": 95
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "100 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"} * 100)",
+          "legendFormat": "{{ instance }}",
+          "refId": "A"
+        }
+      ],
+      "title": "Root Disk Usage",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "irate(node_network_receive_bytes_total[5m])",
+          "legendFormat": "rx {{ instance }} {{ device }}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "irate(node_network_transmit_bytes_total[5m])",
+          "legendFormat": "tx {{ instance }} {{ device }}",
+          "refId": "B"
+        }
+      ],
+      "title": "Network Traffic",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "tags": ["manage", "node-exporter"],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Metrics",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Node Exporter Overview",
+  "uid": "node-exporter-overview",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/grafana/provisioning/datasources/datasources.yml

@@ -0,0 +1,19 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      timeInterval: 15s
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
+    jsonData:
+      maxLines: 1000

monitoring/loki/loki.yml

@@ -0,0 +1,51 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+
+common:
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /loki/tsdb-index
+    cache_location: /loki/tsdb-cache
+
+compactor:
+  working_directory: /loki/compactor
+  retention_enabled: true
+  retention_delete_delay: 2h
+  retention_delete_worker_count: 150
+  compaction_interval: 10m
+
+limits_config:
+  retention_period: 720h
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,53 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    monitor: manage
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets:
+          - localhost:9090
+
+  - job_name: manage-backend
+    static_configs:
+      - targets:
+          - backend:8000
+    metrics_path: /metrics
+    scrape_interval: 15s
+
+  - job_name: node-exporter
+    static_configs:
+      - targets:
+          - node-exporter:9100
+
+  - job_name: loki
+    static_configs:
+      - targets:
+          - loki:3100
+
+  - job_name: alertmanager
+    static_configs:
+      - targets:
+          - alertmanager:9093
+
+  - job_name: grafana
+    static_configs:
+      - targets:
+          - grafana:3000
+
+  - job_name: node-exporter-remote
+    file_sd_configs:
+      - files:
+          - /etc/prometheus/file-sd/node_exporter_targets.json
+        refresh_interval: 30s

monitoring/prometheus/rules/backup_alerts.yml

@@ -0,0 +1,54 @@
+groups:
+  - name: backup
+    rules:
+      - alert: BackupJobFailed
+        expr: increase(manage_backup_runs_total{status="failed"}[1h]) > 0
+        for: 0m
+        labels:
+          severity: critical
+          category: backup
+        annotations:
+          summary: "Backup job {{ $labels.job_name }} failed"
+          description: "At least one failed backup run was recorded for job {{ $labels.job_name }} within the last hour."
+
+      - alert: BackupJobStuck
+        expr: time() - manage_backup_runs_last_success_timestamp > 86400
+        for: 5m
+        labels:
+          severity: warning
+          category: backup
+        annotations:
+          summary: "Backup job {{ $labels.job_name }} has not succeeded in 24h"
+          description: "No successful backup run has been recorded for job {{ $labels.job_name }} in the last 24 hours."
+
+  - name: observability_health
+    rules:
+      - alert: PrometheusTargetMissing
+        expr: up == 0
+        for: 2m
+        labels:
+          severity: warning
+          category: observability
+        annotations:
+          summary: "Prometheus target {{ $labels.job }} / {{ $labels.instance }} is down"
+          description: "The Prometheus scrape target for {{ $labels.job }} on {{ $labels.instance }} has been unreachable for more than 2 minutes."
+
+      - alert: AlertmanagerDown
+        expr: up{job="alertmanager"} == 0
+        for: 2m
+        labels:
+          severity: critical
+          category: observability
+        annotations:
+          summary: "Alertmanager is down"
+          description: "Alertmanager has been unreachable for more than 2 minutes. Alerts may not be delivered."
+
+      - alert: GrafanaDown
+        expr: up{job="grafana"} == 0
+        for: 2m
+        labels:
+          severity: warning
+          category: observability
+        annotations:
+          summary: "Grafana is down"
+          description: "Grafana has been unreachable for more than 2 minutes. Dashboards are unavailable."