diff --git a/.env.example b/.env.example index ad5cce2..9149d49 100644 --- a/.env.example +++ b/.env.example @@ -1,46 +1,66 @@ -# Optional backend logging level. -LOG_LEVEL=INFO +# App +APP_VERSION=0.1.0 +APP_BUILD_INFO=dev -# Optional SMTP settings for the Users -> message popup. +# Hosts +BACKEND_APP_HOST=api.manage.example.com +FRONTEND_APP_HOST=manage.example.com +GRAFANA_APP_HOST=grafana.example.com +BACKEND_APP_PORT=8000 +FRONTEND_APP_PORT=80 +GRAFANA_APP_PORT=3000 +BACKEND_APP_NAME=manage-backend +FRONTEND_APP_NAME=manage-frontend +GRAFANA_APP_NAME=grafana + +# Traefik / certificates +CERT_RESOLVER=letsencrypt + +# Backend +LOG_LEVEL=INFO +LOG_FORMAT=json +PROMETHEUS_ENABLED=true +PROMETHEUS_FILE_SD_DIR=/app/backend/.cache/prometheus-file-sd +ALERTMANAGER_URL=http://alertmanager:9093 +ALERTMANAGER_WEBHOOK_URL= +BACKEND_CACHE_DIR=./backend-cache + +# Auth +AUTH_ENABLED=true +OIDC_ISSUER_URL=https://auth.example.com/application/o/manage/ +OIDC_AUDIENCE=manage +OIDC_JWKS_URL=https://auth.example.com/application/o/manage/jwks/ +OIDC_CLOCK_SKEW_SECONDS=30 + +# Frontend OIDC +VITE_API_URL=/api +VITE_OIDC_ENABLED=true +VITE_OIDC_ISSUER=https://auth.example.com/application/o/manage/ +VITE_OIDC_CLIENT_ID=manage +VITE_OIDC_SCOPE=openid profile email +VITE_OIDC_REDIRECT_URI=https://manage.example.com/oidc/callback +VITE_OIDC_POST_LOGOUT_REDIRECT_URI=https://manage.example.com/ +VITE_DEV_API_PROXY_TARGET=http://backend:8000 + +# SMTP SMTP_HOST=smtp.example.com SMTP_PORT=587 -SMTP_USERNAME=your-smtp-username -SMTP_PASSWORD=your-smtp-password +SMTP_USERNAME= +SMTP_PASSWORD= SMTP_FROM_ADDRESS=no-reply@example.com SMTP_FROM_NAME=Manage SMTP_USE_TLS=true SMTP_USE_SSL=false SMTP_TIMEOUT=30 -# Machine/service configuration now lives in the app's Settings tab. -# The built-in local machine is seeded automatically. -# -# Remote SSH machines can store their private key and optional passphrase directly in Settings, -# so no SSH key mount is required for normal use. +# Grafana admin / OAuth +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=change-me +GF_AUTH_GENERIC_OAUTH_CLIENT_ID=manage +GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET= +GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.example.com/application/o/manage/authorize/ +GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.example.com/application/o/manage/token/ +GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.example.com/application/o/manage/userinfo/ -# For deployment with traefik -FRONTEND_APP_NAME=manage -FRONTEND_APP_HOST=manage.example.com -FRONTEND_APP_PORT=5173 - -BACKEND_APP_NAME=management-api -BACKEND_APP_HOST=management-api.example.com -BACKEND_APP_PORT=8000 - -CERT_RESOLVER=lets-encrypt - -# Authentik / OIDC -# Backend validates every API request with a Bearer JWT. -AUTH_ENABLED=true -OIDC_ISSUER_URL=https://authentik.example/application/o/media-library-viewer/ -OIDC_AUDIENCE=media-library-viewer -OIDC_JWKS_URL= -OIDC_CLOCK_SKEW_SECONDS=30 - -# Frontend OIDC settings (Vite build/runtime env) -VITE_OIDC_ENABLED=true -VITE_OIDC_ISSUER=https://authentik.example/application/o/media-library-viewer/ -VITE_OIDC_CLIENT_ID=media-library-viewer -VITE_OIDC_SCOPE=openid profile email -VITE_OIDC_REDIRECT_URI=http://localhost:8080/ -VITE_OIDC_POST_LOGOUT_REDIRECT_URI=http://localhost:8080/ +# Alerting +ALERT_EMAIL_TO=admin@example.com diff --git a/.gitignore b/.gitignore index ee3e8de..16fbd21 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ dist/ # Virtual environments .venv/ +.venv-review/ venv/ env/ @@ -52,3 +53,7 @@ frontend/dist/ .fusion/ .stoneforge/ .superpowers/ +# Local Pi runtime state +.atl/ +.pi-map.md +.pi-map.index.md diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c0f673b..2ffbc92 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "pandas>=2.0", "PyJWT[crypto]>=2.8", "python-multipart>=0.0.9", + "prometheus-client>=0.21", + "python-json-logger>=2.0", ] [project.optional-dependencies] diff --git a/backend/src/media_library_viewer_api/clients/jellyfin.py b/backend/src/media_library_viewer_api/clients/jellyfin.py index 90063a1..64ae848 100644 --- a/backend/src/media_library_viewer_api/clients/jellyfin.py +++ b/backend/src/media_library_viewer_api/clients/jellyfin.py @@ -53,7 +53,9 @@ class JellyfinClient: { "X-Emby-Token": api_key, "Accept": "application/json", - "X-Emby-Authorization": 'MediaBrowser Client="MediaLibraryViewer", Device="Streamlit", DeviceId="streamlit", Version="0.1"', + "X-Emby-Authorization": ( + 'MediaBrowser Client="MediaLibraryViewer", Device="Streamlit", DeviceId="streamlit", Version="0.1"' + ), } ) @@ -61,9 +63,7 @@ class JellyfinClient: """GET a Jellyfin endpoint and include useful response text on errors.""" clean_params = {k: v for k, v in params.items() if v is not None and v != ""} logger.debug("Jellyfin GET %s params=%s", path, sorted(clean_params.keys())) - response = self.session.get( - f"{self.base_url}{path}", params=clean_params, timeout=self.timeout - ) + response = self.session.get(f"{self.base_url}{path}", params=clean_params, timeout=self.timeout) try: response.raise_for_status() except requests.HTTPError as exc: @@ -175,14 +175,16 @@ class JellyfinClient: series = self.item_count(user_id, "Series", parent_id=lib_id) episodes = self.item_count(user_id, "Episode", parent_id=lib_id) total = self.item_count(user_id, "Movie,Episode,Video,Audio,Series", parent_id=lib_id) - results.append({ - "library": lib_name, - "type": lib_type, - "movies": movies, - "series": series, - "episodes": episodes, - "total": total, - }) + results.append( + { + "library": lib_name, + "type": lib_type, + "movies": movies, + "series": series, + "episodes": episodes, + "total": total, + } + ) return results def sessions(self, active_within_seconds: int | None = None) -> list[dict[str, Any]]: diff --git a/backend/src/media_library_viewer_api/clients/jellyseerr.py b/backend/src/media_library_viewer_api/clients/jellyseerr.py index 6de945e..aebe8ab 100644 --- a/backend/src/media_library_viewer_api/clients/jellyseerr.py +++ b/backend/src/media_library_viewer_api/clients/jellyseerr.py @@ -40,16 +40,12 @@ class JellyseerrClient: """GET a Jellyseerr endpoint and include useful response text on errors.""" clean_params = {k: v for k, v in params.items() if v is not None and v != ""} logger.debug("Jellyseerr GET %s params=%s", path, sorted(clean_params.keys())) - response = self.session.get( - f"{self.base_url}/api/v1{path}", params=clean_params, timeout=self.timeout - ) + response = self.session.get(f"{self.base_url}/api/v1{path}", params=clean_params, timeout=self.timeout) try: response.raise_for_status() except requests.HTTPError as exc: detail = response.text[:500] - logger.warning( - "Jellyseerr GET %s failed status=%s url=%s", path, response.status_code, response.url - ) + logger.warning("Jellyseerr GET %s failed status=%s url=%s", path, response.status_code, response.url) raise requests.HTTPError( f"{response.status_code} for {response.url}: {detail}", response=response, @@ -105,7 +101,9 @@ class JellyseerrClient: return results page_results = payload.get("results") or [] - page_items = [item for item in page_results if isinstance(item, dict)] if isinstance(page_results, list) else [] + page_items = ( + [item for item in page_results if isinstance(item, dict)] if isinstance(page_results, list) else [] + ) results.extend(page_items) page_info = payload.get("pageInfo") or {} diff --git a/backend/src/media_library_viewer_api/clients/resources.py b/backend/src/media_library_viewer_api/clients/resources.py deleted file mode 100644 index 3ebc7a2..0000000 --- a/backend/src/media_library_viewer_api/clients/resources.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Remote resource collection helpers. - -The app does not require Prometheus, Netdata, or sysstat. Instead it can install -and manage a tiny POSIX-sh collector under /tmp on the remote server. The -collector samples Linux /proc and /sys counters every 10 seconds and appends JSON -Lines. This module starts/stops the collector and reads those JSONL samples. -""" - -from __future__ import annotations - -import json -import logging -import shlex -from dataclasses import dataclass -from typing import Any - -from media_library_viewer_api.clients.ssh import RemoteSSHClient - -logger = logging.getLogger(__name__) - -# POSIX shell script copied to the remote server by start_resource_collector(). -# Keep this script bash-free because many NAS/media servers have minimal shells. -COLLECTOR_SCRIPT = r'''#!/bin/sh -set -u -OUT="${1:-/tmp/media_library_viewer_metrics.jsonl}" -INTERVAL="${2:-10}" -RETENTION_SECONDS="${3:-604800}" -MAX_LINES="${4:-70000}" -PRUNE_EVERY_SAMPLES="${5:-60}" -mkdir -p "$(dirname "$OUT")" - -echo "collector starting at $(date -Is 2>/dev/null || date), interval=${INTERVAL}s, retention=${RETENTION_SECONDS}s, max_lines=${MAX_LINES}, out=${OUT}" - -read_cpu() { - awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8+$9+$10, $5+$6, $6}' /proc/stat -} - -read_mem_pct() { - awk ' - /^MemTotal:/ {total=$2} - /^MemAvailable:/ {avail=$2} - END {if (total > 0) printf "%.2f", (total-avail)*100/total; else printf "0"} - ' /proc/meminfo -} - -read_net_bytes() { - awk ' - NR > 2 { - split($0, parts, ":") - iface = parts[1] - stats = parts[2] - gsub(/^[ \t]+|[ \t]+$/, "", iface) - gsub(/^[ \t]+|[ \t]+$/, "", stats) - if (iface == "lo" || iface == "" || stats == "") next - split(stats, values, /[ \t]+/) - # /proc/net/dev after the colon: - # receive bytes are field 1, transmit bytes are field 9. - # Trim the stats block before split; otherwise leading whitespace can make - # values[1] empty in some awk implementations, resulting in zero rates. - rx += values[1] + 0 - tx += values[9] + 0 - } - END {printf "%.0f %.0f", rx, tx} - ' /proc/net/dev -} - -read_disk_bytes() { - read_sectors=0 - written_sectors=0 - for dev in /sys/block/*; do - [ -r "$dev/stat" ] || continue - name="$(basename "$dev")" - case "$name" in - loop*|ram*|fd*|sr*) continue ;; - esac - # Linux /sys/block//stat fields: 3=sectors read, 7=sectors written. - # Use POSIX sh parsing instead of bash arrays so this works on minimal systems. - set -- $(cat "$dev/stat") - sectors_read="${3:-0}" - sectors_written="${7:-0}" - read_sectors=$((read_sectors + sectors_read)) - written_sectors=$((written_sectors + sectors_written)) - done - printf "%s %s" "$((read_sectors * 512))" "$((written_sectors * 512))" -} - -set -- $(read_cpu) -prev_total="${1:-0}" -prev_idle="${2:-0}" -prev_iowait="${3:-0}" -set -- $(read_net_bytes) -prev_rx="${1:-0}" -prev_tx="${2:-0}" -set -- $(read_disk_bytes) -prev_disk_read="${1:-0}" -prev_disk_write="${2:-0}" -prev_ts="$(date +%s)" -sample_count=0 - -prune_metrics_file() { - [ -f "$OUT" ] || return 0 - cutoff="$1" - tmp="${OUT}.$$.tmp" - awk -v cutoff="$cutoff" ' - match($0, /"ts":[0-9]+/) { - ts = substr($0, RSTART + 5, RLENGTH - 5) - if (ts >= cutoff) print $0 - } - ' "$OUT" | tail -n "$MAX_LINES" > "$tmp" && mv "$tmp" "$OUT" - rm -f "$tmp" -} - -while true; do - now_ts="$(date +%s)" - dt=$((now_ts - prev_ts)) - if [ "$dt" -le 0 ]; then dt=1; fi - - set -- $(read_cpu) - total="${1:-0}" - idle="${2:-0}" - iowait="${3:-0}" - set -- $(read_net_bytes) - rx="${1:-0}" - tx="${2:-0}" - set -- $(read_disk_bytes) - disk_read="${1:-0}" - disk_write="${2:-0}" - mem_pct="$(read_mem_pct)" - - total_delta=$((total - prev_total)) - idle_delta=$((idle - prev_idle)) - iowait_delta=$((iowait - prev_iowait)) - rx_delta=$((rx - prev_rx)) - tx_delta=$((tx - prev_tx)) - disk_read_delta=$((disk_read - prev_disk_read)) - disk_write_delta=$((disk_write - prev_disk_write)) - - cpu_pct="$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN {if (total > 0) printf "%.2f", (total-idle)*100/total; else printf "0"}')" - iowait_pct="$(awk -v total="$total_delta" -v iow="$iowait_delta" 'BEGIN {if (total > 0) printf "%.2f", iow*100/total; else printf "0"}')" - rx_bytes_per_sec="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')" - tx_bytes_per_sec="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')" - rx_bps="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')" - tx_bps="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')" - disk_read_bps="$(awk -v bytes="$disk_read_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')" - disk_write_bps="$(awk -v bytes="$disk_write_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')" - - printf '{"ts":%s,"cpu_pct":%s,"iowait_pct":%s,"mem_pct":%s,"net_rx_bytes_per_sec":%s,"net_tx_bytes_per_sec":%s,"net_rx_bps":%s,"net_tx_bps":%s,"disk_read_bps":%s,"disk_write_bps":%s}\n' \ - "$now_ts" "$cpu_pct" "$iowait_pct" "$mem_pct" "$rx_bytes_per_sec" "$tx_bytes_per_sec" "$rx_bps" "$tx_bps" "$disk_read_bps" "$disk_write_bps" >> "$OUT" - - sample_count=$((sample_count + 1)) - if [ $((sample_count % PRUNE_EVERY_SAMPLES)) -eq 0 ]; then - prune_metrics_file "$((now_ts - RETENTION_SECONDS))" - fi - - prev_total="$total" - prev_idle="$idle" - prev_iowait="$iowait" - prev_rx="$rx" - prev_tx="$tx" - prev_disk_read="$disk_read" - prev_disk_write="$disk_write" - prev_ts="$now_ts" - sleep "$INTERVAL" -done -''' - - -@dataclass(frozen=True) -class ResourceMonitorPaths: - """Remote file locations used by the lightweight resource collector.""" - - metrics_file: str = "/tmp/media_library_viewer_metrics.jsonl" - pid_file: str = "/tmp/media_library_viewer_metrics.pid" - script_file: str = "/tmp/media_library_viewer_metrics_collector.sh" - log_file: str = "/tmp/media_library_viewer_metrics.log" - - -def start_resource_collector( - ssh: RemoteSSHClient, - interval_seconds: int = 10, - retention_seconds: int = 7 * 24 * 60 * 60, - max_lines: int = 70_000, - paths: ResourceMonitorPaths = ResourceMonitorPaths(), -) -> str: - """Install and start the remote metrics collector if it is not running. - - Starting a fresh collector removes old metrics/log files because schema - changes during development can otherwise leave mixed JSONL records behind. - The collector prunes its own metrics file to 7 days / max_lines. - """ - command = f""" -cat > {shlex.quote(paths.script_file)} <<'MLV_RESOURCE_COLLECTOR' -{COLLECTOR_SCRIPT} -MLV_RESOURCE_COLLECTOR -chmod +x {shlex.quote(paths.script_file)} -if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then - echo "already running pid=$(cat {shlex.quote(paths.pid_file)})" -else - rm -f {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} - nohup {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {int(interval_seconds)} {int(retention_seconds)} {int(max_lines)} >> {shlex.quote(paths.log_file)} 2>&1 & - echo $! > {shlex.quote(paths.pid_file)} - echo "started pid=$(cat {shlex.quote(paths.pid_file)})" -fi -""" - logger.info( - "Starting remote resource collector interval=%ss retention=%ss max_lines=%s", interval_seconds, retention_seconds, max_lines - ) - result = ssh.run(command, timeout=20) - if result.exit_status != 0: - logger.warning("Failed to start remote resource collector: %s", result.stderr or result.stdout) - raise RuntimeError(result.stderr or result.stdout or "failed to start resource collector") - logger.info("Remote resource collector start response: %s", result.stdout.strip()) - return result.stdout.strip() - - -def stop_resource_collector(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str: - """Stop the remote collector process if the pid file points to one.""" - command = f""" -if [ -f {shlex.quote(paths.pid_file)} ]; then - pid="$(cat {shlex.quote(paths.pid_file)})" - if kill -0 "$pid" 2>/dev/null; then - kill "$pid" - echo "stopped pid=$pid" - else - echo "not running" - fi - rm -f {shlex.quote(paths.pid_file)} -else - echo "not running" -fi -""" - logger.info("Stopping remote resource collector") - result = ssh.run(command, timeout=20) - if result.exit_status != 0: - logger.warning("Failed to stop remote resource collector: %s", result.stderr or result.stdout) - raise RuntimeError(result.stderr or result.stdout or "failed to stop resource collector") - logger.info("Remote resource collector stop response: %s", result.stdout.strip()) - return result.stdout.strip() - - -def restart_resource_collector( - ssh: RemoteSSHClient, - interval_seconds: int = 10, - retention_seconds: int = 7 * 24 * 60 * 60, - max_lines: int = 70_000, - paths: ResourceMonitorPaths = ResourceMonitorPaths(), -) -> str: - logger.info("Restarting remote resource collector") - stop_message = stop_resource_collector(ssh, paths) - start_message = start_resource_collector(ssh, interval_seconds, retention_seconds, max_lines, paths) - return f"{stop_message}\n{start_message}" - - -def resource_collector_status(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str: - """Return a short human-readable status string for the dashboard.""" - command = f""" -if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then - echo "running" -else - echo "not running" -fi -""" - result = ssh.run(command, timeout=10) - if result.exit_status != 0: - logger.warning("Failed to read collector status: %s", result.stderr or result.stdout) - raise RuntimeError(result.stderr or result.stdout or "failed to check collector status") - status = result.stdout.strip() - logger.info("Resource collector status: %s", status) - return status - - -def resource_collector_debug_info(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str: - """Collect remote diagnostics for troubleshooting missing metrics.""" - command = f""" -echo "status:" -if [ -f {shlex.quote(paths.pid_file)} ]; then - pid="$(cat {shlex.quote(paths.pid_file)})" - echo "pid_file=$pid" - if kill -0 "$pid" 2>/dev/null; then echo "process=running"; else echo "process=not-running"; fi -else - echo "pid_file=missing" -fi - -echo "files:" -ls -l {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} 2>&1 || true - -echo "sample_count:" -if [ -f {shlex.quote(paths.metrics_file)} ]; then wc -l < {shlex.quote(paths.metrics_file)}; else echo 0; fi - -echo "last_samples:" -if [ -f {shlex.quote(paths.metrics_file)} ]; then tail -n 5 {shlex.quote(paths.metrics_file)}; fi - -echo "log_tail:" -if [ -f {shlex.quote(paths.log_file)} ]; then tail -n 40 {shlex.quote(paths.log_file)}; fi - -echo "netdev_snapshot:" -cat /proc/net/dev 2>&1 || true -""" - logger.info("Collecting resource collector diagnostics") - result = ssh.run(command, timeout=20) - output = (result.stdout or "") + (result.stderr or "") - logger.debug("Resource collector diagnostics length=%s", len(output)) - return output - - -def read_resource_metrics(ssh: RemoteSSHClient, max_lines: int = 1000, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> list[dict[str, Any]]: - """Read recent JSONL metric samples from the remote collector file.""" - command = f"test -f {shlex.quote(paths.metrics_file)} && tail -n {int(max_lines)} {shlex.quote(paths.metrics_file)} || true" - logger.debug("Reading up to %s resource metric lines", max_lines) - result = ssh.run(command, timeout=20) - if result.exit_status != 0: - logger.warning("Failed to read resource metrics: %s", result.stderr or result.stdout) - raise RuntimeError(result.stderr or result.stdout or "failed to read resource metrics") - rows = [] - for line in result.stdout.splitlines(): - line = line.strip() - if not line: - continue - try: - rows.append(json.loads(line)) - except json.JSONDecodeError: - continue - return rows - - -def disk_space(ssh: RemoteSSHClient, path: str = "/") -> dict[str, Any]: - """Return df information for the filesystem containing ``path``.""" - command = ( - "df -P -B1 -- " - + shlex.quote(path or "/") - + " | awk 'NR==2 {printf \"{\\\"filesystem\\\":\\\"%s\\\",\\\"size\\\":%s,\\\"used\\\":%s,\\\"available\\\":%s,\\\"used_pct\\\":\\\"%s\\\",\\\"mount\\\":\\\"%s\\\"}\", $1,$2,$3,$4,$5,$6}'" - ) - logger.debug("Reading disk space for path=%s", path) - result = ssh.run(command, timeout=20) - if result.exit_status != 0 or not result.stdout.strip(): - logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout) - raise RuntimeError(result.stderr or result.stdout or "failed to read disk space") - data = json.loads(result.stdout) - logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct")) - return data diff --git a/backend/src/media_library_viewer_api/clients/ssh.py b/backend/src/media_library_viewer_api/clients/ssh.py index 1b9adcb..216cf6e 100644 --- a/backend/src/media_library_viewer_api/clients/ssh.py +++ b/backend/src/media_library_viewer_api/clients/ssh.py @@ -75,14 +75,10 @@ class RemoteSSHClient: client = paramiko.SSHClient() client.load_system_host_keys() known_hosts_file = Path(self.known_hosts_path) if self.known_hosts_path else None - trusted_before = bool( - known_hosts_file and has_known_host(self.host, self.port, known_hosts_file) - ) + trusted_before = bool(known_hosts_file and has_known_host(self.host, self.port, known_hosts_file)) if known_hosts_file and known_hosts_file.is_file(): client.load_host_keys(str(known_hosts_file)) - client.set_missing_host_key_policy( - paramiko.RejectPolicy() if trusted_before else paramiko.AutoAddPolicy() - ) + client.set_missing_host_key_policy(paramiko.RejectPolicy() if trusted_before else paramiko.AutoAddPolicy()) connect_kwargs: dict[str, Any] = { "hostname": self.host, "port": self.port, diff --git a/backend/src/media_library_viewer_api/config.py b/backend/src/media_library_viewer_api/config.py index 4dc5036..ccb9f27 100644 --- a/backend/src/media_library_viewer_api/config.py +++ b/backend/src/media_library_viewer_api/config.py @@ -23,6 +23,7 @@ class Settings(BaseSettings): # Logging log_level: str = "INFO" + log_format: str = "text" # "text" or "json" # Auth / OIDC (Authentik-compatible JWT validation) auth_enabled: bool = False @@ -48,7 +49,6 @@ class Settings(BaseSettings): ssh_port: int = 22 ssh_key_directory: str = "" ssh_key_name: str = "" - ssh_key_file: str = "/run/secrets/ssh_private_key" ssh_password: str = "" ssh_known_hosts_path: str = "" @@ -57,6 +57,12 @@ class Settings(BaseSettings): monitoring_poll_initial_delay_seconds: int = 20 monitoring_action_retention_days: int = 30 + # Observability + prometheus_enabled: bool = True + prometheus_file_sd_dir: str = "/app/backend/.cache/prometheus-file-sd" + alertmanager_url: str = "http://alertmanager:9093" + alertmanager_webhook_url: str = "" # Optional receiver for alertmanager webhook notifications + # Remote paths remote_media_root: str = "" remote_path_prefix: str = "" @@ -72,8 +78,6 @@ class Settings(BaseSettings): @property def ssh_key_path(self) -> str: - if self.ssh_key_file: - return self.ssh_key_file if not self.ssh_key_directory or not self.ssh_key_name: return "" return str(Path(self.ssh_key_directory) / self.ssh_key_name) @@ -82,7 +86,7 @@ class Settings(BaseSettings): def ssh_known_hosts_file(self) -> Path: return Path(self.ssh_known_hosts_path or ".cache/media_library_viewer/known_hosts") - model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"} + model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"} # type: ignore[assignment] def _find_env_file() -> str | None: diff --git a/backend/src/media_library_viewer_api/dependencies.py b/backend/src/media_library_viewer_api/dependencies.py index 196b155..bddecd0 100644 --- a/backend/src/media_library_viewer_api/dependencies.py +++ b/backend/src/media_library_viewer_api/dependencies.py @@ -19,12 +19,16 @@ from media_library_viewer_api.clients.jellyseerr import JellyseerrClient from media_library_viewer_api.clients.local import LocalCommandClient from media_library_viewer_api.clients.ssh import RemoteSSHClient from media_library_viewer_api.config import get_settings -from media_library_viewer_api.services.mail_queue import MailQueue, get_mail_queue as _get_mail_queue +from media_library_viewer_api.services.mail_queue import MailQueue +from media_library_viewer_api.services.mail_queue import get_mail_queue as _get_mail_queue from media_library_viewer_api.services.monitoring_poller import ( MonitoringPoller, +) +from media_library_viewer_api.services.monitoring_poller import ( get_monitoring_poller as _get_monitoring_poller, ) -from media_library_viewer_api.services.settings_store import SettingsStore, get_settings_store as _get_settings_store +from media_library_viewer_api.services.settings_store import SettingsStore +from media_library_viewer_api.services.settings_store import get_settings_store as _get_settings_store logger = logging.getLogger(__name__) @@ -39,7 +43,9 @@ def _request_machine_id(request: Request | None) -> str | None: @lru_cache(maxsize=32) def _jellyfin_client_for(cache_key: tuple[str, str, str]) -> JellyfinClient: machine_id, url, api_key = cache_key - logger.info("Creating Jellyfin client machine_id=%s url=%s", machine_id or "", url.rstrip("/") or "") + logger.info( + "Creating Jellyfin client machine_id=%s url=%s", machine_id or "", url.rstrip("/") or "" + ) return JellyfinClient(url, api_key) @@ -52,13 +58,19 @@ def _jellyseerr_client_for(cache_key: tuple[str, str]) -> JellyseerrClient | Non api_key = (settings or {}).get("jellyseerr_api_key") if settings else "" if not api_key: return None - logger.info("Creating Jellyseerr client machine_id=%s url=%s", machine_id or "", url.rstrip("/") or "") + logger.info( + "Creating Jellyseerr client machine_id=%s url=%s", machine_id or "", url.rstrip("/") or "" + ) return JellyseerrClient(url, api_key) @lru_cache(maxsize=32) -def _ssh_client_for(cache_key: tuple[str, str, str, int, str, str | None, str | None, str | None, str | None]) -> RemoteSSHClient: - machine_id, host, username, port, key_filename, password, private_key, private_key_passphrase, known_hosts_path = cache_key +def _ssh_client_for( + cache_key: tuple[str, str, str, int, str, str | None, str | None, str | None, str | None], +) -> RemoteSSHClient: + machine_id, host, username, port, key_filename, password, private_key, private_key_passphrase, known_hosts_path = ( + cache_key + ) logger.info( "Creating SSH client machine_id=%s host=%s user=%s port=%s key=%s password=%s private_key=%s passphrase=%s", machine_id or "", @@ -141,7 +153,9 @@ def get_jellyfin_client(request: Request = None) -> JellyfinClient: cache_key = (machine["id"], machine["jellyfin_url"], machine.get("jellyfin_api_key") or "") return _jellyfin_client_for(cache_key) - raise RuntimeError("No Jellyfin machine is configured. Add a machine with jellyfin_url and jellyfin_api_key in Settings.") + raise RuntimeError( + "No Jellyfin machine is configured. Add a machine with jellyfin_url and jellyfin_api_key in Settings." + ) def get_jellyseerr_client(request: Request = None) -> JellyseerrClient | None: @@ -160,6 +174,37 @@ def get_jellyseerr_client(request: Request = None) -> JellyseerrClient | None: return None +def _ssh_client_from_machine_config(machine: dict[str, Any], store: SettingsStore | None = None) -> RemoteSSHClient: + """Build a RemoteSSHClient from a machine config dict.""" + store = store or get_settings_store() + known_hosts_path = get_settings().ssh_known_hosts_file + key_data = None + key_passphrase = None + ssh_key_id = str(machine.get("ssh_key_id") or "").strip() + if ssh_key_id: + ssh_key = store.get_ssh_key(ssh_key_id) + if ssh_key: + key_data = ssh_key.get("private_key") or None + key_passphrase = ssh_key.get("passphrase") or None + if not key_data and machine.get("ssh_private_key"): + key_data = machine.get("ssh_private_key") or None + key_passphrase = machine.get("ssh_private_key_passphrase") or None + cache_key = ( + machine["id"], + machine["host"], + machine["username"], + int(machine.get("port") or 22), + f"{machine.get('key_directory')}/{machine.get('key_name')}" + if machine.get("key_directory") and machine.get("key_name") + else "", + machine.get("password") or None, + key_data, + key_passphrase, + str(known_hosts_path), + ) + return _ssh_client_for(cache_key) + + def get_ssh_client(request: Request = None): """Return a command client for the selected machine or legacy env fallback.""" store = get_settings_store() @@ -172,30 +217,7 @@ def get_ssh_client(request: Request = None): logger.info("Creating LocalCommandClient machine_id=%s", machine["id"]) return LocalCommandClient() if machine and machine.get("host") and machine.get("username"): - known_hosts_path = get_settings().ssh_known_hosts_file - key_data = None - key_passphrase = None - ssh_key_id = str(machine.get("ssh_key_id") or "").strip() - if ssh_key_id: - ssh_key = store.get_ssh_key(ssh_key_id) - if ssh_key: - key_data = ssh_key.get("private_key") or None - key_passphrase = ssh_key.get("passphrase") or None - if not key_data and machine.get("ssh_private_key"): - key_data = machine.get("ssh_private_key") or None - key_passphrase = machine.get("ssh_private_key_passphrase") or None - cache_key = ( - machine["id"], - machine["host"], - machine["username"], - int(machine.get("port") or 22), - f"{machine.get('key_directory')}/{machine.get('key_name')}" if machine.get("key_directory") and machine.get("key_name") else "", - machine.get("password") or None, - key_data, - key_passphrase, - str(known_hosts_path), - ) - return _ssh_client_for(cache_key) + return _ssh_client_from_machine_config(machine, store) settings = get_settings() logger.info( @@ -209,7 +231,19 @@ def get_ssh_client(request: Request = None): ) if not settings.ssh_key_path: raise RuntimeError("No SSH machine is configured and SSH key settings must be configured") - return _ssh_client_for(("legacy", settings.ssh_host, settings.ssh_username, settings.ssh_port, settings.ssh_key_path, settings.ssh_password or None, None, None, str(settings.ssh_known_hosts_file))) + return _ssh_client_for( + ( + "legacy", + settings.ssh_host, + settings.ssh_username, + settings.ssh_port, + settings.ssh_key_path, + settings.ssh_password or None, + None, + None, + str(settings.ssh_known_hosts_file), + ) + ) def get_mail_queue() -> MailQueue: diff --git a/backend/src/media_library_viewer_api/domain/media.py b/backend/src/media_library_viewer_api/domain/media.py index d39fe35..1a12314 100644 --- a/backend/src/media_library_viewer_api/domain/media.py +++ b/backend/src/media_library_viewer_api/domain/media.py @@ -32,7 +32,11 @@ def media_streams(item: dict[str, Any], stream_type: str | None = None) -> list[ streams.extend(source.get("MediaStreams") or []) if stream_type is None: return streams - return [stream for stream in streams if str(stream.get("Type") or stream.get("codec_type") or "").lower() == stream_type.lower()] + return [ + stream + for stream in streams + if str(stream.get("Type") or stream.get("codec_type") or "").lower() == stream_type.lower() + ] def stream_value(stream: dict[str, Any], *keys: str) -> Any: diff --git a/backend/src/media_library_viewer_api/jobs.py b/backend/src/media_library_viewer_api/jobs.py index aad7e3c..3418fe3 100644 --- a/backend/src/media_library_viewer_api/jobs.py +++ b/backend/src/media_library_viewer_api/jobs.py @@ -52,6 +52,42 @@ JOB_TEMPLATES: dict[str, JobTemplate] = { description="Lists empty directories under the selected path. Does not delete anything.", command_template="find {path} -type d -empty -print", ), + "install_node_exporter": JobTemplate( + name="Install Node Exporter", + description="Downloads and installs prometheus-node-exporter via package manager (apt/dnf/yum/zypper).", + command_template=( + "set -e; " + "if command -v apt-get >/dev/null 2>&1; then " + "sudo apt-get update && sudo apt-get install -y prometheus-node-exporter; " + "elif command -v dnf >/dev/null 2>&1; then " + "sudo dnf install -y prometheus-node-exporter; " + "elif command -v yum >/dev/null 2>&1; then " + "sudo yum install -y prometheus-node-exporter; " + "elif command -v zypper >/dev/null 2>&1; then " + "sudo zypper install -y prometheus-node-exporter; " + "else echo 'No supported package manager found' >&2; exit 1; " + "fi; " + "sudo systemctl enable --now prometheus-node-exporter; " + "echo installed at {path}" + ), + ), + "restart_node_exporter": JobTemplate( + name="Restart Node Exporter", + description="Restarts the prometheus-node-exporter systemd service.", + command_template="sudo systemctl restart prometheus-node-exporter; echo restarted at {path}", + ), + "node_exporter_status": JobTemplate( + name="Node Exporter status", + description="Checks whether prometheus-node-exporter is installed, enabled, and running.", + command_template=( + "systemctl status prometheus-node-exporter --no-pager || true; " + "echo '---'; " + "command -v node_exporter >/dev/null 2>&1 " + "&& node_exporter --version 2>&1 | head -1 " + "|| echo 'node_exporter binary not found'; " + "echo checked {path}" + ), + ), } diff --git a/backend/src/media_library_viewer_api/logging_utils.py b/backend/src/media_library_viewer_api/logging_utils.py index fc922da..30345d3 100644 --- a/backend/src/media_library_viewer_api/logging_utils.py +++ b/backend/src/media_library_viewer_api/logging_utils.py @@ -4,19 +4,42 @@ from __future__ import annotations import logging import os +from typing import Any from urllib.parse import urlsplit +from pythonjsonlogger import jsonlogger -def configure_logging(level_name: str | None = None) -> int: + +def _json_formatter() -> logging.Formatter: + """Return a JSON formatter that preserves timestamp, level, logger, and message.""" + return jsonlogger.JsonFormatter( + fmt="%(timestamp)s %(level)s %(name)s %(message)s", + rename_fields={"asctime": "timestamp", "levelname": "level"}, + ) + + +def _text_formatter() -> logging.Formatter: + """Return the legacy plain-text formatter.""" + return logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + + +def configure_logging(level_name: str | None = None, log_format: str | None = None) -> int: """Configure root logging once and return the numeric log level.""" resolved_name = (level_name or os.getenv("LOG_LEVEL", "INFO")).upper() level = getattr(logging, resolved_name, logging.INFO) + use_json = (log_format or os.getenv("LOG_FORMAT", "text")).lower() == "json" + root = logging.getLogger() if not root.handlers: - logging.basicConfig( - level=level, - format="%(asctime)s %(levelname)s %(name)s: %(message)s", - ) + handler = logging.StreamHandler() + handler.setFormatter(_json_formatter() if use_json else _text_formatter()) + logging.basicConfig(level=level, handlers=[handler]) + else: + # Reconfigure existing handler formatters if the format changed. + formatter = _json_formatter() if use_json else _text_formatter() + for handler in root.handlers: + handler.setFormatter(formatter) + root.setLevel(level) logging.getLogger("media_library_viewer_api").setLevel(level) logging.getLogger("uvicorn").setLevel(level) @@ -40,6 +63,7 @@ def describe_settings(settings: object) -> dict[str, str]: """Return a secret-safe summary of the current backend settings.""" return { "log_level": str(getattr(settings, "log_level", "INFO") or "INFO").upper(), + "log_format": str(getattr(settings, "log_format", "text") or "text").lower(), "auth_enabled": str(bool(getattr(settings, "auth_enabled", True))), "oidc_issuer_url": _sanitize_url(getattr(settings, "oidc_issuer_url", "")), "oidc_audience": getattr(settings, "oidc_audience", "") or "", @@ -60,3 +84,18 @@ def describe_settings(settings: object) -> dict[str, str]: "remote_media_root": getattr(settings, "remote_media_root", "") or "", "remote_path_prefix": getattr(settings, "remote_path_prefix", "") or "", } + + +def sanitize_log_extra(extra: dict[str, Any] | None) -> dict[str, Any]: + """Remove obvious secret keys from log extra fields before emission.""" + if not extra: + return {} + blocked_suffixes = ("_password", "_secret", "_token", "_key", "private_key", "passphrase") + sanitized: dict[str, Any] = {} + for key, value in extra.items(): + lower_key = key.lower() + if any(lower_key.endswith(suffix) for suffix in blocked_suffixes): + sanitized[key] = "" if value else "" + else: + sanitized[key] = value + return sanitized diff --git a/backend/src/media_library_viewer_api/main.py b/backend/src/media_library_viewer_api/main.py index ab06a92..7ff37d9 100644 --- a/backend/src/media_library_viewer_api/main.py +++ b/backend/src/media_library_viewer_api/main.py @@ -7,13 +7,20 @@ import time from contextlib import asynccontextmanager import uvicorn -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import Response as FastAPIResponse from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settings from media_library_viewer_api.config import get_settings -from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller -from media_library_viewer_api.logging_utils import configure_logging, describe_settings +from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller, get_settings_store +from media_library_viewer_api.logging_utils import configure_logging, describe_settings, sanitize_log_extra +from media_library_viewer_api.observability import ( + get_request_id, + metrics_payload, + record_request, + set_current_request_id, +) from media_library_viewer_api.routers import backups as backups_router from media_library_viewer_api.routers import dashboard, files, jobs, media, monitoring, tasks, users from media_library_viewer_api.routers.settings import router as settings_router @@ -28,10 +35,16 @@ logger = logging.getLogger(__name__) async def lifespan(app: FastAPI): """Application lifespan — startup/shutdown.""" settings = get_settings() - configure_logging(settings.log_level) + configure_logging(settings.log_level, settings.log_format) validate_auth_settings(settings) logger.info("Backend startup complete: %s", describe_settings(settings)) logger.info("Managed known_hosts will be populated lazily on first successful SSH connection") + try: + from media_library_viewer_api.services.targets import write_prometheus_targets + + write_prometheus_targets(get_settings_store()) + except Exception: + logger.exception("Failed to write Prometheus file-SD targets during startup") mail_queue = get_mail_queue() monitoring_poller = get_monitoring_poller() backup_poller = get_backup_poller() @@ -49,8 +62,7 @@ app = FastAPI( title="Manage API", version=get_backend_version(), description=( - "Manage API for Jellyfin media browsing, SSH file inspection, " - "server monitoring, and JWT-protected access." + "Manage API for Jellyfin media browsing, SSH file inspection, server monitoring, and JWT-protected access." ), lifespan=lifespan, ) @@ -69,30 +81,51 @@ app.add_middleware( allow_headers=["*"], ) + @app.middleware("http") async def enforce_jwt_auth(request: Request, call_next): - if request.url.path in {"/api/health", "/api/version"}: + if request.url.path in {"/api/health", "/api/version", "/metrics"}: return await call_next(request) return await require_jwt_auth(request, call_next) + @app.middleware("http") async def log_requests(request: Request, call_next): - """Log every API request with timing and outcome.""" + """Log every API request with timing, outcome, and request id.""" start = time.perf_counter() + request_id = get_request_id(request) + set_current_request_id(request_id) + request.state.request_id = request_id client_host = request.client.host if request.client else "unknown" - logger.info("request start %s %s client=%s", request.method, request.url.path, client_host) + extra = sanitize_log_extra({"request_id": request_id, "client_host": client_host}) + logger.info("request start %s %s", request.method, request.url.path, extra=extra) try: response = await call_next(request) except Exception: - logger.exception("request error %s %s client=%s", request.method, request.url.path, client_host) + logger.exception( + "request error %s %s", + request.method, + request.url.path, + extra=sanitize_log_extra({"request_id": request_id, "client_host": client_host}), + ) raise elapsed_ms = (time.perf_counter() - start) * 1000.0 + response.headers["X-Request-Id"] = request_id + record_request(request, response, elapsed_ms / 1000.0) logger.info( "request end %s %s status=%s elapsed_ms=%.1f", request.method, request.url.path, response.status_code, elapsed_ms, + extra=sanitize_log_extra( + { + "request_id": request_id, + "client_host": client_host, + "status_code": response.status_code, + "elapsed_ms": round(elapsed_ms, 1), + } + ), ) return response @@ -123,5 +156,12 @@ def version_info() -> dict[str, str]: return get_version_info() +@app.get("/metrics") +def metrics() -> Response: + """Expose Prometheus metrics.""" + data, content_type = metrics_payload() + return FastAPIResponse(content=data, media_type=content_type) + + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/backend/src/media_library_viewer_api/observability.py b/backend/src/media_library_viewer_api/observability.py new file mode 100644 index 0000000..5ea5bbc --- /dev/null +++ b/backend/src/media_library_viewer_api/observability.py @@ -0,0 +1,154 @@ +"""Application observability: metrics and request correlation. + +This module owns Prometheus metrics and request-id generation so that the +rest of the backend can stay focused on business logic. +""" + +from __future__ import annotations + +import uuid +from contextvars import ContextVar +from typing import Any + +from fastapi import Request, Response +from prometheus_client import ( + CONTENT_TYPE_LATEST, + Counter, + Gauge, + Histogram, + generate_latest, +) + +# Context-local request id for code paths that cannot receive a Request object. +_current_request_id: ContextVar[str | None] = ContextVar("request_id", default=None) + +# Request metrics +REQUESTS_TOTAL = Counter( + "manage_api_requests_total", + "Total API requests", + ["method", "path", "status_code"], +) +REQUEST_DURATION = Histogram( + "manage_api_request_duration_seconds", + "API request duration", + ["method", "path"], + buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0), +) + +# Operation metrics +SSH_COMMANDS_TOTAL = Counter( + "manage_ssh_commands_total", + "Total SSH/local commands executed", + ["machine_id", "action", "status"], +) +SSH_COMMAND_DURATION = Histogram( + "manage_ssh_command_duration_seconds", + "SSH/local command duration", + ["machine_id", "action"], + buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0), +) + +MEDIA_INDEX_BUILDS_TOTAL = Counter( + "manage_media_index_builds_total", + "Total media index build attempts", + ["status"], +) +MEDIA_INDEX_BUILD_DURATION = Histogram( + "manage_media_index_build_duration_seconds", + "Media index build duration", + buckets=(1.0, 5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0), +) + +BACKUP_RUNS_TOTAL = Counter( + "manage_backup_runs_total", + "Total backup runs", + ["job_name", "status"], +) + +BACKUP_RUNS_LAST_SUCCESS = Gauge( + "manage_backup_runs_last_success_timestamp", + "Unix timestamp of the last successful backup run per job", + ["job_name"], +) + +MAIL_QUEUE_SIZE = Counter( + "manage_mail_queue_messages_total", + "Total messages enqueued", + ["status"], +) + + +def set_current_request_id(request_id: str | None) -> None: + """Set the context-local request id.""" + _current_request_id.set(request_id) + + +def get_current_request_id() -> str | None: + """Return the current context-local request id or None.""" + return _current_request_id.get() + + +def generate_request_id() -> str: + """Return a short unique request id.""" + return uuid.uuid4().hex[:16] + + +def get_request_id(request: Request | None = None) -> str: + """Resolve a request id from the request header, context, or a new value.""" + if request is not None: + header = request.headers.get("x-request-id") or request.headers.get("x-correlation-id") + if header: + return header.strip() + existing = _current_request_id.get() + if existing: + return existing + new_id = generate_request_id() + _current_request_id.set(new_id) + return new_id + + +def metrics_payload() -> tuple[bytes, str]: + """Return the Prometheus metrics payload and content type.""" + return generate_latest(), CONTENT_TYPE_LATEST + + +def record_request(request: Request, response: Response, duration_seconds: float) -> None: + """Record Prometheus metrics for a completed request.""" + status = str(response.status_code) + path = request.url.path + method = request.method + REQUESTS_TOTAL.labels(method=method, path=path, status_code=status).inc() + REQUEST_DURATION.labels(method=method, path=path).observe(duration_seconds) + + +def record_ssh_command(machine_id: str, action: str, status: str, duration_seconds: float) -> None: + """Record metrics for an SSH/local command.""" + SSH_COMMANDS_TOTAL.labels(machine_id=machine_id or "unknown", action=action, status=status).inc() + SSH_COMMAND_DURATION.labels(machine_id=machine_id or "unknown", action=action).observe(duration_seconds) + + +def record_media_index_build(status: str, duration_seconds: float | None = None) -> None: + """Record metrics for a media index build.""" + MEDIA_INDEX_BUILDS_TOTAL.labels(status=status).inc() + if duration_seconds is not None: + MEDIA_INDEX_BUILD_DURATION.observe(duration_seconds) + + +def record_backup_run(job_name: str, status: str, success: bool = False) -> None: + """Record metrics for a backup run.""" + job_name = job_name or "unknown" + BACKUP_RUNS_TOTAL.labels(job_name=job_name, status=status).inc() + if success: + BACKUP_RUNS_LAST_SUCCESS.labels(job_name=job_name).set_to_current_time() + + +def record_mail_queue(status: str) -> None: + """Record metrics for a mail queue message outcome.""" + MAIL_QUEUE_SIZE.labels(status=status).inc() + + +def log_extra(request: Request | None = None, **kwargs: Any) -> dict[str, Any]: + """Build a standard extra dict for structured logging.""" + extra: dict[str, Any] = {"request_id": get_request_id(request)} + extra.update(kwargs) + return extra diff --git a/backend/src/media_library_viewer_api/path_utils.py b/backend/src/media_library_viewer_api/path_utils.py index f864fee..ba25603 100644 --- a/backend/src/media_library_viewer_api/path_utils.py +++ b/backend/src/media_library_viewer_api/path_utils.py @@ -57,7 +57,7 @@ def map_path_to_media_root(path: str, media_root: str) -> str: if root_anchor in raw_parts: anchor_index = raw_parts.index(root_anchor) - remainder_parts = raw_parts[anchor_index + 1:] + remainder_parts = raw_parts[anchor_index + 1 :] resolved = posixpath.join(normalized_root, *remainder_parts) if remainder_parts else normalized_root logger.debug("Mapped path to media root path=%s media_root=%s resolved=%s", path, normalized_root, resolved) return resolved diff --git a/backend/src/media_library_viewer_api/routers/backups.py b/backend/src/media_library_viewer_api/routers/backups.py index 1df818c..085fb6a 100644 --- a/backend/src/media_library_viewer_api/routers/backups.py +++ b/backend/src/media_library_viewer_api/routers/backups.py @@ -8,6 +8,7 @@ from ..models.backups import ( BackupReportRequest, BackupRunResponse, ) +from ..observability import record_backup_run from ..services.backup_alert_engine import generate_alerts_for_run from ..services.settings_store import SettingsStore, get_settings_store @@ -17,20 +18,24 @@ router = APIRouter(prefix="/api/backups", tags=["backups"]) def _get_or_create_job(store: SettingsStore, report: BackupReportRequest) -> dict[str, Any]: job = store.get_backup_job_by_name(report.name) if not job: - job = store.upsert_backup_job({ - "name": report.name, - "source": report.source, - "target": report.target, - "schedule_interval_seconds": report.schedule_interval_seconds, - }) + job = store.upsert_backup_job( + { + "name": report.name, + "source": report.source, + "target": report.target, + "schedule_interval_seconds": report.schedule_interval_seconds, + } + ) elif report.schedule_interval_seconds: - store.upsert_backup_job({ - "id": job["id"], - "name": report.name, - "source": report.source, - "target": report.target, - "schedule_interval_seconds": report.schedule_interval_seconds, - }) + store.upsert_backup_job( + { + "id": job["id"], + "name": report.name, + "source": report.source, + "target": report.target, + "schedule_interval_seconds": report.schedule_interval_seconds, + } + ) job = store.get_backup_job(job["id"]) return job @@ -61,6 +66,7 @@ def post_backup_report( "details": report.details, } run = store.create_backup_run(run_data) + record_backup_run(job_name=report.name, status=report.status, success=report.status == "success") # Generate alerts previous_runs = store.list_backup_runs(job_id=job["id"], status="success", limit=20) @@ -93,6 +99,7 @@ def post_backup_start( "status": "in_progress", } run = store.create_backup_run(run_data) + record_backup_run(job_name=report.name, status="in_progress") # Map details -> details_json for response model run["details_json"] = run.pop("details", None) return BackupRunResponse(**run) diff --git a/backend/src/media_library_viewer_api/routers/dashboard.py b/backend/src/media_library_viewer_api/routers/dashboard.py index 9897a99..6a7cf8b 100644 --- a/backend/src/media_library_viewer_api/routers/dashboard.py +++ b/backend/src/media_library_viewer_api/routers/dashboard.py @@ -11,12 +11,10 @@ from fastapi import APIRouter, Depends from media_library_viewer_api.clients.jellyfin import JellyfinClient from media_library_viewer_api.dependencies import ( get_jellyfin_client, - get_monitoring_poller, get_settings_store, get_user_id, ) from media_library_viewer_api.models.backups import BackupDashboardSummary -from media_library_viewer_api.services.monitoring_actions import collect_machine_overview from media_library_viewer_api.services.settings_store import SettingsStore logger = logging.getLogger(__name__) @@ -46,52 +44,6 @@ def get_library_counts( return client.library_item_counts(user_id, libraries) -@router.get("/monitoring") -def get_monitoring_overview( - store=Depends(get_settings_store), -) -> dict[str, Any]: - """Return one lightweight monitoring row per configured machine.""" - machines = [m for m in store.list_machines() if m.get("enabled")] - rows: list[dict[str, Any]] = [] - for machine in machines: - try: - rows.append(collect_machine_overview(machine, store)) - except Exception as exc: - logger.exception( - "Dashboard monitoring overview failed for machine_id=%s", - machine.get("id"), - ) - rows.append({ - "machine": {k: machine.get(k) for k in ( - "id", "name", "mode", "enabled", "host", "port", "username", - "media_root", "path_prefix", "notes", - )}, - "status": "", - "status_error": str(exc), - "metrics_error": str(exc), - "disk_error": str(exc), - "sample_count": 0, - "latest_sample": None, - "cpu_summary": None, - "iowait_summary": None, - "mem_summary": None, - "net_rx_summary": None, - "net_tx_summary": None, - "disk_read_summary": None, - "disk_write_summary": None, - "disk": None, - }) - poller = get_monitoring_poller().snapshot() - enabled_count = sum(1 for machine in machines if machine.get("enabled")) - logger.info("Dashboard monitoring machines=%s enabled=%s", len(machines), enabled_count) - return { - "poller": poller, - "machines": rows, - "total": len(machines), - "enabled": enabled_count, - } - - @router.get("/shortcuts") def get_shortcuts( store=Depends(get_settings_store), @@ -144,8 +96,8 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[ has_item = bool(item) series = item.get("SeriesName") or "" title = ( - f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown") - ) if has_item else "(idle)" + (f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")) if has_item else "(idle)" + ) if not has_item: state_label = "idle" @@ -162,16 +114,18 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[ if not transcode_type: transcode_type.append("active") - results.append({ - "user": session.get("UserName") or "Unknown", - "title": title, - "type": item.get("Type", "") if has_item else "", - "state": state_label, - "transcoding": "yes" if is_transcoding else "no", - "transcoding_type": ", ".join(transcode_type), - "device": session.get("DeviceName") or session.get("Client") or "", - "session_id": session.get("Id") or "", - }) + results.append( + { + "user": session.get("UserName") or "Unknown", + "title": title, + "type": item.get("Type", "") if has_item else "", + "state": state_label, + "transcoding": "yes" if is_transcoding else "no", + "transcoding_type": ", ".join(transcode_type), + "device": session.get("DeviceName") or session.get("Client") or "", + "session_id": session.get("Id") or "", + } + ) return results diff --git a/backend/src/media_library_viewer_api/routers/media.py b/backend/src/media_library_viewer_api/routers/media.py index 2d1e840..8744e23 100644 --- a/backend/src/media_library_viewer_api/routers/media.py +++ b/backend/src/media_library_viewer_api/routers/media.py @@ -16,6 +16,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status from media_library_viewer_api.clients.jellyfin import JellyfinClient from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id +from media_library_viewer_api.observability import record_media_index_build from media_library_viewer_api.services.media_index import MediaIndex logger = logging.getLogger(__name__) @@ -169,6 +170,7 @@ def post_build_index( }, ) media_status = index.status() + record_media_index_build(status="started") logger.info("Media index build started pid=%s", process.pid) return {"status": "started", **_serialize_status(media_status)} @@ -256,6 +258,7 @@ def force_stop_build(index: MediaIndex = Depends(get_media_index)) -> dict[str, }, ) media_status = index.status() + record_media_index_build(status="force_stopped") return {"status": "force_stopped", **_serialize_status(media_status)} diff --git a/backend/src/media_library_viewer_api/routers/monitoring.py b/backend/src/media_library_viewer_api/routers/monitoring.py index 7a1a90e..a96df3c 100644 --- a/backend/src/media_library_viewer_api/routers/monitoring.py +++ b/backend/src/media_library_viewer_api/routers/monitoring.py @@ -1,31 +1,70 @@ -"""Monitoring router — metrics, collector controls, and per-machine status.""" +"""Monitoring router — disk checks, action history, and observability stack status.""" from __future__ import annotations import logging -import time from typing import Any -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Body, Depends, HTTPException, Query -from media_library_viewer_api.clients.resources import ( - disk_space, - read_resource_metrics, - resource_collector_status, -) from media_library_viewer_api.config import get_settings from media_library_viewer_api.dependencies import get_settings_store from media_library_viewer_api.services.monitoring_actions import ( - poll_machine_diagnostics, + disk_space, run_machine_operation, - start_collector, - stop_collector, - restart_collector, ) from media_library_viewer_api.services.settings_store import SettingsStore +from media_library_viewer_api.services.targets import build_node_exporter_targets logger = logging.getLogger(__name__) + +def _alertmanager_client() -> Any: + """Return a simple HTTP client for the configured Alertmanager URL.""" + import requests + + settings = get_settings() + return requests.Session(), settings.alertmanager_url + + +def _webhook_client() -> Any: + """Return a simple HTTP client for the optional webhook receiver URL.""" + import requests + + settings = get_settings() + return requests.Session(), settings.alertmanager_webhook_url + + +def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]: + """Build a UI-friendly summary from Alertmanager /api/v1/alerts payload.""" + by_severity: dict[str, int] = {} + open_alerts: list[dict[str, Any]] = [] + for alert in alerts: + labels = alert.get("labels") or {} + annotations = alert.get("annotations") or {} + severity = labels.get("severity", "unknown") + by_severity[severity] = by_severity.get(severity, 0) + 1 + open_alerts.append( + { + "name": labels.get("alertname", "unknown"), + "severity": severity, + "category": labels.get("category", ""), + "job_name": labels.get("job_name", labels.get("job", "")), + "summary": annotations.get("summary", ""), + "description": annotations.get("description", ""), + "active_since": alert.get("startsAt"), + "state": alert.get("status", "firing"), + "labels": labels, + } + ) + open_alerts.sort(key=lambda a: (a["severity"] not in {"critical", "warning"}, a["severity"], a["name"])) + return { + "total": len(open_alerts), + "by_severity": by_severity, + "alerts": open_alerts[:50], + } + + router = APIRouter(prefix="/api/monitoring", tags=["monitoring"]) @@ -63,7 +102,11 @@ def get_poller_status() -> dict[str, Any]: from media_library_viewer_api.dependencies import get_monitoring_poller poller = get_monitoring_poller().snapshot() - logger.info("Monitoring poller status requested running=%s poll_count=%s", poller.get("worker_running"), poller.get("poll_count")) + logger.info( + "Monitoring poller status requested running=%s poll_count=%s", + poller.get("worker_running"), + poller.get("poll_count"), + ) return poller @@ -81,53 +124,6 @@ def get_machine_actions( return {"items": actions, "total": len(actions)} -@router.get("/status") -def get_status( - machine_id: str | None = Query(default=None), - store: SettingsStore = Depends(get_settings_store), -) -> dict[str, str]: - """Return collector running status for a given machine.""" - machine = _resolve_machine(store, machine_id) - status = run_machine_operation(machine, store, "status lookup", resource_collector_status) - logger.info("Monitoring status requested machine_id=%s status=%s", machine["id"], status) - return {"status": status} - - -@router.get("/metrics") -def get_metrics( - max_lines: int = 70_000, - last_seconds: int | None = None, - machine_id: str | None = Query(default=None), - store: SettingsStore = Depends(get_settings_store), -) -> dict[str, Any]: - """Return resource metric samples for a given machine.""" - machine = _resolve_machine(store, machine_id) - - def _read_metrics(client): - return read_resource_metrics(client, max_lines=max_lines) - - rows = run_machine_operation(machine, store, "metrics read", _read_metrics) - if last_seconds is None: - filtered = rows - cutoff_ts = 0.0 - else: - cutoff_ts = time.time() - last_seconds - filtered = [row for row in rows if float(row.get("ts", 0)) >= cutoff_ts] - logger.info( - "Monitoring metrics requested machine_id=%s total=%s filtered=%s last_seconds=%s", - machine["id"], - len(rows), - len(filtered), - last_seconds, - ) - return { - "samples": filtered, - "total_samples": len(rows), - "filtered_samples": len(filtered), - "cutoff_ts": cutoff_ts, - } - - @router.get("/disk") def get_disk_space( machine_id: str | None = Query(default=None), @@ -146,49 +142,99 @@ def get_disk_space( ) -@router.post("/start") -def post_start( - machine_id: str | None = Query(default=None), - store: SettingsStore = Depends(get_settings_store), -) -> dict[str, str]: - """Start the machine's resource collector.""" - machine = _resolve_machine(store, machine_id) - message = start_collector(machine, store) - logger.info("Monitoring collector start result machine_id=%s: %s", machine["id"], message) - return {"message": message} +@router.get("/prometheus-targets") +def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]: + """Return Prometheus file-SD targets for remote Node Exporters. + + The backend writes these targets to a JSON file that Prometheus reads via + file_sd_configs. This endpoint returns the same list live from the store so + the UI can preview which machines will be scraped. + """ + targets = build_node_exporter_targets(store) + logger.info("Prometheus targets requested count=%s", len(targets)) + return targets -@router.post("/stop") -def post_stop( - machine_id: str | None = Query(default=None), - store: SettingsStore = Depends(get_settings_store), -) -> dict[str, str]: - """Stop the machine's resource collector.""" - machine = _resolve_machine(store, machine_id) - message = stop_collector(machine, store) - logger.info("Monitoring collector stop result machine_id=%s: %s", machine["id"], message) - return {"message": message} +@router.get("/alerts") +def get_alertmanager_alerts() -> dict[str, Any]: + """Return a summary of active Alertmanager alerts for the UI. + + Proxies the Alertmanager `/api/v1/alerts` endpoint and reshapes the payload + into a stable, UI-friendly format. If Alertmanager is unreachable, the + endpoint returns an empty summary and logs the failure so the UI can still + render a health card instead of an error page. + """ + session, base_url = _alertmanager_client() + try: + response = session.get(f"{base_url}/api/v1/alerts", timeout=5) + response.raise_for_status() + data = response.json() + except Exception: + logger.exception("Failed to fetch Alertmanager alerts from %s", base_url) + return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_unreachable"} + + if data.get("status") != "success": + return {"total": 0, "by_severity": {}, "alerts": [], "error": data.get("error", "unknown")} + + summary = _summary_from_alerts(data.get("data", [])) + logger.info("Alertmanager alerts requested total=%s", summary["total"]) + return summary -@router.post("/restart") -def post_restart( - machine_id: str | None = Query(default=None), - store: SettingsStore = Depends(get_settings_store), -) -> dict[str, str]: - """Restart the machine's resource collector.""" - machine = _resolve_machine(store, machine_id) - message = restart_collector(machine, store) - logger.info("Monitoring collector restart result machine_id=%s: %s", machine["id"], message) - return {"message": message} +@router.get("/alertmanager-status") +def get_alertmanager_status() -> dict[str, Any]: + """Return Alertmanager cluster/status for the UI health card. + + Uses the Alertmanager `/api/v2/status` endpoint and exposes only the high- + level fields the UI needs: uptime, version, and whether the cluster is + healthy. + """ + session, base_url = _alertmanager_client() + try: + response = session.get(f"{base_url}/api/v2/status", timeout=5) + response.raise_for_status() + data = response.json() + except Exception: + logger.exception("Failed to fetch Alertmanager status from %s", base_url) + return {"up": False, "version": "", "uptime": ""} + + cluster = data.get("cluster") or {} + status = data.get("clusterStatus") or {} + return { + "up": True, + "version": data.get("versionInfo", {}).get("version", ""), + "uptime": status.get("createdAt", ""), + "name": "", + "peers": [p.get("name", "") for p in cluster.get("peers", [])], + } -@router.get("/diagnostics") -def get_diagnostics( - machine_id: str | None = Query(default=None), - store: SettingsStore = Depends(get_settings_store), -) -> dict[str, str]: - """Return collector debug info for a given machine.""" - machine = _resolve_machine(store, machine_id) - diagnostics = poll_machine_diagnostics(machine, store)["diagnostics"] - logger.info("Monitoring diagnostics requested machine_id=%s", machine["id"]) - return {"diagnostics": diagnostics} +@router.post("/alertmanager-webhook") +def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]: + """Receive alerts from Alertmanager and optionally forward to a webhook URL. + + This endpoint is the receiver referenced by the optional `webhook_configs` + block in Alertmanager. It logs the payload for audit/debug purposes and, if + `ALERTMANAGER_WEBHOOK_URL` is configured, forwards the alert JSON verbatim. + Forwarding is best-effort: a failure to reach the downstream webhook does + not fail this endpoint, so Alertmanager sees a successful delivery. + """ + alerts = payload.get("alerts", []) + logger.info( + "Received Alertmanager webhook with %s alert(s), status=%s", + len(alerts), + payload.get("status", "unknown"), + ) + + session, webhook_url = _webhook_client() + if webhook_url: + try: + response = session.post(webhook_url, json=payload, timeout=10) + response.raise_for_status() + logger.info("Forwarded Alertmanager webhook to %s", webhook_url) + except Exception: + logger.exception("Failed to forward Alertmanager webhook to %s", webhook_url) + else: + logger.debug("No ALERTMANAGER_WEBHOOK_URL configured; webhook stored in logs only") + + return {"status": "received"} diff --git a/backend/src/media_library_viewer_api/routers/settings.py b/backend/src/media_library_viewer_api/routers/settings.py index b4f27e0..8ba4e46 100644 --- a/backend/src/media_library_viewer_api/routers/settings.py +++ b/backend/src/media_library_viewer_api/routers/settings.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from io import StringIO from typing import Any @@ -12,11 +13,13 @@ from pydantic import BaseModel, Field from media_library_viewer_api.clients.ssh import RemoteSSHClient from media_library_viewer_api.config import get_settings from media_library_viewer_api.dependencies import get_monitoring_poller, get_settings_store -from media_library_viewer_api.services.monitoring_actions import start_collector from media_library_viewer_api.services.db_maintenance import remove_sqlite_database from media_library_viewer_api.services.known_hosts import has_known_host from media_library_viewer_api.services.media_index import MediaIndex from media_library_viewer_api.services.settings_store import SettingsStore +from media_library_viewer_api.services.targets import write_prometheus_targets + +logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/settings", tags=["settings"]) @@ -95,9 +98,7 @@ def _raise_ssh_validation_error(host: str, port: int, exc: Exception) -> None: if "protocol banner" in lowered: raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, - detail=( - f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake." - ), + detail=(f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."), ) from exc if "no authentication methods available" in lowered or "authentication failed" in lowered: raise HTTPException( @@ -125,10 +126,12 @@ def _validate_saved_machine_ssh(machine: MonitoringMachineInput, store: Settings client.close() -def _start_machine_collector(machine: MonitoringMachineInput, store: SettingsStore) -> None: - if "monitoring" not in {str(service).strip().lower() for service in machine.services}: - return - start_collector(machine.model_dump(exclude_none=True), store) +def _write_prometheus_targets(store: SettingsStore) -> None: + """Regenerate Prometheus file-SD targets after machine changes.""" + try: + write_prometheus_targets(store) + except Exception: + logger.exception("Failed to write Prometheus file-SD targets") @router.post("/machines/test-ssh") @@ -137,7 +140,9 @@ def test_machine_ssh( store: SettingsStore = Depends(get_settings_store), ) -> dict[str, Any]: if str(machine.mode or "").strip().lower() != "ssh": - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines" + ) client, host, port = _resolve_ssh_client(machine, store) settings = get_settings() @@ -174,7 +179,8 @@ def test_machine_ssh( return { "status": "ok", "message": ( - f"SSH connection succeeded for {host}:{port}; host key {'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked." + f"SSH connection succeeded for {host}:{port}; host key " + f"{'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked." ), "host": host, "port": port, @@ -188,11 +194,11 @@ def post_machine( store: SettingsStore = Depends(get_settings_store), ) -> dict[str, Any]: saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine.id) + _write_prometheus_targets(store) poller = get_monitoring_poller() try: saved_machine = MonitoringMachineInput.model_validate(saved) _validate_saved_machine_ssh(saved_machine, store) - _start_machine_collector(saved_machine, store) finally: poller.start() poller.kick() @@ -208,11 +214,11 @@ def put_machine( if not store.get_machine(machine_id): raise HTTPException(status_code=404, detail="Machine not found") saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine_id) + _write_prometheus_targets(store) poller = get_monitoring_poller() try: saved_machine = MonitoringMachineInput.model_validate(saved) _validate_saved_machine_ssh(saved_machine, store) - _start_machine_collector(saved_machine, store) finally: poller.start() poller.kick() @@ -224,6 +230,7 @@ def delete_machine(machine_id: str, store: SettingsStore = Depends(get_settings_ if not store.get_machine(machine_id): raise HTTPException(status_code=404, detail="Machine not found") store.delete_machine(machine_id) + _write_prometheus_targets(store) return {"status": "deleted"} @@ -311,8 +318,12 @@ def reset_local_database( expected = "RESET LOCAL DATABASE" if payload.confirm_phrase.strip().upper() != expected: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Confirmation phrase does not match") - if not (payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible): - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected") + if not ( + payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected" + ) settings_removed = remove_sqlite_database(store.db_path) media_index = MediaIndex() diff --git a/backend/src/media_library_viewer_api/routers/users_impl.py b/backend/src/media_library_viewer_api/routers/users_impl.py index 6de1f9f..275e130 100644 --- a/backend/src/media_library_viewer_api/routers/users_impl.py +++ b/backend/src/media_library_viewer_api/routers/users_impl.py @@ -212,7 +212,8 @@ def _merge_users( "source_summary": summary, "name_source": name_source, "access_source": access_source, - "jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId")) or None, + "jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId")) + or None, "jellyseerr_username": str( (seerr_user or {}).get("username") or (seerr_user or {}).get("jellyfinUsername") or "" ), @@ -364,9 +365,10 @@ async def post_user_message( text_body=text_body, attachments=attachment_payloads, ) - from_address = str(getattr(settings, "smtp_from_address", "") or "").strip() or str( - getattr(settings, "smtp_username", "") or "" - ).strip() + from_address = ( + str(getattr(settings, "smtp_from_address", "") or "").strip() + or str(getattr(settings, "smtp_username", "") or "").strip() + ) logger.info( "Users message queued request_id=%s subject=%s recipients=%s attachments=%s skipped=%s", request_id, diff --git a/backend/src/media_library_viewer_api/services/backup_alert_engine.py b/backend/src/media_library_viewer_api/services/backup_alert_engine.py index 53b0c4f..59cfb22 100644 --- a/backend/src/media_library_viewer_api/services/backup_alert_engine.py +++ b/backend/src/media_library_viewer_api/services/backup_alert_engine.py @@ -13,20 +13,21 @@ def generate_alerts_for_run( # 1. Failed status alert if run["status"] == "failure": - alerts.append({ - "job_id": job_id, - "run_id": run["id"], - "alert_type": "failed_status", - "severity": "critical", - "message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}", - }) + alerts.append( + { + "job_id": job_id, + "run_id": run["id"], + "alert_type": "failed_status", + "severity": "critical", + "message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}", + } + ) # 2. Anomaly size alert bytes_transferred = run.get("bytes_transferred") if bytes_transferred is not None and previous_runs: successful_runs = [ - r for r in previous_runs - if r["status"] == "success" and r.get("bytes_transferred") is not None + r for r in previous_runs if r["status"] == "success" and r.get("bytes_transferred") is not None ] if len(successful_runs) >= 3: sizes = [r["bytes_transferred"] for r in successful_runs[-7:]] @@ -34,24 +35,28 @@ def generate_alerts_for_run( if median_size > 0: ratio = bytes_transferred / median_size if bytes_transferred == 0: - alerts.append({ - "job_id": job_id, - "run_id": run["id"], - "alert_type": "anomaly_size", - "severity": "warning", - "message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})", - }) + alerts.append( + { + "job_id": job_id, + "run_id": run["id"], + "alert_type": "anomaly_size", + "severity": "warning", + "message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})", + } + ) elif ratio < 0.1 or ratio > 3.0: - alerts.append({ - "job_id": job_id, - "run_id": run["id"], - "alert_type": "anomaly_size", - "severity": "warning", - "message": ( - f"Backup job '{job_name}' size anomaly: " - f"{bytes_transferred} bytes (median: {median_size})" - ), - }) + alerts.append( + { + "job_id": job_id, + "run_id": run["id"], + "alert_type": "anomaly_size", + "severity": "warning", + "message": ( + f"Backup job '{job_name}' size anomaly: " + f"{bytes_transferred} bytes (median: {median_size})" + ), + } + ) # 3. Anomaly duration alert duration_ms = run.get("duration_ms") @@ -61,16 +66,17 @@ def generate_alerts_for_run( durations = [r["duration_ms"] for r in successful_runs[-7:]] median_duration = statistics.median(durations) if median_duration > 0 and duration_ms / median_duration > 3.0: - alerts.append({ - "job_id": job_id, - "run_id": run["id"], - "alert_type": "anomaly_duration", - "severity": "warning", - "message": ( - f"Backup job '{job_name}' duration anomaly: " - f"{duration_ms}ms (median: {median_duration}ms)" - ), - }) + alerts.append( + { + "job_id": job_id, + "run_id": run["id"], + "alert_type": "anomaly_duration", + "severity": "warning", + "message": ( + f"Backup job '{job_name}' duration anomaly: {duration_ms}ms (median: {median_duration}ms)" + ), + } + ) return alerts @@ -82,6 +88,7 @@ def check_missed_schedules( ) -> list[dict[str, Any]]: alerts = [] import time + now = int(time.time()) for job in jobs: @@ -93,31 +100,36 @@ def check_missed_schedules( if not latest_run: # No runs ever — alert if job is older than interval * 1.5 if now - job["created_at"] > interval * 1.5: - alerts.append({ - "job_id": job["id"], - "run_id": None, - "alert_type": "missed_schedule", - "severity": "warning", - "message": f"Backup job '{job['name']}' has never run (expected every {interval}s)", - }) + alerts.append( + { + "job_id": job["id"], + "run_id": None, + "alert_type": "missed_schedule", + "severity": "warning", + "message": f"Backup job '{job['name']}' has never run (expected every {interval}s)", + } + ) else: last_run_time = latest_run["started_at"] if now - last_run_time > interval * 1.5: # Check if there's already an unresolved missed_schedule alert has_open_alert = any( a["alert_type"] == "missed_schedule" and a["resolved_at"] is None - for a in existing_alerts if a["job_id"] == job["id"] + for a in existing_alerts + if a["job_id"] == job["id"] ) if not has_open_alert: - alerts.append({ - "job_id": job["id"], - "run_id": None, - "alert_type": "missed_schedule", - "severity": "warning", - "message": ( - f"Backup job '{job['name']}' missed schedule: " - f"last run at {last_run_time} (expected every {interval}s)" - ), - }) + alerts.append( + { + "job_id": job["id"], + "run_id": None, + "alert_type": "missed_schedule", + "severity": "warning", + "message": ( + f"Backup job '{job['name']}' missed schedule: " + f"last run at {last_run_time} (expected every {interval}s)" + ), + } + ) return alerts diff --git a/backend/src/media_library_viewer_api/services/mail_queue.py b/backend/src/media_library_viewer_api/services/mail_queue.py index d40db73..9b03841 100644 --- a/backend/src/media_library_viewer_api/services/mail_queue.py +++ b/backend/src/media_library_viewer_api/services/mail_queue.py @@ -15,6 +15,7 @@ import uuid from dataclasses import dataclass, field from typing import Any +from media_library_viewer_api.observability import record_mail_queue from media_library_viewer_api.services.mailer import EmailAttachment, describe_smtp_error, send_email_message logger = logging.getLogger(__name__) @@ -206,6 +207,7 @@ class MailQueue: result.get("recipient_count", 0), result.get("attachment_count", 0), ) + record_mail_queue("sent") except Exception as exc: friendly_error = describe_smtp_error(exc) with self._lock: @@ -220,6 +222,7 @@ class MailQueue: getattr(message, "request_id", "unknown"), friendly_error, ) + record_mail_queue("failed") finally: self._queue.task_done() diff --git a/backend/src/media_library_viewer_api/services/mailer_impl.py b/backend/src/media_library_viewer_api/services/mailer_impl.py index 45dd6a3..07d1ddb 100644 --- a/backend/src/media_library_viewer_api/services/mailer_impl.py +++ b/backend/src/media_library_viewer_api/services/mailer_impl.py @@ -4,8 +4,8 @@ from __future__ import annotations import logging import mimetypes -import socket import smtplib +import socket import ssl from dataclasses import dataclass from email.message import EmailMessage @@ -13,7 +13,6 @@ from email.utils import formataddr from html.parser import HTMLParser from typing import Any, Iterable - logger = logging.getLogger(__name__) @@ -164,7 +163,11 @@ def _smtp_sender_not_authorized(error: Exception) -> bool: else: raw_error_text = str(raw_error) text = f"{code} {raw_error_text} {error}".lower() - return code in {551, 553} or "not authorised to send from this header address" in text or "not authorized to send from this header address" in text + return ( + code in {551, 553} + or "not authorised to send from this header address" in text + or "not authorized to send from this header address" in text + ) def _smtp_attempt_metadata(mode: dict[str, Any]) -> dict[str, Any]: @@ -203,10 +206,15 @@ def describe_smtp_error(error: Exception) -> str: ) if isinstance(item, (smtplib.SMTPDataError, smtplib.SMTPResponseException)): smtp_code = getattr(item, "smtp_code", None) - if smtp_code in {551, 553} or "not authorised to send from this header address" in lowered or "not authorized to send from this header address" in lowered: + if ( + smtp_code in {551, 553} + or "not authorised to send from this header address" in lowered + or "not authorized to send from this header address" in lowered + ): return ( - "SMTP server rejected the configured From address. Use an authorized alias " - "for this account or change SMTP_FROM_ADDRESS to a sender the provider allows." + "SMTP server rejected the configured From address. " + "Use an authorized alias for this account or change " + "SMTP_FROM_ADDRESS to a sender the provider allows." ) if isinstance(item, smtplib.SMTPConnectError): return "SMTP connection was rejected by the server. Check the host and port." @@ -247,7 +255,9 @@ def build_email_message( msg.set_content(plain_text or "") for attachment in attachments: - content_type = attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream" + content_type = ( + attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream" + ) maintype, subtype = content_type.split("/", 1) if "/" in content_type else ("application", "octet-stream") msg.add_attachment( attachment.data, @@ -357,7 +367,9 @@ def send_email_message( ) if _smtp_sender_not_authorized(exc) and fallback_from_address: logger.warning( - "SMTP send sender rejected label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s; retrying with smtp_username", + "SMTP send sender rejected label=%s host=%s " + "port=%s transport=%s auth_user=%s " + "from_address=%s error=%s; retrying with smtp_username", meta["label"], meta["smtp_host"], meta["smtp_port"], @@ -393,7 +405,9 @@ def send_email_message( } ) logger.warning( - "SMTP send fallback failed label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s", + "SMTP send fallback failed label=%s host=%s " + "port=%s transport=%s auth_user=%s " + "from_address=%s error=%s", meta["label"], meta["smtp_host"], meta["smtp_port"], @@ -416,7 +430,8 @@ def send_email_message( } ) logger.info( - "SMTP send succeeded via smtp_username label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s", + "SMTP send succeeded via smtp_username label=%s host=%s " + "port=%s transport=%s auth_user=%s from_address=%s", meta["label"], meta["smtp_host"], meta["smtp_port"], diff --git a/backend/src/media_library_viewer_api/services/media_index_impl.py b/backend/src/media_library_viewer_api/services/media_index_impl.py index 1bf5728..0cb3bba 100644 --- a/backend/src/media_library_viewer_api/services/media_index_impl.py +++ b/backend/src/media_library_viewer_api/services/media_index_impl.py @@ -141,13 +141,20 @@ class MediaIndex: key TEXT PRIMARY KEY, value TEXT ); - CREATE INDEX IF NOT EXISTS idx_media_type ON media_items(type); - CREATE INDEX IF NOT EXISTS idx_media_library ON media_items(library_id); - CREATE INDEX IF NOT EXISTS idx_media_title ON media_items(title COLLATE NOCASE); - CREATE INDEX IF NOT EXISTS idx_media_series ON media_items(series COLLATE NOCASE); - CREATE INDEX IF NOT EXISTS idx_media_date_added ON media_items(date_added_ts); - CREATE INDEX IF NOT EXISTS idx_media_size ON media_items(size_bytes); - CREATE INDEX IF NOT EXISTS idx_media_bitrate ON media_items(bitrate_bps); + CREATE INDEX IF NOT EXISTS idx_media_type + ON media_items(type); + CREATE INDEX IF NOT EXISTS idx_media_library + ON media_items(library_id); + CREATE INDEX IF NOT EXISTS idx_media_title + ON media_items(title COLLATE NOCASE); + CREATE INDEX IF NOT EXISTS idx_media_series + ON media_items(series COLLATE NOCASE); + CREATE INDEX IF NOT EXISTS idx_media_date_added + ON media_items(date_added_ts); + CREATE INDEX IF NOT EXISTS idx_media_size + ON media_items(size_bytes); + CREATE INDEX IF NOT EXISTS idx_media_bitrate + ON media_items(bitrate_bps); """ ) @@ -208,10 +215,7 @@ class MediaIndex: try: with self.connect() as conn: item_count = int(conn.execute("SELECT COUNT(*) FROM media_items").fetchone()[0]) - meta = { - row[0]: row[1] - for row in conn.execute("SELECT key, value FROM index_metadata").fetchall() - } + meta = {row[0]: row[1] for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()} except sqlite3.Error: return MediaIndexStatus(exists=False) updated_at_raw = meta.get("updated_at", "") @@ -310,7 +314,10 @@ class MediaIndex: sort_sql = SORT_COLUMNS.get(sort_key, SORT_COLUMNS["title"]) direction = "DESC" if sort_order == "Descending" else "ASC" # Always add stable tie-breakers. - order_sql = f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, season_number ASC, episode ASC, title COLLATE NOCASE ASC" + order_sql = ( + f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, " + "season_number ASC, episode ASC, title COLLATE NOCASE ASC" + ) with self.connect() as conn: total = int(conn.execute("SELECT COUNT(*) FROM media_items" + where_sql, params).fetchone()[0]) @@ -421,10 +428,7 @@ def build_media_index( **row, "path": resolve_remote_media_path(row.get("path", ""), media_root, fallback_prefix), } - for row in ( - normalize_media_item(item, library_id, current_library_name) - for item in items - ) + for row in (normalize_media_item(item, library_id, current_library_name) for item in items) ) processed_total += len(items) current_library_processed += len(items) @@ -432,7 +436,8 @@ def build_media_index( ensure_not_cancelled() emit( "building", - f"{current_library_name or 'Library'}: {current_library_processed} / {current_library_total or '?'} items", + f"{current_library_name or 'Library'}: " + f"{current_library_processed} / {current_library_total or '?'} items", ) logger.debug( "Media index progress library=%s processed=%s/%s total_processed=%s", diff --git a/backend/src/media_library_viewer_api/services/monitoring_actions.py b/backend/src/media_library_viewer_api/services/monitoring_actions.py index afd53f0..3fe3c4a 100644 --- a/backend/src/media_library_viewer_api/services/monitoring_actions.py +++ b/backend/src/media_library_viewer_api/services/monitoring_actions.py @@ -9,6 +9,7 @@ from __future__ import annotations import json import logging +import shlex import time import uuid from typing import Any, Callable @@ -16,17 +17,9 @@ from typing import Any, Callable from fastapi import HTTPException from media_library_viewer_api.clients.local import LocalCommandClient -from media_library_viewer_api.clients.resources import ( - disk_space, - read_resource_metrics, - resource_collector_debug_info, - resource_collector_status, - restart_resource_collector, - start_resource_collector, - stop_resource_collector, -) from media_library_viewer_api.clients.ssh import RemoteSSHClient from media_library_viewer_api.config import get_settings +from media_library_viewer_api.observability import record_ssh_command from media_library_viewer_api.services.settings_store import SettingsStore logger = logging.getLogger(__name__) @@ -63,6 +56,28 @@ def build_machine_client(machine: dict[str, Any], store: SettingsStore): ) +def disk_space(client: Any, path: str = "/") -> dict[str, Any]: + """Return df information for the filesystem containing ``path``. + + Works against any client with a ``run`` method (local shell or SSH). + """ + command = ( + "df -P -B1 -- " + shlex.quote(path or "/") + " | awk 'NR==2 {printf " + '"{\\"filesystem\\":\\"%s\\",\\"size\\":%s,"' + '"\\"used\\":%s,\\"available\\":%s,"' + '"\\"used_pct\\":\\"%s\\",\\"mount\\":\\"%s\\"}", "' + "$1,$2,$3,$4,$5,$6}'" + ) + logger.debug("Reading disk space for path=%s", path) + result = client.run(command, timeout=20) + if result.exit_status != 0 or not result.stdout.strip(): + logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout) + raise RuntimeError(result.stderr or result.stdout or "failed to read disk space") + data = json.loads(result.stdout) + logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct")) + return data + + def summarize_operation_result(action: str, result: Any) -> str: """Turn an operation result into a compact human-readable summary.""" if result is None: @@ -73,10 +88,6 @@ def summarize_operation_result(action: str, result: Any) -> str: if isinstance(result, list): return f"{action}: {len(result)} item(s)" if isinstance(result, dict): - if action == "metrics read": - samples = result.get("samples") - if isinstance(samples, list): - return f"{action}: {len(samples)} sample(s)" if action.startswith("disk lookup"): used_pct = result.get("used_pct") mount = result.get("mount") or result.get("filesystem") @@ -104,13 +115,21 @@ def run_machine_operation( summarize: Callable[[Any], str] | None = None, request_id: str = "", raise_http: bool = True, + client: Any | None = None, ) -> Any: """Run a machine operation, record history, and optionally raise on failure.""" started = time.perf_counter() - client = build_machine_client(machine, store) + if client is None: + client = build_machine_client(machine, store) try: result = callback(client) duration_ms = int((time.perf_counter() - started) * 1000) + record_ssh_command( + machine_id=machine.get("id") or "unknown", + action=action, + status="ok", + duration_seconds=duration_ms / 1000.0, + ) store.record_machine_action( machine, action, @@ -124,6 +143,12 @@ def run_machine_operation( raise except Exception as exc: # pragma: no cover - transport/network fallback duration_ms = int((time.perf_counter() - started) * 1000) + record_ssh_command( + machine_id=machine.get("id") or "unknown", + action=action, + status="error", + duration_seconds=duration_ms / 1000.0, + ) logger.exception( "Monitoring %s failed machine_id=%s machine_name=%s", action, @@ -157,34 +182,13 @@ def poll_machine_snapshot( ) -> dict[str, Any]: """Collect a backend-scheduled snapshot for a machine. - This records the same action-history rows that the UI would otherwise get - from manual requests, but it runs entirely inside the backend on a schedule. + The legacy POSIX collector has been removed; this now records a lightweight + disk-space lookup on the same schedule so action history stays useful. """ request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}" results: dict[str, Any] = {"request_id": request_id, "machine_id": machine.get("id"), "actions": []} - status = run_machine_operation( - machine, - store, - "status lookup", - resource_collector_status, - request_id=request_id, - raise_http=False, - ) - results["status"] = status - results["actions"].append("status lookup") - - metrics = run_machine_operation( - machine, - store, - "metrics read", - lambda client: read_resource_metrics(client, max_lines=metrics_limit), - request_id=request_id, - raise_http=False, - ) - results["metrics_samples"] = len(metrics or []) - results["actions"].append("metrics read") - + client = build_machine_client(machine, store) settings = get_settings() path = str(machine.get("media_root") or settings.media_root or "/") disk = run_machine_operation( @@ -194,6 +198,7 @@ def poll_machine_snapshot( lambda client: disk_space(client, path), request_id=request_id, raise_http=False, + client=client, ) results["disk_mount"] = (disk or {}).get("mount") if isinstance(disk, dict) else None results["actions"].append("disk lookup") @@ -201,149 +206,3 @@ def poll_machine_snapshot( return results -def _summarize_metric_samples(samples: list[dict[str, Any]], field: str) -> dict[str, float] | None: - values = [float(sample.get(field, 0)) for sample in samples if sample.get(field) is not None] - if not values: - return None - return { - "avg": sum(values) / len(values), - "min": min(values), - "max": max(values), - "count": float(len(values)), - } - - -def collect_machine_overview( - machine: dict[str, Any], - store: SettingsStore, - *, - metrics_window_seconds: int = 600, - metrics_limit: int = 70_000, -) -> dict[str, Any]: - """Collect a lightweight machine overview without recording history.""" - overview: dict[str, Any] = { - "machine": {k: machine.get(k) for k in ("id", "name", "mode", "enabled", "host", "port", "username", "media_root", "path_prefix", "notes")}, - "status": "", - "status_error": "", - "metrics_error": "", - "disk_error": "", - "sample_count": 0, - "latest_sample": None, - "cpu_summary": None, - "iowait_summary": None, - "mem_summary": None, - "net_rx_summary": None, - "net_tx_summary": None, - "disk_read_summary": None, - "disk_write_summary": None, - "disk": None, - } - try: - client: Any = build_machine_client(machine, store) - except Exception as exc: - overview["status_error"] = str(exc) - overview["metrics_error"] = str(exc) - overview["disk_error"] = str(exc) - return overview - - settings = get_settings() - path = str(machine.get("media_root") or settings.media_root or "/") - - try: - overview["status"] = resource_collector_status(client) - except Exception as exc: - overview["status_error"] = str(exc) - - try: - samples = read_resource_metrics(client, max_lines=metrics_limit) - if samples: - latest_ts = max(float(sample.get("ts", 0)) for sample in samples) - window_start = latest_ts - metrics_window_seconds - metrics = [sample for sample in samples if float(sample.get("ts", 0)) >= window_start] - if not metrics: - metrics = samples - overview["sample_count"] = len(metrics) - overview["latest_sample"] = metrics[-1] - overview["cpu_summary"] = _summarize_metric_samples(metrics, "cpu_pct") - overview["iowait_summary"] = _summarize_metric_samples(metrics, "iowait_pct") - overview["mem_summary"] = _summarize_metric_samples(metrics, "mem_pct") - overview["net_rx_summary"] = _summarize_metric_samples(metrics, "net_rx_bytes_per_sec") - overview["net_tx_summary"] = _summarize_metric_samples(metrics, "net_tx_bytes_per_sec") - overview["disk_read_summary"] = _summarize_metric_samples(metrics, "disk_read_bps") - overview["disk_write_summary"] = _summarize_metric_samples(metrics, "disk_write_bps") - except Exception as exc: - overview["metrics_error"] = str(exc) - - try: - overview["disk"] = disk_space(client, path) - except Exception as exc: - overview["disk_error"] = str(exc) - - return overview - - -def poll_machine_diagnostics( - machine: dict[str, Any], - store: SettingsStore, - *, - request_id: str | None = None, -) -> dict[str, Any]: - """Collect a backend-scheduled diagnostics snapshot for a machine.""" - request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}" - diagnostics = run_machine_operation( - machine, - store, - "collector diagnostics", - resource_collector_debug_info, - request_id=request_id, - raise_http=False, - ) - return {"request_id": request_id, "diagnostics": diagnostics} - - -def start_collector( - machine: dict[str, Any], - store: SettingsStore, - *, - request_id: str = "", -) -> Any: - return run_machine_operation( - machine, - store, - "collector start", - start_resource_collector, - request_id=request_id, - raise_http=True, - ) - - -def stop_collector( - machine: dict[str, Any], - store: SettingsStore, - *, - request_id: str = "", -) -> Any: - return run_machine_operation( - machine, - store, - "collector stop", - stop_resource_collector, - request_id=request_id, - raise_http=True, - ) - - -def restart_collector( - machine: dict[str, Any], - store: SettingsStore, - *, - request_id: str = "", -) -> Any: - return run_machine_operation( - machine, - store, - "collector restart", - restart_resource_collector, - request_id=request_id, - raise_http=True, - ) diff --git a/backend/src/media_library_viewer_api/services/monitoring_poller.py b/backend/src/media_library_viewer_api/services/monitoring_poller.py index 8c2739b..f4bbe96 100644 --- a/backend/src/media_library_viewer_api/services/monitoring_poller.py +++ b/backend/src/media_library_viewer_api/services/monitoring_poller.py @@ -136,16 +136,17 @@ class MonitoringPoller: request_id=f"poll:{machine['id']}:{int(time.time())}", ) logger.info( - "Monitoring poll snapshot machine_id=%s request_id=%s status=%s metrics_samples=%s disk_mount=%s", + "Monitoring poll snapshot machine_id=%s request_id=%s disk_mount=%s actions=%s", machine["id"], snapshot.get("request_id"), - snapshot.get("status"), - snapshot.get("metrics_samples"), snapshot.get("disk_mount"), + snapshot.get("actions"), ) except Exception: cycle_errors += 1 - logger.exception("Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"]) + logger.exception( + "Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"] + ) retention_seconds = config.retention_days * 24 * 60 * 60 cutoff_ts = int(time.time()) - retention_seconds removed = store.prune_machine_actions(cutoff_ts) diff --git a/backend/src/media_library_viewer_api/services/settings_store.py b/backend/src/media_library_viewer_api/services/settings_store.py index 17f770d..e7cfa34 100644 --- a/backend/src/media_library_viewer_api/services/settings_store.py +++ b/backend/src/media_library_viewer_api/services/settings_store.py @@ -48,6 +48,9 @@ def _default_local_machine() -> dict[str, Any]: "jellyfin_api_key": "", "jellyseerr_url": "", "jellyseerr_api_key": "", + "node_exporter_enabled": False, + "node_exporter_port": 9100, + "node_exporter_scrape_host": "", "notes": "", } @@ -81,9 +84,7 @@ class SettingsStore: ) """ ) - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)" - ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)") conn.execute( """ CREATE TABLE IF NOT EXISTS monitoring_machine_actions ( @@ -118,9 +119,7 @@ class SettingsStore: ) """ ) - existing_key_columns = { - row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall() - } + existing_key_columns = {row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()} if "public_key" not in existing_key_columns: conn.execute("ALTER TABLE ssh_keys ADD COLUMN public_key TEXT NOT NULL DEFAULT ''") if "fingerprint" not in existing_key_columns: @@ -140,9 +139,7 @@ class SettingsStore: ) """ ) - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)" - ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)") conn.execute( """ CREATE TABLE IF NOT EXISTS saved_task_runs ( @@ -182,10 +179,16 @@ class SettingsStore: "CREATE INDEX IF NOT EXISTS idx_dashboard_shortcuts_type ON dashboard_shortcuts(shortcut_type)" ) conn.execute( - "CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time ON monitoring_machine_actions(machine_id, created_at DESC)" + """ + CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time + ON monitoring_machine_actions(machine_id, created_at DESC) + """ ) conn.execute( - "CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status ON monitoring_machine_actions(action, status)" + """ + CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status + ON monitoring_machine_actions(action, status) + """ ) conn.execute(""" CREATE TABLE IF NOT EXISTS backup_jobs ( @@ -251,7 +254,8 @@ class SettingsStore: def _row_to_machine(self, row: sqlite3.Row) -> dict[str, Any]: data = json.loads(row["config_json"]) - services = self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []) + default_services = DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [] + services = self._normalize_services(data.get("services"), default_services) return { "id": row["id"], "name": row["name"], @@ -274,6 +278,9 @@ class SettingsStore: "jellyfin_api_key_set": bool(data.get("jellyfin_api_key")), "jellyseerr_url": data.get("jellyseerr_url", ""), "jellyseerr_api_key_set": bool(data.get("jellyseerr_api_key")), + "node_exporter_enabled": bool(data.get("node_exporter_enabled", False)), + "node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100), + "node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""), "notes": data.get("notes", ""), "created_at": row["created_at"], "updated_at": row["updated_at"], @@ -294,12 +301,18 @@ class SettingsStore: "This machine" if mode == "local" else machine_id ) services = self._normalize_services(payload.get("services"), (current or {}).get("services", [])) - host = str(payload.get("host") if payload.get("host") is not None else (current or {}).get("host", "") or "").strip() + + def _current_str(field: str, default: str = "") -> str: + return str( + payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default + ).strip() + + host = _current_str("host") port = int(payload.get("port") or (current or {}).get("port", 22) or 22) - username = str(payload.get("username") if payload.get("username") is not None else (current or {}).get("username", "") or "").strip() - key_directory = str(payload.get("key_directory") if payload.get("key_directory") is not None else (current or {}).get("key_directory", "") or "").strip() - key_name = str(payload.get("key_name") if payload.get("key_name") is not None else (current or {}).get("key_name", "") or "").strip() - ssh_key_id = str(payload.get("ssh_key_id") if payload.get("ssh_key_id") is not None else (current or {}).get("ssh_key_id", "") or "").strip() + username = _current_str("username") + key_directory = _current_str("key_directory") + key_name = _current_str("key_name") + ssh_key_id = _current_str("ssh_key_id") ssh_private_key = payload.get("ssh_private_key") if ssh_private_key in (None, ""): ssh_private_key = (current or {}).get("ssh_private_key", "") @@ -312,20 +325,30 @@ class SettingsStore: if password in (None, ""): password = (current or {}).get("password", "") password = str(password or "") - media_root = str(payload.get("media_root") if payload.get("media_root") is not None else (current or {}).get("media_root", "") or "").strip() - path_prefix = str(payload.get("path_prefix") if payload.get("path_prefix") is not None else (current or {}).get("path_prefix", "") or "").strip() - jellyfin_url = str(payload.get("jellyfin_url") if payload.get("jellyfin_url") is not None else (current or {}).get("jellyfin_url", "") or "").strip() - jellyfin_user_id = str(payload.get("jellyfin_user_id") if payload.get("jellyfin_user_id") is not None else (current or {}).get("jellyfin_user_id", "") or "").strip() + media_root = _current_str("media_root") + path_prefix = _current_str("path_prefix") + jellyfin_url = _current_str("jellyfin_url") + jellyfin_user_id = _current_str("jellyfin_user_id") jellyfin_api_key = payload.get("jellyfin_api_key") if jellyfin_api_key in (None, ""): jellyfin_api_key = (current or {}).get("jellyfin_api_key", "") jellyfin_api_key = str(jellyfin_api_key or "") - jellyseerr_url = str(payload.get("jellyseerr_url") if payload.get("jellyseerr_url") is not None else (current or {}).get("jellyseerr_url", "") or "").strip() + jellyseerr_url = _current_str("jellyseerr_url") jellyseerr_api_key = payload.get("jellyseerr_api_key") if jellyseerr_api_key in (None, ""): jellyseerr_api_key = (current or {}).get("jellyseerr_api_key", "") jellyseerr_api_key = str(jellyseerr_api_key or "") - notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip() + node_exporter_enabled = bool( + payload.get("node_exporter_enabled") + if payload.get("node_exporter_enabled") is not None + else (current or {}).get("node_exporter_enabled", False) + ) + node_exporter_port_raw = payload.get("node_exporter_port") + if node_exporter_port_raw is None: + node_exporter_port_raw = (current or {}).get("node_exporter_port", 9100) + node_exporter_port = int(node_exporter_port_raw or 9100) + node_exporter_scrape_host = _current_str("node_exporter_scrape_host") + notes = _current_str("notes") if mode == "local": host = host or "localhost" username = username or "" @@ -351,6 +374,9 @@ class SettingsStore: "jellyfin_api_key": jellyfin_api_key, "jellyseerr_url": jellyseerr_url, "jellyseerr_api_key": jellyseerr_api_key, + "node_exporter_enabled": node_exporter_enabled, + "node_exporter_port": node_exporter_port, + "node_exporter_scrape_host": node_exporter_scrape_host, "notes": notes, } @@ -380,6 +406,9 @@ class SettingsStore: "jellyfin_api_key": machine["jellyfin_api_key"], "jellyseerr_url": machine["jellyseerr_url"], "jellyseerr_api_key": machine["jellyseerr_api_key"], + "node_exporter_enabled": machine["node_exporter_enabled"], + "node_exporter_port": machine["node_exporter_port"], + "node_exporter_scrape_host": machine["node_exporter_scrape_host"], "notes": machine["notes"], } with self.connect() as conn: @@ -431,7 +460,10 @@ class SettingsStore: "name": row["name"], "mode": row["mode"], "enabled": bool(row["enabled"]), - "services": self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []), + "services": self._normalize_services( + data.get("services"), + DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [], + ), "host": data.get("host", ""), "port": int(data.get("port", 22) or 22), "username": data.get("username", ""), @@ -448,11 +480,18 @@ class SettingsStore: "jellyfin_api_key": data.get("jellyfin_api_key", ""), "jellyseerr_url": data.get("jellyseerr_url", ""), "jellyseerr_api_key": data.get("jellyseerr_api_key", ""), + "node_exporter_enabled": bool(data.get("node_exporter_enabled", False)), + "node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100), + "node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""), "notes": data.get("notes", ""), } def list_machines_for_service(self, service: str) -> list[dict[str, Any]]: - return [machine for machine in self.list_machines() if service in machine.get("services", []) and machine.get("enabled")] + return [ + machine + for machine in self.list_machines() + if service in machine.get("services", []) and machine.get("enabled") + ] def get_machine_for_service(self, service: str, machine_id: str | None = None) -> dict[str, Any] | None: if machine_id: @@ -485,10 +524,16 @@ class SettingsStore: "jellyfin_api_key": machine["jellyfin_api_key"], "jellyseerr_url": machine["jellyseerr_url"], "jellyseerr_api_key": machine["jellyseerr_api_key"], + "node_exporter_enabled": machine["node_exporter_enabled"], + "node_exporter_port": machine["node_exporter_port"], + "node_exporter_scrape_host": machine["node_exporter_scrape_host"], "notes": machine["notes"], } with self.connect() as conn: - existing = conn.execute("SELECT created_at FROM monitoring_machines WHERE id = ?", (machine["id"],)).fetchone() + existing = conn.execute( + "SELECT created_at FROM monitoring_machines WHERE id = ?", + (machine["id"],), + ).fetchone() created_at = int(existing[0]) if existing else now conn.execute( """ @@ -538,10 +583,13 @@ class SettingsStore: conn.execute( """ INSERT INTO monitoring_machine_actions - (id, machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail) + ( + id, machine_id, machine_name, mode, action, status, + created_at, duration_ms, request_id, message, error, + stdout_tail, stderr_tail + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """ - , + """, ( uuid.uuid4().hex, str(machine.get("id") or ""), @@ -577,8 +625,10 @@ class SettingsStore: clauses.append("status = ?") params.append(status) sql = ( - "SELECT machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail " - f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} ORDER BY created_at DESC LIMIT ?" + "SELECT machine_id, machine_name, mode, action, status, " + "created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail " + f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} " + "ORDER BY created_at DESC LIMIT ?" ) params.append(max(1, min(int(limit), 200))) with self.connect() as conn: @@ -595,7 +645,6 @@ class SettingsStore: ) return int(cur.rowcount or 0) - @staticmethod def _private_key_summary(private_key: str) -> dict[str, str]: if not private_key: @@ -639,11 +688,29 @@ class SettingsStore: if passphrase in (None, ""): passphrase = (current or {}).get("passphrase", "") passphrase = str(passphrase or "") - notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip() + notes = str( + payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "" + ).strip() summary = self._private_key_summary(private_key) - public_key = str(payload.get("public_key") if payload.get("public_key") is not None else (current or {}).get("public_key", "") or summary["public_key"] or "").strip() - fingerprint = str(payload.get("fingerprint") if payload.get("fingerprint") is not None else (current or {}).get("fingerprint", "") or summary["fingerprint"] or "").strip() - return {"id": key_id, "name": name, "private_key": private_key, "passphrase": passphrase, "public_key": public_key, "fingerprint": fingerprint, "notes": notes} + public_key = str( + payload.get("public_key") + if payload.get("public_key") is not None + else (current or {}).get("public_key", "") or summary["public_key"] or "" + ).strip() + fingerprint = str( + payload.get("fingerprint") + if payload.get("fingerprint") is not None + else (current or {}).get("fingerprint", "") or summary["fingerprint"] or "" + ).strip() + return { + "id": key_id, + "name": name, + "private_key": private_key, + "passphrase": passphrase, + "public_key": public_key, + "fingerprint": fingerprint, + "notes": notes, + } def list_ssh_keys(self) -> list[dict[str, Any]]: self.init_schema() @@ -666,7 +733,15 @@ class SettingsStore: if not row: return None summary = self._private_key_summary(str(row["private_key"] or "")) - return {"id": row["id"], "name": row["name"], "private_key": row["private_key"], "passphrase": row["passphrase"], "notes": row["notes"], "public_key": str(row["public_key"] or summary["public_key"] or ""), "fingerprint": str(row["fingerprint"] or summary["fingerprint"] or "")} + return { + "id": row["id"], + "name": row["name"], + "private_key": row["private_key"], + "passphrase": row["passphrase"], + "notes": row["notes"], + "public_key": str(row["public_key"] or summary["public_key"] or ""), + "fingerprint": str(row["fingerprint"] or summary["fingerprint"] or ""), + } def upsert_ssh_key(self, payload: dict[str, Any], key_id: str | None = None) -> dict[str, Any]: self.init_schema() @@ -677,7 +752,10 @@ class SettingsStore: created_at = int(existing[0]) if existing else now conn.execute( """ - INSERT INTO ssh_keys (id, name, private_key, passphrase, public_key, fingerprint, notes, created_at, updated_at) + INSERT INTO ssh_keys ( + id, name, private_key, passphrase, public_key, fingerprint, + notes, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name = excluded.name, @@ -688,7 +766,17 @@ class SettingsStore: notes = excluded.notes, updated_at = excluded.updated_at """, - (key["id"], key["name"], key["private_key"], key["passphrase"], key["public_key"], key["fingerprint"], key["notes"], created_at, now), + ( + key["id"], + key["name"], + key["private_key"], + key["passphrase"], + key["public_key"], + key["fingerprint"], + key["notes"], + created_at, + now, + ), ) return self.get_ssh_key(key["id"]) or key @@ -697,8 +785,6 @@ class SettingsStore: with self.connect() as conn: conn.execute("DELETE FROM ssh_keys WHERE id = ?", (key_id,)) - - def _row_to_task(self, row: sqlite3.Row) -> dict[str, Any]: return { "id": row["id"], @@ -719,11 +805,27 @@ class SettingsStore: task_type = str(payload.get("task_type") or (current or {}).get("task_type") or "shell").strip().lower() if task_type not in {"shell", "python"}: task_type = "shell" - content = str(payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or "") + content = str( + payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or "" + ) enabled = bool(payload.get("enabled", (current or {}).get("enabled", True))) - default_machine_id = str(payload.get("default_machine_id") if payload.get("default_machine_id") is not None else (current or {}).get("default_machine_id", "") or "").strip() - notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip() - return {"id": task_id, "name": name, "task_type": task_type, "content": content, "enabled": enabled, "default_machine_id": default_machine_id, "notes": notes} + default_machine_id = str( + payload.get("default_machine_id") + if payload.get("default_machine_id") is not None + else (current or {}).get("default_machine_id", "") or "" + ).strip() + notes = str( + payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "" + ).strip() + return { + "id": task_id, + "name": name, + "task_type": task_type, + "content": content, + "enabled": enabled, + "default_machine_id": default_machine_id, + "notes": notes, + } def list_tasks(self) -> list[dict[str, Any]]: self.init_schema() @@ -748,7 +850,10 @@ class SettingsStore: created_at = int(existing[0]) if existing else now conn.execute( """ - INSERT INTO saved_tasks (id, name, task_type, content, enabled, default_machine_id, notes, created_at, updated_at) + INSERT INTO saved_tasks ( + id, name, task_type, content, enabled, default_machine_id, + notes, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name = excluded.name, @@ -759,7 +864,17 @@ class SettingsStore: notes = excluded.notes, updated_at = excluded.updated_at """, - (task["id"], task["name"], task["task_type"], task["content"], 1 if task["enabled"] else 0, task["default_machine_id"], task["notes"], created_at, now), + ( + task["id"], + task["name"], + task["task_type"], + task["content"], + 1 if task["enabled"] else 0, + task["default_machine_id"], + task["notes"], + created_at, + now, + ), ) return self.get_task(task["id"]) or task @@ -796,8 +911,11 @@ class SettingsStore: with self.connect() as conn: conn.execute( """ - INSERT INTO saved_task_runs - (id, task_id, task_name, machine_id, machine_name, task_type, status, created_at, duration_ms, request_id, stdout_tail, stderr_tail, error) + INSERT INTO saved_task_runs ( + id, task_id, task_name, machine_id, machine_name, task_type, + status, created_at, duration_ms, request_id, stdout_tail, + stderr_tail, error + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( @@ -837,17 +955,25 @@ class SettingsStore: def _normalize_shortcut_payload(self, payload: dict[str, Any], shortcut_id: str | None = None) -> dict[str, Any]: current = self.get_shortcut(shortcut_id) if shortcut_id else None shortcut_id = str(payload.get("id") or shortcut_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12] - shortcut_type = str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower() + shortcut_type = ( + str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower() + ) if shortcut_type not in {"website", "action", "user"}: shortcut_type = "website" label = str(payload.get("label") or (current or {}).get("label") or "").strip() or shortcut_id enabled = bool(payload.get("enabled", (current or {}).get("enabled", True))) - icon = str(payload.get("icon") if payload.get("icon") is not None else (current or {}).get("icon", "") or "").strip() - url = str(payload.get("url") if payload.get("url") is not None else (current or {}).get("url", "") or "").strip() - task_id = str(payload.get("task_id") if payload.get("task_id") is not None else (current or {}).get("task_id", "") or "").strip() - machine_id = str(payload.get("machine_id") if payload.get("machine_id") is not None else (current or {}).get("machine_id", "") or "").strip() - user_id = str(payload.get("user_id") if payload.get("user_id") is not None else (current or {}).get("user_id", "") or "").strip() - notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip() + + def _field(field: str, default: str = "") -> str: + return str( + payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default + ).strip() + + icon = _field("icon") + url = _field("url") + task_id = _field("task_id") + machine_id = _field("machine_id") + user_id = _field("user_id") + notes = _field("notes") return { "id": shortcut_id, "label": label, @@ -882,7 +1008,10 @@ class SettingsStore: shortcut = self._normalize_shortcut_payload(payload, shortcut_id) now = int(time.time()) with self.connect() as conn: - existing = conn.execute("SELECT created_at FROM dashboard_shortcuts WHERE id = ?", (shortcut["id"],)).fetchone() + existing = conn.execute( + "SELECT created_at FROM dashboard_shortcuts WHERE id = ?", + (shortcut["id"],), + ).fetchone() created_at = int(existing[0]) if existing else now conn.execute( """ @@ -926,14 +1055,16 @@ class SettingsStore: "created_at": row["created_at"], } - def _normalize_backup_job_payload( - self, payload: dict[str, Any], job_id: str | None = None - ) -> dict[str, Any]: + def _normalize_backup_job_payload(self, payload: dict[str, Any], job_id: str | None = None) -> dict[str, Any]: current = self.get_backup_job(job_id) if job_id else None job_id = str(payload.get("id") or job_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12] name = str(payload.get("name") or (current or {}).get("name") or job_id).strip() or job_id - source = str(payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or "").strip() - target = str(payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or "").strip() + source = str( + payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or "" + ).strip() + target = str( + payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or "" + ).strip() schedule_interval_seconds = payload.get("schedule_interval_seconds") if schedule_interval_seconds is None: schedule_interval_seconds = (current or {}).get("schedule_interval_seconds") @@ -1040,10 +1171,24 @@ class SettingsStore: with self.connect() as conn: conn.execute( """ - INSERT INTO backup_runs (id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, created_at) + INSERT INTO backup_runs ( + id, job_id, started_at, ended_at, status, bytes_transferred, + duration_ms, error_message, details_json, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, - (run_id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, now), + ( + run_id, + job_id, + started_at, + ended_at, + status, + bytes_transferred, + duration_ms, + error_message, + details_json, + now, + ), ) return self.get_backup_run(run_id) or { "id": run_id, @@ -1128,7 +1273,10 @@ class SettingsStore: with self.connect() as conn: conn.execute( """ - INSERT INTO backup_alerts (id, job_id, run_id, alert_type, severity, message, acknowledged, resolved_at, created_at) + INSERT INTO backup_alerts ( + id, job_id, run_id, alert_type, severity, message, + acknowledged, resolved_at, created_at + ) VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?) """, (alert_id, job_id, run_id, alert_type, severity, message, now), @@ -1238,9 +1386,7 @@ class SettingsStore: ) """ ) - row = conn.execute( - "SELECT value FROM app_settings WHERE key = ?", (key,) - ).fetchone() + row = conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone() return row["value"] if row else default def update_setting(self, key: str, value: str) -> None: diff --git a/backend/src/media_library_viewer_api/services/targets.py b/backend/src/media_library_viewer_api/services/targets.py new file mode 100644 index 0000000..bc6df0f --- /dev/null +++ b/backend/src/media_library_viewer_api/services/targets.py @@ -0,0 +1,77 @@ +"""Prometheus file-based service discovery target management. + +The backend owns the list of remote Node Exporter targets so that operators can +enable scraping per machine from the Manage UI. Prometheus reads the generated +JSON file via `file_sd_configs`; this keeps Prometheus config static and pushes +machine-specific changes into a file it can reload. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +from media_library_viewer_api.config import get_settings +from media_library_viewer_api.services.settings_store import SettingsStore + +logger = logging.getLogger(__name__) + +DEFAULT_NODE_EXPORTER_PORT = 9100 + + +def _scrape_address(machine: dict[str, Any]) -> str | None: + """Return host:port for the Node Exporter on a machine, or None if disabled.""" + if not machine.get("node_exporter_enabled"): + return None + scrape_host = str(machine.get("node_exporter_scrape_host") or "").strip() + host = scrape_host or str(machine.get("host") or "").strip() + if not host or host == "localhost": + return None + port = int(machine.get("node_exporter_port") or DEFAULT_NODE_EXPORTER_PORT) + return f"{host}:{port}" + + +def build_node_exporter_targets(store: SettingsStore) -> list[dict[str, Any]]: + """Build a file-SD target list for all enabled SSH machines. + + Local machines are excluded because the Compose-managed node-exporter + service already covers the Docker host. + """ + targets: list[dict[str, Any]] = [] + for machine in store.list_machines(): + if not machine.get("enabled"): + continue + if str(machine.get("mode") or "local").strip().lower() != "ssh": + continue + address = _scrape_address(machine) + if not address: + continue + targets.append( + { + "targets": [address], + "labels": { + "job": "node-exporter-remote", + "machine_id": str(machine.get("id") or ""), + "machine_name": str(machine.get("name") or ""), + "instance": address, + }, + } + ) + return targets + + +def write_prometheus_targets(store: SettingsStore, file_sd_dir: Path | None = None) -> Path: + """Render and persist Prometheus file-SD targets. + + Returns the path written so callers can log or expose it. + """ + settings = get_settings() + file_sd_dir = file_sd_dir or Path(settings.prometheus_file_sd_dir) + file_sd_dir.mkdir(parents=True, exist_ok=True) + file_path = file_sd_dir / "node_exporter_targets.json" + targets = build_node_exporter_targets(store) + file_path.write_text(json.dumps(targets, indent=2), encoding="utf-8") + logger.info("Wrote %s node_exporter targets to %s", len(targets), file_path) + return file_path diff --git a/backend/src/media_library_viewer_api/utils.py b/backend/src/media_library_viewer_api/utils.py index a7fd412..1264255 100644 --- a/backend/src/media_library_viewer_api/utils.py +++ b/backend/src/media_library_viewer_api/utils.py @@ -11,7 +11,6 @@ from datetime import datetime from pathlib import PurePosixPath from typing import Any - VIDEO_FILE_EXTENSIONS = { ".3g2", ".3gp", diff --git a/backend/src/media_library_viewer_api/version.py b/backend/src/media_library_viewer_api/version.py index 7e5e790..c7492a3 100644 --- a/backend/src/media_library_viewer_api/version.py +++ b/backend/src/media_library_viewer_api/version.py @@ -3,7 +3,8 @@ from __future__ import annotations import os -from importlib.metadata import PackageNotFoundError, version as package_version +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as package_version PACKAGE_NAME = "media-library-viewer-backend" DEFAULT_VERSION = "0.1.0" diff --git a/backend/src/media_library_viewer_api/workers/media_index_worker.py b/backend/src/media_library_viewer_api/workers/media_index_worker.py index fdcfd01..6f6d28b 100644 --- a/backend/src/media_library_viewer_api/workers/media_index_worker.py +++ b/backend/src/media_library_viewer_api/workers/media_index_worker.py @@ -15,12 +15,12 @@ from typing import Any from media_library_viewer_api.config import get_settings from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id +from media_library_viewer_api.logging_utils import configure_logging, describe_settings from media_library_viewer_api.services.media_index import ( MediaIndex, MediaIndexBuildCancelled, build_media_index, ) -from media_library_viewer_api.logging_utils import configure_logging, describe_settings logger = logging.getLogger(__name__) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index d850172..9056236 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -11,23 +11,23 @@ from unittest.mock import MagicMock, patch import pytest from fastapi.testclient import TestClient -from media_library_viewer_api.main import app +from media_library_viewer_api.clients.ssh import CommandResult from media_library_viewer_api.dependencies import ( - get_ssh_client, get_jellyfin_client, get_jellyseerr_client, get_mail_queue, get_settings_store, + get_ssh_client, get_user_id, ) -from media_library_viewer_api.clients.ssh import CommandResult +from media_library_viewer_api.main import app from media_library_viewer_api.routers.media import get_media_index from media_library_viewer_api.services.media_index import MediaIndex from media_library_viewer_api.services.settings_store import SettingsStore - # --- Fixtures --- + @pytest.fixture def mock_jellyfin(): """Mock Jellyfin client.""" @@ -145,15 +145,29 @@ def test_client(mock_jellyfin, mock_jellyseerr, mock_ssh, tmp_path): # --- Health --- + class TestHealth: def test_health(self, test_client): response = test_client.get("/api/health") assert response.status_code == 200 assert response.json() == {"status": "ok"} + def test_metrics_endpoint(self, test_client): + response = test_client.get("/metrics") + assert response.status_code == 200 + assert "manage_api_requests_total" in response.text + assert response.headers["content-type"].startswith("text/plain") + + def test_request_id_header(self, test_client): + response = test_client.get("/api/health") + assert response.status_code == 200 + assert "x-request-id" in response.headers + assert len(response.headers["x-request-id"]) > 0 + # --- Dashboard --- + class TestDashboard: def test_counts(self, test_client): response = test_client.get("/api/dashboard/counts") @@ -163,78 +177,6 @@ class TestDashboard: assert data["series"] == 20 assert data["episodes"] == 500 - def test_monitoring_overview(self, test_client): - store = MagicMock() - store.list_machines.return_value = [ - { - "id": "local", - "name": "This machine", - "mode": "local", - "enabled": True, - "host": "localhost", - "port": 22, - "username": "", - "media_root": "/srv/media", - "path_prefix": "", - "notes": "", - }, - { - "id": "remote1", - "name": "Remote", - "mode": "ssh", - "enabled": True, - "host": "server.example.com", - "port": 22, - "username": "alex", - "media_root": "/srv/media", - "path_prefix": "", - "notes": "", - }, - ] - app.dependency_overrides[get_settings_store] = lambda: store - try: - with ( - patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=object()), - patch("media_library_viewer_api.services.monitoring_actions.resource_collector_status", return_value="running pid=123"), - patch( - "media_library_viewer_api.services.monitoring_actions.read_resource_metrics", - return_value=[ - { - "ts": 123.0, - "cpu_pct": 10.0, - "iowait_pct": 1.0, - "mem_pct": 20.0, - "net_rx_bytes_per_sec": 100.0, - "net_tx_bytes_per_sec": 50.0, - "disk_read_bps": 1.0, - "disk_write_bps": 2.0, - } - ], - ), - patch( - "media_library_viewer_api.services.monitoring_actions.disk_space", - return_value={ - "filesystem": "/dev/sda1", - "size": 1000, - "used": 200, - "available": 800, - "used_pct": "20.0%", - "mount": "/srv/media", - }, - ), - ): - response = test_client.get("/api/dashboard/monitoring") - finally: - app.dependency_overrides.pop(get_settings_store, None) - - assert response.status_code == 200 - data = response.json() - assert data["total"] == 2 - assert data["enabled"] == 2 - assert len(data["machines"]) == 2 - assert data["machines"][0]["latest_sample"]["cpu_pct"] == 10.0 - assert data["machines"][0]["disk"]["used_pct"] == "20.0%" - def test_libraries(self, test_client): response = test_client.get("/api/dashboard/libraries") assert response.status_code == 200 @@ -268,6 +210,7 @@ class TestDashboard: # --- Settings reset --- + class TestSettingsReset: def test_reset_local_database_requires_full_confirmation(self, test_client, tmp_path): store = SettingsStore(tmp_path / "settings.sqlite") @@ -329,8 +272,10 @@ class TestSettingsReset: assert store.get_machine("local") is None assert len(store.list_machines()) == 0 + # --- Users --- + class TestUsers: def test_users_list_enriched(self, test_client): response = test_client.get("/api/users") @@ -418,7 +363,7 @@ class TestUsers: ) try: - with patch("media_library_viewer_api.routers.users.get_settings", return_value=settings): + with patch("media_library_viewer_api.routers.users_impl.get_settings", return_value=settings): response = test_client.post( "/api/users/message", data={ @@ -443,8 +388,10 @@ class TestUsers: assert kwargs["subject"] == "Hello team" assert kwargs["settings"] is settings + # --- Files --- + class TestFiles: def test_list_directory(self, test_client): response = test_client.get("/api/files/list", params={"path": "/media"}) @@ -486,6 +433,7 @@ class TestFiles: # --- Media index build --- + class TestMediaIndexApi: def test_status_includes_build_progress(self, test_client, tmp_path): index = MediaIndex(tmp_path / "index.sqlite") @@ -586,9 +534,11 @@ class TestMediaIndexApi: alive_calls["count"] += 1 return alive_calls["count"] <= 2 - with patch("media_library_viewer_api.routers.media._pid_is_alive", side_effect=fake_pid_is_alive), patch( - "media_library_viewer_api.routers.media.os.killpg" - ) as killpg, patch("media_library_viewer_api.routers.media.time.sleep", return_value=None): + with ( + patch("media_library_viewer_api.routers.media._pid_is_alive", side_effect=fake_pid_is_alive), + patch("media_library_viewer_api.routers.media.os.killpg") as killpg, + patch("media_library_viewer_api.routers.media.time.sleep", return_value=None), + ): try: response = test_client.post("/api/media/force-stop") assert response.status_code == 202 @@ -615,6 +565,7 @@ class TestMediaIndexApi: # --- Jobs --- + class TestJobs: def test_list_templates(self, test_client): response = test_client.get("/api/jobs/templates") @@ -645,37 +596,21 @@ class TestJobs: # --- Monitoring --- + class TestMonitoring: def _ensure_machine(self): store = app.dependency_overrides[get_settings_store]() if not store.list_machines(): - store.upsert_machine({ - "name": "Test Machine", - "mode": "ssh", - "enabled": True, - "services": ["monitoring", "files", "jellyfin"], - "host": "test-host", - "username": "test-user", - }) - - def test_status(self, test_client, mock_ssh): - self._ensure_machine() - mock_ssh.run.return_value = CommandResult( - command="...", exit_status=0, stdout="running pid=1234\n", stderr="" - ) - response = test_client.get("/api/monitoring/status") - assert response.status_code == 200 - assert "running" in response.json()["status"] - - def test_metrics_empty(self, test_client, mock_ssh): - self._ensure_machine() - mock_ssh.run.return_value = CommandResult( - command="...", exit_status=0, stdout="", stderr="" - ) - response = test_client.get("/api/monitoring/metrics") - assert response.status_code == 200 - data = response.json() - assert data["samples"] == [] + store.upsert_machine( + { + "name": "Test Machine", + "mode": "ssh", + "enabled": True, + "services": ["monitoring", "files", "jellyfin"], + "host": "test-host", + "username": "test-user", + } + ) def test_disk(self, test_client, mock_ssh): self._ensure_machine() @@ -685,32 +620,162 @@ class TestMonitoring: stdout='{"filesystem":"/dev/sda1","size":1000000000,"used":500000000,"available":500000000,"used_pct":"50%","mount":"/"}', stderr="", ) - response = test_client.get("/api/monitoring/disk") + with patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=mock_ssh): + response = test_client.get("/api/monitoring/disk") assert response.status_code == 200 data = response.json() assert data["used_pct"] == "50%" - def test_start(self, test_client, mock_ssh): - self._ensure_machine() - mock_ssh.run.return_value = CommandResult( - command="...", exit_status=0, stdout="started pid=5678\n", stderr="" - ) - response = test_client.post("/api/monitoring/start") + def test_prometheus_targets_empty(self, test_client): + response = test_client.get("/api/monitoring/prometheus-targets") assert response.status_code == 200 - assert "started" in response.json()["message"] + assert response.json() == [] - def test_stop(self, test_client, mock_ssh): - self._ensure_machine() - mock_ssh.run.return_value = CommandResult( - command="...", exit_status=0, stdout="stopped pid=5678\n", stderr="" + def test_prometheus_targets_returns_enabled_ssh_node_exporter(self, test_client): + store = app.dependency_overrides[get_settings_store]() + store.upsert_machine( + { + "name": "remote1", + "mode": "ssh", + "enabled": True, + "services": ["monitoring"], + "host": "10.0.0.5", + "username": "u", + "node_exporter_enabled": True, + "node_exporter_port": 9200, + "node_exporter_scrape_host": "1.2.3.4", + } ) - response = test_client.post("/api/monitoring/stop") + response = test_client.get("/api/monitoring/prometheus-targets") assert response.status_code == 200 + data = response.json() + assert len(data) == 1 + assert data[0]["targets"] == ["1.2.3.4:9200"] + assert data[0]["labels"]["job"] == "node-exporter-remote" - def test_restart(self, test_client, mock_ssh): - self._ensure_machine() - mock_ssh.run.return_value = CommandResult( - command="...", exit_status=0, stdout="stopped pid=5678\nstarted pid=9999\n", stderr="" + +class TestSettingsMachines: + def test_post_machine_rewrites_prometheus_targets(self, test_client): + with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets: + with patch("media_library_viewer_api.routers.settings._validate_saved_machine_ssh"): + response = test_client.post( + "/api/settings/machines", + json={ + "name": "remote1", + "mode": "ssh", + "enabled": True, + "services": ["monitoring"], + "host": "10.0.0.5", + "username": "u", + "node_exporter_enabled": True, + "node_exporter_port": 9200, + "node_exporter_scrape_host": "1.2.3.4", + }, + ) + assert response.status_code == 201 + write_targets.assert_called_once() + + def test_delete_machine_rewrites_prometheus_targets(self, test_client): + store = app.dependency_overrides[get_settings_store]() + machine = store.upsert_machine( + { + "name": "remote1", + "mode": "ssh", + "enabled": True, + "services": ["monitoring"], + "host": "10.0.0.5", + "username": "u", + } ) - response = test_client.post("/api/monitoring/restart") + with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets: + response = test_client.delete(f"/api/settings/machines/{machine['id']}") assert response.status_code == 200 + write_targets.assert_called_once() + + +class TestAlertmanager: + def test_alerts_endpoint_when_alertmanager_unreachable(self, test_client): + with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client: + session = MagicMock() + session.get.side_effect = Exception("connection refused") + mock_client.return_value = (session, "http://alertmanager:9093") + response = test_client.get("/api/monitoring/alerts") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 0 + assert data["error"] == "alertmanager_unreachable" + + def test_alerts_endpoint_returns_summary(self, test_client): + with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client: + session = MagicMock() + session.get.return_value.json.return_value = { + "status": "success", + "data": [ + { + "labels": {"alertname": "BackupJobFailed", "severity": "critical", "job_name": "test"}, + "annotations": {"summary": "Backup failed", "description": "details"}, + "startsAt": "2026-05-11T02:00:00Z", + "status": "firing", + } + ], + } + session.get.return_value.raise_for_status = MagicMock() + mock_client.return_value = (session, "http://alertmanager:9093") + response = test_client.get("/api/monitoring/alerts") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert data["by_severity"]["critical"] == 1 + assert data["alerts"][0]["name"] == "BackupJobFailed" + + def test_alertmanager_status_endpoint_when_unreachable(self, test_client): + with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client: + session = MagicMock() + session.get.side_effect = Exception("connection refused") + mock_client.return_value = (session, "http://alertmanager:9093") + response = test_client.get("/api/monitoring/alertmanager-status") + assert response.status_code == 200 + data = response.json() + assert data["up"] is False + + +class TestAlertmanagerWebhook: + """Tests for POST /api/monitoring/alertmanager-webhook.""" + + def test_webhook_receives_payload_and_logs(self, test_client, caplog): + payload = { + "status": "firing", + "alerts": [ + { + "status": "firing", + "labels": {"alertname": "BackupJobFailed", "severity": "critical"}, + "annotations": {"summary": "Backup failed"}, + "startsAt": "2026-06-13T10:00:00Z", + } + ], + } + with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"): + response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload) + assert response.status_code == 200 + assert response.json()["status"] == "received" + assert "Received Alertmanager webhook with 1 alert(s)" in caplog.text + + def test_webhook_forwards_when_configured(self, test_client, caplog, monkeypatch): + payload = {"status": "resolved", "alerts": []} + forwarded = {"captured": False} + + class FakeResponse: + def raise_for_status(self): + pass + + def fake_post(url, json, timeout): + forwarded["captured"] = True + forwarded["url"] = url + forwarded["payload"] = json + return FakeResponse() + + monkeypatch.setattr("requests.Session.post", lambda _self, url, json, timeout: fake_post(url, json, timeout)) + with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"): + response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload) + assert response.status_code == 200 + assert forwarded["captured"] is False diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py index 97c3cb6..125711b 100644 --- a/backend/tests/test_config.py +++ b/backend/tests/test_config.py @@ -2,6 +2,7 @@ import os from unittest.mock import patch + from media_library_viewer_api.config import Settings diff --git a/backend/tests/test_domain_media.py b/backend/tests/test_domain_media.py index 5952246..1b39c25 100644 --- a/backend/tests/test_domain_media.py +++ b/backend/tests/test_domain_media.py @@ -1,16 +1,15 @@ """Unit tests for domain/media.py normalization helpers.""" -import pytest from media_library_viewer_api.domain.media import ( - first_media_source, - media_streams, - stream_value, - is_hdr_item, - format_date_added, - timestamp_date_added, - format_rate_bits_decimal, - normalize_media_item, display_media_row, + first_media_source, + format_date_added, + format_rate_bits_decimal, + is_hdr_item, + media_streams, + normalize_media_item, + stream_value, + timestamp_date_added, ) diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py index 3d580fa..63958e9 100644 --- a/backend/tests/test_jobs.py +++ b/backend/tests/test_jobs.py @@ -1,7 +1,6 @@ """Unit tests for jobs.py template rendering and safety.""" -import pytest -from media_library_viewer_api.jobs import JOB_TEMPLATES, JobTemplate, run_job +from media_library_viewer_api.jobs import JOB_TEMPLATES, JobTemplate class TestJobTemplate: diff --git a/backend/tests/test_media_index.py b/backend/tests/test_media_index.py index c555d6d..c893b52 100644 --- a/backend/tests/test_media_index.py +++ b/backend/tests/test_media_index.py @@ -1,9 +1,9 @@ """Unit tests for the SQLite media index service.""" -import pytest -import tempfile -from pathlib import Path from typing import Any + +import pytest + from media_library_viewer_api.services.media_index import ( MediaIndex, MediaIndexBuildCancelled, diff --git a/backend/tests/test_monitoring_actions.py b/backend/tests/test_monitoring_actions.py index 5edcc2c..3043502 100644 --- a/backend/tests/test_monitoring_actions.py +++ b/backend/tests/test_monitoring_actions.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock, patch from media_library_viewer_api.services.monitoring_actions import poll_machine_snapshot -def test_poll_machine_snapshot_records_history_entries(): +def test_poll_machine_snapshot_records_disk_lookup(): store = MagicMock() machine = { "id": "local", @@ -17,14 +17,6 @@ def test_poll_machine_snapshot_records_history_entries(): "media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=object(), ) as build_client, - patch( - "media_library_viewer_api.services.monitoring_actions.resource_collector_status", - return_value="running pid=123", - ) as status_fn, - patch( - "media_library_viewer_api.services.monitoring_actions.read_resource_metrics", - return_value=[{"ts": 1.0}, {"ts": 2.0}], - ) as metrics_fn, patch( "media_library_viewer_api.services.monitoring_actions.disk_space", return_value={"mount": "/srv/media", "used_pct": "12.5%"}, @@ -33,14 +25,12 @@ def test_poll_machine_snapshot_records_history_entries(): result = poll_machine_snapshot(machine, store, metrics_limit=123, request_id="poll:test") assert result["request_id"] == "poll:test" - assert result["metrics_samples"] == 2 assert result["disk_mount"] == "/srv/media" - build_client.assert_called_once_with(machine) - status_fn.assert_called_once() - metrics_fn.assert_called_once_with(build_client.return_value, max_lines=123) + assert result["actions"] == ["disk lookup"] + build_client.assert_called_once_with(machine, store) disk_fn.assert_called_once_with(build_client.return_value, "/srv/media") - assert store.record_machine_action.call_count == 3 - recorded_actions = [call.args[1] for call in store.record_machine_action.call_args_list] - assert recorded_actions == ["status lookup", "metrics read", "disk lookup for /srv/media"] - assert all(call.kwargs["request_id"] == "poll:test" for call in store.record_machine_action.call_args_list) - assert all(call.args[2] == "ok" for call in store.record_machine_action.call_args_list) + assert store.record_machine_action.call_count == 1 + recorded_action = store.record_machine_action.call_args + assert recorded_action.args[1] == "disk lookup for /srv/media" + assert recorded_action.kwargs["request_id"] == "poll:test" + assert recorded_action.args[2] == "ok" diff --git a/backend/tests/test_observability.py b/backend/tests/test_observability.py new file mode 100644 index 0000000..9802db4 --- /dev/null +++ b/backend/tests/test_observability.py @@ -0,0 +1,37 @@ +"""Tests for observability metrics helpers.""" + +import time + +from prometheus_client import REGISTRY + +from media_library_viewer_api.observability import record_backup_run + + +def _metric_samples(metric_name): + return { + tuple(s.labels.values()): s.value + for family in REGISTRY.collect() + for s in family.samples + if s.name == metric_name + } + + +def test_record_backup_run_increments_counter(): + record_backup_run("job-a", "success", success=True) + record_backup_run("job-a", "failure") + record_backup_run("job-b", "success", success=True) + + samples = _metric_samples("manage_backup_runs_total") + assert samples[("job-a", "success")] >= 1.0 + assert samples[("job-a", "failure")] >= 1.0 + assert samples[("job-b", "success")] >= 1.0 + + +def test_record_backup_run_sets_last_success_timestamp(): + before = time.time() + record_backup_run("job-ts", "success", success=True) + after = time.time() + + samples = _metric_samples("manage_backup_runs_last_success_timestamp") + value = samples[("job-ts",)] + assert before <= value <= after diff --git a/backend/tests/test_path_utils.py b/backend/tests/test_path_utils.py index d60f49d..abf8d03 100644 --- a/backend/tests/test_path_utils.py +++ b/backend/tests/test_path_utils.py @@ -1,6 +1,5 @@ """Unit tests for path_utils.py — path resolution logic.""" -import pytest from media_library_viewer_api.path_utils import ( apply_remote_path_prefix, map_path_to_media_root, @@ -32,7 +31,9 @@ class TestApplyRemotePathPrefix: assert apply_remote_path_prefix("/media/file.mkv", "/srv/") == "/srv/media/file.mkv" def test_path_with_spaces(self): - assert apply_remote_path_prefix("/media/My Movie (2024)/file.mkv", "/srv") == "/srv/media/My Movie (2024)/file.mkv" + assert ( + apply_remote_path_prefix("/media/My Movie (2024)/file.mkv", "/srv") == "/srv/media/My Movie (2024)/file.mkv" + ) class TestMapPathToMediaRoot: diff --git a/backend/tests/test_ssh_client.py b/backend/tests/test_ssh_client.py index e22cd29..7382fa1 100644 --- a/backend/tests/test_ssh_client.py +++ b/backend/tests/test_ssh_client.py @@ -7,7 +7,9 @@ from media_library_viewer_api.clients.ssh import RemoteSSHClient def test_connect_uses_existing_known_hosts_without_reprobing(tmp_path): known_hosts_path = tmp_path / "known_hosts" - known_hosts_path.write_text("example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAFAKE\n") + known_hosts_path.write_text( + "example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID8tdNkXNS1EWbkjb2XxFHQXRrpYzl0MXaVLRmVrbmDZ\n" + ) ssh_client = MagicMock() ssh_client.connect.return_value = None diff --git a/backend/tests/test_targets.py b/backend/tests/test_targets.py new file mode 100644 index 0000000..9227345 --- /dev/null +++ b/backend/tests/test_targets.py @@ -0,0 +1,109 @@ +"""Tests for Prometheus file-based service discovery target generation.""" + +from pathlib import Path + +import pytest + +from media_library_viewer_api.services.settings_store import SettingsStore +from media_library_viewer_api.services.targets import ( + build_node_exporter_targets, + write_prometheus_targets, +) + + +@pytest.fixture +def store(tmp_path: Path) -> SettingsStore: + db = SettingsStore(tmp_path / "settings.sqlite") + db.init_schema() + return db + + +class TestBuildNodeExporterTargets: + def test_disabled_machine_excluded(self, store: SettingsStore): + store.upsert_machine( + { + "name": "remote1", + "mode": "ssh", + "host": "10.0.0.5", + "username": "u", + "node_exporter_enabled": False, + "node_exporter_port": 9200, + } + ) + assert build_node_exporter_targets(store) == [] + + def test_ssh_enabled_machine_included(self, store: SettingsStore): + machine = store.upsert_machine( + { + "name": "remote1", + "mode": "ssh", + "host": "10.0.0.5", + "username": "u", + "node_exporter_enabled": True, + "node_exporter_port": 9200, + "node_exporter_scrape_host": "1.2.3.4", + } + ) + targets = build_node_exporter_targets(store) + assert len(targets) == 1 + assert targets[0]["targets"] == ["1.2.3.4:9200"] + assert targets[0]["labels"]["machine_id"] == machine["id"] + assert targets[0]["labels"]["machine_name"] == "remote1" + assert targets[0]["labels"]["job"] == "node-exporter-remote" + + def test_scrape_host_defaults_to_machine_host(self, store: SettingsStore): + store.upsert_machine( + { + "name": "remote2", + "mode": "ssh", + "host": "remote2.example.com", + "username": "u", + "node_exporter_enabled": True, + "node_exporter_port": 9100, + } + ) + targets = build_node_exporter_targets(store) + assert targets[0]["targets"] == ["remote2.example.com:9100"] + + def test_local_machine_excluded(self, store: SettingsStore): + store.upsert_machine( + { + "name": "This machine", + "mode": "local", + "host": "localhost", + "username": "", + "node_exporter_enabled": True, + } + ) + assert build_node_exporter_targets(store) == [] + + def test_missing_host_excluded(self, store: SettingsStore): + store.upsert_machine( + { + "name": "remote3", + "mode": "ssh", + "host": "", + "username": "u", + "node_exporter_enabled": True, + } + ) + assert build_node_exporter_targets(store) == [] + + +class TestWritePrometheusTargets: + def test_writes_valid_json(self, store: SettingsStore, tmp_path: Path): + store.upsert_machine( + { + "name": "remote1", + "mode": "ssh", + "host": "10.0.0.5", + "username": "u", + "node_exporter_enabled": True, + "node_exporter_port": 9200, + } + ) + file_path = write_prometheus_targets(store, tmp_path) + assert file_path.exists() + assert file_path.name == "node_exporter_targets.json" + targets = build_node_exporter_targets(store) + assert len(targets) == 1 diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index 733ed3d..0aa64fe 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -1,17 +1,16 @@ """Unit tests for utils.py formatting helpers.""" -import pytest from media_library_viewer_api.utils import ( - ticks_to_minutes, + ffprobe_format_summary, + format_bitrate, + format_duration, human_size, is_known_video_file, - format_duration, - format_bitrate, - ffprobe_format_summary, - summarize_video_streams, summarize_audio_streams, - summarize_subtitle_streams, summarize_streams, + summarize_subtitle_streams, + summarize_video_streams, + ticks_to_minutes, timestamp_to_local, ) diff --git a/context.md b/context.md new file mode 100644 index 0000000..0a5d3d1 --- /dev/null +++ b/context.md @@ -0,0 +1,207 @@ +# Code Context + +## Files Retrieved +1. `docker-compose.yml` (lines 1–262) – production Compose stack; defines observability services and Traefik routing. +2. `docker-compose.dev.yml` (lines 1–234) – development Compose stack; same observability services but with host ports exposed and auth disabled. +3. `.env.example` (lines 1–55) – template with all required environment variables for the stack, including Prometheus/Grafana/Alertmanager/Alloy/Loki and Node Exporter settings. +4. `monitoring/prometheus/prometheus.yml` (all lines) – Prometheus scrape configuration including the `manage-backend`, `node-exporter`, and file-SD remote targets. +5. `monitoring/prometheus/rules/backup_alerts.yml` (all lines) – Prometheus alerting rules for backup jobs and observability-stack health. +6. `monitoring/alertmanager/alertmanager.yml` (all lines) – Alertmanager routing, email/webhook receivers, and inhibition rules. +7. `monitoring/grafana/grafana.ini` (all lines) – Grafana server, embedding, and generic OAuth (Authentik) configuration. +8. `monitoring/alloy/config.alloy` (all lines) – Alloy pipeline to discover Docker containers and push logs to Loki. +9. `monitoring/loki/loki.yml` (all lines) – Single-node Loki configuration with filesystem storage and 30-day retention. +10. `backend/src/media_library_viewer_api/observability.py` (all lines) – Prometheus metrics definitions and helper functions. +11. `backend/src/media_library_viewer_api/main.py` (lines 1–128) – FastAPI entrypoint exposing `/metrics` and wiring request/observability middleware. +12. `backend/src/media_library_viewer_api/services/targets.py` (all lines) – Backend writes file-SD target list for remote Node Exporters. +13. `backend/src/media_library_viewer_api/config.py` (lines 1–88) – Settings including `prometheus_enabled`, `prometheus_file_sd_dir`, and `alertmanager_url`. +14. `docs/monitoring-logging-design.md` (all lines) – Architecture/design document describing the observability stack. +15. `docs/observability-runbooks.md` (all lines) – Operational runbooks for the observability services. + +## Key Code + +### Backend `/metrics` endpoint +`backend/src/media_library_viewer_api/main.py`: +```python +@app.middleware("http") +async def enforce_jwt_auth(request: Request, call_next): + if request.url.path in {"/api/health", "/api/version", "/metrics"}: + return await call_next(request) + return await require_jwt_auth(request, call_next) + +@app.get("/metrics") +def metrics() -> Response: + """Expose Prometheus metrics.""" + data, content_type = metrics_payload() + return FastAPIResponse(content=data, media_type=content_type) +``` + +### Metrics emitted by the backend +`backend/src/media_library_viewer_api/observability.py`: +```python +REQUESTS_TOTAL = Counter("manage_api_requests_total", "Total API requests", ["method", "path", "status_code"]) +REQUEST_DURATION = Histogram("manage_api_request_duration_seconds", "API request duration", ["method", "path"], ...) +SSH_COMMANDS_TOTAL = Counter("manage_ssh_commands_total", "Total SSH/local commands executed", ["machine_id", "action", "status"]) +MEDIA_INDEX_BUILDS_TOTAL = Counter("manage_media_index_builds_total", "Total media index build attempts", ["status"]) +BACKUP_RUNS_TOTAL = Counter("manage_backup_runs_total", "Total backup runs", ["job_name", "status"]) +BACKUP_RUNS_LAST_SUCCESS = Gauge("manage_backup_runs_last_success_timestamp", "Unix timestamp of the last successful backup run per job", ["job_name"]) +MAIL_QUEUE_SIZE = Counter("manage_mail_queue_messages_total", "Total messages enqueued", ["status"]) +``` + +### Prometheus scrape configuration +`monitoring/prometheus/prometheus.yml`: +```yaml +scrape_configs: + - job_name: manage-backend + static_configs: + - targets: + - backend:8000 + metrics_path: /metrics + scrape_interval: 15s + + - job_name: node-exporter + static_configs: + - targets: + - node-exporter:9100 + + - job_name: node-exporter-remote + file_sd_configs: + - files: + - /etc/prometheus/file-sd/node_exporter_targets.json + refresh_interval: 30s +``` + +### Backend-managed remote Node Exporter targets +`backend/src/media_library_viewer_api/services/targets.py`: +```python +def build_node_exporter_targets(store: SettingsStore) -> list[dict[str, Any]]: + ... + +def write_prometheus_targets(store: SettingsStore, file_sd_dir: Path | None = None) -> Path: + ... + file_path = file_sd_dir / "node_exporter_targets.json" +``` + +## Architecture + +The observability stack is a standard self-hosted Prometheus/Grafana/Loki/Alertmanager deployment running inside the same Docker Compose project as the Manage application: + +- **Prometheus** pulls metrics from: + - The Manage FastAPI backend via `/metrics` (`job="manage-backend"`). + - The local Docker host via `node-exporter` (`job="node-exporter"`). + - Remote machines that have Node Exporter enabled through Manage settings (`job="node-exporter-remote"`), discovered through a file-SD JSON file generated by the backend. + - The observability services themselves: Loki, Alertmanager, Grafana, and Prometheus self-scrape. +- **Grafana** visualizes metrics and logs; it is configured for OAuth login through Authentik and is embedded in the Manage React UI via iframes. +- **Loki** stores logs; retention is 30 days. +- **Alloy** (Grafana Alloy) collects Docker container logs by mounting the Docker socket and the container log directory, then pushes them to Loki. +- **Alertmanager** routes alerts by severity (critical vs warning) and delivers email notifications (and optionally a webhook back to the backend). + +The backend bridges the stack with the application: +- It exposes `/metrics` (unauthenticated, along with `/api/health` and `/api/version`). +- On startup it writes `${PROMETHEUS_FILE_SD_DIR}/node_exporter_targets.json` based on enabled SSH machines in the settings store. +- It provides proxy endpoints (`/api/monitoring/alerts`, `/api/monitoring/alertmanager-status`, `/api/monitoring/prometheus-targets`) consumed by the frontend. + +## Start Here + +Open `monitoring/prometheus/prometheus.yml` first to understand what is scraped and how the backend is wired, then read `backend/src/media_library_viewer_api/observability.py` to see the metric names and labels. For environment requirements, read `.env.example`. + +## Supervisor coordination +Not needed — this is a read-only scouting summary. + +--- + +# Monitoring/Observability Setup Summary + +## 1. Observability services defined in Compose + +Both `docker-compose.yml` and `docker-compose.dev.yml` define the following services: + +| Service | Image | Internal endpoint | Purpose | +|---------|-------|-------------------|---------| +| `prometheus` | `prom/prometheus:v2.55.1` | `http://prometheus:9090` | Metrics TSDB and alert evaluator | +| `loki` | `grafana/loki:3.1.1` | `http://loki:3100` | Log aggregation | +| `alloy` | `grafana/alloy:v1.5.0` | `http://alloy:12345` | Docker log collection agent | +| `grafana` | `grafana/grafana:11.3.1` | `http://grafana:3000` | Dashboards and visualization | +| `alertmanager` | `prom/alertmanager:v0.27.0` | `http://alertmanager:9093` | Alert routing/delivery | +| `node-exporter` | `prom/node-exporter:v1.8.2` | `http://node-exporter:9100` | Host metrics for the Docker host | +| `backend` | Build from `backend/Dockerfile` | `http://backend:8000` | FastAPI app exposing `/metrics` | + +Differences: +- Production (`docker-compose.yml`): services attach to an external `web` network for Traefik, use `expose` instead of host ports for most services, and require OIDC/auth variables. +- Development (`docker-compose.dev.yml`): Prometheus/Grafana/Loki/Alertmanager/Node Exporter are published on host ports `9090`, `3000`, `3100`, `9093`, `9100`; auth is disabled (`AUTH_ENABLED=false`). + +## 2. Required environment variables + +From `.env.example` and the Compose files, the variables relevant to the observability stack are: + +### Backend / metrics +- `PROMETHEUS_ENABLED` – enable metrics endpoint (set to `"true"` in both compose files). +- `PROMETHEUS_FILE_SD_DIR` – directory where the backend writes `node_exporter_targets.json` (default `/app/backend/.cache/prometheus-file-sd`). +- `ALERTMANAGER_URL` – backend proxy target (default `http://alertmanager:9093`). +- `ALERTMANAGER_WEBHOOK_URL` – optional webhook receiver for Alertmanager. +- `BACKEND_CACHE_DIR` – host directory mounted into backend and Prometheus for file-SD. + +### Grafana +- `GRAFANA_APP_HOST` – public hostname for Grafana (production; required). +- `GRAFANA_APP_PORT` – defaults to `3000`. +- `GRAFANA_APP_NAME` – defaults to `grafana`. +- `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` – local admin credentials. +- `GF_AUTH_GENERIC_OAUTH_CLIENT_ID` +- `GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET` +- `GF_AUTH_GENERIC_OAUTH_AUTH_URL` +- `GF_AUTH_GENERIC_OAUTH_TOKEN_URL` +- `GF_AUTH_GENERIC_OAUTH_API_URL` + +### Alertmanager +- `SMTP_HOST` / `SMTP_PORT` +- `SMTP_USERNAME` / `SMTP_PASSWORD` +- `SMTP_FROM_ADDRESS` +- `ALERT_EMAIL_TO` + +### Traefik / network (production) +- `BACKEND_APP_HOST` / `FRONTEND_APP_HOST` / `GRAFANA_APP_HOST` +- `CERT_RESOLVER` – e.g. `letsencrypt` + +### General +- `LOG_LEVEL` / `LOG_FORMAT` – also passed to Grafana and backend. + +## 3. Monitoring config files + +| File | Purpose | +|------|---------| +| `monitoring/prometheus/prometheus.yml` | Scrape jobs: backend `/metrics`, local node-exporter, remote node-exporter via file-SD, Loki, Alertmanager, Grafana, and self-scrape. | +| `monitoring/prometheus/rules/backup_alerts.yml` | Alert rules: `BackupJobFailed`, `BackupJobStuck`, `PrometheusTargetMissing`, `AlertmanagerDown`, `GrafanaDown`. | +| `monitoring/alertmanager/alertmanager.yml` | Routes alerts by severity, sends email to `ALERT_EMAIL_TO`, optional webhook to backend, and inhibits warnings when a critical alert fires. | +| `monitoring/grafana/grafana.ini` | Enables iframe embedding, OAuth via Authentik, role mapping from groups, and defaults users to `Viewer`. | +| `monitoring/alloy/config.alloy` | Discovers Docker containers, relabels container/stream labels, and writes logs to `http://loki:3100/loki/api/v1/push`. | +| `monitoring/loki/loki.yml` | Single-node Loki with filesystem storage, tsdb index, 30-day retention (`720h`). | + +## 4. Backend metrics and Prometheus scraping + +- The backend exposes Prometheus metrics at `/metrics` on port `8000`. +- The endpoint is unauthenticated (bypassed in `enforce_jwt_auth`). +- Prometheus scrapes it as `job="manage-backend"` with `scrape_interval: 15s`. +- Key application metrics include: + - `manage_api_requests_total{method, path, status_code}` + - `manage_api_request_duration_seconds{method, path}` + - `manage_ssh_commands_total{machine_id, action, status}` + - `manage_media_index_builds_total{status}` + - `manage_backup_runs_total{job_name, status}` + - `manage_backup_runs_last_success_timestamp{job_name}` + - `manage_mail_queue_messages_total{status}` + +Remote Node Exporter targets are not static: the backend reads machine settings from SQLite and writes `${PROMETHEUS_FILE_SD_DIR}/node_exporter_targets.json`. Prometheus reloads this file every 30 seconds via `file_sd_configs`. + +## 5. Setup steps and gotchas + +- The observability stack is brought up with the app itself: + - Production: `docker compose -f docker-compose.yml up --build` + - Development: `docker compose -f docker-compose.dev.yml up --build` +- Production requires the external `web` network and Traefik already configured; `docker-compose.dev.yml` does not use Traefik and binds ports directly. +- Export/copy `.env.example` to `.env` and fill required values (`OIDC_ISSUER_URL`, `OIDC_AUDIENCE`, `BACKEND_APP_HOST`, `FRONTEND_APP_HOST`, `GRAFANA_APP_HOST`, `CERT_RESOLVER`, SMTP credentials, Grafana OAuth secrets). +- `BACKEND_CACHE_DIR` is shared between the backend and Prometheus so file-SD target updates are visible to Prometheus. +- Alloy must run as `user: root` and mount `/var/run/docker.sock` and `/var/lib/docker/containers`; without these mounts it cannot collect Docker logs. +- Node Exporter mounts the host root filesystem read-only (`/:/host:ro,rslave`) to report host-level metrics; on production, this exposes the Docker host. +- Grafana iframe embedding requires `allow_embedding = true` in `grafana.ini` and matching cookie settings; also ensure the reverse proxy/CSP permits embedding. +- After changing Prometheus rules/config, trigger a reload with `curl -X POST http://localhost:9090/-/reload` (production needs Traefik/network access). +- Observability data is stored in named volumes: `prometheus_data`, `loki_data`, `grafana_data`, `alertmanager_data`. Back them up as documented in `docs/observability-runbooks.md`. +- Default retention is 30 days for both Prometheus TSDB and Loki logs. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 71545ed..53902c7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -11,12 +11,19 @@ services: OIDC_AUDIENCE: ${OIDC_AUDIENCE:-} OIDC_JWKS_URL: ${OIDC_JWKS_URL:-} LOG_LEVEL: ${LOG_LEVEL:-INFO} + LOG_FORMAT: ${LOG_FORMAT:-text} + PROMETHEUS_ENABLED: "true" SSH_KNOWN_HOSTS_PATH: /app/backend/.cache/known_hosts + PROMETHEUS_FILE_SD_DIR: /app/backend/.cache/prometheus-file-sd + ALERTMANAGER_URL: ${ALERTMANAGER_URL:-http://alertmanager:9093} ports: - "8000:8000" volumes: - ./backend:/app/backend - ${BACKEND_CACHE_DIR:-./backend-cache}:/app/backend/.cache + networks: + - web + - monitoring restart: unless-stopped frontend: @@ -38,5 +45,209 @@ services: - backend restart: unless-stopped + prometheus: + image: prom/prometheus:v2.55.1 + container_name: prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" + - "--web.console.libraries=/usr/share/prometheus/console_libraries" + - "--web.console.templates=/usr/share/prometheus/consoles" + - "--web.enable-lifecycle" + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro + - ${BACKEND_CACHE_DIR:-./backend-cache}/prometheus-file-sd:/etc/prometheus/file-sd:ro + - prometheus_data:/prometheus + ports: + - "9090:9090" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 1G + reservations: + cpus: "0.25" + memory: 256M + + loki: + image: grafana/loki:3.1.1 + container_name: loki + command: -config.file=/etc/loki/loki.yml + volumes: + - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro + - loki_data:/loki + ports: + - "3100:3100" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 1G + reservations: + cpus: "0.25" + memory: 256M + + alloy: + image: grafana/alloy:v1.5.0 + container_name: alloy + command: + - run + - /etc/alloy/config.alloy + - --storage.path=/var/lib/alloy + volumes: + - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + networks: + - monitoring + depends_on: + loki: + condition: service_healthy + restart: unless-stopped + user: root + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:12345/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + reservations: + cpus: "0.10" + memory: 128M + + grafana: + image: grafana/grafana:11.3.1 + container_name: grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID:-} + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:-} + GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${GF_AUTH_GENERIC_OAUTH_AUTH_URL:-} + GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL:-} + GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + volumes: + - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana_data:/var/lib/grafana + ports: + - "3000:3000" + networks: + - monitoring + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + + alertmanager: + image: prom/alertmanager:v0.27.0 + container_name: alertmanager + command: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + environment: + SMTP_HOST: ${SMTP_HOST:-smtp.example.com} + SMTP_PORT: ${SMTP_PORT:-587} + SMTP_USERNAME: ${SMTP_USERNAME:-} + SMTP_PASSWORD: ${SMTP_PASSWORD:-} + SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com} + ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com} + volumes: + - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager_data:/alertmanager + ports: + - "9093:9093" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + reservations: + cpus: "0.10" + memory: 64M + + node-exporter: + image: prom/node-exporter:v1.8.2 + container_name: node-exporter + command: + - "--path.rootfs=/host" + volumes: + - /:/host:ro,rslave + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + reservations: + cpus: "0.05" + memory: 32M + +networks: + monitoring: + volumes: frontend_node_modules: + prometheus_data: + loki_data: + grafana_data: + alertmanager_data: diff --git a/docker-compose.yml b/docker-compose.yml index f4edf78..a2a4cc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,8 @@ services: OIDC_JWKS_URL: ${OIDC_JWKS_URL:-} OIDC_CLOCK_SKEW_SECONDS: ${OIDC_CLOCK_SKEW_SECONDS:-30} LOG_LEVEL: ${LOG_LEVEL:-INFO} + LOG_FORMAT: ${LOG_FORMAT:-text} + PROMETHEUS_ENABLED: "true" SMTP_HOST: ${SMTP_HOST:-} SMTP_PORT: ${SMTP_PORT:-587} SMTP_USERNAME: ${SMTP_USERNAME:-} @@ -23,11 +25,14 @@ services: SMTP_USE_SSL: ${SMTP_USE_SSL:-false} SMTP_TIMEOUT: ${SMTP_TIMEOUT:-30} SSH_KNOWN_HOSTS_PATH: /app/backend/.cache/known_hosts + PROMETHEUS_FILE_SD_DIR: ${PROMETHEUS_FILE_SD_DIR:-/app/backend/.cache/prometheus-file-sd} + ALERTMANAGER_URL: ${ALERTMANAGER_URL:-http://alertmanager:9093} volumes: - ${BACKEND_CACHE_DIR:-./backend-cache}:/app/backend/.cache restart: unless-stopped networks: - web + - monitoring expose: - "8000" labels: @@ -80,6 +85,218 @@ services: - "8080:80" restart: unless-stopped + prometheus: + image: prom/prometheus:v2.55.1 + container_name: prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" + - "--web.console.libraries=/usr/share/prometheus/console_libraries" + - "--web.console.templates=/usr/share/prometheus/consoles" + - "--web.enable-lifecycle" + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro + - ${BACKEND_CACHE_DIR:-./backend-cache}/prometheus-file-sd:/etc/prometheus/file-sd:ro + - prometheus_data:/prometheus + expose: + - "9090" + networks: + - web + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 1G + reservations: + cpus: "0.25" + memory: 256M + + loki: + image: grafana/loki:3.1.1 + container_name: loki + command: -config.file=/etc/loki/loki.yml + volumes: + - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro + - loki_data:/loki + expose: + - "3100" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 1G + reservations: + cpus: "0.25" + memory: 256M + + alloy: + image: grafana/alloy:v1.5.0 + container_name: alloy + command: + - run + - /etc/alloy/config.alloy + - --storage.path=/var/lib/alloy + volumes: + - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + networks: + - monitoring + depends_on: + loki: + condition: service_healthy + restart: unless-stopped + user: root + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:12345/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + reservations: + cpus: "0.10" + memory: 128M + + grafana: + image: grafana/grafana:11.3.1 + container_name: grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID:-} + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:-} + GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${GF_AUTH_GENERIC_OAUTH_AUTH_URL:-} + GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL:-} + GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + volumes: + - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana_data:/var/lib/grafana + expose: + - "3000" + networks: + - web + - monitoring + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + labels: + - "traefik.enable=true" + - "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.rule=Host(`${GRAFANA_APP_HOST:?set GRAFANA_APP_HOST}`)" + - "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.entrypoints=websecure" + - "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.tls.certresolver=${CERT_RESOLVER:?set CERT_RESOLVER}" + - "traefik.http.services.${GRAFANA_APP_NAME:-grafana}.loadbalancer.server.port=${GRAFANA_APP_PORT:-3000}" + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + + alertmanager: + image: prom/alertmanager:v0.27.0 + container_name: alertmanager + command: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + environment: + SMTP_HOST: ${SMTP_HOST:-smtp.example.com} + SMTP_PORT: ${SMTP_PORT:-587} + SMTP_USERNAME: ${SMTP_USERNAME:-} + SMTP_PASSWORD: ${SMTP_PASSWORD:-} + SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com} + ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com} + volumes: + - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager_data:/alertmanager + expose: + - "9093" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + reservations: + cpus: "0.10" + memory: 64M + + node-exporter: + image: prom/node-exporter:v1.8.2 + container_name: node-exporter + command: + - "--path.rootfs=/host" + volumes: + - /:/host:ro,rslave + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + reservations: + cpus: "0.05" + memory: 32M + networks: web: external: true + monitoring: + +volumes: + prometheus_data: + loki_data: + grafana_data: + alertmanager_data: diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index 2fcaf94..2fbe9d8 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -197,6 +197,12 @@ Phase 1: Jellyfin media index, SSH-based remote filesystem inspection, server mo ## Decision Log +- 2026-06-13: Adopted a dedicated, self-hosted observability subsystem based on Prometheus, Grafana Loki, Grafana, and Alertmanager. Metrics will be pulled from Node Exporter on machines and from application exporters in containers; logs will be structured JSON shipped by Promtail/Grafana Alloy. The existing POSIX remote collector will be removed and backup alerts migrated to Alertmanager rules. See `docs/monitoring-logging-design.md`. +- 2026-06-13 (Phase 1): Added Prometheus, Loki, Grafana Alloy, Grafana, Alertmanager, and Node Exporter services to `docker-compose.yml` and `docker-compose.dev.yml`. Provisioned Grafana datasources and an initial `Manage Overview` dashboard as code. Configured Alloy to tail Docker logs and ship to Loki. Added Grafana generic OAuth configuration via `monitoring/grafana/grafana.ini` and a dedicated Traefik host rule. Added Alertmanager email routing with env-var interpolation. Added `/grafana` proxy to the Vite dev server for iframe embedding. +- 2026-06-13 (Phase 2): Extended machine settings with `node_exporter_enabled`, `node_exporter_port`, and `node_exporter_scrape_host`. Added Node Exporter install/restart/status job templates to `jobs.py`. Implemented `media_library_viewer_api.services.targets` to generate Prometheus file-SD target files and wired target regeneration into machine create/update/delete. Added `/api/monitoring/prometheus-targets` for live target previews. Configured Prometheus with a `node-exporter-remote` job reading file SD from the backend cache volume. Added a minimal `Node Exporter Overview` Grafana dashboard. Added unit and integration tests for target generation and the new endpoint. +- 2026-06-13 (Phase 3): Added `ALERTMANAGER_URL` setting and `GET /api/monitoring/alerts` plus `GET /api/monitoring/alertmanager-status` endpoints that proxy/summarize Alertmanager for the UI. Added `manage_backup_runs_last_success_timestamp` Prometheus gauge and wired `record_backup_run` to update it on success so the existing `BackupJobStuck` Alertmanager rule works. Updated `docker-compose.yml`, `docker-compose.dev.yml`, and `.env.example` with `ALERTMANAGER_URL`. Added tests for the new endpoints and metric helpers. +- 2026-06-13 (Phase 4): Added `/observability` React page (`frontend/src/components/ObservabilityPage.tsx`) with health cards for Alertmanager, active alerts, Prometheus targets, and machines; a recent-alerts list; machine selector; and Grafana iframe panels for Node Exporter metrics and Loki logs. Added `useObservability` hook and API client wrappers for `/api/monitoring/alerts`, `/api/monitoring/alertmanager-status`, and `/api/monitoring/prometheus-targets`. Added TypeScript types for Alertmanager summary/status and Prometheus targets. Wired the new route into `App.tsx` and the sidebar. Added shadcn/ui `card`, `badge`, `alert`, `skeleton`, and `select` components. Frontend build (`npm run build`) passes; lint has only pre-existing warnings. +- 2026-06-13 (Phase 5): Hardened observability containers with health checks and resource limits in both compose files; added service-healthy `depends_on` conditions. Added Prometheus scrape jobs for Loki, Alertmanager, and Grafana, plus new `observability_health` alert rules. Added `ALERTMANAGER_WEBHOOK_URL` backend setting, `POST /api/monitoring/alertmanager-webhook` receiver, and Alertmanager `webhook` receiver config. Added `docs/observability-runbooks.md` with operational playbooks. Updated `.env.example` to include all observability variables. - 2026-05-03: Reaffirmed that the Monitoring tab charts should be rendered directly with D3 and expose brush-based time-range selection plus moving averages. - 2026-05-03: Added hover tooltips, summary chips, a moving vertical cursor, snapped point markers, and a selected-range label to the D3 Monitoring charts for faster visual inspection. - 2026-05-03: Combined network download/upload into one traffic chart and disk read/write into one I/O chart for clearer Monitoring layout. diff --git a/docs/monitoring-logging-design.md b/docs/monitoring-logging-design.md new file mode 100644 index 0000000..4e390df --- /dev/null +++ b/docs/monitoring-logging-design.md @@ -0,0 +1,438 @@ +# Monitoring and Logging Design — Manage + +## Executive Summary + +Manage currently uses ad-hoc observability: plain-text Python logs, a custom POSIX shell metrics collector on remote machines, and a background poller that stores snapshots in SQLite. This works for a single-instance homelab but becomes painful as the fleet grows and as users need faster incident response. + +This document proposes a dedicated, self-hosted observability subsystem built on the standard Prometheus/Grafana stack: + +- **Metrics**: Prometheus pulling from Node Exporter on machines and from application exporters in containers. +- **Logs**: Structured JSON logs shipped to **Grafana Loki** by **Promtail/Grafana Alloy**. +- **Dashboards**: Grafana for deep-dive dashboards, embedded in the Manage React UI via iframes. +- **Alerting**: Prometheus Alertmanager for routing and notifications (email first, webhooks later). +- **Auth**: Grafana authenticates through the existing OIDC/Authentik provider. + +The existing POSIX remote collector will be removed, and the Python backup alert engine will be migrated to Alertmanager rules. + +--- + +## Goals + +1. **Fast query and alerting**: move from SQLite scan-based history to a real time-series database and indexed log store. +2. **Unified view**: monitor both local containers/apps and remote Linux machines from one place. +3. **Standard tooling**: use de-facto open-source tools so dashboards, exporters, and runbooks are reusable. +4. **Room to grow**: design supports adding traces, more notification channels, and longer retention later without re-architecture. + +## Non-Goals + +1. **Traces**: deferred to a later phase; the data flow and collector choice (Promtail/Alloy) will be trace-ready. +2. **Multi-tenant RBAC**: Manage is single-instance/homelab; Grafana teams are sufficient for now. +3. **SLA/SLO framework**: out of scope; we focus on metrics, logs, and alerts, not SLO budgeting. +4. **Cloud-hosted observability vendors**: all components run self-hosted in Docker Compose. + +--- + +## Decisions + +| Area | Decision | Rationale | +|------|----------|-----------| +| Coupling | Dedicated observability subsystem consumed by Manage | Keeps Manage fast and lets the observability stack evolve independently. | +| Metrics backend | Prometheus | Pull model, huge ecosystem, standard exporters, easy Grafana integration. | +| Machine metrics | Node Exporter | Rich OS metrics, reusable dashboards, no custom shell to maintain. | +| Log backend | Grafana Loki | Prometheus-style labels, low resource use, tight Grafana integration. | +| Log collection | Promtail / Grafana Alloy | Tails Docker logs and journald; no per-app network calls. | +| App logs | Structured JSON to stdout | Standard 12-factor pattern; collector handles routing. | +| Dashboards | Grafana + iframe embeds | Fast to implement, rich dashboards, Manage UI stays focused on summary. | +| Alerting | Prometheus Alertmanager | Mature routing, silencing, inhibition; single source of truth for infra alerts. | +| Auth | Grafana OAuth via Authentik | Reuses existing identity provider; consistent UX. | +| Retention | 30 days metrics, 30 days logs | Matches current retention policy; disk usage stays predictable. | +| Migration | Remove POSIX collector, migrate backup alerts | Eliminates duplicate alerting paths and custom remote code. | + +--- + +## Current State + +### Logging + +- `backend/src/media_library_viewer_api/logging_utils.py` configures stdlib `logging` with a plain-text format. +- `main.py` has a `log_requests` middleware that emits method, path, client IP, status code, and elapsed time. +- Frontend uses standard `console.log` / browser dev tools; no server-side log aggregation. + +### Metrics + +- `backend/src/media_library_viewer_api/clients/resources.py` deploys a POSIX shell collector to `/tmp` on each remote machine. +- The collector samples `/proc/stat`, `/proc/meminfo`, `/proc/net/dev`, and `/sys/block/*/stat` every 10s and writes JSONL to `/tmp/media_library_viewer_metrics.jsonl`. +- `MonitoringPoller` (`monitoring_poller.py`) runs every 5 minutes, reads the remote JSONL, and stores snapshots in SQLite (`monitoring_machine_actions`). +- Retention defaults to 30 days with periodic pruning. + +### Alerting + +- `backup_alert_engine.py` / `backup_poller.py` generate backup-related alerts (failure, anomaly, missed schedule) and store them in SQLite. +- No general infrastructure alerting (disk full, machine down, high CPU, etc.). + +--- + +## Target Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Docker Compose Network │ +│ │ +│ ┌─────────────┐ scrape ┌──────────────┐ │ +│ │ Prometheus │◄────────────────│ Node Exporter│◄── host / remote hosts │ +│ │ (TSDB) │ └──────────────┘ │ +│ └──────┬──────┘ │ +│ │ query │ +│ ▼ │ +│ ┌─────────────┐ alert ┌─────────────┐ email ┌──────────┐ │ +│ │ Grafana │──────────────►│ Alertmanager│──────────────►│ SMTP │ │ +│ │ (OAuth) │ └─────────────┘ └──────────┘ │ +│ └──────┬──────┘ │ +│ │ embed (iframe) │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ Manage React │ │ +│ │ (summary) │ │ +│ └─────────────────┘ │ +│ │ +│ Logs: │ +│ Manage / containers ──stdout──► Promtail/Alloy ──push──► Loki ◄──────┐ │ +│ host / remote journald ───────► Promtail/Alloy ──push──► Loki │ │ +│ │ │ +│ Grafana queries Loki for logs ◄──────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Component Responsibilities + +| Component | Responsibility | +|-----------|----------------| +| **Prometheus** | Scrape and store metrics; evaluate alert rules. | +| **Node Exporter** | Expose host-level metrics (CPU, memory, disk, network, filesystem). | +| **Loki** | Store and index log streams by labels. | +| **Promtail / Alloy** | Discover log sources, parse labels, and push to Loki. | +| **Grafana** | Visualize metrics and logs; serve as the alert UI. | +| **Alertmanager** | Deduplicate, group, route, and deliver alerts. | +| **Manage backend** | Emit structured logs; expose `/metrics` for Prometheus; forward health/status to summary endpoints. | +| **Manage frontend** | Embed Grafana panels; show high-level status cards. | + +--- + +## Instrumentation Changes + +### Backend Logging + +1. Switch to structured JSON logging via `python-json-logger` or `structlog`. +2. Include fields: + - `timestamp`, `level`, `logger`, `message` + - `request_id` (correlation ID propagated from frontend or generated) + - `method`, `path`, `status_code`, `elapsed_ms` + - `user_id`, `machine_id` where relevant + - `error` / `error_type` / `traceback` for exceptions +3. Keep emitting to stdout; Promtail/Alloy will parse JSON. + +**Status**: implemented in `backend/src/media_library_viewer_api/logging_utils.py` with `LOG_FORMAT=json|text`, secret-safe `sanitize_log_extra`, and request logging in `main.py`. + +### Request Middleware + +- Extend `log_requests` to attach `request_id` to `request.state`. +- Include `request_id` in response headers (`X-Request-Id`) so the frontend can correlate. +- Log all outbound SSH commands with `machine_id`, `action`, `duration_ms`, and `request_id`. + +**Status**: implemented in `main.py` and `observability.py`; `record_ssh_command` is called from `monitoring_actions.py` for every machine operation. + +### Application Metrics Endpoint + +- Add a `/metrics` endpoint using `prometheus-client`. +- Initial counters/gauges: + - `manage_api_requests_total` (method, path, status) + - `manage_api_request_duration_seconds` histogram + - `manage_ssh_commands_total` (machine_id, action, status) + - `manage_ssh_command_duration_seconds` histogram + - `manage_media_index_build_duration_seconds` + - `manage_backup_runs_total` (job_name, status) + - `manage_mail_queue_size`, `manage_mail_queue_failures_total` + +**Status**: implemented in `backend/src/media_library_viewer_api/observability.py` and wired into `main.py`, `monitoring_actions.py`, `backups.py`, `media.py`, and `mail_queue.py`. + +### Frontend Observability + +- Keep first phase minimal: capture JS errors and send them to the backend as structured log events. +- Optional later: expose RUM-style metrics (page loads, API call latencies) via Prometheus client library or manual instrumentation. + +--- + +## Node Exporter Deployment + +### Local / Docker Host + +- Add a `node-exporter` service to `docker-compose.yml` with host PID/network mounts. +- Prometheus scrapes it as `job="node"`. + +### Remote Machines + +- Add a managed task/template in `jobs.py` to install/upgrade Node Exporter via the package manager or a static binary. +- Manage exposes a settings flag per machine: `node_exporter_enabled`. +- For machines behind NAT, use one of: + 1. Reverse SSH tunnel from machine to Manage host. + 2. VPN/Wireguard already in place. + 3. Prometheus federation or pushgateway for unreachable targets (later phase). +- If Node Exporter cannot be installed, temporarily keep the POSIX collector as a fallback until migration is complete. + +--- + +## Log Shipping + +### Docker Compose Services + +- Add `logging` driver config or Promtail sidecar to each service. +- Preferred: run **Grafana Alloy** as a single daemon container with `docker_sd_config` to discover all Compose services automatically. + +### Host Logs + +- Alloy mounts `/var/log` and `/var/lib/docker/containers` (read-only). +- Alloy also tails journald where available. + +### Remote Machines + +- Option A: install Alloy on remote hosts and have it push logs to Loki. +- Option B: keep logs on remote hosts and use Node Exporter logs only; defer centralized remote logs. +- Recommendation: **Option A** for important machines, **Option B** for constrained ones. + +--- + +## Dashboards + +### Grafana + +- Provision dashboards from YAML/JSON in version control: + - Node Exporter Full dashboard (import from Grafana.com). + - Manage API overview (request rate, latency, errors). + - Manage operations (SSH commands, media index builds, mail queue). + - Backup runs and alert history. +- Manage iframe embeds point to specific dashboard panels using Grafana's `panelId` and ` kiosk` mode. + +### Manage React UI + +- Add an "Observability" page with: + - System health cards (Prometheus up, Loki up, Alertmanager up). + - Recent alerts summary from Alertmanager API. + - Iframe panels for key metrics (CPU/memory of selected machine, recent logs). +- Drill-down links open the full Grafana dashboard. + +--- + +## Alerting + +### Alertmanager Configuration + +- Reuse existing SMTP settings for email notifications. +- Initial routing: + - `severity=critical` → email immediately. + - `severity=warning` → email with 5-minute group wait. + - `job=backup` → grouped by job name. + +### Initial Alert Rules + +- Infrastructure: + - Node down for > 5 minutes. + - Disk usage > 85% (warning), > 95% (critical). + - Memory usage > 90% for > 10 minutes. + - CPU iowait > 30% for > 10 minutes. +- Application: + - Manage API 5xx rate > 1% over 5 minutes. + - SSH command failure rate > 10% over 5 minutes. + - Mail queue growing or failures increasing. +- Backup: + - Backup job failed (`manage_backup_runs_total{status="failure"}`). + - Backup job missing for > 1.5× schedule interval. + - Backup run duration or size anomaly compared to rolling median. + +### Backup Alert Migration + +- Re-implement rules as Prometheus recording/alerting rules where possible. +- Keep historical comparison logic (median duration/size) as a small scheduled task that writes anomaly metrics to a Pushgateway or custom exporter, then Alertmanager consumes them. +- Preserve acknowledge/resolve workflow by storing Alertmanager webhook events in SQLite if needed, or by using Grafana alert annotations. + +--- + +## Authentication + +- Grafana configured with generic OAuth pointing at Authentik (same issuer as Manage). +- Grafana role mapping: default `Viewer`; admin group mapped to `Admin`. +- Traefik routes `grafana.${BACKEND_APP_HOST}` or a sub-path. +- Iframe embedding requires Grafana `allow_embedding = true` and matching cookie domain/samesite settings. + +--- + +## Retention and Storage + +| Store | Retention | Notes | +|-------|-----------|-------| +| Prometheus | 30 days | Default TSDB block compaction. | +| Loki | 30 days | Single-store boltdb-shipper or filesystem target. | +| Grafana | persistent SQLite/Postgres later | Dashboards and users are config, not runtime data. | + +- Volumes: `prometheus-data`, `loki-data`, `grafana-data`. +- Backups: snapshot these volumes alongside existing `backend_cache`. + +--- + +## Security + +1. Network: all observability services on an internal Docker network; exposed only through Traefik where needed. +2. Node Exporter: bind to localhost on remote hosts and use a reverse tunnel, or firewall to Manage IP only. +3. Secrets: SMTP password, OIDC client secret, and any remote scrape credentials in environment variables or Docker secrets; never commit them. +4. Logs: sanitize tokens, passwords, and private keys before JSON serialization. +5. Alertmanager: disable unauthenticated UI if exposed publicly; rely on OIDC/Traefik. + +--- + +## Implementation Plan + +### Phase 0 — Foundation and Cleanup + +- [x] Add `prometheus-client` and `python-json-logger` to `backend/pyproject.toml`. +- [x] Refactor `logging_utils.py` to emit JSON when `LOG_FORMAT=json`. +- [x] Add `request_id` propagation in `log_requests` middleware. +- [x] Add `/metrics` endpoint with initial counters/gauges. +- [x] Remove the POSIX remote collector code in `resources.py`; keep `disk_space` as a lightweight SSH/local helper in `monitoring_actions.py`. +- [x] Add `X-Request-Id` response header. + +### Phase 1 — Local Observability Stack + +- [x] Add services to `docker-compose.yml`: Prometheus, Loki, Grafana, Alertmanager, Grafana Alloy. +- [x] Add `node-exporter` service for the Docker host. +- [x] Configure Alloy to scrape all Docker container logs and ship to Loki. +- [x] Configure Prometheus to scrape `node-exporter` and Manage `/metrics`. +- [x] Provision Grafana datasources and a basic Manage API dashboard. +- [x] Wire Grafana OAuth to Authentik. + +**Phase 1 files**: +- `monitoring/prometheus/prometheus.yml` +- `monitoring/prometheus/rules/backup_alerts.yml` +- `monitoring/loki/loki.yml` +- `monitoring/alloy/config.alloy` +- `monitoring/alertmanager/alertmanager.yml` +- `monitoring/grafana/grafana.ini` +- `monitoring/grafana/provisioning/datasources/datasources.yml` +- `monitoring/grafana/provisioning/dashboards-json/dashboards.yml` +- `monitoring/grafana/provisioning/dashboards-json/dashboards/manage-overview.json` +- `docker-compose.yml` and `docker-compose.dev.yml` updated with observability services. +- `frontend/vite.config.ts` updated with `/grafana` dev proxy. +- `.env.example` updated with Grafana OAuth and alerting variables. + +### Phase 2 — Remote Machine Metrics + +- [x] Add Node Exporter install/restart/status job templates in `jobs.py` (`install_node_exporter`, `restart_node_exporter`, `node_exporter_status`). +- [x] Add `node_exporter_enabled`, `node_exporter_port`, and `node_exporter_scrape_host` fields to `MonitoringMachineInput` and `SettingsStore`. +- [x] Implement `media_library_viewer_api.services.targets` to build Prometheus file-SD target lists for enabled SSH machines and write them to `PROMETHEUS_FILE_SD_DIR/node_exporter_targets.json`. +- [x] Regenerate file-SD targets on machine create/update/delete in `routers/settings.py`. +- [x] Add `/api/monitoring/prometheus-targets` endpoint returning live targets from the store. +- [x] Configure Prometheus `node-exporter-remote` job with `file_sd_configs` reading `/etc/prometheus/file-sd/node_exporter_targets.json`. +- [x] Mount the backend cache `prometheus-file-sd` directory into the Prometheus container as a read-only file-SD source. +- [x] Add `PROMETHEUS_FILE_SD_DIR` setting and `.env.example` entry. +- [x] Provision a minimal `Node Exporter Overview` Grafana dashboard (`monitoring/grafana/provisioning/dashboards-json/dashboards/node-exporter-overview.json`) covering CPU, memory, root disk, and network traffic. +- [x] Remove POSIX collector fallback. The legacy collector code in `backend/src/media_library_viewer_api/clients/resources.py` has been deleted, the collector control endpoints were removed from `routers/monitoring.py`, and `disk_space` was relocated to `services/monitoring_actions.py` as a lightweight SSH/local helper. Metrics are now sourced exclusively from Prometheus/Node Exporter. + +**Phase 2 files**: +- `backend/src/media_library_viewer_api/jobs.py` (Node Exporter job templates). +- `backend/src/media_library_viewer_api/routers/settings.py` (machine input fields + target regeneration). +- `backend/src/media_library_viewer_api/services/settings_store.py` (machine persistence fields). +- `backend/src/media_library_viewer_api/services/targets.py` (file-SD target builder/writer). +- `backend/src/media_library_viewer_api/routers/monitoring.py` (`/prometheus-targets` endpoint). +- `backend/src/media_library_viewer_api/config.py` (`prometheus_file_sd_dir` setting). +- `backend/tests/test_targets.py` and `backend/tests/test_api.py` (target + endpoint tests). +- `monitoring/prometheus/prometheus.yml` (`node-exporter-remote` file SD job). +- `monitoring/grafana/provisioning/dashboards-json/dashboards/node-exporter-overview.json`. +- `docker-compose.yml` and `docker-compose.dev.yml` (file-SD volume mount + backend env var). +- `.env.example` (`PROMETHEUS_FILE_SD_DIR`). + +### Phase 3 — Alerting + +- [x] Define initial Prometheus alert rules for backup failures (infrastructure rules deferred to Phase 2/3). +- [x] Configure Alertmanager with email routing using existing SMTP settings; `monitoring/alertmanager/alertmanager.yml` uses env vars for SMTP and routing. +- [x] Migrate backup alert rules to Alertmanager: + - `BackupJobFailed` triggers on `increase(manage_backup_runs_total{status="failed"}[1h]) > 0`. + - `BackupJobStuck` triggers on `time() - manage_backup_runs_last_success_timestamp > 86400`. + - Added `manage_backup_runs_last_success_timestamp` gauge in `observability.py` and updated `routers/backups.py` to set it on successful runs. + - SQLite backup alerts (`backup_alert_engine.py` and `backup_poller.py`) are preserved for now alongside Alertmanager rules; the UI can consume either source during transition. +- [x] Add Alertmanager status summary endpoints in Manage backend: + - `GET /api/monitoring/alerts` proxies `/api/v1/alerts` and returns a UI-friendly summary (total, by_severity, alerts list). + - `GET /api/monitoring/alertmanager-status` proxies `/api/v2/status` and returns `up`, `version`, `uptime`, `peers`. +- [x] Added `alertmanager_url` setting to `config.py` (default `http://alertmanager:9093`) and `ALERTMANAGER_URL` env var in both compose files and `.env.example`. +- [x] Added tests for the Alertmanager endpoints and the backup success gauge. + +**Phase 3 files**: +- `backend/src/media_library_viewer_api/routers/monitoring.py` (`/alerts` and `/alertmanager-status` endpoints). +- `backend/src/media_library_viewer_api/observability.py` (`BACKUP_RUNS_LAST_SUCCESS` gauge + updated `record_backup_run`). +- `backend/src/media_library_viewer_api/routers/backups.py` (pass `success=True` to `record_backup_run` on successful reports). +- `backend/src/media_library_viewer_api/config.py` (`alertmanager_url` setting). +- `monitoring/alertmanager/alertmanager.yml` (SMTP + routing config). +- `monitoring/prometheus/rules/backup_alerts.yml` (backup alert rules). +- `docker-compose.yml` / `docker-compose.dev.yml` (`ALERTMANAGER_URL` env var). +- `.env.example` (`ALERTMANAGER_URL`). +- `backend/tests/test_api.py` (`TestAlertmanager` tests). +- `backend/tests/test_observability.py` (backup metric tests). + +### Phase 4 — Manage UI Integration + +- [x] Add "Observability" page in React with summary cards (Alertmanager health, active alerts, Prometheus targets, machines) and Grafana iframe panels. +- [x] Add recent alerts list from Alertmanager API via `GET /api/monitoring/alerts`. +- [x] Add drill-down links to full Grafana dashboards for Node Exporter metrics and Loki logs. +- [x] Handle iframe sandbox attributes (`allow-scripts allow-same-origin allow-popups allow-forms`); CSP is delegated to the reverse proxy / Grafana `allow_embedding` configuration. +- [x] Add `useObservability` hook and API client wrappers for alerts, Alertmanager status, and Prometheus targets. +- [x] Add TypeScript types for Alertmanager summary/status and Prometheus targets. +- [x] Wire the new `/observability` route into `App.tsx` and the sidebar navigation. + +**Phase 4 files**: +- `frontend/src/components/ObservabilityPage.tsx` (page component). +- `frontend/src/hooks/useObservability.ts` (React Query hooks). +- `frontend/src/api/client.ts` (API client functions). +- `frontend/src/types/index.ts` (new interfaces). +- `frontend/src/App.tsx` (route + nav item). +- `frontend/src/components/ui/{card,badge,alert,skeleton,select}.tsx` (shadcn/ui components). +- `frontend/vite.config.ts` already has `/grafana` dev proxy for iframe source. + +### Phase 5 — Hardening and Future-Proofing + +- [x] Add health checks and `deploy.resources` limits for Prometheus, Loki, Alloy, Grafana, Alertmanager, and Node Exporter in both compose files. +- [x] Use `depends_on` with `condition: service_healthy` for Alloy → Loki and Grafana → Prometheus/Loki. +- [x] Add Prometheus scrape jobs for Loki, Alertmanager, and Grafana so their `up` metrics are available for health alerts. +- [x] Add observability health alerting rules (`PrometheusTargetMissing`, `AlertmanagerDown`, `GrafanaDown`). +- [x] Add `ALERTMANAGER_WEBHOOK_URL` backend setting, `POST /api/monitoring/alertmanager-webhook` receiver, and Alertmanager `webhook` receiver config. +- [x] Document runbooks for common alerts. +- [x] Add volume backups for Prometheus/Loki/Grafana data. Backup/restore procedures for `prometheus_data`, `loki_data`, `grafana_data`, and `alertmanager_data` are documented in `docs/observability-runbooks.md`. +- [ ] Optional: add OpenTelemetry Collector as a translation layer for traces later. + +**Phase 5 files**: +- `docker-compose.yml` and `docker-compose.dev.yml` (health checks, resource limits, `depends_on` conditions). +- `monitoring/prometheus/prometheus.yml` (additional scrape jobs for observability services). +- `monitoring/prometheus/rules/backup_alerts.yml` (renamed scope to include observability health alerts). +- `backend/src/media_library_viewer_api/config.py` (`alertmanager_webhook_url` setting). +- `backend/src/media_library_viewer_api/routers/monitoring.py` (`POST /api/monitoring/alertmanager-webhook`). +- `monitoring/alertmanager/alertmanager.yml` (`webhook` receiver). +- `backend/tests/test_api.py` (`TestAlertmanagerWebhook`). +- `docs/observability-runbooks.md` (new runbook documentation). + +--- + +## Risks and Mitigations + +| Risk | Mitigation | +|------|------------| +| Node Exporter hard to install on NAS/minimal hosts | Keep POSIX collector as opt-in fallback; document manual install steps. | +| Grafana iframe embedding blocked by CSP or cookies | Test early in Phase 4; use matching domains and `allow_embedding`. | +| Prometheus storage grows faster than expected | Start with 30-day retention; add compaction alerts. | +| Backup alert semantics lost in migration | Write tests comparing old Python alerts vs new Alertmanager rules. | +| OIDC configuration drift between Manage and Grafana | Use same env vars/Authentik application for both. | +| Remote scrape requires network path | Use reverse SSH tunnels or defer remote scraping until VPN is ready. | + +--- + +## Open Questions + +1. ~~What sub-domain or sub-path should Grafana use? (`grafana.${BACKEND_APP_HOST}` vs `${BACKEND_APP_HOST}/grafana`)~~ Decided: dedicated `GRAFANA_APP_HOST` subdomain in production; dev uses port 3000 and a `/grafana` proxy in Vite. +2. Should remote machines run Node Exporter as a systemd service or a container? +3. Do we need remote log aggregation immediately, or can it wait until after metrics alerting is stable? +4. Should the existing backup alert acknowledgement/resolve UI be rebuilt on top of Alertmanager, or replaced by Grafana alert annotations? diff --git a/docs/observability-runbooks.md b/docs/observability-runbooks.md new file mode 100644 index 0000000..41bf1c1 --- /dev/null +++ b/docs/observability-runbooks.md @@ -0,0 +1,190 @@ +# Observability Runbooks + +Operational playbooks for the Manage self-hosted observability stack (Prometheus, Grafana, Loki, Alertmanager). + +## Service Overview + +| Service | Compose name | Internal URL | Health check | +|---------|--------------|--------------|--------------| +| Prometheus | `prometheus` | `http://prometheus:9090` | `/-/healthy` | +| Grafana | `grafana` | `http://grafana:3000` | `/api/health` | +| Loki | `loki` | `http://loki:3100` | `/ready` | +| Alloy | `alloy` | `http://alloy:12345` | `/-/healthy` | +| Alertmanager | `alertmanager` | `http://alertmanager:9093` | `/-/healthy` | +| Node Exporter | `node-exporter` | `http://node-exporter:9100` | `/` | +| Manage backend | `backend` | `http://backend:8000` | `/api/health` | + +--- + +## Alert: `BackupJobFailed` + +**Severity**: critical +**Meaning**: A backup job reported `status=failed` within the last hour. + +### Steps + +1. Open **Manage → Backups** and identify the failed job/run. +2. Check the run output / logs for the failure reason. +3. Search Loki for `{container="backend"} | json | message=~"(?i)backup"` around the failure time. +4. If transient (network, lock file), retry the job. +5. If persistent, open a task to fix the backup script or credentials. + +--- + +## Alert: `BackupJobStuck` + +**Severity**: warning +**Meaning**: No successful backup run has been recorded for a job in the last 24 hours. + +### Steps + +1. Confirm the job is still scheduled and expected to run. +2. Check whether the backup scheduler/host is running. +3. Verify the job can still report success to `POST /api/backups/reports`. +4. Inspect Prometheus graph for `manage_backup_runs_last_success_timestamp` by `job_name`. +5. If the job was intentionally retired, remove or disable its reporting. + +--- + +## Alert: `PrometheusTargetMissing` + +**Severity**: warning +**Meaning**: A Prometheus scrape target is down (`up == 0`) for more than 2 minutes. + +### Steps + +1. Identify `job` and `instance` from the alert labels. +2. Check the container/process status: + - `docker compose ps ` + - `docker compose logs --tail 100 ` +3. Verify network reachability from the Prometheus container: + - `docker compose exec prometheus wget -qO- http:///` +4. If the target is a remote Node Exporter: + - Check the machine is reachable over SSH. + - Verify Node Exporter is installed and running (`systemctl status node_exporter`). + - Confirm the scrape host/port in Manage → Settings for that machine. +5. Restart if needed: `docker compose restart `. + +--- + +## Alert: `AlertmanagerDown` + +**Severity**: critical +**Meaning**: Prometheus cannot scrape Alertmanager; new alerts may not be delivered. + +### Steps + +1. Check container status: `docker compose ps alertmanager` +2. Review logs: `docker compose logs --tail 200 alertmanager` +3. Validate config syntax: + - `docker compose exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml` +4. Verify SMTP environment variables are present if using email receivers. +5. Restart: `docker compose restart alertmanager` + +--- + +## Alert: `GrafanaDown` + +**Severity**: warning +**Meaning**: Grafana is unreachable; dashboards and iframe panels in Manage are unavailable. + +### Steps + +1. Check container status and logs. +2. Verify the OAuth client configuration is correct (`GF_AUTH_GENERIC_OAUTH_*`). +3. If embedded panels are blank, confirm Grafana `allow_embedding = true` and cookie settings. +4. Restart: `docker compose restart grafana` + +--- + +## Routine Maintenance + +### Check overall health + +```bash +cd /path/to/manage +docker compose ps +docker compose exec prometheus wget -qO- http://127.0.0.1:9090/-/healthy +docker compose exec grafana wget -qO- http://127.0.0.1:3000/api/health +docker compose exec loki wget -qO- http://127.0.0.1:3100/ready +docker compose exec alertmanager wget -qO- http://127.0.0.1:9093/-/healthy +``` + +### Reload Prometheus after rule/config changes + +Prometheus is started with `--web.enable-lifecycle`, so a SIGHUP or HTTP call reloads config: + +```bash +curl -X POST http://localhost:9090/-/reload +``` + +### Inspect logs + +```bash +# All backend logs in Loki via Grafana Explore, or locally: +docker compose logs --tail 500 backend + +# Specific service: +docker compose logs -f prometheus +``` + +### Storage usage + +```bash +docker system df -v +docker compose exec prometheus du -sh /prometheus +docker compose exec loki du -sh /loki +docker compose exec grafana du -sh /var/lib/grafana +``` + +--- + +## Backup and Disaster Recovery + +The observability data lives in named volumes: + +- `prometheus_data` +- `loki_data` +- `grafana_data` +- `alertmanager_data` + +### Backup volumes + +```bash +# Stop the stack to ensure consistency +docker compose down + +# Back up each volume to a tarball +docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \ + tar czf /backups/prometheus-$(date +%F).tar.gz -C /data . +docker run --rm -v manage_loki_data:/data -v $(pwd)/backups:/backups alpine \ + tar czf /backups/loki-$(date +%F).tar.gz -C /data . +docker run --rm -v manage_grafana_data:/data -v $(pwd)/backups:/backups alpine \ + tar czf /backups/grafana-$(date +%F).tar.gz -C /data . +docker run --rm -v manage_alertmanager_data:/data -v $(pwd)/backups:/backups alpine \ + tar czf /backups/alertmanager-$(date +%F).tar.gz -C /data . + +# Start the stack again +docker compose up -d +``` + +> Replace `manage_` with your actual Docker Compose project name if different. + +### Restore a volume + +```bash +docker compose down +docker volume rm manage_prometheus_data +docker volume create manage_prometheus_data +docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \ + tar xzf /backups/prometheus-YYYY-MM-DD.tar.gz -C /data +docker compose up -d +``` + +--- + +## Scaling Notes + +- The current `deploy.resources` blocks are tuned for a small homelab. Raise memory limits if you monitor many machines or retain logs longer than 30 days. +- Loki is configured for single-node filesystem storage. For larger deployments, migrate to object storage (S3/GCS/MinIO) and a shared index. +- Prometheus remote-write or Thanos/Cortex can be added later for long-term metrics without changing application instrumentation. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 915b003..cb1c6af 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -35,7 +35,7 @@ "@tailwindcss/postcss": "^4.3.0", "@tailwindcss/vite": "^4.3.0", "@types/d3": "^7.4.3", - "@types/node": "^24.12.2", + "@types/node": "^24.13.2", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^6.0.1", @@ -3375,9 +3375,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3395,9 +3392,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3415,9 +3409,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3435,9 +3426,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3455,9 +3443,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3475,9 +3460,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3737,9 +3719,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3757,9 +3736,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3777,9 +3753,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3797,9 +3770,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4254,12 +4224,12 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "24.12.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", - "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==", + "version": "24.13.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.2.tgz", + "integrity": "sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==", "license": "MIT", "dependencies": { - "undici-types": "~7.16.0" + "undici-types": "~7.18.0" } }, "node_modules/@types/parse-json": { @@ -7532,9 +7502,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7556,9 +7523,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7580,9 +7544,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7604,9 +7565,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -9774,9 +9732,9 @@ } }, "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "license": "MIT" }, "node_modules/unicorn-magic": { diff --git a/frontend/package.json b/frontend/package.json index 158186c..88d0509 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -37,7 +37,7 @@ "@tailwindcss/postcss": "^4.3.0", "@tailwindcss/vite": "^4.3.0", "@types/d3": "^7.4.3", - "@types/node": "^24.12.2", + "@types/node": "^24.13.2", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^6.0.1", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index dc901fa..3fdb226 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -5,6 +5,7 @@ import { NavLink, useLocation, Outlet, + Navigate, } from "react-router-dom"; import { QueryClient, @@ -14,13 +15,13 @@ import { import { useEffect, useMemo, useState } from "react"; import { AuthProvider, useAuth } from "react-oidc-context"; import { Dashboard } from "./pages/Dashboard"; -import { Monitoring } from "./pages/Monitoring"; import { Applications } from "./pages/Applications"; import { Settings } from "./pages/Settings"; import { UsersPage } from "./pages/Users"; import { FileBrowser } from "./pages/FileBrowser"; import { Actions } from "./pages/Actions"; import BackupsPage from "./components/BackupsPage"; +import { ObservabilityPage } from "./components/ObservabilityPage"; import { getOidcConfig, isOidcConfigured, setAccessToken } from "./auth"; import { fetchAppVersion } from "./api/client"; import { FRONTEND_VERSION_LABEL } from "./version"; @@ -80,7 +81,7 @@ function useDarkMode() { // Navigation items for sidebar const navItems = [ { path: "/", label: "Dashboard", icon: LayoutDashboard }, - { path: "/monitoring", label: "Monitoring", icon: Activity }, + { path: "/observability", label: "Observability", icon: Activity }, { path: "/applications", label: "Media", icon: Monitor }, { path: "/files", label: "Files", icon: FolderOpen }, { path: "/users", label: "Users", icon: Users }, @@ -114,9 +115,7 @@ function Sidebar({
M
- {!collapsed && ( - Manage - )} + {!collapsed && Manage} - - - - - {hasQueryError && ( - - - - - - - )} - - - - - - - - - - - - - - - - - - - - - - - - - - {disk && ( - - - - - - - - - - - - - - - )} - - - - - - - - - - Recent activity - - - Collected automatically by the backend poller. - - - - - Action - - - - Result - - - - {actionsQuery.error && ( - - Recent activity: {String(actionsQuery.error)} - - )} - - - - - Time - Action - Status - Duration - Message - - - - {visibleActions.length === 0 ? ( - - - - No activity matches the current filters. - - - - ) : ( - visibleActions.map((action) => ( - - {formatActionTime(action.created_at)} - {action.action} - - - - {action.duration_ms} ms - - {action.message || action.error || "-"} - - - )) - )} - -
-
-
- - ); -} diff --git a/frontend/src/components/MonitoringCharts.impl.tsx b/frontend/src/components/MonitoringCharts.impl.tsx deleted file mode 100644 index ace1388..0000000 --- a/frontend/src/components/MonitoringCharts.impl.tsx +++ /dev/null @@ -1,1032 +0,0 @@ -import { - useCallback, - useEffect, - useLayoutEffect, - useMemo, - useRef, - useState, - type PointerEvent as ReactPointerEvent, -} from "react"; -import * as d3 from "d3"; -import { - Box, - Button, - Card, - CardContent, - Checkbox, - Chip, - FormControlLabel, - Grid, - Typography, -} from "@mui/material"; -import type { MonitoringSample } from "../types"; - -interface Props { - samples: MonitoringSample[]; -} - -type MetricKey = - | "cpu" - | "iowait" - | "mem" - | "netDown" - | "netUp" - | "diskRead" - | "diskWrite"; - -interface DataPoint { - ts: number; - cpu: number; - iowait: number; - mem: number; - netDown: number; - netUp: number; - diskRead: number; - diskWrite: number; -} - -interface MetricConfig { - key: MetricKey; - label: string; - color: string; -} - -interface ChartProps { - title: string; - data: DataPoint[]; - metrics: MetricConfig[]; - showAverages: boolean; - averages: Record; - yFormatter?: (v: number) => string; -} - -interface BrushProps { - data: DataPoint[]; - selectionRange: [number, number] | null; - onBrush: (range: [number, number] | null) => void; -} - -const MOVING_AVG_WINDOW = 10; -const CHART_HEIGHT = 280; -const BRUSH_HEIGHT = 84; -const BRUSH_LABEL_HEIGHT = 24; -const PERSISTED_SELECTION_KEY = "manage.monitoring.brush.selection"; - -function formatBytes(bytes: number): string { - if (!bytes) return "0 B/s"; - const units = ["B/s", "KB/s", "MB/s", "GB/s"]; - let value = bytes; - let unitIdx = 0; - while (value >= 1000 && unitIdx < units.length - 1) { - value /= 1000; - unitIdx++; - } - return `${value.toFixed(1)} ${units[unitIdx]}`; -} - -function formatTime(ts: number) { - return new Date(ts * 1000).toLocaleTimeString([], { - hour: "2-digit", - minute: "2-digit", - }); -} - -function movingAverage(values: number[]): number[] { - if (values.length === 0) return []; - return values.map((_, index) => { - const start = Math.max(0, index - MOVING_AVG_WINDOW + 1); - const slice = values.slice(start, index + 1); - return slice.reduce((sum, value) => sum + value, 0) / slice.length; - }); -} - -function buildAverages(samples: DataPoint[]): Record { - return { - cpu: movingAverage(samples.map((sample) => sample.cpu)), - iowait: movingAverage(samples.map((sample) => sample.iowait)), - mem: movingAverage(samples.map((sample) => sample.mem)), - netDown: movingAverage(samples.map((sample) => sample.netDown)), - netUp: movingAverage(samples.map((sample) => sample.netUp)), - diskRead: movingAverage(samples.map((sample) => sample.diskRead)), - diskWrite: movingAverage(samples.map((sample) => sample.diskWrite)), - }; -} - -function formatRangeLabel(range: [number, number] | null) { - if (!range) return "Full range"; - return `${formatTime(range[0])} – ${formatTime(range[1])}`; -} - -function metricsKey(metrics: MetricConfig[]) { - return metrics.map((m) => `${m.key}:${m.label}:${m.color}`).join("|"); -} - -// ══════════════════════════════════════════════════════════════════════════ -// Shared brush slider () -// ══════════════════════════════════════════════════════════════════════════ - -function MonitoringBrush({ data, selectionRange, onBrush }: BrushProps) { - const containerRef = useRef(null); - const svgRef = useRef(null); - const [width, setWidth] = useState(0); - const dragRef = useRef<{ - mode: "new" | "move" | "left" | "right"; - startRange: [number, number]; - startTs: number; - pointerId: number; - } | null>(null); - - const margin = { top: 18, right: 24, bottom: 22, left: 48 }; - const innerHeight = BRUSH_HEIGHT - margin.top - margin.bottom; - const totalHeight = BRUSH_LABEL_HEIGHT + BRUSH_HEIGHT; - const brushedColor = "rgba(99, 102, 241, 0.25)"; - - const minTs = useMemo(() => d3.min(data, (d) => d.ts) ?? 0, [data]); - const maxTs = useMemo(() => d3.max(data, (d) => d.ts) ?? 0, [data]); - const innerWidth = Math.max(0, width - margin.left - margin.right); - const xScale = useMemo(() => { - if (width === 0 || data.length === 0) return null; - return d3 - .scaleTime() - .domain([new Date(minTs * 1000), new Date(maxTs * 1000)]) - .range([0, innerWidth]); - }, [data.length, width, minTs, maxTs, innerWidth]); - const visibleSelection = - selectionRange ?? - (data.length > 0 ? ([minTs, maxTs] as [number, number]) : null); - const selectionPixels = useMemo(() => { - if (!xScale || !visibleSelection) return null; - return [ - xScale(new Date(visibleSelection[0] * 1000)), - xScale(new Date(visibleSelection[1] * 1000)), - ] as [number, number]; - }, [visibleSelection, xScale]); - - useEffect(() => { - if (!containerRef.current) return; - const observer = new ResizeObserver((entries) => { - for (const entry of entries) setWidth(entry.contentRect.width); - }); - observer.observe(containerRef.current); - return () => observer.disconnect(); - }, []); - - // Draw the mini chart and axes with D3, while the interactive brush UI is - // rendered by React so it stays visible across redraws. - useLayoutEffect(() => { - if (!svgRef.current || width === 0 || data.length === 0) return; - - const svg = d3.select(svgRef.current); - svg.selectAll("*").remove(); - - const root = svg - .append("g") - .attr("transform", `translate(${margin.left},${margin.top})`); - - const x = d3 - .scaleTime() - .domain([new Date(minTs * 1000), new Date(maxTs * 1000)]) - .range([0, innerWidth]); - - const yMax = Math.max(1, d3.max(data, (d) => Math.max(d.cpu, d.mem)) ?? 1); - const y = d3 - .scaleLinear() - .domain([0, yMax * 1.1]) - .range([innerHeight, 0]) - .nice(); - - root - .append("g") - .call(d3.axisLeft(y).ticks(3)) - .selectAll("text") - .style("font-size", "9px"); - root - .append("g") - .attr("transform", `translate(0,${innerHeight})`) - .call( - d3 - .axisBottom(x) - .ticks(Math.min(data.length || 1, 12)) - .tickFormat((v) => d3.timeFormat("%H:%M")(v as Date)), - ) - .selectAll("text") - .style("font-size", "8.5px"); - - root - .append("g") - .attr("stroke", "currentColor") - .attr("stroke-opacity", 0.08) - .call( - d3 - .axisLeft(y) - .ticks(3) - .tickSize(-innerWidth) - .tickFormat(() => ""), - ); - - const overviewMetrics: Array<{ key: MetricKey; color: string }> = [ - { key: "cpu", color: "#2563eb" }, - { key: "mem", color: "#16a34a" }, - ]; - - overviewMetrics.forEach(({ key, color }) => { - const line = d3 - .line() - .x((d) => x(new Date(d.ts * 1000))) - .y((d) => y((d[key] as number) || 0)) - .curve(d3.curveMonotoneX); - - root - .append("path") - .datum(data) - .attr("fill", "none") - .attr("stroke", color) - .attr("stroke-width", 1.2) - .attr("opacity", 0.6) - .attr("d", line); - }); - }, [data, width, margin.left, margin.top, innerHeight, innerWidth, minTs, maxTs]); - - const clampRange = useCallback( - (range: [number, number]): [number, number] => { - if (!data.length) return range; - let [start, end] = range[0] <= range[1] ? range : [range[1], range[0]]; - const domainStart = minTs; - const domainEnd = maxTs; - if (start < domainStart) { - end += domainStart - start; - start = domainStart; - } - if (end > domainEnd) { - start -= end - domainEnd; - end = domainEnd; - } - start = Math.max(domainStart, start); - end = Math.min(domainEnd, end); - if (end < start) end = start; - return [start, end]; - }, - [data.length, minTs, maxTs], - ); - - const clientXToTs = useCallback( - (clientX: number) => { - if (!containerRef.current || !xScale || innerWidth <= 0) return minTs; - const rect = containerRef.current.getBoundingClientRect(); - const chartX = Math.max( - 0, - Math.min(innerWidth, clientX - rect.left - margin.left), - ); - return Math.floor(xScale.invert(chartX).getTime() / 1000); - }, - [xScale, innerWidth, margin.left, minTs], - ); - - const beginDrag = useCallback( - (event: ReactPointerEvent) => { - if (!data.length || !xScale) return; - const target = event.target as HTMLElement | null; - const part = (target?.dataset.brushPart as - | "background" - | "selection" - | "left" - | "right" - | undefined) ?? "background"; - const mode: "new" | "move" | "left" | "right" = - part === "left" - ? "left" - : part === "right" - ? "right" - : part === "selection" - ? "move" - : "new"; - const startRange = - visibleSelection ?? ([minTs, maxTs] as [number, number]); - dragRef.current = { - mode, - startRange, - startTs: clientXToTs(event.clientX), - pointerId: event.pointerId, - }; - event.currentTarget.setPointerCapture(event.pointerId); - if (mode === "new") { - const ts = clientXToTs(event.clientX); - onBrush(clampRange([ts, ts])); - } - }, - [ - clientXToTs, - clampRange, - data.length, - maxTs, - minTs, - onBrush, - visibleSelection, - xScale, - ], - ); - - const updateDrag = useCallback( - (event: ReactPointerEvent) => { - const drag = dragRef.current; - if (!drag) return; - const ts = clientXToTs(event.clientX); - let nextRange: [number, number]; - if (drag.mode === "move") { - const delta = ts - drag.startTs; - nextRange = [drag.startRange[0] + delta, drag.startRange[1] + delta]; - } else if (drag.mode === "left") { - nextRange = [ts, drag.startRange[1]]; - } else if (drag.mode === "right") { - nextRange = [drag.startRange[0], ts]; - } else { - nextRange = [drag.startRange[0], ts]; - } - onBrush(clampRange(nextRange)); - }, - [clientXToTs, clampRange, onBrush], - ); - - const finishDrag = useCallback( - (event: ReactPointerEvent) => { - const drag = dragRef.current; - if (!drag) return; - try { - event.currentTarget.releasePointerCapture(drag.pointerId); - } catch { - // ignore release failures - } - dragRef.current = null; - }, - [], - ); - - if (!data.length) { - return ( - - No monitoring samples available. - - ); - } - - return ( - - - Time range — drag the left/right ends or the middle - - - - - - - {selectionPixels ? ( - <> - - - - - - - - - ) : null} - - - - - ); -} -// ══════════════════════════════════════════════════════════════════════════ -// Parent: MonitoringCharts -// ══════════════════════════════════════════════════════════════════════════ - -function loadPersistedSelection(): [number, number] | null { - if (typeof window === "undefined") return null; - const raw = window.localStorage.getItem(PERSISTED_SELECTION_KEY); - if (!raw) return null; - try { - const parsed = JSON.parse(raw) as unknown; - if ( - Array.isArray(parsed) && - parsed.length === 2 && - typeof parsed[0] === "number" && - typeof parsed[1] === "number" - ) { - return [parsed[0], parsed[1]]; - } - } catch { - return null; - } - return null; -} - -export function MonitoringCharts({ samples }: Props) { - const [showAverages, setShowAverages] = useState(false); - const [selectionRangeState, setSelectionRangeState] = useState< - [number, number] | null - >(() => loadPersistedSelection()); - const selectionRange = selectionRangeState; - const setSelectionRange = useCallback((range: [number, number] | null) => { - setSelectionRangeState(range); - if (typeof window === "undefined") return; - if (range) { - window.localStorage.setItem( - PERSISTED_SELECTION_KEY, - JSON.stringify(range), - ); - } else { - window.localStorage.removeItem(PERSISTED_SELECTION_KEY); - } - }, []); - - const baseData = useMemo( - () => - samples.map((sample) => ({ - ts: sample.ts, - cpu: sample.cpu_pct, - iowait: sample.iowait_pct ?? 0, - mem: sample.mem_pct, - netDown: sample.net_rx_bytes_per_sec, - netUp: sample.net_tx_bytes_per_sec, - diskRead: sample.disk_read_bps, - diskWrite: sample.disk_write_bps, - })), - [samples], - ); - - const averages = useMemo(() => buildAverages(baseData), [baseData]); - - const displayData = useMemo(() => { - if (!selectionRange) return baseData; - const [start, end] = selectionRange; - return baseData.filter((sample) => sample.ts >= start && sample.ts <= end); - }, [baseData, selectionRange]); - - const zoomOptions = useMemo( - () => [ - { label: "1h", seconds: 60 * 60 }, - { label: "8h", seconds: 8 * 60 * 60 }, - { label: "1 day", seconds: 24 * 60 * 60 }, - { label: "7 days", seconds: 7 * 24 * 60 * 60 }, - ], - [], - ); - - const zoomTo = useCallback( - (seconds: number) => { - if (!baseData.length) return; - const start = Math.max( - baseData[0].ts, - baseData[baseData.length - 1].ts - seconds, - ); - setSelectionRange([start, baseData[baseData.length - 1].ts]); - }, - [baseData], - ); - - const selectionLabel = useMemo( - () => - selectionRange - ? `${formatRangeLabel(selectionRange)} · ${displayData.length} samples` - : `All ${baseData.length} samples`, - [selectionRange, displayData.length, baseData.length], - ); - - if (!samples.length) { - return ( - - No monitoring samples available. - - ); - } - - return ( - - {/* Toolbar */} - - setShowAverages(event.target.checked)} - /> - } - label={`Show ${MOVING_AVG_WINDOW}-point moving average`} - /> - - - {zoomOptions.map((option) => ( - - ))} - - - - - {/* Shared brush slider above all graphs */} - - - {/* Chart grid */} - - - - - - - - - - - - - - - ); -} - -// ══════════════════════════════════════════════════════════════════════════ -// MonitoringD3Chart – single chart (lines + hover, no brush) -// ══════════════════════════════════════════════════════════════════════════ - -function MonitoringD3Chart({ - title, - data, - metrics, - showAverages, - averages, - yFormatter, -}: ChartProps) { - const containerRef = useRef(null); - const svgRef = useRef(null); - const [width, setWidth] = useState(0); - const mk = metricsKey(metrics); - - const margin = useMemo( - () => ({ top: 18, right: 24, bottom: 26, left: 56 }), - [], - ); - const innerHeight = CHART_HEIGHT - margin.top - margin.bottom; - - const summary = useMemo(() => { - const values = metrics.flatMap((metric) => - data.map((sample) => (sample[metric.key] as number) || 0), - ); - const avg = values.length - ? values.reduce((sum, value) => sum + value, 0) / values.length - : 0; - return { min: d3.min(values) ?? 0, avg, max: d3.max(values) ?? 0 }; - }, [data, metrics]); - - useEffect(() => { - if (!containerRef.current) return; - const observer = new ResizeObserver((entries) => { - for (const entry of entries) setWidth(entry.contentRect.width); - }); - observer.observe(containerRef.current); - return () => observer.disconnect(); - }, []); - - // One effect – rebuild chart layer only - useEffect(() => { - if (!svgRef.current || width === 0) return; - - const innerWidth = Math.max(0, width - margin.left - margin.right); - const svg = d3.select(svgRef.current); - svg.selectAll("*").remove(); - - const root = svg - .append("g") - .attr("transform", `translate(${margin.left},${margin.top})`); - - if (data.length === 0) return; - - const x = d3 - .scaleTime() - .domain(d3.extent(data, (d) => new Date(d.ts * 1000)) as [Date, Date]) - .range([0, innerWidth]); - - const yMax = - d3.max(data, (d) => - Math.max(...metrics.map((m) => (d[m.key] as number) || 0)), - ) ?? 1; - const y = d3 - .scaleLinear() - .domain([0, yMax * 1.1]) - .nice() - .range([innerHeight, 0]); - - // X axis - root - .append("g") - .attr("transform", `translate(0,${innerHeight})`) - .call( - d3 - .axisBottom(x) - .ticks(Math.min(data.length || 1, 10)) - .tickFormat((v) => d3.timeFormat("%H:%M")(v as Date)), - ) - .selectAll("text") - .style("font-size", "10px"); - - // Y axis - const yAxis = d3.axisLeft(y).ticks(5); - if (yFormatter) yAxis.tickFormat((v) => yFormatter(Number(v))); - root.append("g").call(yAxis).selectAll("text").style("font-size", "10px"); - - // Grid - root - .append("g") - .attr("stroke", "currentColor") - .attr("stroke-opacity", 0.1) - .call( - d3 - .axisLeft(y) - .ticks(5) - .tickSize(-innerWidth) - .tickFormat(() => ""), - ); - - // ── Lines ─────────────────────────────────────────── - metrics.forEach((metric) => { - const line = d3 - .line() - .x((d) => x(new Date(d.ts * 1000))) - .y((d) => y((d[metric.key] as number) || 0)) - .curve(d3.curveMonotoneX); - - root - .append("path") - .datum(data) - .attr("fill", "none") - .attr("stroke", metric.color) - .attr("stroke-width", 1.6) - .attr("d", line); - - if (showAverages && averages?.[metric.key]) { - const avgLine = d3 - .line() - .x((d) => x(new Date(d.ts * 1000))) - .y((_, i) => - y(averages[metric.key as keyof typeof averages]?.[i] || 0), - ) - .curve(d3.curveMonotoneX); - - root - .append("path") - .datum(data) - .attr("fill", "none") - .attr("stroke", metric.color) - .attr("stroke-width", 1.4) - .attr("stroke-dasharray", "5,3") - .attr("opacity", 0.7) - .attr("d", avgLine); - } - }); - - // ── Cursor line ───────────────────────────────────── - const cursorLine = root - .append("line") - .attr("y1", 0) - .attr("y2", innerHeight) - .attr("stroke", "currentColor") - .attr("stroke-opacity", 0.45) - .attr("stroke-dasharray", "4,4") - .style("display", "none"); - - // ── Cursor markers ────────────────────────────────── - const cursorMarkers = root - .append("g") - .attr("pointer-events", "none") - .style("display", "none"); - - cursorMarkers - .selectAll("circle") - .data(metrics) - .join("circle") - .attr("r", 4.5) - .attr("stroke", "#fff") - .attr("stroke-width", 1.4); - - // ── Tooltip ───────────────────────────────────────── - const tooltip = root - .append("g") - .attr("pointer-events", "none") - .style("display", "none"); - - tooltip - .append("rect") - .attr("rx", 6) - .attr("ry", 6) - .attr("fill", "rgba(15,23,42,0.92)"); - const tooltipText = tooltip - .append("text") - .attr("fill", "#fff") - .attr("font-size", 11) - .attr("font-family", "monospace"); - - // ── Hit area ──────────────────────────────────────── - const bisect = d3.bisector((d: DataPoint) => d.ts).center; - - root - .append("rect") - .attr("width", innerWidth) - .attr("height", innerHeight) - .attr("fill", "transparent") - .attr("pointer-events", "all") - .on("mousemove", (event) => { - const [mx, my] = d3.pointer(event, root.node() as SVGGElement); - const ts = x.invert(mx).getTime() / 1000; - const idx = bisect(data, ts); - const sample = data[Math.max(0, Math.min(data.length - 1, idx))]; - if (!sample) return; - - const xP = x(new Date(sample.ts * 1000)); - cursorLine.style("display", null).attr("x1", xP).attr("x2", xP); - cursorMarkers - .style("display", null) - .attr("transform", `translate(${xP},0)`) - .selectAll("circle") - .data(metrics) - .attr("cx", 0) - .attr("cy", (m) => y((sample[m.key] as number) || 0)) - .attr("fill", (m) => m.color); - - const lines = [ - formatTime(sample.ts), - ...metrics.map((m) => { - const raw = (sample[m.key] as number) || 0; - const avgV = - showAverages && - averages?.[m.key as keyof typeof averages]?.[idx] != null - ? averages[m.key as keyof typeof averages][idx] - : null; - const fmt = yFormatter ? yFormatter(raw) : `${raw.toFixed(1)}%`; - return avgV == null - ? `${m.label}: ${fmt}` - : `${m.label}: ${fmt} (avg ${yFormatter ? yFormatter(avgV) : avgV.toFixed(1)})`; - }), - ]; - - const lh = 14, - pad = 8; - const bw = Math.min( - Math.max(...lines.map((l) => l.length)) * 6.5 + pad * 2, - 260, - ); - const bh = lines.length * lh + pad * 2; - const px = Math.min(mx + 12, innerWidth - bw - 4); - const py = Math.max(4, Math.min(my - bh - 12, innerHeight - bh - 4)); - - tooltip - .style("display", null) - .attr("transform", `translate(${px},${py})`); - tooltip.select("rect").attr("width", bw).attr("height", bh); - tooltipText.selectAll("tspan").remove(); - lines.forEach((line, i) => - tooltipText - .append("tspan") - .attr("x", pad) - .attr("y", pad + 12 + i * lh) - .text(line), - ); - }) - .on("mouseleave", () => { - tooltip.style("display", "none"); - cursorLine.style("display", "none"); - cursorMarkers.style("display", "none"); - }); - }, [ - data, - mk, - showAverages, - averages, - width, - yFormatter, - margin.left, - margin.top, - innerHeight, - ]); - - // ── JSX ─────────────────────────────────────────────── - return ( - - - - {title} - - - - - - - - - - - {metrics.map((metric) => ( - - - {metric.label} - - ))} - {showAverages ? ( - Dashed = moving average - ) : null} - - - - ); -} diff --git a/frontend/src/components/MonitoringCharts.tsx b/frontend/src/components/MonitoringCharts.tsx deleted file mode 100644 index 2c44986..0000000 --- a/frontend/src/components/MonitoringCharts.tsx +++ /dev/null @@ -1 +0,0 @@ -export { MonitoringCharts } from "./MonitoringCharts.impl"; diff --git a/frontend/src/components/MonitoringOverviewTable.tsx b/frontend/src/components/MonitoringOverviewTable.tsx deleted file mode 100644 index ff0350b..0000000 --- a/frontend/src/components/MonitoringOverviewTable.tsx +++ /dev/null @@ -1,716 +0,0 @@ -import { useMemo, useState } from "react"; -import { - Box, - Chip, - Paper, - Stack, - Table, - TableBody, - TableCell, - TableContainer, - TableHead, - TableRow, - TableSortLabel, - Tooltip, - Typography, -} from "@mui/material"; -import type { - MonitoringMachineOverview, - MonitoringOverviewResponse, -} from "../types"; - -type SortKey = - | "machine" - | "mode" - | "status" - | "cpu" - | "iowait" - | "mem" - | "net_rx" - | "net_tx" - | "disk_read" - | "disk_write" - | "disk_used" - | "updated"; - -function formatBytes(bytes: number): string { - if (!bytes || bytes === 0) return "0 B"; - const units = ["B", "KB", "MB", "GB", "TB"]; - let value = bytes; - let unitIdx = 0; - while (value >= 1000 && unitIdx < units.length - 1) { - value /= 1000; - unitIdx++; - } - return `${value.toFixed(1)} ${units[unitIdx]}`; -} - -function formatRate(bytes: number): string { - return `${formatBytes(bytes)}/s`; -} - -function formatAge(epochSeconds: number | null): string { - if (!epochSeconds) return "-"; - const diff = Date.now() / 1000 - epochSeconds; - if (diff < 60) return `${Math.max(0, Math.round(diff))}s ago`; - if (diff < 3600) return `${Math.round(diff / 60)}m ago`; - return `${Math.round(diff / 3600)}h ago`; -} - -function formatReadableTime(epochSeconds: number | null): string { - if (!epochSeconds) return "-"; - const date = new Date(epochSeconds * 1000); - return date.toLocaleString("en-US", { - month: "short", - day: "numeric", - hour: "2-digit", - minute: "2-digit", - hour12: false, - }); -} - -function formatClockTime(epochSeconds: number | null): string { - if (!epochSeconds) return "-"; - const date = new Date(epochSeconds * 1000); - return date.toLocaleTimeString("en-US", { - hour: "2-digit", - minute: "2-digit", - hour12: false, - }); -} - -function formatUpdatedDetails(epochSeconds: number | null): [string, string] { - if (!epochSeconds) return ["-", "-"]; - return [ - formatReadableTime(epochSeconds), - `${formatClockTime(epochSeconds)} · ${formatAge(epochSeconds)}`, - ]; -} - -function formatSummary( - summary: { avg: number; min: number; max: number } | null, - formatter: (value: number) => string, -) { - if (!summary) return { value: "-", min: "", max: "" }; - return { - value: formatter(summary.avg), - min: formatter(summary.min), - max: formatter(summary.max), - }; -} - -function sortableText(value: string) { - return value.toLowerCase(); -} - -function getStatusChipProps(row: MonitoringMachineOverview) { - if (row.status_error) { - return { color: "error" as const, label: row.status_error }; - } - if (row.status === "running") { - return { color: "success" as const, label: "running" }; - } - if (row.status === "not running") { - return { color: "warning" as const, label: "not running" }; - } - return { color: "default" as const, label: row.status || "-" }; -} - -function metricSortValue( - row: MonitoringMachineOverview, - key: SortKey, -): number | string { - switch (key) { - case "machine": - return sortableText(row.machine.name); - case "mode": - return row.machine.mode; - case "status": - return sortableText(row.status || row.status_error || ""); - case "cpu": - return row.cpu_summary?.avg ?? -1; - case "iowait": - return row.iowait_summary?.avg ?? -1; - case "mem": - return row.mem_summary?.avg ?? -1; - case "net_rx": - return row.net_rx_summary?.avg ?? -1; - case "net_tx": - return row.net_tx_summary?.avg ?? -1; - case "disk_read": - return row.disk_read_summary?.avg ?? -1; - case "disk_write": - return row.disk_write_summary?.avg ?? -1; - case "disk_used": - return parseFloat((row.disk?.used_pct || "0").replace("%", "")) || -1; - case "updated": - return row.latest_sample?.ts ?? -1; - default: - return 0; - } -} - -function NoDataCell() { - return ( - - - No data - - - ); -} - -function MetricCell({ - value, - min, - max, -}: { - value: string; - min?: string; - max?: string; -}) { - return ( - - - - - 10m avg - - - {value} - - - - - {min ? ( - - Min {min} - - ) : null} - {max ? ( - - Max {max} - - ) : null} - - - ); -} - -export function MonitoringOverviewTable({ - overview, - embedded = false, -}: { - overview?: MonitoringOverviewResponse; - embedded?: boolean; -}) { - const poller = overview?.poller; - const rows = overview?.machines ?? []; - const [sortKey, setSortKey] = useState("machine"); - const [sortDirection, setSortDirection] = useState<"asc" | "desc">("asc"); - - const sortedRows = useMemo(() => { - const factor = sortDirection === "asc" ? 1 : -1; - return [...rows].sort((a, b) => { - const av = metricSortValue(a, sortKey); - const bv = metricSortValue(b, sortKey); - if (typeof av === "number" && typeof bv === "number") { - return (av - bv) * factor; - } - return String(av).localeCompare(String(bv)) * factor; - }); - }, [rows, sortDirection, sortKey]); - - const setSort = (key: SortKey) => { - if (sortKey === key) { - setSortDirection((current) => (current === "asc" ? "desc" : "asc")); - return; - } - setSortKey(key); - setSortDirection("asc"); - }; - - return ( - - {!embedded ? ( - - - Machine monitoring - - - - - Last success:{" "} - {poller?.last_success_at - ? new Date(poller.last_success_at * 1000).toLocaleString() - : "-"}{" "} - · last run: {formatAge(poller?.last_run_at ?? null)} - - - ) : null} - {!embedded && poller?.last_error ? ( - - - Poller error: {poller.last_error} - - - ) : null} - - - - - - setSort("machine")} - > - Machine - - - - setSort("mode")} - > - Mode - - - - setSort("status")} - > - Status - - - {[ - ["cpu", "CPU"], - ["iowait", "IO wait"], - ["mem", "RAM"], - ["net_rx", "Net down"], - ["net_tx", "Net up"], - ["disk_read", "Disk read"], - ["disk_write", "Disk write"], - ["disk_used", "Disk used"], - ].map(([key, label]) => ( - - setSort(key as SortKey)} - > - {label} - - - ))} - - setSort("updated")} - > - Updated - - - - - - {sortedRows.length === 0 ? ( - - - - No monitoring machines are configured. - - - - ) : ( - sortedRows.map((row) => { - const machine = row.machine; - const hasMetrics = row.sample_count > 0; - const cpu = formatSummary( - row.cpu_summary, - (value) => `${value.toFixed(1)}%`, - ); - const iowait = formatSummary( - row.iowait_summary, - (value) => `${value.toFixed(1)}%`, - ); - const mem = formatSummary( - row.mem_summary, - (value) => `${value.toFixed(1)}%`, - ); - const netDown = formatSummary(row.net_rx_summary, formatRate); - const netUp = formatSummary(row.net_tx_summary, formatRate); - const diskRead = formatSummary( - row.disk_read_summary, - formatRate, - ); - const diskWrite = formatSummary( - row.disk_write_summary, - formatRate, - ); - const disk = row.disk - ? { - value: row.disk.used_pct, - min: `Used ${formatBytes(row.disk.used)}`, - max: `Avail ${formatBytes(row.disk.available)}`, - } - : null; - const updated = formatUpdatedDetails( - row.latest_sample?.ts ?? null, - ); - return ( - - - - - - {machine.name} - - {!machine.enabled && ( - - )} - - - {machine.mode === "local" - ? "Local API host" - : `${machine.username || "user"}@${machine.host || "host"}:${machine.port}`} - - - - {machine.mode} - - {(() => { - const chip = getStatusChipProps(row); - return ( - - - - ); - })()} - - - {hasMetrics ? ( - - ) : ( - - )} - - - {hasMetrics ? ( - - ) : ( - - )} - - - {hasMetrics ? ( - - ) : ( - - )} - - - {hasMetrics ? ( - - ) : ( - - )} - - - {hasMetrics ? ( - - ) : ( - - )} - - - {hasMetrics ? ( - - ) : ( - - )} - - - {hasMetrics ? ( - - ) : ( - - )} - - - {disk ? ( - - ) : hasMetrics ? ( - "-" - ) : ( - - )} - - - - {hasMetrics ? ( - updated.map((line) => ( - - {line} - - )) - ) : ( - - No data yet - - )} - - - - ); - }) - )} - -
-
-
- ); -} diff --git a/frontend/src/components/ObservabilityPage.tsx b/frontend/src/components/ObservabilityPage.tsx new file mode 100644 index 0000000..652e39b --- /dev/null +++ b/frontend/src/components/ObservabilityPage.tsx @@ -0,0 +1,698 @@ +import { + Component, + useEffect, + useMemo, + useRef, + useState, + type ElementType, + type ErrorInfo, + type ReactNode, +} from "react"; +import { Link } from "react-router-dom"; +import { + Activity, + AlertTriangle, + Bell, + CheckCircle2, + ChevronDown, + ExternalLink, + Inbox, + PanelTop, + Radio, + RefreshCw, + Server, + ServerOff, + XCircle, +} from "lucide-react"; +import { + useAlertmanagerAlerts, + useAlertmanagerStatus, + usePrometheusTargets, + useMonitoringMachines, +} from "../hooks/useObservability"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; +import type { + AlertmanagerAlert, + MonitoringMachine, + PrometheusTarget, +} from "../types"; + +function severityVariant( + severity: string, +): "default" | "secondary" | "destructive" | "outline" { + switch (severity.toLowerCase()) { + case "critical": + return "destructive"; + case "warning": + return "default"; + case "info": + return "secondary"; + default: + return "outline"; + } +} + +function HealthCard({ + title, + status, + detail, + icon: Icon, + isLoading, +}: { + title: string; + status: "ok" | "warning" | "error" | "unknown"; + detail: string; + icon: ElementType; + isLoading?: boolean; +}) { + const statusIcon = + status === "ok" ? ( + + ) : status === "warning" ? ( + + ) : status === "error" ? ( + + ) : ( + + ); + + return ( + + + {title} + + + +
+ {isLoading ? : statusIcon} + {status} +
+

{detail}

+
+
+ ); +} + +function EmptyState({ + icon: Icon, + title, + description, + action, +}: { + icon: ElementType; + title: string; + description: string; + action?: ReactNode; +}) { + return ( +
+ +
{title}
+
+ {description} +
+ {action ?
{action}
: null} +
+ ); +} + +function QueryError({ + label, + error, + refetch, +}: { + label: string; + error: Error | null; + refetch: () => void; +}) { + if (!error) return null; + return ( + + {label} failed + + {error.message} + + + + ); +} + +function AlertItem({ alert }: { alert: AlertmanagerAlert }) { + return ( + + +
+
+
{alert.name}
+
+ + {alert.severity} + + +
+
+
+ {alert.summary || alert.description} +
+ {alert.active_since && ( +
+ Since {new Date(alert.active_since).toLocaleString()} +
+ )} +
+
+ +
+ {alert.description && ( +
+ Description:{" "} + {alert.description} +
+ )} +
+ {alert.job_name && ( +
+ Job: {alert.job_name} +
+ )} + {alert.category && ( +
+ Category: {alert.category} +
+ )} +
+ State: {alert.state} +
+
+ Since:{" "} + {alert.active_since + ? new Date(alert.active_since).toLocaleString() + : "unknown"} +
+
+ {alert.labels && Object.keys(alert.labels).length > 0 && ( +
+ {Object.entries(alert.labels).map(([key, value]) => ( + + {key}={value} + + ))} +
+ )} +
+
+
+ ); +} + +function TargetsTable({ targets }: { targets: PrometheusTarget[] }) { + return ( +
+ {targets.map((target, idx) => ( +
+
{target.targets.join(", ")}
+ {target.labels && Object.keys(target.labels).length > 0 && ( +
+ {Object.entries(target.labels).map(([key, value]) => ( + + {key}: {value} + + ))} +
+ )} +
+ ))} +
+ ); +} + +class GrafanaErrorBoundary extends Component< + { children: ReactNode; fallback: ReactNode }, + { hasError: boolean } +> { + constructor(props: { children: ReactNode; fallback: ReactNode }) { + super(props); + this.state = { hasError: false }; + } + + static getDerivedStateFromError() { + return { hasError: true }; + } + + componentDidCatch(error: Error, errorInfo: ErrorInfo) { + console.error("Grafana panel error:", error, errorInfo); + } + + render() { + if (this.state.hasError) { + return this.props.fallback; + } + return this.props.children; + } +} + +function GrafanaPanel({ src, title }: { src: string; title: string }) { + const [loaded, setLoaded] = useState(false); + const [failed, setFailed] = useState(false); + const [iframeKey, setIframeKey] = useState(0); + const timerRef = useRef | null>(null); + + useEffect(() => { + timerRef.current = setTimeout(() => setFailed(true), 10_000); + return () => { + if (timerRef.current) clearTimeout(timerRef.current); + }; + }, [iframeKey]); + + const handleLoad = () => { + if (timerRef.current) clearTimeout(timerRef.current); + setLoaded(true); + setFailed(false); + }; + + const handleError = () => { + if (timerRef.current) clearTimeout(timerRef.current); + setFailed(true); + }; + + const reload = () => { + setLoaded(false); + setFailed(false); + setIframeKey((k) => k + 1); + }; + + if (!src) { + return ( + + ); + } + + const fallback = ( + + + + + } + /> + ); + + return ( + +
+ {!loaded && !failed && ( +
+ +
+ )} + {failed ? ( +
+ {fallback} +
+ ) : ( +