alex/manage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

Browse Source
This commit is contained in:
Developer
2026-06-16 11:08:32 +00:00
parent 615e02e970
commit e2ad731b5f
80 changed files with 5020 additions and 3665 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+56 -36
View File
@@ -1,46 +1,66 @@
# Optional backend logging level.
LOG_LEVEL=INFO
# App
APP_VERSION=0.1.0
APP_BUILD_INFO=dev
# Optional SMTP settings for the Users -> message popup.
# Hosts
BACKEND_APP_HOST=api.manage.example.com
FRONTEND_APP_HOST=manage.example.com
GRAFANA_APP_HOST=grafana.example.com
BACKEND_APP_PORT=8000
FRONTEND_APP_PORT=80
GRAFANA_APP_PORT=3000
BACKEND_APP_NAME=manage-backend
FRONTEND_APP_NAME=manage-frontend
GRAFANA_APP_NAME=grafana
# Traefik / certificates
CERT_RESOLVER=letsencrypt
# Backend
LOG_LEVEL=INFO
LOG_FORMAT=json
PROMETHEUS_ENABLED=true
PROMETHEUS_FILE_SD_DIR=/app/backend/.cache/prometheus-file-sd
ALERTMANAGER_URL=http://alertmanager:9093
ALERTMANAGER_WEBHOOK_URL=
BACKEND_CACHE_DIR=./backend-cache
# Auth
AUTH_ENABLED=true
OIDC_ISSUER_URL=https://auth.example.com/application/o/manage/
OIDC_AUDIENCE=manage
OIDC_JWKS_URL=https://auth.example.com/application/o/manage/jwks/
OIDC_CLOCK_SKEW_SECONDS=30
# Frontend OIDC
VITE_API_URL=/api
VITE_OIDC_ENABLED=true
VITE_OIDC_ISSUER=https://auth.example.com/application/o/manage/
VITE_OIDC_CLIENT_ID=manage
VITE_OIDC_SCOPE=openid profile email
VITE_OIDC_REDIRECT_URI=https://manage.example.com/oidc/callback
VITE_OIDC_POST_LOGOUT_REDIRECT_URI=https://manage.example.com/
VITE_DEV_API_PROXY_TARGET=http://backend:8000
# SMTP
SMTP_HOST=smtp.example.com
SMTP_PORT=587
SMTP_USERNAME=your-smtp-username
SMTP_PASSWORD=your-smtp-password
SMTP_USERNAME=
SMTP_PASSWORD=
SMTP_FROM_ADDRESS=no-reply@example.com
SMTP_FROM_NAME=Manage
SMTP_USE_TLS=true
SMTP_USE_SSL=false
SMTP_TIMEOUT=30
# Machine/service configuration now lives in the app's Settings tab.
# The built-in local machine is seeded automatically.
#
# Remote SSH machines can store their private key and optional passphrase directly in Settings,
# so no SSH key mount is required for normal use.
# Grafana admin / OAuth
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=change-me
GF_AUTH_GENERIC_OAUTH_CLIENT_ID=manage
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=
GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.example.com/application/o/manage/authorize/
GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.example.com/application/o/manage/token/
GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.example.com/application/o/manage/userinfo/
# For deployment with traefik
FRONTEND_APP_NAME=manage
FRONTEND_APP_HOST=manage.example.com
FRONTEND_APP_PORT=5173
BACKEND_APP_NAME=management-api
BACKEND_APP_HOST=management-api.example.com
BACKEND_APP_PORT=8000
CERT_RESOLVER=lets-encrypt
# Authentik / OIDC
# Backend validates every API request with a Bearer JWT.
AUTH_ENABLED=true
OIDC_ISSUER_URL=https://authentik.example/application/o/media-library-viewer/
OIDC_AUDIENCE=media-library-viewer
OIDC_JWKS_URL=
OIDC_CLOCK_SKEW_SECONDS=30
# Frontend OIDC settings (Vite build/runtime env)
VITE_OIDC_ENABLED=true
VITE_OIDC_ISSUER=https://authentik.example/application/o/media-library-viewer/
VITE_OIDC_CLIENT_ID=media-library-viewer
VITE_OIDC_SCOPE=openid profile email
VITE_OIDC_REDIRECT_URI=http://localhost:8080/
VITE_OIDC_POST_LOGOUT_REDIRECT_URI=http://localhost:8080/
# Alerting
ALERT_EMAIL_TO=admin@example.com
.gitignore
+5
View File
@@ -15,6 +15,7 @@ dist/
# Virtual environments
.venv/
.venv-review/
venv/
env/
@@ -52,3 +53,7 @@ frontend/dist/
.fusion/
.stoneforge/
.superpowers/
# Local Pi runtime state
.atl/
.pi-map.md
.pi-map.index.md
backend/pyproject.toml
+2
View File
@@ -13,6 +13,8 @@ dependencies = [
"pandas>=2.0",
"PyJWT[crypto]>=2.8",
"python-multipart>=0.0.9",
"prometheus-client>=0.21",
"python-json-logger>=2.0",
]
[project.optional-dependencies]
backend/src/media_library_viewer_api/clients/jellyfin.py
+14 -12
View File
@@ -53,7 +53,9 @@ class JellyfinClient:
{
"X-Emby-Token": api_key,
"Accept": "application/json",
"X-Emby-Authorization": 'MediaBrowser Client="MediaLibraryViewer", Device="Streamlit", DeviceId="streamlit", Version="0.1"',
"X-Emby-Authorization": (
'MediaBrowser Client="MediaLibraryViewer", Device="Streamlit", DeviceId="streamlit", Version="0.1"'
),
}
)
@@ -61,9 +63,7 @@ class JellyfinClient:
"""GET a Jellyfin endpoint and include useful response text on errors."""
clean_params = {k: v for k, v in params.items() if v is not None and v != ""}
logger.debug("Jellyfin GET %s params=%s", path, sorted(clean_params.keys()))
response = self.session.get(
f"{self.base_url}{path}", params=clean_params, timeout=self.timeout
)
response = self.session.get(f"{self.base_url}{path}", params=clean_params, timeout=self.timeout)
try:
response.raise_for_status()
except requests.HTTPError as exc:
@@ -175,14 +175,16 @@ class JellyfinClient:
series = self.item_count(user_id, "Series", parent_id=lib_id)
episodes = self.item_count(user_id, "Episode", parent_id=lib_id)
total = self.item_count(user_id, "Movie,Episode,Video,Audio,Series", parent_id=lib_id)
results.append({
"library": lib_name,
"type": lib_type,
"movies": movies,
"series": series,
"episodes": episodes,
"total": total,
})
results.append(
{
"library": lib_name,
"type": lib_type,
"movies": movies,
"series": series,
"episodes": episodes,
"total": total,
}
)
return results
def sessions(self, active_within_seconds: int | None = None) -> list[dict[str, Any]]:
backend/src/media_library_viewer_api/clients/jellyseerr.py
+5 -7
View File
@@ -40,16 +40,12 @@ class JellyseerrClient:
"""GET a Jellyseerr endpoint and include useful response text on errors."""
clean_params = {k: v for k, v in params.items() if v is not None and v != ""}
logger.debug("Jellyseerr GET %s params=%s", path, sorted(clean_params.keys()))
response = self.session.get(
f"{self.base_url}/api/v1{path}", params=clean_params, timeout=self.timeout
)
response = self.session.get(f"{self.base_url}/api/v1{path}", params=clean_params, timeout=self.timeout)
try:
response.raise_for_status()
except requests.HTTPError as exc:
detail = response.text[:500]
logger.warning(
"Jellyseerr GET %s failed status=%s url=%s", path, response.status_code, response.url
)
logger.warning("Jellyseerr GET %s failed status=%s url=%s", path, response.status_code, response.url)
raise requests.HTTPError(
f"{response.status_code} for {response.url}: {detail}",
response=response,
@@ -105,7 +101,9 @@ class JellyseerrClient:
return results
page_results = payload.get("results") or []
page_items = [item for item in page_results if isinstance(item, dict)] if isinstance(page_results, list) else []
page_items = (
[item for item in page_results if isinstance(item, dict)] if isinstance(page_results, list) else []
)
results.extend(page_items)
page_info = payload.get("pageInfo") or {}
backend/src/media_library_viewer_api/clients/resources.py
-340
View File
@@ -1,340 +0,0 @@
"""Remote resource collection helpers.
The app does not require Prometheus, Netdata, or sysstat. Instead it can install
and manage a tiny POSIX-sh collector under /tmp on the remote server. The
collector samples Linux /proc and /sys counters every 10 seconds and appends JSON
Lines. This module starts/stops the collector and reads those JSONL samples.
"""
from __future__ import annotations
import json
import logging
import shlex
from dataclasses import dataclass
from typing import Any
from media_library_viewer_api.clients.ssh import RemoteSSHClient
logger = logging.getLogger(__name__)
# POSIX shell script copied to the remote server by start_resource_collector().
# Keep this script bash-free because many NAS/media servers have minimal shells.
COLLECTOR_SCRIPT = r'''#!/bin/sh
set -u
OUT="${1:-/tmp/media_library_viewer_metrics.jsonl}"
INTERVAL="${2:-10}"
RETENTION_SECONDS="${3:-604800}"
MAX_LINES="${4:-70000}"
PRUNE_EVERY_SAMPLES="${5:-60}"
mkdir -p "$(dirname "$OUT")"
echo "collector starting at $(date -Is 2>/dev/null || date), interval=${INTERVAL}s, retention=${RETENTION_SECONDS}s, max_lines=${MAX_LINES}, out=${OUT}"
read_cpu() {
awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8+$9+$10, $5+$6, $6}' /proc/stat
}
read_mem_pct() {
awk '
/^MemTotal:/ {total=$2}
/^MemAvailable:/ {avail=$2}
END {if (total > 0) printf "%.2f", (total-avail)*100/total; else printf "0"}
' /proc/meminfo
}
read_net_bytes() {
awk '
NR > 2 {
split($0, parts, ":")
iface = parts[1]
stats = parts[2]
gsub(/^[ \t]+|[ \t]+$/, "", iface)
gsub(/^[ \t]+|[ \t]+$/, "", stats)
if (iface == "lo" || iface == "" || stats == "") next
split(stats, values, /[ \t]+/)
# /proc/net/dev after the colon:
# receive bytes are field 1, transmit bytes are field 9.
# Trim the stats block before split; otherwise leading whitespace can make
# values[1] empty in some awk implementations, resulting in zero rates.
rx += values[1] + 0
tx += values[9] + 0
}
END {printf "%.0f %.0f", rx, tx}
' /proc/net/dev
}
read_disk_bytes() {
read_sectors=0
written_sectors=0
for dev in /sys/block/*; do
[ -r "$dev/stat" ] || continue
name="$(basename "$dev")"
case "$name" in
loop*|ram*|fd*|sr*) continue ;;
esac
# Linux /sys/block/<dev>/stat fields: 3=sectors read, 7=sectors written.
# Use POSIX sh parsing instead of bash arrays so this works on minimal systems.
set -- $(cat "$dev/stat")
sectors_read="${3:-0}"
sectors_written="${7:-0}"
read_sectors=$((read_sectors + sectors_read))
written_sectors=$((written_sectors + sectors_written))
done
printf "%s %s" "$((read_sectors * 512))" "$((written_sectors * 512))"
}
set -- $(read_cpu)
prev_total="${1:-0}"
prev_idle="${2:-0}"
prev_iowait="${3:-0}"
set -- $(read_net_bytes)
prev_rx="${1:-0}"
prev_tx="${2:-0}"
set -- $(read_disk_bytes)
prev_disk_read="${1:-0}"
prev_disk_write="${2:-0}"
prev_ts="$(date +%s)"
sample_count=0
prune_metrics_file() {
[ -f "$OUT" ] || return 0
cutoff="$1"
tmp="${OUT}.$$.tmp"
awk -v cutoff="$cutoff" '
match($0, /"ts":[0-9]+/) {
ts = substr($0, RSTART + 5, RLENGTH - 5)
if (ts >= cutoff) print $0
}
' "$OUT" | tail -n "$MAX_LINES" > "$tmp" && mv "$tmp" "$OUT"
rm -f "$tmp"
}
while true; do
now_ts="$(date +%s)"
dt=$((now_ts - prev_ts))
if [ "$dt" -le 0 ]; then dt=1; fi
set -- $(read_cpu)
total="${1:-0}"
idle="${2:-0}"
iowait="${3:-0}"
set -- $(read_net_bytes)
rx="${1:-0}"
tx="${2:-0}"
set -- $(read_disk_bytes)
disk_read="${1:-0}"
disk_write="${2:-0}"
mem_pct="$(read_mem_pct)"
total_delta=$((total - prev_total))
idle_delta=$((idle - prev_idle))
iowait_delta=$((iowait - prev_iowait))
rx_delta=$((rx - prev_rx))
tx_delta=$((tx - prev_tx))
disk_read_delta=$((disk_read - prev_disk_read))
disk_write_delta=$((disk_write - prev_disk_write))
cpu_pct="$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN {if (total > 0) printf "%.2f", (total-idle)*100/total; else printf "0"}')"
iowait_pct="$(awk -v total="$total_delta" -v iow="$iowait_delta" 'BEGIN {if (total > 0) printf "%.2f", iow*100/total; else printf "0"}')"
rx_bytes_per_sec="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
tx_bytes_per_sec="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
rx_bps="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
tx_bps="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
disk_read_bps="$(awk -v bytes="$disk_read_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
disk_write_bps="$(awk -v bytes="$disk_write_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
printf '{"ts":%s,"cpu_pct":%s,"iowait_pct":%s,"mem_pct":%s,"net_rx_bytes_per_sec":%s,"net_tx_bytes_per_sec":%s,"net_rx_bps":%s,"net_tx_bps":%s,"disk_read_bps":%s,"disk_write_bps":%s}\n' \
"$now_ts" "$cpu_pct" "$iowait_pct" "$mem_pct" "$rx_bytes_per_sec" "$tx_bytes_per_sec" "$rx_bps" "$tx_bps" "$disk_read_bps" "$disk_write_bps" >> "$OUT"
sample_count=$((sample_count + 1))
if [ $((sample_count % PRUNE_EVERY_SAMPLES)) -eq 0 ]; then
prune_metrics_file "$((now_ts - RETENTION_SECONDS))"
fi
prev_total="$total"
prev_idle="$idle"
prev_iowait="$iowait"
prev_rx="$rx"
prev_tx="$tx"
prev_disk_read="$disk_read"
prev_disk_write="$disk_write"
prev_ts="$now_ts"
sleep "$INTERVAL"
done
'''
@dataclass(frozen=True)
class ResourceMonitorPaths:
"""Remote file locations used by the lightweight resource collector."""
metrics_file: str = "/tmp/media_library_viewer_metrics.jsonl"
pid_file: str = "/tmp/media_library_viewer_metrics.pid"
script_file: str = "/tmp/media_library_viewer_metrics_collector.sh"
log_file: str = "/tmp/media_library_viewer_metrics.log"
def start_resource_collector(
ssh: RemoteSSHClient,
interval_seconds: int = 10,
retention_seconds: int = 7 * 24 * 60 * 60,
max_lines: int = 70_000,
paths: ResourceMonitorPaths = ResourceMonitorPaths(),
) -> str:
"""Install and start the remote metrics collector if it is not running.
Starting a fresh collector removes old metrics/log files because schema
changes during development can otherwise leave mixed JSONL records behind.
The collector prunes its own metrics file to 7 days / max_lines.
"""
command = f"""
cat > {shlex.quote(paths.script_file)} <<'MLV_RESOURCE_COLLECTOR'
{COLLECTOR_SCRIPT}
MLV_RESOURCE_COLLECTOR
chmod +x {shlex.quote(paths.script_file)}
if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
echo "already running pid=$(cat {shlex.quote(paths.pid_file)})"
else
rm -f {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)}
nohup {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {int(interval_seconds)} {int(retention_seconds)} {int(max_lines)} >> {shlex.quote(paths.log_file)} 2>&1 &
echo $! > {shlex.quote(paths.pid_file)}
echo "started pid=$(cat {shlex.quote(paths.pid_file)})"
fi
"""
logger.info(
"Starting remote resource collector interval=%ss retention=%ss max_lines=%s", interval_seconds, retention_seconds, max_lines
)
result = ssh.run(command, timeout=20)
if result.exit_status != 0:
logger.warning("Failed to start remote resource collector: %s", result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to start resource collector")
logger.info("Remote resource collector start response: %s", result.stdout.strip())
return result.stdout.strip()
def stop_resource_collector(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
"""Stop the remote collector process if the pid file points to one."""
command = f"""
if [ -f {shlex.quote(paths.pid_file)} ]; then
pid="$(cat {shlex.quote(paths.pid_file)})"
if kill -0 "$pid" 2>/dev/null; then
kill "$pid"
echo "stopped pid=$pid"
else
echo "not running"
fi
rm -f {shlex.quote(paths.pid_file)}
else
echo "not running"
fi
"""
logger.info("Stopping remote resource collector")
result = ssh.run(command, timeout=20)
if result.exit_status != 0:
logger.warning("Failed to stop remote resource collector: %s", result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to stop resource collector")
logger.info("Remote resource collector stop response: %s", result.stdout.strip())
return result.stdout.strip()
def restart_resource_collector(
ssh: RemoteSSHClient,
interval_seconds: int = 10,
retention_seconds: int = 7 * 24 * 60 * 60,
max_lines: int = 70_000,
paths: ResourceMonitorPaths = ResourceMonitorPaths(),
) -> str:
logger.info("Restarting remote resource collector")
stop_message = stop_resource_collector(ssh, paths)
start_message = start_resource_collector(ssh, interval_seconds, retention_seconds, max_lines, paths)
return f"{stop_message}\n{start_message}"
def resource_collector_status(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
"""Return a short human-readable status string for the dashboard."""
command = f"""
if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
echo "running"
else
echo "not running"
fi
"""
result = ssh.run(command, timeout=10)
if result.exit_status != 0:
logger.warning("Failed to read collector status: %s", result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to check collector status")
status = result.stdout.strip()
logger.info("Resource collector status: %s", status)
return status
def resource_collector_debug_info(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
"""Collect remote diagnostics for troubleshooting missing metrics."""
command = f"""
echo "status:"
if [ -f {shlex.quote(paths.pid_file)} ]; then
pid="$(cat {shlex.quote(paths.pid_file)})"
echo "pid_file=$pid"
if kill -0 "$pid" 2>/dev/null; then echo "process=running"; else echo "process=not-running"; fi
else
echo "pid_file=missing"
fi
echo "files:"
ls -l {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} 2>&1 || true
echo "sample_count:"
if [ -f {shlex.quote(paths.metrics_file)} ]; then wc -l < {shlex.quote(paths.metrics_file)}; else echo 0; fi
echo "last_samples:"
if [ -f {shlex.quote(paths.metrics_file)} ]; then tail -n 5 {shlex.quote(paths.metrics_file)}; fi
echo "log_tail:"
if [ -f {shlex.quote(paths.log_file)} ]; then tail -n 40 {shlex.quote(paths.log_file)}; fi
echo "netdev_snapshot:"
cat /proc/net/dev 2>&1 || true
"""
logger.info("Collecting resource collector diagnostics")
result = ssh.run(command, timeout=20)
output = (result.stdout or "") + (result.stderr or "")
logger.debug("Resource collector diagnostics length=%s", len(output))
return output
def read_resource_metrics(ssh: RemoteSSHClient, max_lines: int = 1000, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> list[dict[str, Any]]:
"""Read recent JSONL metric samples from the remote collector file."""
command = f"test -f {shlex.quote(paths.metrics_file)} && tail -n {int(max_lines)} {shlex.quote(paths.metrics_file)} || true"
logger.debug("Reading up to %s resource metric lines", max_lines)
result = ssh.run(command, timeout=20)
if result.exit_status != 0:
logger.warning("Failed to read resource metrics: %s", result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to read resource metrics")
rows = []
for line in result.stdout.splitlines():
line = line.strip()
if not line:
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
continue
return rows
def disk_space(ssh: RemoteSSHClient, path: str = "/") -> dict[str, Any]:
"""Return df information for the filesystem containing ``path``."""
command = (
"df -P -B1 -- "
+ shlex.quote(path or "/")
+ " | awk 'NR==2 {printf \"{\\\"filesystem\\\":\\\"%s\\\",\\\"size\\\":%s,\\\"used\\\":%s,\\\"available\\\":%s,\\\"used_pct\\\":\\\"%s\\\",\\\"mount\\\":\\\"%s\\\"}\", $1,$2,$3,$4,$5,$6}'"
)
logger.debug("Reading disk space for path=%s", path)
result = ssh.run(command, timeout=20)
if result.exit_status != 0 or not result.stdout.strip():
logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
data = json.loads(result.stdout)
logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct"))
return data
backend/src/media_library_viewer_api/clients/ssh.py
+2 -6
View File
@@ -75,14 +75,10 @@ class RemoteSSHClient:
client = paramiko.SSHClient()
client.load_system_host_keys()
known_hosts_file = Path(self.known_hosts_path) if self.known_hosts_path else None
trusted_before = bool(
known_hosts_file and has_known_host(self.host, self.port, known_hosts_file)
)
trusted_before = bool(known_hosts_file and has_known_host(self.host, self.port, known_hosts_file))
if known_hosts_file and known_hosts_file.is_file():
client.load_host_keys(str(known_hosts_file))
client.set_missing_host_key_policy(
paramiko.RejectPolicy() if trusted_before else paramiko.AutoAddPolicy()
)
client.set_missing_host_key_policy(paramiko.RejectPolicy() if trusted_before else paramiko.AutoAddPolicy())
connect_kwargs: dict[str, Any] = {
"hostname": self.host,
"port": self.port,
backend/src/media_library_viewer_api/config.py
+8 -4
View File
@@ -23,6 +23,7 @@ class Settings(BaseSettings):
# Logging
log_level: str = "INFO"
log_format: str = "text" # "text" or "json"
# Auth / OIDC (Authentik-compatible JWT validation)
auth_enabled: bool = False
@@ -48,7 +49,6 @@ class Settings(BaseSettings):
ssh_port: int = 22
ssh_key_directory: str = ""
ssh_key_name: str = ""
ssh_key_file: str = "/run/secrets/ssh_private_key"
ssh_password: str = ""
ssh_known_hosts_path: str = ""
@@ -57,6 +57,12 @@ class Settings(BaseSettings):
monitoring_poll_initial_delay_seconds: int = 20
monitoring_action_retention_days: int = 30
# Observability
prometheus_enabled: bool = True
prometheus_file_sd_dir: str = "/app/backend/.cache/prometheus-file-sd"
alertmanager_url: str = "http://alertmanager:9093"
alertmanager_webhook_url: str = "" # Optional receiver for alertmanager webhook notifications
# Remote paths
remote_media_root: str = ""
remote_path_prefix: str = ""
@@ -72,8 +78,6 @@ class Settings(BaseSettings):
@property
def ssh_key_path(self) -> str:
if self.ssh_key_file:
return self.ssh_key_file
if not self.ssh_key_directory or not self.ssh_key_name:
return ""
return str(Path(self.ssh_key_directory) / self.ssh_key_name)
@@ -82,7 +86,7 @@ class Settings(BaseSettings):
def ssh_known_hosts_file(self) -> Path:
return Path(self.ssh_known_hosts_path or ".cache/media_library_viewer/known_hosts")
model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"}
model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"} # type: ignore[assignment]
def _find_env_file() -> str | None:
backend/src/media_library_viewer_api/dependencies.py
+66 -32
View File
@@ -19,12 +19,16 @@ from media_library_viewer_api.clients.jellyseerr import JellyseerrClient
from media_library_viewer_api.clients.local import LocalCommandClient
from media_library_viewer_api.clients.ssh import RemoteSSHClient
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.services.mail_queue import MailQueue, get_mail_queue as _get_mail_queue
from media_library_viewer_api.services.mail_queue import MailQueue
from media_library_viewer_api.services.mail_queue import get_mail_queue as _get_mail_queue
from media_library_viewer_api.services.monitoring_poller import (
MonitoringPoller,
)
from media_library_viewer_api.services.monitoring_poller import (
get_monitoring_poller as _get_monitoring_poller,
)
from media_library_viewer_api.services.settings_store import SettingsStore, get_settings_store as _get_settings_store
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.settings_store import get_settings_store as _get_settings_store
logger = logging.getLogger(__name__)
@@ -39,7 +43,9 @@ def _request_machine_id(request: Request | None) -> str | None:
@lru_cache(maxsize=32)
def _jellyfin_client_for(cache_key: tuple[str, str, str]) -> JellyfinClient:
machine_id, url, api_key = cache_key
logger.info("Creating Jellyfin client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>")
logger.info(
"Creating Jellyfin client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>"
)
return JellyfinClient(url, api_key)
@@ -52,13 +58,19 @@ def _jellyseerr_client_for(cache_key: tuple[str, str]) -> JellyseerrClient | Non
api_key = (settings or {}).get("jellyseerr_api_key") if settings else ""
if not api_key:
return None
logger.info("Creating Jellyseerr client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>")
logger.info(
"Creating Jellyseerr client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>"
)
return JellyseerrClient(url, api_key)
@lru_cache(maxsize=32)
def _ssh_client_for(cache_key: tuple[str, str, str, int, str, str | None, str | None, str | None, str | None]) -> RemoteSSHClient:
machine_id, host, username, port, key_filename, password, private_key, private_key_passphrase, known_hosts_path = cache_key
def _ssh_client_for(
cache_key: tuple[str, str, str, int, str, str | None, str | None, str | None, str | None],
) -> RemoteSSHClient:
machine_id, host, username, port, key_filename, password, private_key, private_key_passphrase, known_hosts_path = (
cache_key
)
logger.info(
"Creating SSH client machine_id=%s host=%s user=%s port=%s key=%s password=%s private_key=%s passphrase=%s",
machine_id or "<default>",
@@ -141,7 +153,9 @@ def get_jellyfin_client(request: Request = None) -> JellyfinClient:
cache_key = (machine["id"], machine["jellyfin_url"], machine.get("jellyfin_api_key") or "")
return _jellyfin_client_for(cache_key)
raise RuntimeError("No Jellyfin machine is configured. Add a machine with jellyfin_url and jellyfin_api_key in Settings.")
raise RuntimeError(
"No Jellyfin machine is configured. Add a machine with jellyfin_url and jellyfin_api_key in Settings."
)
def get_jellyseerr_client(request: Request = None) -> JellyseerrClient | None:
@@ -160,6 +174,37 @@ def get_jellyseerr_client(request: Request = None) -> JellyseerrClient | None:
return None
def _ssh_client_from_machine_config(machine: dict[str, Any], store: SettingsStore | None = None) -> RemoteSSHClient:
"""Build a RemoteSSHClient from a machine config dict."""
store = store or get_settings_store()
known_hosts_path = get_settings().ssh_known_hosts_file
key_data = None
key_passphrase = None
ssh_key_id = str(machine.get("ssh_key_id") or "").strip()
if ssh_key_id:
ssh_key = store.get_ssh_key(ssh_key_id)
if ssh_key:
key_data = ssh_key.get("private_key") or None
key_passphrase = ssh_key.get("passphrase") or None
if not key_data and machine.get("ssh_private_key"):
key_data = machine.get("ssh_private_key") or None
key_passphrase = machine.get("ssh_private_key_passphrase") or None
cache_key = (
machine["id"],
machine["host"],
machine["username"],
int(machine.get("port") or 22),
f"{machine.get('key_directory')}/{machine.get('key_name')}"
if machine.get("key_directory") and machine.get("key_name")
else "",
machine.get("password") or None,
key_data,
key_passphrase,
str(known_hosts_path),
)
return _ssh_client_for(cache_key)
def get_ssh_client(request: Request = None):
"""Return a command client for the selected machine or legacy env fallback."""
store = get_settings_store()
@@ -172,30 +217,7 @@ def get_ssh_client(request: Request = None):
logger.info("Creating LocalCommandClient machine_id=%s", machine["id"])
return LocalCommandClient()
if machine and machine.get("host") and machine.get("username"):
known_hosts_path = get_settings().ssh_known_hosts_file
key_data = None
key_passphrase = None
ssh_key_id = str(machine.get("ssh_key_id") or "").strip()
if ssh_key_id:
ssh_key = store.get_ssh_key(ssh_key_id)
if ssh_key:
key_data = ssh_key.get("private_key") or None
key_passphrase = ssh_key.get("passphrase") or None
if not key_data and machine.get("ssh_private_key"):
key_data = machine.get("ssh_private_key") or None
key_passphrase = machine.get("ssh_private_key_passphrase") or None
cache_key = (
machine["id"],
machine["host"],
machine["username"],
int(machine.get("port") or 22),
f"{machine.get('key_directory')}/{machine.get('key_name')}" if machine.get("key_directory") and machine.get("key_name") else "",
machine.get("password") or None,
key_data,
key_passphrase,
str(known_hosts_path),
)
return _ssh_client_for(cache_key)
return _ssh_client_from_machine_config(machine, store)
settings = get_settings()
logger.info(
@@ -209,7 +231,19 @@ def get_ssh_client(request: Request = None):
)
if not settings.ssh_key_path:
raise RuntimeError("No SSH machine is configured and SSH key settings must be configured")
return _ssh_client_for(("legacy", settings.ssh_host, settings.ssh_username, settings.ssh_port, settings.ssh_key_path, settings.ssh_password or None, None, None, str(settings.ssh_known_hosts_file)))
return _ssh_client_for(
(
"legacy",
settings.ssh_host,
settings.ssh_username,
settings.ssh_port,
settings.ssh_key_path,
settings.ssh_password or None,
None,
None,
str(settings.ssh_known_hosts_file),
)
)
def get_mail_queue() -> MailQueue:
backend/src/media_library_viewer_api/domain/media.py
+5 -1
View File
@@ -32,7 +32,11 @@ def media_streams(item: dict[str, Any], stream_type: str | None = None) -> list[
streams.extend(source.get("MediaStreams") or [])
if stream_type is None:
return streams
return [stream for stream in streams if str(stream.get("Type") or stream.get("codec_type") or "").lower() == stream_type.lower()]
return [
stream
for stream in streams
if str(stream.get("Type") or stream.get("codec_type") or "").lower() == stream_type.lower()
]
def stream_value(stream: dict[str, Any], *keys: str) -> Any:
backend/src/media_library_viewer_api/jobs.py
+36
View File
@@ -52,6 +52,42 @@ JOB_TEMPLATES: dict[str, JobTemplate] = {
description="Lists empty directories under the selected path. Does not delete anything.",
command_template="find {path} -type d -empty -print",
),
"install_node_exporter": JobTemplate(
name="Install Node Exporter",
description="Downloads and installs prometheus-node-exporter via package manager (apt/dnf/yum/zypper).",
command_template=(
"set -e; "
"if command -v apt-get >/dev/null 2>&1; then "
"sudo apt-get update && sudo apt-get install -y prometheus-node-exporter; "
"elif command -v dnf >/dev/null 2>&1; then "
"sudo dnf install -y prometheus-node-exporter; "
"elif command -v yum >/dev/null 2>&1; then "
"sudo yum install -y prometheus-node-exporter; "
"elif command -v zypper >/dev/null 2>&1; then "
"sudo zypper install -y prometheus-node-exporter; "
"else echo 'No supported package manager found' >&2; exit 1; "
"fi; "
"sudo systemctl enable --now prometheus-node-exporter; "
"echo installed at {path}"
),
),
"restart_node_exporter": JobTemplate(
name="Restart Node Exporter",
description="Restarts the prometheus-node-exporter systemd service.",
command_template="sudo systemctl restart prometheus-node-exporter; echo restarted at {path}",
),
"node_exporter_status": JobTemplate(
name="Node Exporter status",
description="Checks whether prometheus-node-exporter is installed, enabled, and running.",
command_template=(
"systemctl status prometheus-node-exporter --no-pager || true; "
"echo '---'; "
"command -v node_exporter >/dev/null 2>&1 "
"&& node_exporter --version 2>&1 | head -1 "
"|| echo 'node_exporter binary not found'; "
"echo checked {path}"
),
),
}
backend/src/media_library_viewer_api/logging_utils.py
+44 -5
View File
@@ -4,19 +4,42 @@ from __future__ import annotations
import logging
import os
from typing import Any
from urllib.parse import urlsplit
from pythonjsonlogger import jsonlogger
def configure_logging(level_name: str | None = None) -> int:
def _json_formatter() -> logging.Formatter:
"""Return a JSON formatter that preserves timestamp, level, logger, and message."""
return jsonlogger.JsonFormatter(
fmt="%(timestamp)s %(level)s %(name)s %(message)s",
rename_fields={"asctime": "timestamp", "levelname": "level"},
)
def _text_formatter() -> logging.Formatter:
"""Return the legacy plain-text formatter."""
return logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
def configure_logging(level_name: str | None = None, log_format: str | None = None) -> int:
"""Configure root logging once and return the numeric log level."""
resolved_name = (level_name or os.getenv("LOG_LEVEL", "INFO")).upper()
level = getattr(logging, resolved_name, logging.INFO)
use_json = (log_format or os.getenv("LOG_FORMAT", "text")).lower() == "json"
root = logging.getLogger()
if not root.handlers:
logging.basicConfig(
level=level,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
handler = logging.StreamHandler()
handler.setFormatter(_json_formatter() if use_json else _text_formatter())
logging.basicConfig(level=level, handlers=[handler])
else:
# Reconfigure existing handler formatters if the format changed.
formatter = _json_formatter() if use_json else _text_formatter()
for handler in root.handlers:
handler.setFormatter(formatter)
root.setLevel(level)
logging.getLogger("media_library_viewer_api").setLevel(level)
logging.getLogger("uvicorn").setLevel(level)
@@ -40,6 +63,7 @@ def describe_settings(settings: object) -> dict[str, str]:
"""Return a secret-safe summary of the current backend settings."""
return {
"log_level": str(getattr(settings, "log_level", "INFO") or "INFO").upper(),
"log_format": str(getattr(settings, "log_format", "text") or "text").lower(),
"auth_enabled": str(bool(getattr(settings, "auth_enabled", True))),
"oidc_issuer_url": _sanitize_url(getattr(settings, "oidc_issuer_url", "")),
"oidc_audience": getattr(settings, "oidc_audience", "") or "<unset>",
@@ -60,3 +84,18 @@ def describe_settings(settings: object) -> dict[str, str]:
"remote_media_root": getattr(settings, "remote_media_root", "") or "<unset>",
"remote_path_prefix": getattr(settings, "remote_path_prefix", "") or "<unset>",
}
def sanitize_log_extra(extra: dict[str, Any] | None) -> dict[str, Any]:
"""Remove obvious secret keys from log extra fields before emission."""
if not extra:
return {}
blocked_suffixes = ("_password", "_secret", "_token", "_key", "private_key", "passphrase")
sanitized: dict[str, Any] = {}
for key, value in extra.items():
lower_key = key.lower()
if any(lower_key.endswith(suffix) for suffix in blocked_suffixes):
sanitized[key] = "<redacted>" if value else ""
else:
sanitized[key] = value
return sanitized
backend/src/media_library_viewer_api/main.py
+50 -10
View File
@@ -7,13 +7,20 @@ import time
from contextlib import asynccontextmanager
import uvicorn
from fastapi import FastAPI, Request
from fastapi import FastAPI, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import Response as FastAPIResponse
from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settings
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller, get_settings_store
from media_library_viewer_api.logging_utils import configure_logging, describe_settings, sanitize_log_extra
from media_library_viewer_api.observability import (
get_request_id,
metrics_payload,
record_request,
set_current_request_id,
)
from media_library_viewer_api.routers import backups as backups_router
from media_library_viewer_api.routers import dashboard, files, jobs, media, monitoring, tasks, users
from media_library_viewer_api.routers.settings import router as settings_router
@@ -28,10 +35,16 @@ logger = logging.getLogger(__name__)
async def lifespan(app: FastAPI):
"""Application lifespan — startup/shutdown."""
settings = get_settings()
configure_logging(settings.log_level)
configure_logging(settings.log_level, settings.log_format)
validate_auth_settings(settings)
logger.info("Backend startup complete: %s", describe_settings(settings))
logger.info("Managed known_hosts will be populated lazily on first successful SSH connection")
try:
from media_library_viewer_api.services.targets import write_prometheus_targets
write_prometheus_targets(get_settings_store())
except Exception:
logger.exception("Failed to write Prometheus file-SD targets during startup")
mail_queue = get_mail_queue()
monitoring_poller = get_monitoring_poller()
backup_poller = get_backup_poller()
@@ -49,8 +62,7 @@ app = FastAPI(
title="Manage API",
version=get_backend_version(),
description=(
"Manage API for Jellyfin media browsing, SSH file inspection, "
"server monitoring, and JWT-protected access."
"Manage API for Jellyfin media browsing, SSH file inspection, server monitoring, and JWT-protected access."
),
lifespan=lifespan,
)
@@ -69,30 +81,51 @@ app.add_middleware(
allow_headers=["*"],
)
@app.middleware("http")
async def enforce_jwt_auth(request: Request, call_next):
if request.url.path in {"/api/health", "/api/version"}:
if request.url.path in {"/api/health", "/api/version", "/metrics"}:
return await call_next(request)
return await require_jwt_auth(request, call_next)
@app.middleware("http")
async def log_requests(request: Request, call_next):
"""Log every API request with timing and outcome."""
"""Log every API request with timing, outcome, and request id."""
start = time.perf_counter()
request_id = get_request_id(request)
set_current_request_id(request_id)
request.state.request_id = request_id
client_host = request.client.host if request.client else "unknown"
logger.info("request start %s %s client=%s", request.method, request.url.path, client_host)
extra = sanitize_log_extra({"request_id": request_id, "client_host": client_host})
logger.info("request start %s %s", request.method, request.url.path, extra=extra)
try:
response = await call_next(request)
except Exception:
logger.exception("request error %s %s client=%s", request.method, request.url.path, client_host)
logger.exception(
"request error %s %s",
request.method,
request.url.path,
extra=sanitize_log_extra({"request_id": request_id, "client_host": client_host}),
)
raise
elapsed_ms = (time.perf_counter() - start) * 1000.0
response.headers["X-Request-Id"] = request_id
record_request(request, response, elapsed_ms / 1000.0)
logger.info(
"request end %s %s status=%s elapsed_ms=%.1f",
request.method,
request.url.path,
response.status_code,
elapsed_ms,
extra=sanitize_log_extra(
{
"request_id": request_id,
"client_host": client_host,
"status_code": response.status_code,
"elapsed_ms": round(elapsed_ms, 1),
}
),
)
return response
@@ -123,5 +156,12 @@ def version_info() -> dict[str, str]:
return get_version_info()
@app.get("/metrics")
def metrics() -> Response:
"""Expose Prometheus metrics."""
data, content_type = metrics_payload()
return FastAPIResponse(content=data, media_type=content_type)
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)
backend/src/media_library_viewer_api/observability.py
+154
View File
@@ -0,0 +1,154 @@
"""Application observability: metrics and request correlation.
This module owns Prometheus metrics and request-id generation so that the
rest of the backend can stay focused on business logic.
"""
from __future__ import annotations
import uuid
from contextvars import ContextVar
from typing import Any
from fastapi import Request, Response
from prometheus_client import (
CONTENT_TYPE_LATEST,
Counter,
Gauge,
Histogram,
generate_latest,
)
# Context-local request id for code paths that cannot receive a Request object.
_current_request_id: ContextVar[str | None] = ContextVar("request_id", default=None)
# Request metrics
REQUESTS_TOTAL = Counter(
"manage_api_requests_total",
"Total API requests",
["method", "path", "status_code"],
)
REQUEST_DURATION = Histogram(
"manage_api_request_duration_seconds",
"API request duration",
["method", "path"],
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0),
)
# Operation metrics
SSH_COMMANDS_TOTAL = Counter(
"manage_ssh_commands_total",
"Total SSH/local commands executed",
["machine_id", "action", "status"],
)
SSH_COMMAND_DURATION = Histogram(
"manage_ssh_command_duration_seconds",
"SSH/local command duration",
["machine_id", "action"],
buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0),
)
MEDIA_INDEX_BUILDS_TOTAL = Counter(
"manage_media_index_builds_total",
"Total media index build attempts",
["status"],
)
MEDIA_INDEX_BUILD_DURATION = Histogram(
"manage_media_index_build_duration_seconds",
"Media index build duration",
buckets=(1.0, 5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0),
)
BACKUP_RUNS_TOTAL = Counter(
"manage_backup_runs_total",
"Total backup runs",
["job_name", "status"],
)
BACKUP_RUNS_LAST_SUCCESS = Gauge(
"manage_backup_runs_last_success_timestamp",
"Unix timestamp of the last successful backup run per job",
["job_name"],
)
MAIL_QUEUE_SIZE = Counter(
"manage_mail_queue_messages_total",
"Total messages enqueued",
["status"],
)
def set_current_request_id(request_id: str | None) -> None:
"""Set the context-local request id."""
_current_request_id.set(request_id)
def get_current_request_id() -> str | None:
"""Return the current context-local request id or None."""
return _current_request_id.get()
def generate_request_id() -> str:
"""Return a short unique request id."""
return uuid.uuid4().hex[:16]
def get_request_id(request: Request | None = None) -> str:
"""Resolve a request id from the request header, context, or a new value."""
if request is not None:
header = request.headers.get("x-request-id") or request.headers.get("x-correlation-id")
if header:
return header.strip()
existing = _current_request_id.get()
if existing:
return existing
new_id = generate_request_id()
_current_request_id.set(new_id)
return new_id
def metrics_payload() -> tuple[bytes, str]:
"""Return the Prometheus metrics payload and content type."""
return generate_latest(), CONTENT_TYPE_LATEST
def record_request(request: Request, response: Response, duration_seconds: float) -> None:
"""Record Prometheus metrics for a completed request."""
status = str(response.status_code)
path = request.url.path
method = request.method
REQUESTS_TOTAL.labels(method=method, path=path, status_code=status).inc()
REQUEST_DURATION.labels(method=method, path=path).observe(duration_seconds)
def record_ssh_command(machine_id: str, action: str, status: str, duration_seconds: float) -> None:
"""Record metrics for an SSH/local command."""
SSH_COMMANDS_TOTAL.labels(machine_id=machine_id or "unknown", action=action, status=status).inc()
SSH_COMMAND_DURATION.labels(machine_id=machine_id or "unknown", action=action).observe(duration_seconds)
def record_media_index_build(status: str, duration_seconds: float | None = None) -> None:
"""Record metrics for a media index build."""
MEDIA_INDEX_BUILDS_TOTAL.labels(status=status).inc()
if duration_seconds is not None:
MEDIA_INDEX_BUILD_DURATION.observe(duration_seconds)
def record_backup_run(job_name: str, status: str, success: bool = False) -> None:
"""Record metrics for a backup run."""
job_name = job_name or "unknown"
BACKUP_RUNS_TOTAL.labels(job_name=job_name, status=status).inc()
if success:
BACKUP_RUNS_LAST_SUCCESS.labels(job_name=job_name).set_to_current_time()
def record_mail_queue(status: str) -> None:
"""Record metrics for a mail queue message outcome."""
MAIL_QUEUE_SIZE.labels(status=status).inc()
def log_extra(request: Request | None = None, **kwargs: Any) -> dict[str, Any]:
"""Build a standard extra dict for structured logging."""
extra: dict[str, Any] = {"request_id": get_request_id(request)}
extra.update(kwargs)
return extra
backend/src/media_library_viewer_api/path_utils.py
+1 -1
View File
@@ -57,7 +57,7 @@ def map_path_to_media_root(path: str, media_root: str) -> str:
if root_anchor in raw_parts:
anchor_index = raw_parts.index(root_anchor)
remainder_parts = raw_parts[anchor_index + 1:]
remainder_parts = raw_parts[anchor_index + 1 :]
resolved = posixpath.join(normalized_root, *remainder_parts) if remainder_parts else normalized_root
logger.debug("Mapped path to media root path=%s media_root=%s resolved=%s", path, normalized_root, resolved)
return resolved
backend/src/media_library_viewer_api/routers/backups.py
+20 -13
View File
@@ -8,6 +8,7 @@ from ..models.backups import (
BackupReportRequest,
BackupRunResponse,
)
from ..observability import record_backup_run
from ..services.backup_alert_engine import generate_alerts_for_run
from ..services.settings_store import SettingsStore, get_settings_store
@@ -17,20 +18,24 @@ router = APIRouter(prefix="/api/backups", tags=["backups"])
def _get_or_create_job(store: SettingsStore, report: BackupReportRequest) -> dict[str, Any]:
job = store.get_backup_job_by_name(report.name)
if not job:
job = store.upsert_backup_job({
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
})
job = store.upsert_backup_job(
{
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
}
)
elif report.schedule_interval_seconds:
store.upsert_backup_job({
"id": job["id"],
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
})
store.upsert_backup_job(
{
"id": job["id"],
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
}
)
job = store.get_backup_job(job["id"])
return job
@@ -61,6 +66,7 @@ def post_backup_report(
"details": report.details,
}
run = store.create_backup_run(run_data)
record_backup_run(job_name=report.name, status=report.status, success=report.status == "success")
# Generate alerts
previous_runs = store.list_backup_runs(job_id=job["id"], status="success", limit=20)
@@ -93,6 +99,7 @@ def post_backup_start(
"status": "in_progress",
}
run = store.create_backup_run(run_data)
record_backup_run(job_name=report.name, status="in_progress")
# Map details -> details_json for response model
run["details_json"] = run.pop("details", None)
return BackupRunResponse(**run)
backend/src/media_library_viewer_api/routers/dashboard.py
+14 -60
View File
@@ -11,12 +11,10 @@ from fastapi import APIRouter, Depends
from media_library_viewer_api.clients.jellyfin import JellyfinClient
from media_library_viewer_api.dependencies import (
get_jellyfin_client,
get_monitoring_poller,
get_settings_store,
get_user_id,
)
from media_library_viewer_api.models.backups import BackupDashboardSummary
from media_library_viewer_api.services.monitoring_actions import collect_machine_overview
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
@@ -46,52 +44,6 @@ def get_library_counts(
return client.library_item_counts(user_id, libraries)
@router.get("/monitoring")
def get_monitoring_overview(
store=Depends(get_settings_store),
) -> dict[str, Any]:
"""Return one lightweight monitoring row per configured machine."""
machines = [m for m in store.list_machines() if m.get("enabled")]
rows: list[dict[str, Any]] = []
for machine in machines:
try:
rows.append(collect_machine_overview(machine, store))
except Exception as exc:
logger.exception(
"Dashboard monitoring overview failed for machine_id=%s",
machine.get("id"),
)
rows.append({
"machine": {k: machine.get(k) for k in (
"id", "name", "mode", "enabled", "host", "port", "username",
"media_root", "path_prefix", "notes",
)},
"status": "",
"status_error": str(exc),
"metrics_error": str(exc),
"disk_error": str(exc),
"sample_count": 0,
"latest_sample": None,
"cpu_summary": None,
"iowait_summary": None,
"mem_summary": None,
"net_rx_summary": None,
"net_tx_summary": None,
"disk_read_summary": None,
"disk_write_summary": None,
"disk": None,
})
poller = get_monitoring_poller().snapshot()
enabled_count = sum(1 for machine in machines if machine.get("enabled"))
logger.info("Dashboard monitoring machines=%s enabled=%s", len(machines), enabled_count)
return {
"poller": poller,
"machines": rows,
"total": len(machines),
"enabled": enabled_count,
}
@router.get("/shortcuts")
def get_shortcuts(
store=Depends(get_settings_store),
@@ -144,8 +96,8 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[
has_item = bool(item)
series = item.get("SeriesName") or ""
title = (
f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")
) if has_item else "(idle)"
(f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")) if has_item else "(idle)"
)
if not has_item:
state_label = "idle"
@@ -162,16 +114,18 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[
if not transcode_type:
transcode_type.append("active")
results.append({
"user": session.get("UserName") or "Unknown",
"title": title,
"type": item.get("Type", "") if has_item else "",
"state": state_label,
"transcoding": "yes" if is_transcoding else "no",
"transcoding_type": ", ".join(transcode_type),
"device": session.get("DeviceName") or session.get("Client") or "",
"session_id": session.get("Id") or "",
})
results.append(
{
"user": session.get("UserName") or "Unknown",
"title": title,
"type": item.get("Type", "") if has_item else "",
"state": state_label,
"transcoding": "yes" if is_transcoding else "no",
"transcoding_type": ", ".join(transcode_type),
"device": session.get("DeviceName") or session.get("Client") or "",
"session_id": session.get("Id") or "",
}
)
return results
backend/src/media_library_viewer_api/routers/media.py
+3
View File
@@ -16,6 +16,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status
from media_library_viewer_api.clients.jellyfin import JellyfinClient
from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id
from media_library_viewer_api.observability import record_media_index_build
from media_library_viewer_api.services.media_index import MediaIndex
logger = logging.getLogger(__name__)
@@ -169,6 +170,7 @@ def post_build_index(
},
)
media_status = index.status()
record_media_index_build(status="started")
logger.info("Media index build started pid=%s", process.pid)
return {"status": "started", **_serialize_status(media_status)}
@@ -256,6 +258,7 @@ def force_stop_build(index: MediaIndex = Depends(get_media_index)) -> dict[str,
},
)
media_status = index.status()
record_media_index_build(status="force_stopped")
return {"status": "force_stopped", **_serialize_status(media_status)}
backend/src/media_library_viewer_api/routers/monitoring.py
+146 -100
View File
@@ -1,31 +1,70 @@
"""Monitoring router — metrics, collector controls, and per-machine status."""
"""Monitoring router — disk checks, action history, and observability stack status."""
from __future__ import annotations
import logging
import time
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Body, Depends, HTTPException, Query
from media_library_viewer_api.clients.resources import (
disk_space,
read_resource_metrics,
resource_collector_status,
)
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_settings_store
from media_library_viewer_api.services.monitoring_actions import (
poll_machine_diagnostics,
disk_space,
run_machine_operation,
start_collector,
stop_collector,
restart_collector,
)
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import build_node_exporter_targets
logger = logging.getLogger(__name__)
def _alertmanager_client() -> Any:
"""Return a simple HTTP client for the configured Alertmanager URL."""
import requests
settings = get_settings()
return requests.Session(), settings.alertmanager_url
def _webhook_client() -> Any:
"""Return a simple HTTP client for the optional webhook receiver URL."""
import requests
settings = get_settings()
return requests.Session(), settings.alertmanager_webhook_url
def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
"""Build a UI-friendly summary from Alertmanager /api/v1/alerts payload."""
by_severity: dict[str, int] = {}
open_alerts: list[dict[str, Any]] = []
for alert in alerts:
labels = alert.get("labels") or {}
annotations = alert.get("annotations") or {}
severity = labels.get("severity", "unknown")
by_severity[severity] = by_severity.get(severity, 0) + 1
open_alerts.append(
{
"name": labels.get("alertname", "unknown"),
"severity": severity,
"category": labels.get("category", ""),
"job_name": labels.get("job_name", labels.get("job", "")),
"summary": annotations.get("summary", ""),
"description": annotations.get("description", ""),
"active_since": alert.get("startsAt"),
"state": alert.get("status", "firing"),
"labels": labels,
}
)
open_alerts.sort(key=lambda a: (a["severity"] not in {"critical", "warning"}, a["severity"], a["name"]))
return {
"total": len(open_alerts),
"by_severity": by_severity,
"alerts": open_alerts[:50],
}
router = APIRouter(prefix="/api/monitoring", tags=["monitoring"])
@@ -63,7 +102,11 @@ def get_poller_status() -> dict[str, Any]:
from media_library_viewer_api.dependencies import get_monitoring_poller
poller = get_monitoring_poller().snapshot()
logger.info("Monitoring poller status requested running=%s poll_count=%s", poller.get("worker_running"), poller.get("poll_count"))
logger.info(
"Monitoring poller status requested running=%s poll_count=%s",
poller.get("worker_running"),
poller.get("poll_count"),
)
return poller
@@ -81,53 +124,6 @@ def get_machine_actions(
return {"items": actions, "total": len(actions)}
@router.get("/status")
def get_status(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Return collector running status for a given machine."""
machine = _resolve_machine(store, machine_id)
status = run_machine_operation(machine, store, "status lookup", resource_collector_status)
logger.info("Monitoring status requested machine_id=%s status=%s", machine["id"], status)
return {"status": status}
@router.get("/metrics")
def get_metrics(
max_lines: int = 70_000,
last_seconds: int | None = None,
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Return resource metric samples for a given machine."""
machine = _resolve_machine(store, machine_id)
def _read_metrics(client):
return read_resource_metrics(client, max_lines=max_lines)
rows = run_machine_operation(machine, store, "metrics read", _read_metrics)
if last_seconds is None:
filtered = rows
cutoff_ts = 0.0
else:
cutoff_ts = time.time() - last_seconds
filtered = [row for row in rows if float(row.get("ts", 0)) >= cutoff_ts]
logger.info(
"Monitoring metrics requested machine_id=%s total=%s filtered=%s last_seconds=%s",
machine["id"],
len(rows),
len(filtered),
last_seconds,
)
return {
"samples": filtered,
"total_samples": len(rows),
"filtered_samples": len(filtered),
"cutoff_ts": cutoff_ts,
}
@router.get("/disk")
def get_disk_space(
machine_id: str | None = Query(default=None),
@@ -146,49 +142,99 @@ def get_disk_space(
)
@router.post("/start")
def post_start(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Start the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = start_collector(machine, store)
logger.info("Monitoring collector start result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/prometheus-targets")
def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
"""Return Prometheus file-SD targets for remote Node Exporters.
The backend writes these targets to a JSON file that Prometheus reads via
file_sd_configs. This endpoint returns the same list live from the store so
the UI can preview which machines will be scraped.
"""
targets = build_node_exporter_targets(store)
logger.info("Prometheus targets requested count=%s", len(targets))
return targets
@router.post("/stop")
def post_stop(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Stop the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = stop_collector(machine, store)
logger.info("Monitoring collector stop result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/alerts")
def get_alertmanager_alerts() -> dict[str, Any]:
"""Return a summary of active Alertmanager alerts for the UI.
Proxies the Alertmanager `/api/v1/alerts` endpoint and reshapes the payload
into a stable, UI-friendly format. If Alertmanager is unreachable, the
endpoint returns an empty summary and logs the failure so the UI can still
render a health card instead of an error page.
"""
session, base_url = _alertmanager_client()
try:
response = session.get(f"{base_url}/api/v1/alerts", timeout=5)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager alerts from %s", base_url)
return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_unreachable"}
if data.get("status") != "success":
return {"total": 0, "by_severity": {}, "alerts": [], "error": data.get("error", "unknown")}
summary = _summary_from_alerts(data.get("data", []))
logger.info("Alertmanager alerts requested total=%s", summary["total"])
return summary
@router.post("/restart")
def post_restart(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Restart the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = restart_collector(machine, store)
logger.info("Monitoring collector restart result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/alertmanager-status")
def get_alertmanager_status() -> dict[str, Any]:
"""Return Alertmanager cluster/status for the UI health card.
Uses the Alertmanager `/api/v2/status` endpoint and exposes only the high-
level fields the UI needs: uptime, version, and whether the cluster is
healthy.
"""
session, base_url = _alertmanager_client()
try:
response = session.get(f"{base_url}/api/v2/status", timeout=5)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager status from %s", base_url)
return {"up": False, "version": "", "uptime": ""}
cluster = data.get("cluster") or {}
status = data.get("clusterStatus") or {}
return {
"up": True,
"version": data.get("versionInfo", {}).get("version", ""),
"uptime": status.get("createdAt", ""),
"name": "",
"peers": [p.get("name", "") for p in cluster.get("peers", [])],
}
@router.get("/diagnostics")
def get_diagnostics(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Return collector debug info for a given machine."""
machine = _resolve_machine(store, machine_id)
diagnostics = poll_machine_diagnostics(machine, store)["diagnostics"]
logger.info("Monitoring diagnostics requested machine_id=%s", machine["id"])
return {"diagnostics": diagnostics}
@router.post("/alertmanager-webhook")
def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
"""Receive alerts from Alertmanager and optionally forward to a webhook URL.
This endpoint is the receiver referenced by the optional `webhook_configs`
block in Alertmanager. It logs the payload for audit/debug purposes and, if
`ALERTMANAGER_WEBHOOK_URL` is configured, forwards the alert JSON verbatim.
Forwarding is best-effort: a failure to reach the downstream webhook does
not fail this endpoint, so Alertmanager sees a successful delivery.
"""
alerts = payload.get("alerts", [])
logger.info(
"Received Alertmanager webhook with %s alert(s), status=%s",
len(alerts),
payload.get("status", "unknown"),
)
session, webhook_url = _webhook_client()
if webhook_url:
try:
response = session.post(webhook_url, json=payload, timeout=10)
response.raise_for_status()
logger.info("Forwarded Alertmanager webhook to %s", webhook_url)
except Exception:
logger.exception("Failed to forward Alertmanager webhook to %s", webhook_url)
else:
logger.debug("No ALERTMANAGER_WEBHOOK_URL configured; webhook stored in logs only")
return {"status": "received"}
backend/src/media_library_viewer_api/routers/settings.py
+25 -14
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
from io import StringIO
from typing import Any
@@ -12,11 +13,13 @@ from pydantic import BaseModel, Field
from media_library_viewer_api.clients.ssh import RemoteSSHClient
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_monitoring_poller, get_settings_store
from media_library_viewer_api.services.monitoring_actions import start_collector
from media_library_viewer_api.services.db_maintenance import remove_sqlite_database
from media_library_viewer_api.services.known_hosts import has_known_host
from media_library_viewer_api.services.media_index import MediaIndex
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import write_prometheus_targets
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/settings", tags=["settings"])
@@ -95,9 +98,7 @@ def _raise_ssh_validation_error(host: str, port: int, exc: Exception) -> None:
if "protocol banner" in lowered:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail=(
f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."
),
detail=(f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."),
) from exc
if "no authentication methods available" in lowered or "authentication failed" in lowered:
raise HTTPException(
@@ -125,10 +126,12 @@ def _validate_saved_machine_ssh(machine: MonitoringMachineInput, store: Settings
client.close()
def _start_machine_collector(machine: MonitoringMachineInput, store: SettingsStore) -> None:
if "monitoring" not in {str(service).strip().lower() for service in machine.services}:
return
start_collector(machine.model_dump(exclude_none=True), store)
def _write_prometheus_targets(store: SettingsStore) -> None:
"""Regenerate Prometheus file-SD targets after machine changes."""
try:
write_prometheus_targets(store)
except Exception:
logger.exception("Failed to write Prometheus file-SD targets")
@router.post("/machines/test-ssh")
@@ -137,7 +140,9 @@ def test_machine_ssh(
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
if str(machine.mode or "").strip().lower() != "ssh":
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines"
)
client, host, port = _resolve_ssh_client(machine, store)
settings = get_settings()
@@ -174,7 +179,8 @@ def test_machine_ssh(
return {
"status": "ok",
"message": (
f"SSH connection succeeded for {host}:{port}; host key {'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
f"SSH connection succeeded for {host}:{port}; host key "
f"{'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
),
"host": host,
"port": port,
@@ -188,11 +194,11 @@ def post_machine(
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine.id)
_write_prometheus_targets(store)
poller = get_monitoring_poller()
try:
saved_machine = MonitoringMachineInput.model_validate(saved)
_validate_saved_machine_ssh(saved_machine, store)
_start_machine_collector(saved_machine, store)
finally:
poller.start()
poller.kick()
@@ -208,11 +214,11 @@ def put_machine(
if not store.get_machine(machine_id):
raise HTTPException(status_code=404, detail="Machine not found")
saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine_id)
_write_prometheus_targets(store)
poller = get_monitoring_poller()
try:
saved_machine = MonitoringMachineInput.model_validate(saved)
_validate_saved_machine_ssh(saved_machine, store)
_start_machine_collector(saved_machine, store)
finally:
poller.start()
poller.kick()
@@ -224,6 +230,7 @@ def delete_machine(machine_id: str, store: SettingsStore = Depends(get_settings_
if not store.get_machine(machine_id):
raise HTTPException(status_code=404, detail="Machine not found")
store.delete_machine(machine_id)
_write_prometheus_targets(store)
return {"status": "deleted"}
@@ -311,8 +318,12 @@ def reset_local_database(
expected = "RESET LOCAL DATABASE"
if payload.confirm_phrase.strip().upper() != expected:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Confirmation phrase does not match")
if not (payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected")
if not (
payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible
):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected"
)
settings_removed = remove_sqlite_database(store.db_path)
media_index = MediaIndex()
backend/src/media_library_viewer_api/routers/users_impl.py
+6 -4
View File
@@ -212,7 +212,8 @@ def _merge_users(
"source_summary": summary,
"name_source": name_source,
"access_source": access_source,
"jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId")) or None,
"jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId"))
or None,
"jellyseerr_username": str(
(seerr_user or {}).get("username") or (seerr_user or {}).get("jellyfinUsername") or ""
),
@@ -364,9 +365,10 @@ async def post_user_message(
text_body=text_body,
attachments=attachment_payloads,
)
from_address = str(getattr(settings, "smtp_from_address", "") or "").strip() or str(
getattr(settings, "smtp_username", "") or ""
).strip()
from_address = (
str(getattr(settings, "smtp_from_address", "") or "").strip()
or str(getattr(settings, "smtp_username", "") or "").strip()
)
logger.info(
"Users message queued request_id=%s subject=%s recipients=%s attachments=%s skipped=%s",
request_id,
backend/src/media_library_viewer_api/services/backup_alert_engine.py
+66 -54
View File
@@ -13,20 +13,21 @@ def generate_alerts_for_run(
# 1. Failed status alert
if run["status"] == "failure":
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "failed_status",
"severity": "critical",
"message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "failed_status",
"severity": "critical",
"message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
}
)
# 2. Anomaly size alert
bytes_transferred = run.get("bytes_transferred")
if bytes_transferred is not None and previous_runs:
successful_runs = [
r for r in previous_runs
if r["status"] == "success" and r.get("bytes_transferred") is not None
r for r in previous_runs if r["status"] == "success" and r.get("bytes_transferred") is not None
]
if len(successful_runs) >= 3:
sizes = [r["bytes_transferred"] for r in successful_runs[-7:]]
@@ -34,24 +35,28 @@ def generate_alerts_for_run(
if median_size > 0:
ratio = bytes_transferred / median_size
if bytes_transferred == 0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
}
)
elif ratio < 0.1 or ratio > 3.0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": (
f"Backup job '{job_name}' size anomaly: "
f"{bytes_transferred} bytes (median: {median_size})"
),
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": (
f"Backup job '{job_name}' size anomaly: "
f"{bytes_transferred} bytes (median: {median_size})"
),
}
)
# 3. Anomaly duration alert
duration_ms = run.get("duration_ms")
@@ -61,16 +66,17 @@ def generate_alerts_for_run(
durations = [r["duration_ms"] for r in successful_runs[-7:]]
median_duration = statistics.median(durations)
if median_duration > 0 and duration_ms / median_duration > 3.0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_duration",
"severity": "warning",
"message": (
f"Backup job '{job_name}' duration anomaly: "
f"{duration_ms}ms (median: {median_duration}ms)"
),
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_duration",
"severity": "warning",
"message": (
f"Backup job '{job_name}' duration anomaly: {duration_ms}ms (median: {median_duration}ms)"
),
}
)
return alerts
@@ -82,6 +88,7 @@ def check_missed_schedules(
) -> list[dict[str, Any]]:
alerts = []
import time
now = int(time.time())
for job in jobs:
@@ -93,31 +100,36 @@ def check_missed_schedules(
if not latest_run:
# No runs ever — alert if job is older than interval * 1.5
if now - job["created_at"] > interval * 1.5:
alerts.append({
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
})
alerts.append(
{
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
}
)
else:
last_run_time = latest_run["started_at"]
if now - last_run_time > interval * 1.5:
# Check if there's already an unresolved missed_schedule alert
has_open_alert = any(
a["alert_type"] == "missed_schedule" and a["resolved_at"] is None
for a in existing_alerts if a["job_id"] == job["id"]
for a in existing_alerts
if a["job_id"] == job["id"]
)
if not has_open_alert:
alerts.append({
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": (
f"Backup job '{job['name']}' missed schedule: "
f"last run at {last_run_time} (expected every {interval}s)"
),
})
alerts.append(
{
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": (
f"Backup job '{job['name']}' missed schedule: "
f"last run at {last_run_time} (expected every {interval}s)"
),
}
)
return alerts
backend/src/media_library_viewer_api/services/mail_queue.py
+3
View File
@@ -15,6 +15,7 @@ import uuid
from dataclasses import dataclass, field
from typing import Any
from media_library_viewer_api.observability import record_mail_queue
from media_library_viewer_api.services.mailer import EmailAttachment, describe_smtp_error, send_email_message
logger = logging.getLogger(__name__)
@@ -206,6 +207,7 @@ class MailQueue:
result.get("recipient_count", 0),
result.get("attachment_count", 0),
)
record_mail_queue("sent")
except Exception as exc:
friendly_error = describe_smtp_error(exc)
with self._lock:
@@ -220,6 +222,7 @@ class MailQueue:
getattr(message, "request_id", "unknown"),
friendly_error,
)
record_mail_queue("failed")
finally:
self._queue.task_done()
backend/src/media_library_viewer_api/services/mailer_impl.py
+25 -10
View File
@@ -4,8 +4,8 @@ from __future__ import annotations
import logging
import mimetypes
import socket
import smtplib
import socket
import ssl
from dataclasses import dataclass
from email.message import EmailMessage
@@ -13,7 +13,6 @@ from email.utils import formataddr
from html.parser import HTMLParser
from typing import Any, Iterable
logger = logging.getLogger(__name__)
@@ -164,7 +163,11 @@ def _smtp_sender_not_authorized(error: Exception) -> bool:
else:
raw_error_text = str(raw_error)
text = f"{code} {raw_error_text} {error}".lower()
return code in {551, 553} or "not authorised to send from this header address" in text or "not authorized to send from this header address" in text
return (
code in {551, 553}
or "not authorised to send from this header address" in text
or "not authorized to send from this header address" in text
)
def _smtp_attempt_metadata(mode: dict[str, Any]) -> dict[str, Any]:
@@ -203,10 +206,15 @@ def describe_smtp_error(error: Exception) -> str:
)
if isinstance(item, (smtplib.SMTPDataError, smtplib.SMTPResponseException)):
smtp_code = getattr(item, "smtp_code", None)
if smtp_code in {551, 553} or "not authorised to send from this header address" in lowered or "not authorized to send from this header address" in lowered:
if (
smtp_code in {551, 553}
or "not authorised to send from this header address" in lowered
or "not authorized to send from this header address" in lowered
):
return (
"SMTP server rejected the configured From address. Use an authorized alias "
"for this account or change SMTP_FROM_ADDRESS to a sender the provider allows."
"SMTP server rejected the configured From address. "
"Use an authorized alias for this account or change "
"SMTP_FROM_ADDRESS to a sender the provider allows."
)
if isinstance(item, smtplib.SMTPConnectError):
return "SMTP connection was rejected by the server. Check the host and port."
@@ -247,7 +255,9 @@ def build_email_message(
msg.set_content(plain_text or "")
for attachment in attachments:
content_type = attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream"
content_type = (
attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream"
)
maintype, subtype = content_type.split("/", 1) if "/" in content_type else ("application", "octet-stream")
msg.add_attachment(
attachment.data,
@@ -357,7 +367,9 @@ def send_email_message(
)
if _smtp_sender_not_authorized(exc) and fallback_from_address:
logger.warning(
"SMTP send sender rejected label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s; retrying with smtp_username",
"SMTP send sender rejected label=%s host=%s "
"port=%s transport=%s auth_user=%s "
"from_address=%s error=%s; retrying with smtp_username",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
@@ -393,7 +405,9 @@ def send_email_message(
}
)
logger.warning(
"SMTP send fallback failed label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s",
"SMTP send fallback failed label=%s host=%s "
"port=%s transport=%s auth_user=%s "
"from_address=%s error=%s",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
@@ -416,7 +430,8 @@ def send_email_message(
}
)
logger.info(
"SMTP send succeeded via smtp_username label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s",
"SMTP send succeeded via smtp_username label=%s host=%s "
"port=%s transport=%s auth_user=%s from_address=%s",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
backend/src/media_library_viewer_api/services/media_index_impl.py
+22 -17
View File
@@ -141,13 +141,20 @@ class MediaIndex:
key TEXT PRIMARY KEY,
value TEXT
);
CREATE INDEX IF NOT EXISTS idx_media_type ON media_items(type);
CREATE INDEX IF NOT EXISTS idx_media_library ON media_items(library_id);
CREATE INDEX IF NOT EXISTS idx_media_title ON media_items(title COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_series ON media_items(series COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_date_added ON media_items(date_added_ts);
CREATE INDEX IF NOT EXISTS idx_media_size ON media_items(size_bytes);
CREATE INDEX IF NOT EXISTS idx_media_bitrate ON media_items(bitrate_bps);
CREATE INDEX IF NOT EXISTS idx_media_type
ON media_items(type);
CREATE INDEX IF NOT EXISTS idx_media_library
ON media_items(library_id);
CREATE INDEX IF NOT EXISTS idx_media_title
ON media_items(title COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_series
ON media_items(series COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_date_added
ON media_items(date_added_ts);
CREATE INDEX IF NOT EXISTS idx_media_size
ON media_items(size_bytes);
CREATE INDEX IF NOT EXISTS idx_media_bitrate
ON media_items(bitrate_bps);
"""
)
@@ -208,10 +215,7 @@ class MediaIndex:
try:
with self.connect() as conn:
item_count = int(conn.execute("SELECT COUNT(*) FROM media_items").fetchone()[0])
meta = {
row[0]: row[1]
for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()
}
meta = {row[0]: row[1] for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()}
except sqlite3.Error:
return MediaIndexStatus(exists=False)
updated_at_raw = meta.get("updated_at", "")
@@ -310,7 +314,10 @@ class MediaIndex:
sort_sql = SORT_COLUMNS.get(sort_key, SORT_COLUMNS["title"])
direction = "DESC" if sort_order == "Descending" else "ASC"
# Always add stable tie-breakers.
order_sql = f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, season_number ASC, episode ASC, title COLLATE NOCASE ASC"
order_sql = (
f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, "
"season_number ASC, episode ASC, title COLLATE NOCASE ASC"
)
with self.connect() as conn:
total = int(conn.execute("SELECT COUNT(*) FROM media_items" + where_sql, params).fetchone()[0])
@@ -421,10 +428,7 @@ def build_media_index(
**row,
"path": resolve_remote_media_path(row.get("path", ""), media_root, fallback_prefix),
}
for row in (
normalize_media_item(item, library_id, current_library_name)
for item in items
)
for row in (normalize_media_item(item, library_id, current_library_name) for item in items)
)
processed_total += len(items)
current_library_processed += len(items)
@@ -432,7 +436,8 @@ def build_media_index(
ensure_not_cancelled()
emit(
"building",
f"{current_library_name or 'Library'}: {current_library_processed} / {current_library_total or '?'} items",
f"{current_library_name or 'Library'}: "
f"{current_library_processed} / {current_library_total or '?'} items",
)
logger.debug(
"Media index progress library=%s processed=%s/%s total_processed=%s",
backend/src/media_library_viewer_api/services/monitoring_actions.py
+43 -184
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import json
import logging
import shlex
import time
import uuid
from typing import Any, Callable
@@ -16,17 +17,9 @@ from typing import Any, Callable
from fastapi import HTTPException
from media_library_viewer_api.clients.local import LocalCommandClient
from media_library_viewer_api.clients.resources import (
disk_space,
read_resource_metrics,
resource_collector_debug_info,
resource_collector_status,
restart_resource_collector,
start_resource_collector,
stop_resource_collector,
)
from media_library_viewer_api.clients.ssh import RemoteSSHClient
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.observability import record_ssh_command
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
@@ -63,6 +56,28 @@ def build_machine_client(machine: dict[str, Any], store: SettingsStore):
)
def disk_space(client: Any, path: str = "/") -> dict[str, Any]:
"""Return df information for the filesystem containing ``path``.
Works against any client with a ``run`` method (local shell or SSH).
"""
command = (
"df -P -B1 -- " + shlex.quote(path or "/") + " | awk 'NR==2 {printf "
'"{\\"filesystem\\":\\"%s\\",\\"size\\":%s,"'
'"\\"used\\":%s,\\"available\\":%s,"'
'"\\"used_pct\\":\\"%s\\",\\"mount\\":\\"%s\\"}", "'
"$1,$2,$3,$4,$5,$6}'"
)
logger.debug("Reading disk space for path=%s", path)
result = client.run(command, timeout=20)
if result.exit_status != 0 or not result.stdout.strip():
logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
data = json.loads(result.stdout)
logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct"))
return data
def summarize_operation_result(action: str, result: Any) -> str:
"""Turn an operation result into a compact human-readable summary."""
if result is None:
@@ -73,10 +88,6 @@ def summarize_operation_result(action: str, result: Any) -> str:
if isinstance(result, list):
return f"{action}: {len(result)} item(s)"
if isinstance(result, dict):
if action == "metrics read":
samples = result.get("samples")
if isinstance(samples, list):
return f"{action}: {len(samples)} sample(s)"
if action.startswith("disk lookup"):
used_pct = result.get("used_pct")
mount = result.get("mount") or result.get("filesystem")
@@ -104,13 +115,21 @@ def run_machine_operation(
summarize: Callable[[Any], str] | None = None,
request_id: str = "",
raise_http: bool = True,
client: Any | None = None,
) -> Any:
"""Run a machine operation, record history, and optionally raise on failure."""
started = time.perf_counter()
client = build_machine_client(machine, store)
if client is None:
client = build_machine_client(machine, store)
try:
result = callback(client)
duration_ms = int((time.perf_counter() - started) * 1000)
record_ssh_command(
machine_id=machine.get("id") or "unknown",
action=action,
status="ok",
duration_seconds=duration_ms / 1000.0,
)
store.record_machine_action(
machine,
action,
@@ -124,6 +143,12 @@ def run_machine_operation(
raise
except Exception as exc: # pragma: no cover - transport/network fallback
duration_ms = int((time.perf_counter() - started) * 1000)
record_ssh_command(
machine_id=machine.get("id") or "unknown",
action=action,
status="error",
duration_seconds=duration_ms / 1000.0,
)
logger.exception(
"Monitoring %s failed machine_id=%s machine_name=%s",
action,
@@ -157,34 +182,13 @@ def poll_machine_snapshot(
) -> dict[str, Any]:
"""Collect a backend-scheduled snapshot for a machine.
This records the same action-history rows that the UI would otherwise get
from manual requests, but it runs entirely inside the backend on a schedule.
The legacy POSIX collector has been removed; this now records a lightweight
disk-space lookup on the same schedule so action history stays useful.
"""
request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}"
results: dict[str, Any] = {"request_id": request_id, "machine_id": machine.get("id"), "actions": []}
status = run_machine_operation(
machine,
store,
"status lookup",
resource_collector_status,
request_id=request_id,
raise_http=False,
)
results["status"] = status
results["actions"].append("status lookup")
metrics = run_machine_operation(
machine,
store,
"metrics read",
lambda client: read_resource_metrics(client, max_lines=metrics_limit),
request_id=request_id,
raise_http=False,
)
results["metrics_samples"] = len(metrics or [])
results["actions"].append("metrics read")
client = build_machine_client(machine, store)
settings = get_settings()
path = str(machine.get("media_root") or settings.media_root or "/")
disk = run_machine_operation(
@@ -194,6 +198,7 @@ def poll_machine_snapshot(
lambda client: disk_space(client, path),
request_id=request_id,
raise_http=False,
client=client,
)
results["disk_mount"] = (disk or {}).get("mount") if isinstance(disk, dict) else None
results["actions"].append("disk lookup")
@@ -201,149 +206,3 @@ def poll_machine_snapshot(
return results
def _summarize_metric_samples(samples: list[dict[str, Any]], field: str) -> dict[str, float] | None:
values = [float(sample.get(field, 0)) for sample in samples if sample.get(field) is not None]
if not values:
return None
return {
"avg": sum(values) / len(values),
"min": min(values),
"max": max(values),
"count": float(len(values)),
}
def collect_machine_overview(
machine: dict[str, Any],
store: SettingsStore,
*,
metrics_window_seconds: int = 600,
metrics_limit: int = 70_000,
) -> dict[str, Any]:
"""Collect a lightweight machine overview without recording history."""
overview: dict[str, Any] = {
"machine": {k: machine.get(k) for k in ("id", "name", "mode", "enabled", "host", "port", "username", "media_root", "path_prefix", "notes")},
"status": "",
"status_error": "",
"metrics_error": "",
"disk_error": "",
"sample_count": 0,
"latest_sample": None,
"cpu_summary": None,
"iowait_summary": None,
"mem_summary": None,
"net_rx_summary": None,
"net_tx_summary": None,
"disk_read_summary": None,
"disk_write_summary": None,
"disk": None,
}
try:
client: Any = build_machine_client(machine, store)
except Exception as exc:
overview["status_error"] = str(exc)
overview["metrics_error"] = str(exc)
overview["disk_error"] = str(exc)
return overview
settings = get_settings()
path = str(machine.get("media_root") or settings.media_root or "/")
try:
overview["status"] = resource_collector_status(client)
except Exception as exc:
overview["status_error"] = str(exc)
try:
samples = read_resource_metrics(client, max_lines=metrics_limit)
if samples:
latest_ts = max(float(sample.get("ts", 0)) for sample in samples)
window_start = latest_ts - metrics_window_seconds
metrics = [sample for sample in samples if float(sample.get("ts", 0)) >= window_start]
if not metrics:
metrics = samples
overview["sample_count"] = len(metrics)
overview["latest_sample"] = metrics[-1]
overview["cpu_summary"] = _summarize_metric_samples(metrics, "cpu_pct")
overview["iowait_summary"] = _summarize_metric_samples(metrics, "iowait_pct")
overview["mem_summary"] = _summarize_metric_samples(metrics, "mem_pct")
overview["net_rx_summary"] = _summarize_metric_samples(metrics, "net_rx_bytes_per_sec")
overview["net_tx_summary"] = _summarize_metric_samples(metrics, "net_tx_bytes_per_sec")
overview["disk_read_summary"] = _summarize_metric_samples(metrics, "disk_read_bps")
overview["disk_write_summary"] = _summarize_metric_samples(metrics, "disk_write_bps")
except Exception as exc:
overview["metrics_error"] = str(exc)
try:
overview["disk"] = disk_space(client, path)
except Exception as exc:
overview["disk_error"] = str(exc)
return overview
def poll_machine_diagnostics(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str | None = None,
) -> dict[str, Any]:
"""Collect a backend-scheduled diagnostics snapshot for a machine."""
request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}"
diagnostics = run_machine_operation(
machine,
store,
"collector diagnostics",
resource_collector_debug_info,
request_id=request_id,
raise_http=False,
)
return {"request_id": request_id, "diagnostics": diagnostics}
def start_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector start",
start_resource_collector,
request_id=request_id,
raise_http=True,
)
def stop_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector stop",
stop_resource_collector,
request_id=request_id,
raise_http=True,
)
def restart_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector restart",
restart_resource_collector,
request_id=request_id,
raise_http=True,
)
backend/src/media_library_viewer_api/services/monitoring_poller.py
+5 -4
View File
@@ -136,16 +136,17 @@ class MonitoringPoller:
request_id=f"poll:{machine['id']}:{int(time.time())}",
)
logger.info(
"Monitoring poll snapshot machine_id=%s request_id=%s status=%s metrics_samples=%s disk_mount=%s",
"Monitoring poll snapshot machine_id=%s request_id=%s disk_mount=%s actions=%s",
machine["id"],
snapshot.get("request_id"),
snapshot.get("status"),
snapshot.get("metrics_samples"),
snapshot.get("disk_mount"),
snapshot.get("actions"),
)
except Exception:
cycle_errors += 1
logger.exception("Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"])
logger.exception(
"Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"]
)
retention_seconds = config.retention_days * 24 * 60 * 60
cutoff_ts = int(time.time()) - retention_seconds
removed = store.prune_machine_actions(cutoff_ts)
backend/src/media_library_viewer_api/services/settings_store.py
+214 -68
View File
@@ -48,6 +48,9 @@ def _default_local_machine() -> dict[str, Any]:
"jellyfin_api_key": "",
"jellyseerr_url": "",
"jellyseerr_api_key": "",
"node_exporter_enabled": False,
"node_exporter_port": 9100,
"node_exporter_scrape_host": "",
"notes": "",
}
@@ -81,9 +84,7 @@ class SettingsStore:
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)"
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)")
conn.execute(
"""
CREATE TABLE IF NOT EXISTS monitoring_machine_actions (
@@ -118,9 +119,7 @@ class SettingsStore:
)
"""
)
existing_key_columns = {
row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()
}
existing_key_columns = {row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()}
if "public_key" not in existing_key_columns:
conn.execute("ALTER TABLE ssh_keys ADD COLUMN public_key TEXT NOT NULL DEFAULT ''")
if "fingerprint" not in existing_key_columns:
@@ -140,9 +139,7 @@ class SettingsStore:
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)"
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)")
conn.execute(
"""
CREATE TABLE IF NOT EXISTS saved_task_runs (
@@ -182,10 +179,16 @@ class SettingsStore:
"CREATE INDEX IF NOT EXISTS idx_dashboard_shortcuts_type ON dashboard_shortcuts(shortcut_type)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time ON monitoring_machine_actions(machine_id, created_at DESC)"
"""
CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time
ON monitoring_machine_actions(machine_id, created_at DESC)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status ON monitoring_machine_actions(action, status)"
"""
CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status
ON monitoring_machine_actions(action, status)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS backup_jobs (
@@ -251,7 +254,8 @@ class SettingsStore:
def _row_to_machine(self, row: sqlite3.Row) -> dict[str, Any]:
data = json.loads(row["config_json"])
services = self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [])
default_services = DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []
services = self._normalize_services(data.get("services"), default_services)
return {
"id": row["id"],
"name": row["name"],
@@ -274,6 +278,9 @@ class SettingsStore:
"jellyfin_api_key_set": bool(data.get("jellyfin_api_key")),
"jellyseerr_url": data.get("jellyseerr_url", ""),
"jellyseerr_api_key_set": bool(data.get("jellyseerr_api_key")),
"node_exporter_enabled": bool(data.get("node_exporter_enabled", False)),
"node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100),
"node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""),
"notes": data.get("notes", ""),
"created_at": row["created_at"],
"updated_at": row["updated_at"],
@@ -294,12 +301,18 @@ class SettingsStore:
"This machine" if mode == "local" else machine_id
)
services = self._normalize_services(payload.get("services"), (current or {}).get("services", []))
host = str(payload.get("host") if payload.get("host") is not None else (current or {}).get("host", "") or "").strip()
def _current_str(field: str, default: str = "") -> str:
return str(
payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default
).strip()
host = _current_str("host")
port = int(payload.get("port") or (current or {}).get("port", 22) or 22)
username = str(payload.get("username") if payload.get("username") is not None else (current or {}).get("username", "") or "").strip()
key_directory = str(payload.get("key_directory") if payload.get("key_directory") is not None else (current or {}).get("key_directory", "") or "").strip()
key_name = str(payload.get("key_name") if payload.get("key_name") is not None else (current or {}).get("key_name", "") or "").strip()
ssh_key_id = str(payload.get("ssh_key_id") if payload.get("ssh_key_id") is not None else (current or {}).get("ssh_key_id", "") or "").strip()
username = _current_str("username")
key_directory = _current_str("key_directory")
key_name = _current_str("key_name")
ssh_key_id = _current_str("ssh_key_id")
ssh_private_key = payload.get("ssh_private_key")
if ssh_private_key in (None, ""):
ssh_private_key = (current or {}).get("ssh_private_key", "")
@@ -312,20 +325,30 @@ class SettingsStore:
if password in (None, ""):
password = (current or {}).get("password", "")
password = str(password or "")
media_root = str(payload.get("media_root") if payload.get("media_root") is not None else (current or {}).get("media_root", "") or "").strip()
path_prefix = str(payload.get("path_prefix") if payload.get("path_prefix") is not None else (current or {}).get("path_prefix", "") or "").strip()
jellyfin_url = str(payload.get("jellyfin_url") if payload.get("jellyfin_url") is not None else (current or {}).get("jellyfin_url", "") or "").strip()
jellyfin_user_id = str(payload.get("jellyfin_user_id") if payload.get("jellyfin_user_id") is not None else (current or {}).get("jellyfin_user_id", "") or "").strip()
media_root = _current_str("media_root")
path_prefix = _current_str("path_prefix")
jellyfin_url = _current_str("jellyfin_url")
jellyfin_user_id = _current_str("jellyfin_user_id")
jellyfin_api_key = payload.get("jellyfin_api_key")
if jellyfin_api_key in (None, ""):
jellyfin_api_key = (current or {}).get("jellyfin_api_key", "")
jellyfin_api_key = str(jellyfin_api_key or "")
jellyseerr_url = str(payload.get("jellyseerr_url") if payload.get("jellyseerr_url") is not None else (current or {}).get("jellyseerr_url", "") or "").strip()
jellyseerr_url = _current_str("jellyseerr_url")
jellyseerr_api_key = payload.get("jellyseerr_api_key")
if jellyseerr_api_key in (None, ""):
jellyseerr_api_key = (current or {}).get("jellyseerr_api_key", "")
jellyseerr_api_key = str(jellyseerr_api_key or "")
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
node_exporter_enabled = bool(
payload.get("node_exporter_enabled")
if payload.get("node_exporter_enabled") is not None
else (current or {}).get("node_exporter_enabled", False)
)
node_exporter_port_raw = payload.get("node_exporter_port")
if node_exporter_port_raw is None:
node_exporter_port_raw = (current or {}).get("node_exporter_port", 9100)
node_exporter_port = int(node_exporter_port_raw or 9100)
node_exporter_scrape_host = _current_str("node_exporter_scrape_host")
notes = _current_str("notes")
if mode == "local":
host = host or "localhost"
username = username or ""
@@ -351,6 +374,9 @@ class SettingsStore:
"jellyfin_api_key": jellyfin_api_key,
"jellyseerr_url": jellyseerr_url,
"jellyseerr_api_key": jellyseerr_api_key,
"node_exporter_enabled": node_exporter_enabled,
"node_exporter_port": node_exporter_port,
"node_exporter_scrape_host": node_exporter_scrape_host,
"notes": notes,
}
@@ -380,6 +406,9 @@ class SettingsStore:
"jellyfin_api_key": machine["jellyfin_api_key"],
"jellyseerr_url": machine["jellyseerr_url"],
"jellyseerr_api_key": machine["jellyseerr_api_key"],
"node_exporter_enabled": machine["node_exporter_enabled"],
"node_exporter_port": machine["node_exporter_port"],
"node_exporter_scrape_host": machine["node_exporter_scrape_host"],
"notes": machine["notes"],
}
with self.connect() as conn:
@@ -431,7 +460,10 @@ class SettingsStore:
"name": row["name"],
"mode": row["mode"],
"enabled": bool(row["enabled"]),
"services": self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []),
"services": self._normalize_services(
data.get("services"),
DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [],
),
"host": data.get("host", ""),
"port": int(data.get("port", 22) or 22),
"username": data.get("username", ""),
@@ -448,11 +480,18 @@ class SettingsStore:
"jellyfin_api_key": data.get("jellyfin_api_key", ""),
"jellyseerr_url": data.get("jellyseerr_url", ""),
"jellyseerr_api_key": data.get("jellyseerr_api_key", ""),
"node_exporter_enabled": bool(data.get("node_exporter_enabled", False)),
"node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100),
"node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""),
"notes": data.get("notes", ""),
}
def list_machines_for_service(self, service: str) -> list[dict[str, Any]]:
return [machine for machine in self.list_machines() if service in machine.get("services", []) and machine.get("enabled")]
return [
machine
for machine in self.list_machines()
if service in machine.get("services", []) and machine.get("enabled")
]
def get_machine_for_service(self, service: str, machine_id: str | None = None) -> dict[str, Any] | None:
if machine_id:
@@ -485,10 +524,16 @@ class SettingsStore:
"jellyfin_api_key": machine["jellyfin_api_key"],
"jellyseerr_url": machine["jellyseerr_url"],
"jellyseerr_api_key": machine["jellyseerr_api_key"],
"node_exporter_enabled": machine["node_exporter_enabled"],
"node_exporter_port": machine["node_exporter_port"],
"node_exporter_scrape_host": machine["node_exporter_scrape_host"],
"notes": machine["notes"],
}
with self.connect() as conn:
existing = conn.execute("SELECT created_at FROM monitoring_machines WHERE id = ?", (machine["id"],)).fetchone()
existing = conn.execute(
"SELECT created_at FROM monitoring_machines WHERE id = ?",
(machine["id"],),
).fetchone()
created_at = int(existing[0]) if existing else now
conn.execute(
"""
@@ -538,10 +583,13 @@ class SettingsStore:
conn.execute(
"""
INSERT INTO monitoring_machine_actions
(id, machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail)
(
id, machine_id, machine_name, mode, action, status,
created_at, duration_ms, request_id, message, error,
stdout_tail, stderr_tail
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
,
""",
(
uuid.uuid4().hex,
str(machine.get("id") or ""),
@@ -577,8 +625,10 @@ class SettingsStore:
clauses.append("status = ?")
params.append(status)
sql = (
"SELECT machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail "
f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} ORDER BY created_at DESC LIMIT ?"
"SELECT machine_id, machine_name, mode, action, status, "
"created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail "
f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} "
"ORDER BY created_at DESC LIMIT ?"
)
params.append(max(1, min(int(limit), 200)))
with self.connect() as conn:
@@ -595,7 +645,6 @@ class SettingsStore:
)
return int(cur.rowcount or 0)
@staticmethod
def _private_key_summary(private_key: str) -> dict[str, str]:
if not private_key:
@@ -639,11 +688,29 @@ class SettingsStore:
if passphrase in (None, ""):
passphrase = (current or {}).get("passphrase", "")
passphrase = str(passphrase or "")
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
notes = str(
payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or ""
).strip()
summary = self._private_key_summary(private_key)
public_key = str(payload.get("public_key") if payload.get("public_key") is not None else (current or {}).get("public_key", "") or summary["public_key"] or "").strip()
fingerprint = str(payload.get("fingerprint") if payload.get("fingerprint") is not None else (current or {}).get("fingerprint", "") or summary["fingerprint"] or "").strip()
return {"id": key_id, "name": name, "private_key": private_key, "passphrase": passphrase, "public_key": public_key, "fingerprint": fingerprint, "notes": notes}
public_key = str(
payload.get("public_key")
if payload.get("public_key") is not None
else (current or {}).get("public_key", "") or summary["public_key"] or ""
).strip()
fingerprint = str(
payload.get("fingerprint")
if payload.get("fingerprint") is not None
else (current or {}).get("fingerprint", "") or summary["fingerprint"] or ""
).strip()
return {
"id": key_id,
"name": name,
"private_key": private_key,
"passphrase": passphrase,
"public_key": public_key,
"fingerprint": fingerprint,
"notes": notes,
}
def list_ssh_keys(self) -> list[dict[str, Any]]:
self.init_schema()
@@ -666,7 +733,15 @@ class SettingsStore:
if not row:
return None
summary = self._private_key_summary(str(row["private_key"] or ""))
return {"id": row["id"], "name": row["name"], "private_key": row["private_key"], "passphrase": row["passphrase"], "notes": row["notes"], "public_key": str(row["public_key"] or summary["public_key"] or ""), "fingerprint": str(row["fingerprint"] or summary["fingerprint"] or "")}
return {
"id": row["id"],
"name": row["name"],
"private_key": row["private_key"],
"passphrase": row["passphrase"],
"notes": row["notes"],
"public_key": str(row["public_key"] or summary["public_key"] or ""),
"fingerprint": str(row["fingerprint"] or summary["fingerprint"] or ""),
}
def upsert_ssh_key(self, payload: dict[str, Any], key_id: str | None = None) -> dict[str, Any]:
self.init_schema()
@@ -677,7 +752,10 @@ class SettingsStore:
created_at = int(existing[0]) if existing else now
conn.execute(
"""
INSERT INTO ssh_keys (id, name, private_key, passphrase, public_key, fingerprint, notes, created_at, updated_at)
INSERT INTO ssh_keys (
id, name, private_key, passphrase, public_key, fingerprint,
notes, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
name = excluded.name,
@@ -688,7 +766,17 @@ class SettingsStore:
notes = excluded.notes,
updated_at = excluded.updated_at
""",
(key["id"], key["name"], key["private_key"], key["passphrase"], key["public_key"], key["fingerprint"], key["notes"], created_at, now),
(
key["id"],
key["name"],
key["private_key"],
key["passphrase"],
key["public_key"],
key["fingerprint"],
key["notes"],
created_at,
now,
),
)
return self.get_ssh_key(key["id"]) or key
@@ -697,8 +785,6 @@ class SettingsStore:
with self.connect() as conn:
conn.execute("DELETE FROM ssh_keys WHERE id = ?", (key_id,))
def _row_to_task(self, row: sqlite3.Row) -> dict[str, Any]:
return {
"id": row["id"],
@@ -719,11 +805,27 @@ class SettingsStore:
task_type = str(payload.get("task_type") or (current or {}).get("task_type") or "shell").strip().lower()
if task_type not in {"shell", "python"}:
task_type = "shell"
content = str(payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or "")
content = str(
payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or ""
)
enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
default_machine_id = str(payload.get("default_machine_id") if payload.get("default_machine_id") is not None else (current or {}).get("default_machine_id", "") or "").strip()
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
return {"id": task_id, "name": name, "task_type": task_type, "content": content, "enabled": enabled, "default_machine_id": default_machine_id, "notes": notes}
default_machine_id = str(
payload.get("default_machine_id")
if payload.get("default_machine_id") is not None
else (current or {}).get("default_machine_id", "") or ""
).strip()
notes = str(
payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or ""
).strip()
return {
"id": task_id,
"name": name,
"task_type": task_type,
"content": content,
"enabled": enabled,
"default_machine_id": default_machine_id,
"notes": notes,
}
def list_tasks(self) -> list[dict[str, Any]]:
self.init_schema()
@@ -748,7 +850,10 @@ class SettingsStore:
created_at = int(existing[0]) if existing else now
conn.execute(
"""
INSERT INTO saved_tasks (id, name, task_type, content, enabled, default_machine_id, notes, created_at, updated_at)
INSERT INTO saved_tasks (
id, name, task_type, content, enabled, default_machine_id,
notes, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
name = excluded.name,
@@ -759,7 +864,17 @@ class SettingsStore:
notes = excluded.notes,
updated_at = excluded.updated_at
""",
(task["id"], task["name"], task["task_type"], task["content"], 1 if task["enabled"] else 0, task["default_machine_id"], task["notes"], created_at, now),
(
task["id"],
task["name"],
task["task_type"],
task["content"],
1 if task["enabled"] else 0,
task["default_machine_id"],
task["notes"],
created_at,
now,
),
)
return self.get_task(task["id"]) or task
@@ -796,8 +911,11 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO saved_task_runs
(id, task_id, task_name, machine_id, machine_name, task_type, status, created_at, duration_ms, request_id, stdout_tail, stderr_tail, error)
INSERT INTO saved_task_runs (
id, task_id, task_name, machine_id, machine_name, task_type,
status, created_at, duration_ms, request_id, stdout_tail,
stderr_tail, error
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
@@ -837,17 +955,25 @@ class SettingsStore:
def _normalize_shortcut_payload(self, payload: dict[str, Any], shortcut_id: str | None = None) -> dict[str, Any]:
current = self.get_shortcut(shortcut_id) if shortcut_id else None
shortcut_id = str(payload.get("id") or shortcut_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12]
shortcut_type = str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower()
shortcut_type = (
str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower()
)
if shortcut_type not in {"website", "action", "user"}:
shortcut_type = "website"
label = str(payload.get("label") or (current or {}).get("label") or "").strip() or shortcut_id
enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
icon = str(payload.get("icon") if payload.get("icon") is not None else (current or {}).get("icon", "") or "").strip()
url = str(payload.get("url") if payload.get("url") is not None else (current or {}).get("url", "") or "").strip()
task_id = str(payload.get("task_id") if payload.get("task_id") is not None else (current or {}).get("task_id", "") or "").strip()
machine_id = str(payload.get("machine_id") if payload.get("machine_id") is not None else (current or {}).get("machine_id", "") or "").strip()
user_id = str(payload.get("user_id") if payload.get("user_id") is not None else (current or {}).get("user_id", "") or "").strip()
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
def _field(field: str, default: str = "") -> str:
return str(
payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default
).strip()
icon = _field("icon")
url = _field("url")
task_id = _field("task_id")
machine_id = _field("machine_id")
user_id = _field("user_id")
notes = _field("notes")
return {
"id": shortcut_id,
"label": label,
@@ -882,7 +1008,10 @@ class SettingsStore:
shortcut = self._normalize_shortcut_payload(payload, shortcut_id)
now = int(time.time())
with self.connect() as conn:
existing = conn.execute("SELECT created_at FROM dashboard_shortcuts WHERE id = ?", (shortcut["id"],)).fetchone()
existing = conn.execute(
"SELECT created_at FROM dashboard_shortcuts WHERE id = ?",
(shortcut["id"],),
).fetchone()
created_at = int(existing[0]) if existing else now
conn.execute(
"""
@@ -926,14 +1055,16 @@ class SettingsStore:
"created_at": row["created_at"],
}
def _normalize_backup_job_payload(
self, payload: dict[str, Any], job_id: str | None = None
) -> dict[str, Any]:
def _normalize_backup_job_payload(self, payload: dict[str, Any], job_id: str | None = None) -> dict[str, Any]:
current = self.get_backup_job(job_id) if job_id else None
job_id = str(payload.get("id") or job_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12]
name = str(payload.get("name") or (current or {}).get("name") or job_id).strip() or job_id
source = str(payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or "").strip()
target = str(payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or "").strip()
source = str(
payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or ""
).strip()
target = str(
payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or ""
).strip()
schedule_interval_seconds = payload.get("schedule_interval_seconds")
if schedule_interval_seconds is None:
schedule_interval_seconds = (current or {}).get("schedule_interval_seconds")
@@ -1040,10 +1171,24 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO backup_runs (id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, created_at)
INSERT INTO backup_runs (
id, job_id, started_at, ended_at, status, bytes_transferred,
duration_ms, error_message, details_json, created_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(run_id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, now),
(
run_id,
job_id,
started_at,
ended_at,
status,
bytes_transferred,
duration_ms,
error_message,
details_json,
now,
),
)
return self.get_backup_run(run_id) or {
"id": run_id,
@@ -1128,7 +1273,10 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO backup_alerts (id, job_id, run_id, alert_type, severity, message, acknowledged, resolved_at, created_at)
INSERT INTO backup_alerts (
id, job_id, run_id, alert_type, severity, message,
acknowledged, resolved_at, created_at
)
VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?)
""",
(alert_id, job_id, run_id, alert_type, severity, message, now),
@@ -1238,9 +1386,7 @@ class SettingsStore:
)
"""
)
row = conn.execute(
"SELECT value FROM app_settings WHERE key = ?", (key,)
).fetchone()
row = conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone()
return row["value"] if row else default
def update_setting(self, key: str, value: str) -> None:
backend/src/media_library_viewer_api/services/targets.py
+77
View File
@@ -0,0 +1,77 @@
"""Prometheus file-based service discovery target management.
The backend owns the list of remote Node Exporter targets so that operators can
enable scraping per machine from the Manage UI. Prometheus reads the generated
JSON file via `file_sd_configs`; this keeps Prometheus config static and pushes
machine-specific changes into a file it can reload.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import Any
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
DEFAULT_NODE_EXPORTER_PORT = 9100
def _scrape_address(machine: dict[str, Any]) -> str | None:
"""Return host:port for the Node Exporter on a machine, or None if disabled."""
if not machine.get("node_exporter_enabled"):
return None
scrape_host = str(machine.get("node_exporter_scrape_host") or "").strip()
host = scrape_host or str(machine.get("host") or "").strip()
if not host or host == "localhost":
return None
port = int(machine.get("node_exporter_port") or DEFAULT_NODE_EXPORTER_PORT)
return f"{host}:{port}"
def build_node_exporter_targets(store: SettingsStore) -> list[dict[str, Any]]:
"""Build a file-SD target list for all enabled SSH machines.
Local machines are excluded because the Compose-managed node-exporter
service already covers the Docker host.
"""
targets: list[dict[str, Any]] = []
for machine in store.list_machines():
if not machine.get("enabled"):
continue
if str(machine.get("mode") or "local").strip().lower() != "ssh":
continue
address = _scrape_address(machine)
if not address:
continue
targets.append(
{
"targets": [address],
"labels": {
"job": "node-exporter-remote",
"machine_id": str(machine.get("id") or ""),
"machine_name": str(machine.get("name") or ""),
"instance": address,
},
}
)
return targets
def write_prometheus_targets(store: SettingsStore, file_sd_dir: Path | None = None) -> Path:
"""Render and persist Prometheus file-SD targets.
Returns the path written so callers can log or expose it.
"""
settings = get_settings()
file_sd_dir = file_sd_dir or Path(settings.prometheus_file_sd_dir)
file_sd_dir.mkdir(parents=True, exist_ok=True)
file_path = file_sd_dir / "node_exporter_targets.json"
targets = build_node_exporter_targets(store)
file_path.write_text(json.dumps(targets, indent=2), encoding="utf-8")
logger.info("Wrote %s node_exporter targets to %s", len(targets), file_path)
return file_path
backend/src/media_library_viewer_api/utils.py
-1
View File
@@ -11,7 +11,6 @@ from datetime import datetime
from pathlib import PurePosixPath
from typing import Any
VIDEO_FILE_EXTENSIONS = {
".3g2",
".3gp",
backend/src/media_library_viewer_api/version.py
+2 -1
View File
@@ -3,7 +3,8 @@
from __future__ import annotations
import os
from importlib.metadata import PackageNotFoundError, version as package_version
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as package_version
PACKAGE_NAME = "media-library-viewer-backend"
DEFAULT_VERSION = "0.1.0"
backend/src/media_library_viewer_api/workers/media_index_worker.py
+1 -1
View File
@@ -15,12 +15,12 @@ from typing import Any
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
from media_library_viewer_api.services.media_index import (
MediaIndex,
MediaIndexBuildCancelled,
build_media_index,
)
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
logger = logging.getLogger(__name__)
backend/tests/test_api.py
+190 -125
View File
@@ -11,23 +11,23 @@ from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from media_library_viewer_api.main import app
from media_library_viewer_api.clients.ssh import CommandResult
from media_library_viewer_api.dependencies import (
get_ssh_client,
get_jellyfin_client,
get_jellyseerr_client,
get_mail_queue,
get_settings_store,
get_ssh_client,
get_user_id,
)
from media_library_viewer_api.clients.ssh import CommandResult
from media_library_viewer_api.main import app
from media_library_viewer_api.routers.media import get_media_index
from media_library_viewer_api.services.media_index import MediaIndex
from media_library_viewer_api.services.settings_store import SettingsStore
# --- Fixtures ---
@pytest.fixture
def mock_jellyfin():
"""Mock Jellyfin client."""
@@ -145,15 +145,29 @@ def test_client(mock_jellyfin, mock_jellyseerr, mock_ssh, tmp_path):
# --- Health ---
class TestHealth:
def test_health(self, test_client):
response = test_client.get("/api/health")
assert response.status_code == 200
assert response.json() == {"status": "ok"}
def test_metrics_endpoint(self, test_client):
response = test_client.get("/metrics")
assert response.status_code == 200
assert "manage_api_requests_total" in response.text
assert response.headers["content-type"].startswith("text/plain")
def test_request_id_header(self, test_client):
response = test_client.get("/api/health")
assert response.status_code == 200
assert "x-request-id" in response.headers
assert len(response.headers["x-request-id"]) > 0
# --- Dashboard ---
class TestDashboard:
def test_counts(self, test_client):
response = test_client.get("/api/dashboard/counts")
@@ -163,78 +177,6 @@ class TestDashboard:
assert data["series"] == 20
assert data["episodes"] == 500
def test_monitoring_overview(self, test_client):
store = MagicMock()
store.list_machines.return_value = [
{
"id": "local",
"name": "This machine",
"mode": "local",
"enabled": True,
"host": "localhost",
"port": 22,
"username": "",
"media_root": "/srv/media",
"path_prefix": "",
"notes": "",
},
{
"id": "remote1",
"name": "Remote",
"mode": "ssh",
"enabled": True,
"host": "server.example.com",
"port": 22,
"username": "alex",
"media_root": "/srv/media",
"path_prefix": "",
"notes": "",
},
]
app.dependency_overrides[get_settings_store] = lambda: store
try:
with (
patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=object()),
patch("media_library_viewer_api.services.monitoring_actions.resource_collector_status", return_value="running pid=123"),
patch(
"media_library_viewer_api.services.monitoring_actions.read_resource_metrics",
return_value=[
{
"ts": 123.0,
"cpu_pct": 10.0,
"iowait_pct": 1.0,
"mem_pct": 20.0,
"net_rx_bytes_per_sec": 100.0,
"net_tx_bytes_per_sec": 50.0,
"disk_read_bps": 1.0,
"disk_write_bps": 2.0,
}
],
),
patch(
"media_library_viewer_api.services.monitoring_actions.disk_space",
return_value={
"filesystem": "/dev/sda1",
"size": 1000,
"used": 200,
"available": 800,
"used_pct": "20.0%",
"mount": "/srv/media",
},
),
):
response = test_client.get("/api/dashboard/monitoring")
finally:
app.dependency_overrides.pop(get_settings_store, None)
assert response.status_code == 200
data = response.json()
assert data["total"] == 2
assert data["enabled"] == 2
assert len(data["machines"]) == 2
assert data["machines"][0]["latest_sample"]["cpu_pct"] == 10.0
assert data["machines"][0]["disk"]["used_pct"] == "20.0%"
def test_libraries(self, test_client):
response = test_client.get("/api/dashboard/libraries")
assert response.status_code == 200
@@ -268,6 +210,7 @@ class TestDashboard:
# --- Settings reset ---
class TestSettingsReset:
def test_reset_local_database_requires_full_confirmation(self, test_client, tmp_path):
store = SettingsStore(tmp_path / "settings.sqlite")
@@ -329,8 +272,10 @@ class TestSettingsReset:
assert store.get_machine("local") is None
assert len(store.list_machines()) == 0
# --- Users ---
class TestUsers:
def test_users_list_enriched(self, test_client):
response = test_client.get("/api/users")
@@ -418,7 +363,7 @@ class TestUsers:
)
try:
with patch("media_library_viewer_api.routers.users.get_settings", return_value=settings):
with patch("media_library_viewer_api.routers.users_impl.get_settings", return_value=settings):
response = test_client.post(
"/api/users/message",
data={
@@ -443,8 +388,10 @@ class TestUsers:
assert kwargs["subject"] == "Hello team"
assert kwargs["settings"] is settings
# --- Files ---
class TestFiles:
def test_list_directory(self, test_client):
response = test_client.get("/api/files/list", params={"path": "/media"})
@@ -486,6 +433,7 @@ class TestFiles:
# --- Media index build ---
class TestMediaIndexApi:
def test_status_includes_build_progress(self, test_client, tmp_path):
index = MediaIndex(tmp_path / "index.sqlite")
@@ -586,9 +534,11 @@ class TestMediaIndexApi:
alive_calls["count"] += 1
return alive_calls["count"] <= 2
with patch("media_library_viewer_api.routers.media._pid_is_alive", side_effect=fake_pid_is_alive), patch(
"media_library_viewer_api.routers.media.os.killpg"
) as killpg, patch("media_library_viewer_api.routers.media.time.sleep", return_value=None):
with (
patch("media_library_viewer_api.routers.media._pid_is_alive", side_effect=fake_pid_is_alive),
patch("media_library_viewer_api.routers.media.os.killpg") as killpg,
patch("media_library_viewer_api.routers.media.time.sleep", return_value=None),
):
try:
response = test_client.post("/api/media/force-stop")
assert response.status_code == 202
@@ -615,6 +565,7 @@ class TestMediaIndexApi:
# --- Jobs ---
class TestJobs:
def test_list_templates(self, test_client):
response = test_client.get("/api/jobs/templates")
@@ -645,37 +596,21 @@ class TestJobs:
# --- Monitoring ---
class TestMonitoring:
def _ensure_machine(self):
store = app.dependency_overrides[get_settings_store]()
if not store.list_machines():
store.upsert_machine({
"name": "Test Machine",
"mode": "ssh",
"enabled": True,
"services": ["monitoring", "files", "jellyfin"],
"host": "test-host",
"username": "test-user",
})
def test_status(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="running pid=1234\n", stderr=""
)
response = test_client.get("/api/monitoring/status")
assert response.status_code == 200
assert "running" in response.json()["status"]
def test_metrics_empty(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="", stderr=""
)
response = test_client.get("/api/monitoring/metrics")
assert response.status_code == 200
data = response.json()
assert data["samples"] == []
store.upsert_machine(
{
"name": "Test Machine",
"mode": "ssh",
"enabled": True,
"services": ["monitoring", "files", "jellyfin"],
"host": "test-host",
"username": "test-user",
}
)
def test_disk(self, test_client, mock_ssh):
self._ensure_machine()
@@ -685,32 +620,162 @@ class TestMonitoring:
stdout='{"filesystem":"/dev/sda1","size":1000000000,"used":500000000,"available":500000000,"used_pct":"50%","mount":"/"}',
stderr="",
)
response = test_client.get("/api/monitoring/disk")
with patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=mock_ssh):
response = test_client.get("/api/monitoring/disk")
assert response.status_code == 200
data = response.json()
assert data["used_pct"] == "50%"
def test_start(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="started pid=5678\n", stderr=""
)
response = test_client.post("/api/monitoring/start")
def test_prometheus_targets_empty(self, test_client):
response = test_client.get("/api/monitoring/prometheus-targets")
assert response.status_code == 200
assert "started" in response.json()["message"]
assert response.json() == []
def test_stop(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="stopped pid=5678\n", stderr=""
def test_prometheus_targets_returns_enabled_ssh_node_exporter(self, test_client):
store = app.dependency_overrides[get_settings_store]()
store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"enabled": True,
"services": ["monitoring"],
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9200,
"node_exporter_scrape_host": "1.2.3.4",
}
)
response = test_client.post("/api/monitoring/stop")
response = test_client.get("/api/monitoring/prometheus-targets")
assert response.status_code == 200
data = response.json()
assert len(data) == 1
assert data[0]["targets"] == ["1.2.3.4:9200"]
assert data[0]["labels"]["job"] == "node-exporter-remote"
def test_restart(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="stopped pid=5678\nstarted pid=9999\n", stderr=""
class TestSettingsMachines:
def test_post_machine_rewrites_prometheus_targets(self, test_client):
with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets:
with patch("media_library_viewer_api.routers.settings._validate_saved_machine_ssh"):
response = test_client.post(
"/api/settings/machines",
json={
"name": "remote1",
"mode": "ssh",
"enabled": True,
"services": ["monitoring"],
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9200,
"node_exporter_scrape_host": "1.2.3.4",
},
)
assert response.status_code == 201
write_targets.assert_called_once()
def test_delete_machine_rewrites_prometheus_targets(self, test_client):
store = app.dependency_overrides[get_settings_store]()
machine = store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"enabled": True,
"services": ["monitoring"],
"host": "10.0.0.5",
"username": "u",
}
)
response = test_client.post("/api/monitoring/restart")
with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets:
response = test_client.delete(f"/api/settings/machines/{machine['id']}")
assert response.status_code == 200
write_targets.assert_called_once()
class TestAlertmanager:
def test_alerts_endpoint_when_alertmanager_unreachable(self, test_client):
with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
session = MagicMock()
session.get.side_effect = Exception("connection refused")
mock_client.return_value = (session, "http://alertmanager:9093")
response = test_client.get("/api/monitoring/alerts")
assert response.status_code == 200
data = response.json()
assert data["total"] == 0
assert data["error"] == "alertmanager_unreachable"
def test_alerts_endpoint_returns_summary(self, test_client):
with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
session = MagicMock()
session.get.return_value.json.return_value = {
"status": "success",
"data": [
{
"labels": {"alertname": "BackupJobFailed", "severity": "critical", "job_name": "test"},
"annotations": {"summary": "Backup failed", "description": "details"},
"startsAt": "2026-05-11T02:00:00Z",
"status": "firing",
}
],
}
session.get.return_value.raise_for_status = MagicMock()
mock_client.return_value = (session, "http://alertmanager:9093")
response = test_client.get("/api/monitoring/alerts")
assert response.status_code == 200
data = response.json()
assert data["total"] == 1
assert data["by_severity"]["critical"] == 1
assert data["alerts"][0]["name"] == "BackupJobFailed"
def test_alertmanager_status_endpoint_when_unreachable(self, test_client):
with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
session = MagicMock()
session.get.side_effect = Exception("connection refused")
mock_client.return_value = (session, "http://alertmanager:9093")
response = test_client.get("/api/monitoring/alertmanager-status")
assert response.status_code == 200
data = response.json()
assert data["up"] is False
class TestAlertmanagerWebhook:
"""Tests for POST /api/monitoring/alertmanager-webhook."""
def test_webhook_receives_payload_and_logs(self, test_client, caplog):
payload = {
"status": "firing",
"alerts": [
{
"status": "firing",
"labels": {"alertname": "BackupJobFailed", "severity": "critical"},
"annotations": {"summary": "Backup failed"},
"startsAt": "2026-06-13T10:00:00Z",
}
],
}
with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"):
response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload)
assert response.status_code == 200
assert response.json()["status"] == "received"
assert "Received Alertmanager webhook with 1 alert(s)" in caplog.text
def test_webhook_forwards_when_configured(self, test_client, caplog, monkeypatch):
payload = {"status": "resolved", "alerts": []}
forwarded = {"captured": False}
class FakeResponse:
def raise_for_status(self):
pass
def fake_post(url, json, timeout):
forwarded["captured"] = True
forwarded["url"] = url
forwarded["payload"] = json
return FakeResponse()
monkeypatch.setattr("requests.Session.post", lambda _self, url, json, timeout: fake_post(url, json, timeout))
with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"):
response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload)
assert response.status_code == 200
assert forwarded["captured"] is False
backend/tests/test_config.py
+1
View File
@@ -2,6 +2,7 @@
import os
from unittest.mock import patch
from media_library_viewer_api.config import Settings
backend/tests/test_domain_media.py
+8 -9
View File
@@ -1,16 +1,15 @@
"""Unit tests for domain/media.py normalization helpers."""
import pytest
from media_library_viewer_api.domain.media import (
first_media_source,
media_streams,
stream_value,
is_hdr_item,
format_date_added,
timestamp_date_added,
format_rate_bits_decimal,
normalize_media_item,
display_media_row,
first_media_source,
format_date_added,
format_rate_bits_decimal,
is_hdr_item,
media_streams,
normalize_media_item,
stream_value,
timestamp_date_added,
)
backend/tests/test_jobs.py
+1 -2
View File
@@ -1,7 +1,6 @@
"""Unit tests for jobs.py template rendering and safety."""
import pytest
from media_library_viewer_api.jobs import JOB_TEMPLATES, JobTemplate, run_job
from media_library_viewer_api.jobs import JOB_TEMPLATES, JobTemplate
class TestJobTemplate:
backend/tests/test_media_index.py
+3 -3
View File
@@ -1,9 +1,9 @@
"""Unit tests for the SQLite media index service."""
import pytest
import tempfile
from pathlib import Path
from typing import Any
import pytest
from media_library_viewer_api.services.media_index import (
MediaIndex,
MediaIndexBuildCancelled,
backend/tests/test_monitoring_actions.py
+8 -18
View File
@@ -3,7 +3,7 @@ from unittest.mock import MagicMock, patch
from media_library_viewer_api.services.monitoring_actions import poll_machine_snapshot
def test_poll_machine_snapshot_records_history_entries():
def test_poll_machine_snapshot_records_disk_lookup():
store = MagicMock()
machine = {
"id": "local",
@@ -17,14 +17,6 @@ def test_poll_machine_snapshot_records_history_entries():
"media_library_viewer_api.services.monitoring_actions.build_machine_client",
return_value=object(),
) as build_client,
patch(
"media_library_viewer_api.services.monitoring_actions.resource_collector_status",
return_value="running pid=123",
) as status_fn,
patch(
"media_library_viewer_api.services.monitoring_actions.read_resource_metrics",
return_value=[{"ts": 1.0}, {"ts": 2.0}],
) as metrics_fn,
patch(
"media_library_viewer_api.services.monitoring_actions.disk_space",
return_value={"mount": "/srv/media", "used_pct": "12.5%"},
@@ -33,14 +25,12 @@ def test_poll_machine_snapshot_records_history_entries():
result = poll_machine_snapshot(machine, store, metrics_limit=123, request_id="poll:test")
assert result["request_id"] == "poll:test"
assert result["metrics_samples"] == 2
assert result["disk_mount"] == "/srv/media"
build_client.assert_called_once_with(machine)
status_fn.assert_called_once()
metrics_fn.assert_called_once_with(build_client.return_value, max_lines=123)
assert result["actions"] == ["disk lookup"]
build_client.assert_called_once_with(machine, store)
disk_fn.assert_called_once_with(build_client.return_value, "/srv/media")
assert store.record_machine_action.call_count == 3
recorded_actions = [call.args[1] for call in store.record_machine_action.call_args_list]
assert recorded_actions == ["status lookup", "metrics read", "disk lookup for /srv/media"]
assert all(call.kwargs["request_id"] == "poll:test" for call in store.record_machine_action.call_args_list)
assert all(call.args[2] == "ok" for call in store.record_machine_action.call_args_list)
assert store.record_machine_action.call_count == 1
recorded_action = store.record_machine_action.call_args
assert recorded_action.args[1] == "disk lookup for /srv/media"
assert recorded_action.kwargs["request_id"] == "poll:test"
assert recorded_action.args[2] == "ok"
backend/tests/test_observability.py
+37
View File
@@ -0,0 +1,37 @@
"""Tests for observability metrics helpers."""
import time
from prometheus_client import REGISTRY
from media_library_viewer_api.observability import record_backup_run
def _metric_samples(metric_name):
return {
tuple(s.labels.values()): s.value
for family in REGISTRY.collect()
for s in family.samples
if s.name == metric_name
}
def test_record_backup_run_increments_counter():
record_backup_run("job-a", "success", success=True)
record_backup_run("job-a", "failure")
record_backup_run("job-b", "success", success=True)
samples = _metric_samples("manage_backup_runs_total")
assert samples[("job-a", "success")] >= 1.0
assert samples[("job-a", "failure")] >= 1.0
assert samples[("job-b", "success")] >= 1.0
def test_record_backup_run_sets_last_success_timestamp():
before = time.time()
record_backup_run("job-ts", "success", success=True)
after = time.time()
samples = _metric_samples("manage_backup_runs_last_success_timestamp")
value = samples[("job-ts",)]
assert before <= value <= after
backend/tests/test_path_utils.py
+3 -2
View File
@@ -1,6 +1,5 @@
"""Unit tests for path_utils.py — path resolution logic."""
import pytest
from media_library_viewer_api.path_utils import (
apply_remote_path_prefix,
map_path_to_media_root,
@@ -32,7 +31,9 @@ class TestApplyRemotePathPrefix:
assert apply_remote_path_prefix("/media/file.mkv", "/srv/") == "/srv/media/file.mkv"
def test_path_with_spaces(self):
assert apply_remote_path_prefix("/media/My Movie (2024)/file.mkv", "/srv") == "/srv/media/My Movie (2024)/file.mkv"
assert (
apply_remote_path_prefix("/media/My Movie (2024)/file.mkv", "/srv") == "/srv/media/My Movie (2024)/file.mkv"
)
class TestMapPathToMediaRoot:
backend/tests/test_ssh_client.py
+3 -1
View File
@@ -7,7 +7,9 @@ from media_library_viewer_api.clients.ssh import RemoteSSHClient
def test_connect_uses_existing_known_hosts_without_reprobing(tmp_path):
known_hosts_path = tmp_path / "known_hosts"
known_hosts_path.write_text("example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAFAKE\n")
known_hosts_path.write_text(
"example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID8tdNkXNS1EWbkjb2XxFHQXRrpYzl0MXaVLRmVrbmDZ\n"
)
ssh_client = MagicMock()
ssh_client.connect.return_value = None
backend/tests/test_targets.py
+109
View File
@@ -0,0 +1,109 @@
"""Tests for Prometheus file-based service discovery target generation."""
from pathlib import Path
import pytest
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import (
build_node_exporter_targets,
write_prometheus_targets,
)
@pytest.fixture
def store(tmp_path: Path) -> SettingsStore:
db = SettingsStore(tmp_path / "settings.sqlite")
db.init_schema()
return db
class TestBuildNodeExporterTargets:
def test_disabled_machine_excluded(self, store: SettingsStore):
store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": False,
"node_exporter_port": 9200,
}
)
assert build_node_exporter_targets(store) == []
def test_ssh_enabled_machine_included(self, store: SettingsStore):
machine = store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9200,
"node_exporter_scrape_host": "1.2.3.4",
}
)
targets = build_node_exporter_targets(store)
assert len(targets) == 1
assert targets[0]["targets"] == ["1.2.3.4:9200"]
assert targets[0]["labels"]["machine_id"] == machine["id"]
assert targets[0]["labels"]["machine_name"] == "remote1"
assert targets[0]["labels"]["job"] == "node-exporter-remote"
def test_scrape_host_defaults_to_machine_host(self, store: SettingsStore):
store.upsert_machine(
{
"name": "remote2",
"mode": "ssh",
"host": "remote2.example.com",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9100,
}
)
targets = build_node_exporter_targets(store)
assert targets[0]["targets"] == ["remote2.example.com:9100"]
def test_local_machine_excluded(self, store: SettingsStore):
store.upsert_machine(
{
"name": "This machine",
"mode": "local",
"host": "localhost",
"username": "",
"node_exporter_enabled": True,
}
)
assert build_node_exporter_targets(store) == []
def test_missing_host_excluded(self, store: SettingsStore):
store.upsert_machine(
{
"name": "remote3",
"mode": "ssh",
"host": "",
"username": "u",
"node_exporter_enabled": True,
}
)
assert build_node_exporter_targets(store) == []
class TestWritePrometheusTargets:
def test_writes_valid_json(self, store: SettingsStore, tmp_path: Path):
store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9200,
}
)
file_path = write_prometheus_targets(store, tmp_path)
assert file_path.exists()
assert file_path.name == "node_exporter_targets.json"
targets = build_node_exporter_targets(store)
assert len(targets) == 1
backend/tests/test_utils.py
+6 -7
View File
@@ -1,17 +1,16 @@
"""Unit tests for utils.py formatting helpers."""
import pytest
from media_library_viewer_api.utils import (
ticks_to_minutes,
ffprobe_format_summary,
format_bitrate,
format_duration,
human_size,
is_known_video_file,
format_duration,
format_bitrate,
ffprobe_format_summary,
summarize_video_streams,
summarize_audio_streams,
summarize_subtitle_streams,
summarize_streams,
summarize_subtitle_streams,
summarize_video_streams,
ticks_to_minutes,
timestamp_to_local,
)
context.md
+207
View File
@@ -0,0 +1,207 @@
# Code Context
## Files Retrieved
1. `docker-compose.yml` (lines 1–262) – production Compose stack; defines observability services and Traefik routing.
2. `docker-compose.dev.yml` (lines 1–234) – development Compose stack; same observability services but with host ports exposed and auth disabled.
3. `.env.example` (lines 1–55) – template with all required environment variables for the stack, including Prometheus/Grafana/Alertmanager/Alloy/Loki and Node Exporter settings.
4. `monitoring/prometheus/prometheus.yml` (all lines) – Prometheus scrape configuration including the `manage-backend`, `node-exporter`, and file-SD remote targets.
5. `monitoring/prometheus/rules/backup_alerts.yml` (all lines) – Prometheus alerting rules for backup jobs and observability-stack health.
6. `monitoring/alertmanager/alertmanager.yml` (all lines) – Alertmanager routing, email/webhook receivers, and inhibition rules.
7. `monitoring/grafana/grafana.ini` (all lines) – Grafana server, embedding, and generic OAuth (Authentik) configuration.
8. `monitoring/alloy/config.alloy` (all lines) – Alloy pipeline to discover Docker containers and push logs to Loki.
9. `monitoring/loki/loki.yml` (all lines) – Single-node Loki configuration with filesystem storage and 30-day retention.
10. `backend/src/media_library_viewer_api/observability.py` (all lines) – Prometheus metrics definitions and helper functions.
11. `backend/src/media_library_viewer_api/main.py` (lines 1–128) – FastAPI entrypoint exposing `/metrics` and wiring request/observability middleware.
12. `backend/src/media_library_viewer_api/services/targets.py` (all lines) – Backend writes file-SD target list for remote Node Exporters.
13. `backend/src/media_library_viewer_api/config.py` (lines 1–88) – Settings including `prometheus_enabled`, `prometheus_file_sd_dir`, and `alertmanager_url`.
14. `docs/monitoring-logging-design.md` (all lines) – Architecture/design document describing the observability stack.
15. `docs/observability-runbooks.md` (all lines) – Operational runbooks for the observability services.
## Key Code
### Backend `/metrics` endpoint
`backend/src/media_library_viewer_api/main.py`:
```python
@app.middleware("http")
async def enforce_jwt_auth(request: Request, call_next):
if request.url.path in {"/api/health", "/api/version", "/metrics"}:
return await call_next(request)
return await require_jwt_auth(request, call_next)
@app.get("/metrics")
def metrics() -> Response:
"""Expose Prometheus metrics."""
data, content_type = metrics_payload()
return FastAPIResponse(content=data, media_type=content_type)
```
### Metrics emitted by the backend
`backend/src/media_library_viewer_api/observability.py`:
```python
REQUESTS_TOTAL = Counter("manage_api_requests_total", "Total API requests", ["method", "path", "status_code"])
REQUEST_DURATION = Histogram("manage_api_request_duration_seconds", "API request duration", ["method", "path"], ...)
SSH_COMMANDS_TOTAL = Counter("manage_ssh_commands_total", "Total SSH/local commands executed", ["machine_id", "action", "status"])
MEDIA_INDEX_BUILDS_TOTAL = Counter("manage_media_index_builds_total", "Total media index build attempts", ["status"])
BACKUP_RUNS_TOTAL = Counter("manage_backup_runs_total", "Total backup runs", ["job_name", "status"])
BACKUP_RUNS_LAST_SUCCESS = Gauge("manage_backup_runs_last_success_timestamp", "Unix timestamp of the last successful backup run per job", ["job_name"])
MAIL_QUEUE_SIZE = Counter("manage_mail_queue_messages_total", "Total messages enqueued", ["status"])
```
### Prometheus scrape configuration
`monitoring/prometheus/prometheus.yml`:
```yaml
scrape_configs:
- job_name: manage-backend
static_configs:
- targets:
- backend:8000
metrics_path: /metrics
scrape_interval: 15s
- job_name: node-exporter
static_configs:
- targets:
- node-exporter:9100
- job_name: node-exporter-remote
file_sd_configs:
- files:
- /etc/prometheus/file-sd/node_exporter_targets.json
refresh_interval: 30s
```
### Backend-managed remote Node Exporter targets
`backend/src/media_library_viewer_api/services/targets.py`:
```python
def build_node_exporter_targets(store: SettingsStore) -> list[dict[str, Any]]:
...
def write_prometheus_targets(store: SettingsStore, file_sd_dir: Path | None = None) -> Path:
...
file_path = file_sd_dir / "node_exporter_targets.json"
```
## Architecture
The observability stack is a standard self-hosted Prometheus/Grafana/Loki/Alertmanager deployment running inside the same Docker Compose project as the Manage application:
- **Prometheus** pulls metrics from:
- The Manage FastAPI backend via `/metrics` (`job="manage-backend"`).
- The local Docker host via `node-exporter` (`job="node-exporter"`).
- Remote machines that have Node Exporter enabled through Manage settings (`job="node-exporter-remote"`), discovered through a file-SD JSON file generated by the backend.
- The observability services themselves: Loki, Alertmanager, Grafana, and Prometheus self-scrape.
- **Grafana** visualizes metrics and logs; it is configured for OAuth login through Authentik and is embedded in the Manage React UI via iframes.
- **Loki** stores logs; retention is 30 days.
- **Alloy** (Grafana Alloy) collects Docker container logs by mounting the Docker socket and the container log directory, then pushes them to Loki.
- **Alertmanager** routes alerts by severity (critical vs warning) and delivers email notifications (and optionally a webhook back to the backend).
The backend bridges the stack with the application:
- It exposes `/metrics` (unauthenticated, along with `/api/health` and `/api/version`).
- On startup it writes `${PROMETHEUS_FILE_SD_DIR}/node_exporter_targets.json` based on enabled SSH machines in the settings store.
- It provides proxy endpoints (`/api/monitoring/alerts`, `/api/monitoring/alertmanager-status`, `/api/monitoring/prometheus-targets`) consumed by the frontend.
## Start Here
Open `monitoring/prometheus/prometheus.yml` first to understand what is scraped and how the backend is wired, then read `backend/src/media_library_viewer_api/observability.py` to see the metric names and labels. For environment requirements, read `.env.example`.
## Supervisor coordination
Not needed — this is a read-only scouting summary.
---
# Monitoring/Observability Setup Summary
## 1. Observability services defined in Compose
Both `docker-compose.yml` and `docker-compose.dev.yml` define the following services:
| Service | Image | Internal endpoint | Purpose |
|---------|-------|-------------------|---------|
| `prometheus` | `prom/prometheus:v2.55.1` | `http://prometheus:9090` | Metrics TSDB and alert evaluator |
| `loki` | `grafana/loki:3.1.1` | `http://loki:3100` | Log aggregation |
| `alloy` | `grafana/alloy:v1.5.0` | `http://alloy:12345` | Docker log collection agent |
| `grafana` | `grafana/grafana:11.3.1` | `http://grafana:3000` | Dashboards and visualization |
| `alertmanager` | `prom/alertmanager:v0.27.0` | `http://alertmanager:9093` | Alert routing/delivery |
| `node-exporter` | `prom/node-exporter:v1.8.2` | `http://node-exporter:9100` | Host metrics for the Docker host |
| `backend` | Build from `backend/Dockerfile` | `http://backend:8000` | FastAPI app exposing `/metrics` |
Differences:
- Production (`docker-compose.yml`): services attach to an external `web` network for Traefik, use `expose` instead of host ports for most services, and require OIDC/auth variables.
- Development (`docker-compose.dev.yml`): Prometheus/Grafana/Loki/Alertmanager/Node Exporter are published on host ports `9090`, `3000`, `3100`, `9093`, `9100`; auth is disabled (`AUTH_ENABLED=false`).
## 2. Required environment variables
From `.env.example` and the Compose files, the variables relevant to the observability stack are:
### Backend / metrics
- `PROMETHEUS_ENABLED` – enable metrics endpoint (set to `"true"` in both compose files).
- `PROMETHEUS_FILE_SD_DIR` – directory where the backend writes `node_exporter_targets.json` (default `/app/backend/.cache/prometheus-file-sd`).
- `ALERTMANAGER_URL` – backend proxy target (default `http://alertmanager:9093`).
- `ALERTMANAGER_WEBHOOK_URL` – optional webhook receiver for Alertmanager.
- `BACKEND_CACHE_DIR` – host directory mounted into backend and Prometheus for file-SD.
### Grafana
- `GRAFANA_APP_HOST` – public hostname for Grafana (production; required).
- `GRAFANA_APP_PORT` – defaults to `3000`.
- `GRAFANA_APP_NAME` – defaults to `grafana`.
- `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` – local admin credentials.
- `GF_AUTH_GENERIC_OAUTH_CLIENT_ID`
- `GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET`
- `GF_AUTH_GENERIC_OAUTH_AUTH_URL`
- `GF_AUTH_GENERIC_OAUTH_TOKEN_URL`
- `GF_AUTH_GENERIC_OAUTH_API_URL`
### Alertmanager
- `SMTP_HOST` / `SMTP_PORT`
- `SMTP_USERNAME` / `SMTP_PASSWORD`
- `SMTP_FROM_ADDRESS`
- `ALERT_EMAIL_TO`
### Traefik / network (production)
- `BACKEND_APP_HOST` / `FRONTEND_APP_HOST` / `GRAFANA_APP_HOST`
- `CERT_RESOLVER` – e.g. `letsencrypt`
### General
- `LOG_LEVEL` / `LOG_FORMAT` – also passed to Grafana and backend.
## 3. Monitoring config files
| File | Purpose |
|------|---------|
| `monitoring/prometheus/prometheus.yml` | Scrape jobs: backend `/metrics`, local node-exporter, remote node-exporter via file-SD, Loki, Alertmanager, Grafana, and self-scrape. |
| `monitoring/prometheus/rules/backup_alerts.yml` | Alert rules: `BackupJobFailed`, `BackupJobStuck`, `PrometheusTargetMissing`, `AlertmanagerDown`, `GrafanaDown`. |
| `monitoring/alertmanager/alertmanager.yml` | Routes alerts by severity, sends email to `ALERT_EMAIL_TO`, optional webhook to backend, and inhibits warnings when a critical alert fires. |
| `monitoring/grafana/grafana.ini` | Enables iframe embedding, OAuth via Authentik, role mapping from groups, and defaults users to `Viewer`. |
| `monitoring/alloy/config.alloy` | Discovers Docker containers, relabels container/stream labels, and writes logs to `http://loki:3100/loki/api/v1/push`. |
| `monitoring/loki/loki.yml` | Single-node Loki with filesystem storage, tsdb index, 30-day retention (`720h`). |
## 4. Backend metrics and Prometheus scraping
- The backend exposes Prometheus metrics at `/metrics` on port `8000`.
- The endpoint is unauthenticated (bypassed in `enforce_jwt_auth`).
- Prometheus scrapes it as `job="manage-backend"` with `scrape_interval: 15s`.
- Key application metrics include:
- `manage_api_requests_total{method, path, status_code}`
- `manage_api_request_duration_seconds{method, path}`
- `manage_ssh_commands_total{machine_id, action, status}`
- `manage_media_index_builds_total{status}`
- `manage_backup_runs_total{job_name, status}`
- `manage_backup_runs_last_success_timestamp{job_name}`
- `manage_mail_queue_messages_total{status}`
Remote Node Exporter targets are not static: the backend reads machine settings from SQLite and writes `${PROMETHEUS_FILE_SD_DIR}/node_exporter_targets.json`. Prometheus reloads this file every 30 seconds via `file_sd_configs`.
## 5. Setup steps and gotchas
- The observability stack is brought up with the app itself:
- Production: `docker compose -f docker-compose.yml up --build`
- Development: `docker compose -f docker-compose.dev.yml up --build`
- Production requires the external `web` network and Traefik already configured; `docker-compose.dev.yml` does not use Traefik and binds ports directly.
- Export/copy `.env.example` to `.env` and fill required values (`OIDC_ISSUER_URL`, `OIDC_AUDIENCE`, `BACKEND_APP_HOST`, `FRONTEND_APP_HOST`, `GRAFANA_APP_HOST`, `CERT_RESOLVER`, SMTP credentials, Grafana OAuth secrets).
- `BACKEND_CACHE_DIR` is shared between the backend and Prometheus so file-SD target updates are visible to Prometheus.
- Alloy must run as `user: root` and mount `/var/run/docker.sock` and `/var/lib/docker/containers`; without these mounts it cannot collect Docker logs.
- Node Exporter mounts the host root filesystem read-only (`/:/host:ro,rslave`) to report host-level metrics; on production, this exposes the Docker host.
- Grafana iframe embedding requires `allow_embedding = true` in `grafana.ini` and matching cookie settings; also ensure the reverse proxy/CSP permits embedding.
- After changing Prometheus rules/config, trigger a reload with `curl -X POST http://localhost:9090/-/reload` (production needs Traefik/network access).
- Observability data is stored in named volumes: `prometheus_data`, `loki_data`, `grafana_data`, `alertmanager_data`. Back them up as documented in `docs/observability-runbooks.md`.
- Default retention is 30 days for both Prometheus TSDB and Loki logs.
docker-compose.dev.yml
+211
View File
@@ -11,12 +11,19 @@ services:
OIDC_AUDIENCE: ${OIDC_AUDIENCE:-}
OIDC_JWKS_URL: ${OIDC_JWKS_URL:-}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
LOG_FORMAT: ${LOG_FORMAT:-text}
PROMETHEUS_ENABLED: "true"
SSH_KNOWN_HOSTS_PATH: /app/backend/.cache/known_hosts
PROMETHEUS_FILE_SD_DIR: /app/backend/.cache/prometheus-file-sd
ALERTMANAGER_URL: ${ALERTMANAGER_URL:-http://alertmanager:9093}
ports:
- "8000:8000"
volumes:
- ./backend:/app/backend
- ${BACKEND_CACHE_DIR:-./backend-cache}:/app/backend/.cache
networks:
- web
- monitoring
restart: unless-stopped
frontend:
@@ -38,5 +45,209 @@ services:
- backend
restart: unless-stopped
prometheus:
image: prom/prometheus:v2.55.1
container_name: prometheus
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time=30d"
- "--web.console.libraries=/usr/share/prometheus/console_libraries"
- "--web.console.templates=/usr/share/prometheus/consoles"
- "--web.enable-lifecycle"
volumes:
- ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
- ${BACKEND_CACHE_DIR:-./backend-cache}/prometheus-file-sd:/etc/prometheus/file-sd:ro
- prometheus_data:/prometheus
ports:
- "9090:9090"
networks:
- monitoring
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "1.00"
memory: 1G
reservations:
cpus: "0.25"
memory: 256M
loki:
image: grafana/loki:3.1.1
container_name: loki
command: -config.file=/etc/loki/loki.yml
volumes:
- ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
- loki_data:/loki
ports:
- "3100:3100"
networks:
- monitoring
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "1.00"
memory: 1G
reservations:
cpus: "0.25"
memory: 256M
alloy:
image: grafana/alloy:v1.5.0
container_name: alloy
command:
- run
- /etc/alloy/config.alloy
- --storage.path=/var/lib/alloy
volumes:
- ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
- /var/lib/docker/containers:/var/lib/docker/containers:ro
networks:
- monitoring
depends_on:
loki:
condition: service_healthy
restart: unless-stopped
user: root
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:12345/-/healthy"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "0.50"
memory: 512M
reservations:
cpus: "0.10"
memory: 128M
grafana:
image: grafana/grafana:11.3.1
container_name: grafana
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID:-}
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:-}
GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${GF_AUTH_GENERIC_OAUTH_AUTH_URL:-}
GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL:-}
GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
volumes:
- ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
- ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
- grafana_data:/var/lib/grafana
ports:
- "3000:3000"
networks:
- monitoring
depends_on:
prometheus:
condition: service_healthy
loki:
condition: service_healthy
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
reservations:
cpus: "0.25"
memory: 128M
alertmanager:
image: prom/alertmanager:v0.27.0
container_name: alertmanager
command:
- "--config.file=/etc/alertmanager/alertmanager.yml"
- "--storage.path=/alertmanager"
environment:
SMTP_HOST: ${SMTP_HOST:-smtp.example.com}
SMTP_PORT: ${SMTP_PORT:-587}
SMTP_USERNAME: ${SMTP_USERNAME:-}
SMTP_PASSWORD: ${SMTP_PASSWORD:-}
SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com}
ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
volumes:
- ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
- alertmanager_data:/alertmanager
ports:
- "9093:9093"
networks:
- monitoring
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "0.50"
memory: 256M
reservations:
cpus: "0.10"
memory: 64M
node-exporter:
image: prom/node-exporter:v1.8.2
container_name: node-exporter
command:
- "--path.rootfs=/host"
volumes:
- /:/host:ro,rslave
networks:
- monitoring
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
reservations:
cpus: "0.05"
memory: 32M
networks:
monitoring:
volumes:
frontend_node_modules:
prometheus_data:
loki_data:
grafana_data:
alertmanager_data:
docker-compose.yml
+217
View File
@@ -13,6 +13,8 @@ services:
OIDC_JWKS_URL: ${OIDC_JWKS_URL:-}
OIDC_CLOCK_SKEW_SECONDS: ${OIDC_CLOCK_SKEW_SECONDS:-30}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
LOG_FORMAT: ${LOG_FORMAT:-text}
PROMETHEUS_ENABLED: "true"
SMTP_HOST: ${SMTP_HOST:-}
SMTP_PORT: ${SMTP_PORT:-587}
SMTP_USERNAME: ${SMTP_USERNAME:-}
@@ -23,11 +25,14 @@ services:
SMTP_USE_SSL: ${SMTP_USE_SSL:-false}
SMTP_TIMEOUT: ${SMTP_TIMEOUT:-30}
SSH_KNOWN_HOSTS_PATH: /app/backend/.cache/known_hosts
PROMETHEUS_FILE_SD_DIR: ${PROMETHEUS_FILE_SD_DIR:-/app/backend/.cache/prometheus-file-sd}
ALERTMANAGER_URL: ${ALERTMANAGER_URL:-http://alertmanager:9093}
volumes:
- ${BACKEND_CACHE_DIR:-./backend-cache}:/app/backend/.cache
restart: unless-stopped
networks:
- web
- monitoring
expose:
- "8000"
labels:
@@ -80,6 +85,218 @@ services:
- "8080:80"
restart: unless-stopped
prometheus:
image: prom/prometheus:v2.55.1
container_name: prometheus
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time=30d"
- "--web.console.libraries=/usr/share/prometheus/console_libraries"
- "--web.console.templates=/usr/share/prometheus/consoles"
- "--web.enable-lifecycle"
volumes:
- ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./monitoring/prometheus/rules:/etc/prometheus/rules:ro
- ${BACKEND_CACHE_DIR:-./backend-cache}/prometheus-file-sd:/etc/prometheus/file-sd:ro
- prometheus_data:/prometheus
expose:
- "9090"
networks:
- web
- monitoring
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "1.00"
memory: 1G
reservations:
cpus: "0.25"
memory: 256M
loki:
image: grafana/loki:3.1.1
container_name: loki
command: -config.file=/etc/loki/loki.yml
volumes:
- ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro
- loki_data:/loki
expose:
- "3100"
networks:
- monitoring
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "1.00"
memory: 1G
reservations:
cpus: "0.25"
memory: 256M
alloy:
image: grafana/alloy:v1.5.0
container_name: alloy
command:
- run
- /etc/alloy/config.alloy
- --storage.path=/var/lib/alloy
volumes:
- ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
- /var/lib/docker/containers:/var/lib/docker/containers:ro
networks:
- monitoring
depends_on:
loki:
condition: service_healthy
restart: unless-stopped
user: root
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:12345/-/healthy"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "0.50"
memory: 512M
reservations:
cpus: "0.10"
memory: 128M
grafana:
image: grafana/grafana:11.3.1
container_name: grafana
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID:-}
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:-}
GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${GF_AUTH_GENERIC_OAUTH_AUTH_URL:-}
GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL:-}
GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
volumes:
- ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
- ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
- grafana_data:/var/lib/grafana
expose:
- "3000"
networks:
- web
- monitoring
depends_on:
prometheus:
condition: service_healthy
loki:
condition: service_healthy
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
labels:
- "traefik.enable=true"
- "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.rule=Host(`${GRAFANA_APP_HOST:?set GRAFANA_APP_HOST}`)"
- "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.entrypoints=websecure"
- "traefik.http.routers.${GRAFANA_APP_NAME:-grafana}.tls.certresolver=${CERT_RESOLVER:?set CERT_RESOLVER}"
- "traefik.http.services.${GRAFANA_APP_NAME:-grafana}.loadbalancer.server.port=${GRAFANA_APP_PORT:-3000}"
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
reservations:
cpus: "0.25"
memory: 128M
alertmanager:
image: prom/alertmanager:v0.27.0
container_name: alertmanager
command:
- "--config.file=/etc/alertmanager/alertmanager.yml"
- "--storage.path=/alertmanager"
environment:
SMTP_HOST: ${SMTP_HOST:-smtp.example.com}
SMTP_PORT: ${SMTP_PORT:-587}
SMTP_USERNAME: ${SMTP_USERNAME:-}
SMTP_PASSWORD: ${SMTP_PASSWORD:-}
SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com}
ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com}
volumes:
- ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
- alertmanager_data:/alertmanager
expose:
- "9093"
networks:
- monitoring
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
deploy:
resources:
limits:
cpus: "0.50"
memory: 256M
reservations:
cpus: "0.10"
memory: 64M
node-exporter:
image: prom/node-exporter:v1.8.2
container_name: node-exporter
command:
- "--path.rootfs=/host"
volumes:
- /:/host:ro,rslave
networks:
- monitoring
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
reservations:
cpus: "0.05"
memory: 32M
networks:
web:
external: true
monitoring:
volumes:
prometheus_data:
loki_data:
grafana_data:
alertmanager_data:
docs/REQUIREMENTS.md
+6
View File
@@ -197,6 +197,12 @@ Phase 1: Jellyfin media index, SSH-based remote filesystem inspection, server mo
## Decision Log
- 2026-06-13: Adopted a dedicated, self-hosted observability subsystem based on Prometheus, Grafana Loki, Grafana, and Alertmanager. Metrics will be pulled from Node Exporter on machines and from application exporters in containers; logs will be structured JSON shipped by Promtail/Grafana Alloy. The existing POSIX remote collector will be removed and backup alerts migrated to Alertmanager rules. See `docs/monitoring-logging-design.md`.
- 2026-06-13 (Phase 1): Added Prometheus, Loki, Grafana Alloy, Grafana, Alertmanager, and Node Exporter services to `docker-compose.yml` and `docker-compose.dev.yml`. Provisioned Grafana datasources and an initial `Manage Overview` dashboard as code. Configured Alloy to tail Docker logs and ship to Loki. Added Grafana generic OAuth configuration via `monitoring/grafana/grafana.ini` and a dedicated Traefik host rule. Added Alertmanager email routing with env-var interpolation. Added `/grafana` proxy to the Vite dev server for iframe embedding.
- 2026-06-13 (Phase 2): Extended machine settings with `node_exporter_enabled`, `node_exporter_port`, and `node_exporter_scrape_host`. Added Node Exporter install/restart/status job templates to `jobs.py`. Implemented `media_library_viewer_api.services.targets` to generate Prometheus file-SD target files and wired target regeneration into machine create/update/delete. Added `/api/monitoring/prometheus-targets` for live target previews. Configured Prometheus with a `node-exporter-remote` job reading file SD from the backend cache volume. Added a minimal `Node Exporter Overview` Grafana dashboard. Added unit and integration tests for target generation and the new endpoint.
- 2026-06-13 (Phase 3): Added `ALERTMANAGER_URL` setting and `GET /api/monitoring/alerts` plus `GET /api/monitoring/alertmanager-status` endpoints that proxy/summarize Alertmanager for the UI. Added `manage_backup_runs_last_success_timestamp` Prometheus gauge and wired `record_backup_run` to update it on success so the existing `BackupJobStuck` Alertmanager rule works. Updated `docker-compose.yml`, `docker-compose.dev.yml`, and `.env.example` with `ALERTMANAGER_URL`. Added tests for the new endpoints and metric helpers.
- 2026-06-13 (Phase 4): Added `/observability` React page (`frontend/src/components/ObservabilityPage.tsx`) with health cards for Alertmanager, active alerts, Prometheus targets, and machines; a recent-alerts list; machine selector; and Grafana iframe panels for Node Exporter metrics and Loki logs. Added `useObservability` hook and API client wrappers for `/api/monitoring/alerts`, `/api/monitoring/alertmanager-status`, and `/api/monitoring/prometheus-targets`. Added TypeScript types for Alertmanager summary/status and Prometheus targets. Wired the new route into `App.tsx` and the sidebar. Added shadcn/ui `card`, `badge`, `alert`, `skeleton`, and `select` components. Frontend build (`npm run build`) passes; lint has only pre-existing warnings.
- 2026-06-13 (Phase 5): Hardened observability containers with health checks and resource limits in both compose files; added service-healthy `depends_on` conditions. Added Prometheus scrape jobs for Loki, Alertmanager, and Grafana, plus new `observability_health` alert rules. Added `ALERTMANAGER_WEBHOOK_URL` backend setting, `POST /api/monitoring/alertmanager-webhook` receiver, and Alertmanager `webhook` receiver config. Added `docs/observability-runbooks.md` with operational playbooks. Updated `.env.example` to include all observability variables.
- 2026-05-03: Reaffirmed that the Monitoring tab charts should be rendered directly with D3 and expose brush-based time-range selection plus moving averages.
- 2026-05-03: Added hover tooltips, summary chips, a moving vertical cursor, snapped point markers, and a selected-range label to the D3 Monitoring charts for faster visual inspection.
- 2026-05-03: Combined network download/upload into one traffic chart and disk read/write into one I/O chart for clearer Monitoring layout.
docs/monitoring-logging-design.md
+438
View File
@@ -0,0 +1,438 @@
# Monitoring and Logging Design — Manage
## Executive Summary
Manage currently uses ad-hoc observability: plain-text Python logs, a custom POSIX shell metrics collector on remote machines, and a background poller that stores snapshots in SQLite. This works for a single-instance homelab but becomes painful as the fleet grows and as users need faster incident response.
This document proposes a dedicated, self-hosted observability subsystem built on the standard Prometheus/Grafana stack:
- **Metrics**: Prometheus pulling from Node Exporter on machines and from application exporters in containers.
- **Logs**: Structured JSON logs shipped to **Grafana Loki** by **Promtail/Grafana Alloy**.
- **Dashboards**: Grafana for deep-dive dashboards, embedded in the Manage React UI via iframes.
- **Alerting**: Prometheus Alertmanager for routing and notifications (email first, webhooks later).
- **Auth**: Grafana authenticates through the existing OIDC/Authentik provider.
The existing POSIX remote collector will be removed, and the Python backup alert engine will be migrated to Alertmanager rules.
---
## Goals
1. **Fast query and alerting**: move from SQLite scan-based history to a real time-series database and indexed log store.
2. **Unified view**: monitor both local containers/apps and remote Linux machines from one place.
3. **Standard tooling**: use de-facto open-source tools so dashboards, exporters, and runbooks are reusable.
4. **Room to grow**: design supports adding traces, more notification channels, and longer retention later without re-architecture.
## Non-Goals
1. **Traces**: deferred to a later phase; the data flow and collector choice (Promtail/Alloy) will be trace-ready.
2. **Multi-tenant RBAC**: Manage is single-instance/homelab; Grafana teams are sufficient for now.
3. **SLA/SLO framework**: out of scope; we focus on metrics, logs, and alerts, not SLO budgeting.
4. **Cloud-hosted observability vendors**: all components run self-hosted in Docker Compose.
---
## Decisions
| Area | Decision | Rationale |
|------|----------|-----------|
| Coupling | Dedicated observability subsystem consumed by Manage | Keeps Manage fast and lets the observability stack evolve independently. |
| Metrics backend | Prometheus | Pull model, huge ecosystem, standard exporters, easy Grafana integration. |
| Machine metrics | Node Exporter | Rich OS metrics, reusable dashboards, no custom shell to maintain. |
| Log backend | Grafana Loki | Prometheus-style labels, low resource use, tight Grafana integration. |
| Log collection | Promtail / Grafana Alloy | Tails Docker logs and journald; no per-app network calls. |
| App logs | Structured JSON to stdout | Standard 12-factor pattern; collector handles routing. |
| Dashboards | Grafana + iframe embeds | Fast to implement, rich dashboards, Manage UI stays focused on summary. |
| Alerting | Prometheus Alertmanager | Mature routing, silencing, inhibition; single source of truth for infra alerts. |
| Auth | Grafana OAuth via Authentik | Reuses existing identity provider; consistent UX. |
| Retention | 30 days metrics, 30 days logs | Matches current retention policy; disk usage stays predictable. |
| Migration | Remove POSIX collector, migrate backup alerts | Eliminates duplicate alerting paths and custom remote code. |
---
## Current State
### Logging
- `backend/src/media_library_viewer_api/logging_utils.py` configures stdlib `logging` with a plain-text format.
- `main.py` has a `log_requests` middleware that emits method, path, client IP, status code, and elapsed time.
- Frontend uses standard `console.log` / browser dev tools; no server-side log aggregation.
### Metrics
- `backend/src/media_library_viewer_api/clients/resources.py` deploys a POSIX shell collector to `/tmp` on each remote machine.
- The collector samples `/proc/stat`, `/proc/meminfo`, `/proc/net/dev`, and `/sys/block/*/stat` every 10s and writes JSONL to `/tmp/media_library_viewer_metrics.jsonl`.
- `MonitoringPoller` (`monitoring_poller.py`) runs every 5 minutes, reads the remote JSONL, and stores snapshots in SQLite (`monitoring_machine_actions`).
- Retention defaults to 30 days with periodic pruning.
### Alerting
- `backup_alert_engine.py` / `backup_poller.py` generate backup-related alerts (failure, anomaly, missed schedule) and store them in SQLite.
- No general infrastructure alerting (disk full, machine down, high CPU, etc.).
---
## Target Architecture
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ Docker Compose Network │
│ │
│ ┌─────────────┐ scrape ┌──────────────┐ │
│ │ Prometheus │◄────────────────│ Node Exporter│◄── host / remote hosts │
│ │ (TSDB) │ └──────────────┘ │
│ └──────┬──────┘ │
│ │ query │
│ ▼ │
│ ┌─────────────┐ alert ┌─────────────┐ email ┌──────────┐ │
│ │ Grafana │──────────────►│ Alertmanager│──────────────►│ SMTP │ │
│ │ (OAuth) │ └─────────────┘ └──────────┘ │
│ └──────┬──────┘ │
│ │ embed (iframe) │
│ ▼ │
│ ┌─────────────────┐ │
│ │ Manage React │ │
│ │ (summary) │ │
│ └─────────────────┘ │
│ │
│ Logs: │
│ Manage / containers ──stdout──► Promtail/Alloy ──push──► Loki ◄──────┐ │
│ host / remote journald ───────► Promtail/Alloy ──push──► Loki │ │
│ │ │
│ Grafana queries Loki for logs ◄──────────────────────────────────────┘ │
└─────────────────────────────────────────────────────────────────────────────┘
```
### Component Responsibilities
| Component | Responsibility |
|-----------|----------------|
| **Prometheus** | Scrape and store metrics; evaluate alert rules. |
| **Node Exporter** | Expose host-level metrics (CPU, memory, disk, network, filesystem). |
| **Loki** | Store and index log streams by labels. |
| **Promtail / Alloy** | Discover log sources, parse labels, and push to Loki. |
| **Grafana** | Visualize metrics and logs; serve as the alert UI. |
| **Alertmanager** | Deduplicate, group, route, and deliver alerts. |
| **Manage backend** | Emit structured logs; expose `/metrics` for Prometheus; forward health/status to summary endpoints. |
| **Manage frontend** | Embed Grafana panels; show high-level status cards. |
---
## Instrumentation Changes
### Backend Logging
1. Switch to structured JSON logging via `python-json-logger` or `structlog`.
2. Include fields:
- `timestamp`, `level`, `logger`, `message`
- `request_id` (correlation ID propagated from frontend or generated)
- `method`, `path`, `status_code`, `elapsed_ms`
- `user_id`, `machine_id` where relevant
- `error` / `error_type` / `traceback` for exceptions
3. Keep emitting to stdout; Promtail/Alloy will parse JSON.
**Status**: implemented in `backend/src/media_library_viewer_api/logging_utils.py` with `LOG_FORMAT=json|text`, secret-safe `sanitize_log_extra`, and request logging in `main.py`.
### Request Middleware
- Extend `log_requests` to attach `request_id` to `request.state`.
- Include `request_id` in response headers (`X-Request-Id`) so the frontend can correlate.
- Log all outbound SSH commands with `machine_id`, `action`, `duration_ms`, and `request_id`.
**Status**: implemented in `main.py` and `observability.py`; `record_ssh_command` is called from `monitoring_actions.py` for every machine operation.
### Application Metrics Endpoint
- Add a `/metrics` endpoint using `prometheus-client`.
- Initial counters/gauges:
- `manage_api_requests_total` (method, path, status)
- `manage_api_request_duration_seconds` histogram
- `manage_ssh_commands_total` (machine_id, action, status)
- `manage_ssh_command_duration_seconds` histogram
- `manage_media_index_build_duration_seconds`
- `manage_backup_runs_total` (job_name, status)
- `manage_mail_queue_size`, `manage_mail_queue_failures_total`
**Status**: implemented in `backend/src/media_library_viewer_api/observability.py` and wired into `main.py`, `monitoring_actions.py`, `backups.py`, `media.py`, and `mail_queue.py`.
### Frontend Observability
- Keep first phase minimal: capture JS errors and send them to the backend as structured log events.
- Optional later: expose RUM-style metrics (page loads, API call latencies) via Prometheus client library or manual instrumentation.
---
## Node Exporter Deployment
### Local / Docker Host
- Add a `node-exporter` service to `docker-compose.yml` with host PID/network mounts.
- Prometheus scrapes it as `job="node"`.
### Remote Machines
- Add a managed task/template in `jobs.py` to install/upgrade Node Exporter via the package manager or a static binary.
- Manage exposes a settings flag per machine: `node_exporter_enabled`.
- For machines behind NAT, use one of:
1. Reverse SSH tunnel from machine to Manage host.
2. VPN/Wireguard already in place.
3. Prometheus federation or pushgateway for unreachable targets (later phase).
- If Node Exporter cannot be installed, temporarily keep the POSIX collector as a fallback until migration is complete.
---
## Log Shipping
### Docker Compose Services
- Add `logging` driver config or Promtail sidecar to each service.
- Preferred: run **Grafana Alloy** as a single daemon container with `docker_sd_config` to discover all Compose services automatically.
### Host Logs
- Alloy mounts `/var/log` and `/var/lib/docker/containers` (read-only).
- Alloy also tails journald where available.
### Remote Machines
- Option A: install Alloy on remote hosts and have it push logs to Loki.
- Option B: keep logs on remote hosts and use Node Exporter logs only; defer centralized remote logs.
- Recommendation: **Option A** for important machines, **Option B** for constrained ones.
---
## Dashboards
### Grafana
- Provision dashboards from YAML/JSON in version control:
- Node Exporter Full dashboard (import from Grafana.com).
- Manage API overview (request rate, latency, errors).
- Manage operations (SSH commands, media index builds, mail queue).
- Backup runs and alert history.
- Manage iframe embeds point to specific dashboard panels using Grafana's `panelId` and ` kiosk` mode.
### Manage React UI
- Add an "Observability" page with:
- System health cards (Prometheus up, Loki up, Alertmanager up).
- Recent alerts summary from Alertmanager API.
- Iframe panels for key metrics (CPU/memory of selected machine, recent logs).
- Drill-down links open the full Grafana dashboard.
---
## Alerting
### Alertmanager Configuration
- Reuse existing SMTP settings for email notifications.
- Initial routing:
- `severity=critical` → email immediately.
- `severity=warning` → email with 5-minute group wait.
- `job=backup` → grouped by job name.
### Initial Alert Rules
- Infrastructure:
- Node down for > 5 minutes.
- Disk usage > 85% (warning), > 95% (critical).
- Memory usage > 90% for > 10 minutes.
- CPU iowait > 30% for > 10 minutes.
- Application:
- Manage API 5xx rate > 1% over 5 minutes.
- SSH command failure rate > 10% over 5 minutes.
- Mail queue growing or failures increasing.
- Backup:
- Backup job failed (`manage_backup_runs_total{status="failure"}`).
- Backup job missing for > 1.5× schedule interval.
- Backup run duration or size anomaly compared to rolling median.
### Backup Alert Migration
- Re-implement rules as Prometheus recording/alerting rules where possible.
- Keep historical comparison logic (median duration/size) as a small scheduled task that writes anomaly metrics to a Pushgateway or custom exporter, then Alertmanager consumes them.
- Preserve acknowledge/resolve workflow by storing Alertmanager webhook events in SQLite if needed, or by using Grafana alert annotations.
---
## Authentication
- Grafana configured with generic OAuth pointing at Authentik (same issuer as Manage).
- Grafana role mapping: default `Viewer`; admin group mapped to `Admin`.
- Traefik routes `grafana.${BACKEND_APP_HOST}` or a sub-path.
- Iframe embedding requires Grafana `allow_embedding = true` and matching cookie domain/samesite settings.
---
## Retention and Storage
| Store | Retention | Notes |
|-------|-----------|-------|
| Prometheus | 30 days | Default TSDB block compaction. |
| Loki | 30 days | Single-store boltdb-shipper or filesystem target. |
| Grafana | persistent SQLite/Postgres later | Dashboards and users are config, not runtime data. |
- Volumes: `prometheus-data`, `loki-data`, `grafana-data`.
- Backups: snapshot these volumes alongside existing `backend_cache`.
---
## Security
1. Network: all observability services on an internal Docker network; exposed only through Traefik where needed.
2. Node Exporter: bind to localhost on remote hosts and use a reverse tunnel, or firewall to Manage IP only.
3. Secrets: SMTP password, OIDC client secret, and any remote scrape credentials in environment variables or Docker secrets; never commit them.
4. Logs: sanitize tokens, passwords, and private keys before JSON serialization.
5. Alertmanager: disable unauthenticated UI if exposed publicly; rely on OIDC/Traefik.
---
## Implementation Plan
### Phase 0 — Foundation and Cleanup
- [x] Add `prometheus-client` and `python-json-logger` to `backend/pyproject.toml`.
- [x] Refactor `logging_utils.py` to emit JSON when `LOG_FORMAT=json`.
- [x] Add `request_id` propagation in `log_requests` middleware.
- [x] Add `/metrics` endpoint with initial counters/gauges.
- [x] Remove the POSIX remote collector code in `resources.py`; keep `disk_space` as a lightweight SSH/local helper in `monitoring_actions.py`.
- [x] Add `X-Request-Id` response header.
### Phase 1 — Local Observability Stack
- [x] Add services to `docker-compose.yml`: Prometheus, Loki, Grafana, Alertmanager, Grafana Alloy.
- [x] Add `node-exporter` service for the Docker host.
- [x] Configure Alloy to scrape all Docker container logs and ship to Loki.
- [x] Configure Prometheus to scrape `node-exporter` and Manage `/metrics`.
- [x] Provision Grafana datasources and a basic Manage API dashboard.
- [x] Wire Grafana OAuth to Authentik.
**Phase 1 files**:
- `monitoring/prometheus/prometheus.yml`
- `monitoring/prometheus/rules/backup_alerts.yml`
- `monitoring/loki/loki.yml`
- `monitoring/alloy/config.alloy`
- `monitoring/alertmanager/alertmanager.yml`
- `monitoring/grafana/grafana.ini`
- `monitoring/grafana/provisioning/datasources/datasources.yml`
- `monitoring/grafana/provisioning/dashboards-json/dashboards.yml`
- `monitoring/grafana/provisioning/dashboards-json/dashboards/manage-overview.json`
- `docker-compose.yml` and `docker-compose.dev.yml` updated with observability services.
- `frontend/vite.config.ts` updated with `/grafana` dev proxy.
- `.env.example` updated with Grafana OAuth and alerting variables.
### Phase 2 — Remote Machine Metrics
- [x] Add Node Exporter install/restart/status job templates in `jobs.py` (`install_node_exporter`, `restart_node_exporter`, `node_exporter_status`).
- [x] Add `node_exporter_enabled`, `node_exporter_port`, and `node_exporter_scrape_host` fields to `MonitoringMachineInput` and `SettingsStore`.
- [x] Implement `media_library_viewer_api.services.targets` to build Prometheus file-SD target lists for enabled SSH machines and write them to `PROMETHEUS_FILE_SD_DIR/node_exporter_targets.json`.
- [x] Regenerate file-SD targets on machine create/update/delete in `routers/settings.py`.
- [x] Add `/api/monitoring/prometheus-targets` endpoint returning live targets from the store.
- [x] Configure Prometheus `node-exporter-remote` job with `file_sd_configs` reading `/etc/prometheus/file-sd/node_exporter_targets.json`.
- [x] Mount the backend cache `prometheus-file-sd` directory into the Prometheus container as a read-only file-SD source.
- [x] Add `PROMETHEUS_FILE_SD_DIR` setting and `.env.example` entry.
- [x] Provision a minimal `Node Exporter Overview` Grafana dashboard (`monitoring/grafana/provisioning/dashboards-json/dashboards/node-exporter-overview.json`) covering CPU, memory, root disk, and network traffic.
- [x] Remove POSIX collector fallback. The legacy collector code in `backend/src/media_library_viewer_api/clients/resources.py` has been deleted, the collector control endpoints were removed from `routers/monitoring.py`, and `disk_space` was relocated to `services/monitoring_actions.py` as a lightweight SSH/local helper. Metrics are now sourced exclusively from Prometheus/Node Exporter.
**Phase 2 files**:
- `backend/src/media_library_viewer_api/jobs.py` (Node Exporter job templates).
- `backend/src/media_library_viewer_api/routers/settings.py` (machine input fields + target regeneration).
- `backend/src/media_library_viewer_api/services/settings_store.py` (machine persistence fields).
- `backend/src/media_library_viewer_api/services/targets.py` (file-SD target builder/writer).
- `backend/src/media_library_viewer_api/routers/monitoring.py` (`/prometheus-targets` endpoint).
- `backend/src/media_library_viewer_api/config.py` (`prometheus_file_sd_dir` setting).
- `backend/tests/test_targets.py` and `backend/tests/test_api.py` (target + endpoint tests).
- `monitoring/prometheus/prometheus.yml` (`node-exporter-remote` file SD job).
- `monitoring/grafana/provisioning/dashboards-json/dashboards/node-exporter-overview.json`.
- `docker-compose.yml` and `docker-compose.dev.yml` (file-SD volume mount + backend env var).
- `.env.example` (`PROMETHEUS_FILE_SD_DIR`).
### Phase 3 — Alerting
- [x] Define initial Prometheus alert rules for backup failures (infrastructure rules deferred to Phase 2/3).
- [x] Configure Alertmanager with email routing using existing SMTP settings; `monitoring/alertmanager/alertmanager.yml` uses env vars for SMTP and routing.
- [x] Migrate backup alert rules to Alertmanager:
- `BackupJobFailed` triggers on `increase(manage_backup_runs_total{status="failed"}[1h]) > 0`.
- `BackupJobStuck` triggers on `time() - manage_backup_runs_last_success_timestamp > 86400`.
- Added `manage_backup_runs_last_success_timestamp` gauge in `observability.py` and updated `routers/backups.py` to set it on successful runs.
- SQLite backup alerts (`backup_alert_engine.py` and `backup_poller.py`) are preserved for now alongside Alertmanager rules; the UI can consume either source during transition.
- [x] Add Alertmanager status summary endpoints in Manage backend:
- `GET /api/monitoring/alerts` proxies `/api/v1/alerts` and returns a UI-friendly summary (total, by_severity, alerts list).
- `GET /api/monitoring/alertmanager-status` proxies `/api/v2/status` and returns `up`, `version`, `uptime`, `peers`.
- [x] Added `alertmanager_url` setting to `config.py` (default `http://alertmanager:9093`) and `ALERTMANAGER_URL` env var in both compose files and `.env.example`.
- [x] Added tests for the Alertmanager endpoints and the backup success gauge.
**Phase 3 files**:
- `backend/src/media_library_viewer_api/routers/monitoring.py` (`/alerts` and `/alertmanager-status` endpoints).
- `backend/src/media_library_viewer_api/observability.py` (`BACKUP_RUNS_LAST_SUCCESS` gauge + updated `record_backup_run`).
- `backend/src/media_library_viewer_api/routers/backups.py` (pass `success=True` to `record_backup_run` on successful reports).
- `backend/src/media_library_viewer_api/config.py` (`alertmanager_url` setting).
- `monitoring/alertmanager/alertmanager.yml` (SMTP + routing config).
- `monitoring/prometheus/rules/backup_alerts.yml` (backup alert rules).
- `docker-compose.yml` / `docker-compose.dev.yml` (`ALERTMANAGER_URL` env var).
- `.env.example` (`ALERTMANAGER_URL`).
- `backend/tests/test_api.py` (`TestAlertmanager` tests).
- `backend/tests/test_observability.py` (backup metric tests).
### Phase 4 — Manage UI Integration
- [x] Add "Observability" page in React with summary cards (Alertmanager health, active alerts, Prometheus targets, machines) and Grafana iframe panels.
- [x] Add recent alerts list from Alertmanager API via `GET /api/monitoring/alerts`.
- [x] Add drill-down links to full Grafana dashboards for Node Exporter metrics and Loki logs.
- [x] Handle iframe sandbox attributes (`allow-scripts allow-same-origin allow-popups allow-forms`); CSP is delegated to the reverse proxy / Grafana `allow_embedding` configuration.
- [x] Add `useObservability` hook and API client wrappers for alerts, Alertmanager status, and Prometheus targets.
- [x] Add TypeScript types for Alertmanager summary/status and Prometheus targets.
- [x] Wire the new `/observability` route into `App.tsx` and the sidebar navigation.
**Phase 4 files**:
- `frontend/src/components/ObservabilityPage.tsx` (page component).
- `frontend/src/hooks/useObservability.ts` (React Query hooks).
- `frontend/src/api/client.ts` (API client functions).
- `frontend/src/types/index.ts` (new interfaces).
- `frontend/src/App.tsx` (route + nav item).
- `frontend/src/components/ui/{card,badge,alert,skeleton,select}.tsx` (shadcn/ui components).
- `frontend/vite.config.ts` already has `/grafana` dev proxy for iframe source.
### Phase 5 — Hardening and Future-Proofing
- [x] Add health checks and `deploy.resources` limits for Prometheus, Loki, Alloy, Grafana, Alertmanager, and Node Exporter in both compose files.
- [x] Use `depends_on` with `condition: service_healthy` for Alloy → Loki and Grafana → Prometheus/Loki.
- [x] Add Prometheus scrape jobs for Loki, Alertmanager, and Grafana so their `up` metrics are available for health alerts.
- [x] Add observability health alerting rules (`PrometheusTargetMissing`, `AlertmanagerDown`, `GrafanaDown`).
- [x] Add `ALERTMANAGER_WEBHOOK_URL` backend setting, `POST /api/monitoring/alertmanager-webhook` receiver, and Alertmanager `webhook` receiver config.
- [x] Document runbooks for common alerts.
- [x] Add volume backups for Prometheus/Loki/Grafana data. Backup/restore procedures for `prometheus_data`, `loki_data`, `grafana_data`, and `alertmanager_data` are documented in `docs/observability-runbooks.md`.
- [ ] Optional: add OpenTelemetry Collector as a translation layer for traces later.
**Phase 5 files**:
- `docker-compose.yml` and `docker-compose.dev.yml` (health checks, resource limits, `depends_on` conditions).
- `monitoring/prometheus/prometheus.yml` (additional scrape jobs for observability services).
- `monitoring/prometheus/rules/backup_alerts.yml` (renamed scope to include observability health alerts).
- `backend/src/media_library_viewer_api/config.py` (`alertmanager_webhook_url` setting).
- `backend/src/media_library_viewer_api/routers/monitoring.py` (`POST /api/monitoring/alertmanager-webhook`).
- `monitoring/alertmanager/alertmanager.yml` (`webhook` receiver).
- `backend/tests/test_api.py` (`TestAlertmanagerWebhook`).
- `docs/observability-runbooks.md` (new runbook documentation).
---
## Risks and Mitigations
| Risk | Mitigation |
|------|------------|
| Node Exporter hard to install on NAS/minimal hosts | Keep POSIX collector as opt-in fallback; document manual install steps. |
| Grafana iframe embedding blocked by CSP or cookies | Test early in Phase 4; use matching domains and `allow_embedding`. |
| Prometheus storage grows faster than expected | Start with 30-day retention; add compaction alerts. |
| Backup alert semantics lost in migration | Write tests comparing old Python alerts vs new Alertmanager rules. |
| OIDC configuration drift between Manage and Grafana | Use same env vars/Authentik application for both. |
| Remote scrape requires network path | Use reverse SSH tunnels or defer remote scraping until VPN is ready. |
---
## Open Questions
1. ~~What sub-domain or sub-path should Grafana use? (`grafana.${BACKEND_APP_HOST}` vs `${BACKEND_APP_HOST}/grafana`)~~ Decided: dedicated `GRAFANA_APP_HOST` subdomain in production; dev uses port 3000 and a `/grafana` proxy in Vite.
2. Should remote machines run Node Exporter as a systemd service or a container?
3. Do we need remote log aggregation immediately, or can it wait until after metrics alerting is stable?
4. Should the existing backup alert acknowledgement/resolve UI be rebuilt on top of Alertmanager, or replaced by Grafana alert annotations?
docs/observability-runbooks.md
+190
View File
@@ -0,0 +1,190 @@
# Observability Runbooks
Operational playbooks for the Manage self-hosted observability stack (Prometheus, Grafana, Loki, Alertmanager).
## Service Overview
| Service | Compose name | Internal URL | Health check |
|---------|--------------|--------------|--------------|
| Prometheus | `prometheus` | `http://prometheus:9090` | `/-/healthy` |
| Grafana | `grafana` | `http://grafana:3000` | `/api/health` |
| Loki | `loki` | `http://loki:3100` | `/ready` |
| Alloy | `alloy` | `http://alloy:12345` | `/-/healthy` |
| Alertmanager | `alertmanager` | `http://alertmanager:9093` | `/-/healthy` |
| Node Exporter | `node-exporter` | `http://node-exporter:9100` | `/` |
| Manage backend | `backend` | `http://backend:8000` | `/api/health` |
---
## Alert: `BackupJobFailed`
**Severity**: critical
**Meaning**: A backup job reported `status=failed` within the last hour.
### Steps
1. Open **Manage → Backups** and identify the failed job/run.
2. Check the run output / logs for the failure reason.
3. Search Loki for `{container="backend"} | json | message=~"(?i)backup"` around the failure time.
4. If transient (network, lock file), retry the job.
5. If persistent, open a task to fix the backup script or credentials.
---
## Alert: `BackupJobStuck`
**Severity**: warning
**Meaning**: No successful backup run has been recorded for a job in the last 24 hours.
### Steps
1. Confirm the job is still scheduled and expected to run.
2. Check whether the backup scheduler/host is running.
3. Verify the job can still report success to `POST /api/backups/reports`.
4. Inspect Prometheus graph for `manage_backup_runs_last_success_timestamp` by `job_name`.
5. If the job was intentionally retired, remove or disable its reporting.
---
## Alert: `PrometheusTargetMissing`
**Severity**: warning
**Meaning**: A Prometheus scrape target is down (`up == 0`) for more than 2 minutes.
### Steps
1. Identify `job` and `instance` from the alert labels.
2. Check the container/process status:
- `docker compose ps <service>`
- `docker compose logs --tail 100 <service>`
3. Verify network reachability from the Prometheus container:
- `docker compose exec prometheus wget -qO- http://<instance>/`
4. If the target is a remote Node Exporter:
- Check the machine is reachable over SSH.
- Verify Node Exporter is installed and running (`systemctl status node_exporter`).
- Confirm the scrape host/port in Manage → Settings for that machine.
5. Restart if needed: `docker compose restart <service>`.
---
## Alert: `AlertmanagerDown`
**Severity**: critical
**Meaning**: Prometheus cannot scrape Alertmanager; new alerts may not be delivered.
### Steps
1. Check container status: `docker compose ps alertmanager`
2. Review logs: `docker compose logs --tail 200 alertmanager`
3. Validate config syntax:
- `docker compose exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml`
4. Verify SMTP environment variables are present if using email receivers.
5. Restart: `docker compose restart alertmanager`
---
## Alert: `GrafanaDown`
**Severity**: warning
**Meaning**: Grafana is unreachable; dashboards and iframe panels in Manage are unavailable.
### Steps
1. Check container status and logs.
2. Verify the OAuth client configuration is correct (`GF_AUTH_GENERIC_OAUTH_*`).
3. If embedded panels are blank, confirm Grafana `allow_embedding = true` and cookie settings.
4. Restart: `docker compose restart grafana`
---
## Routine Maintenance
### Check overall health
```bash
cd /path/to/manage
docker compose ps
docker compose exec prometheus wget -qO- http://127.0.0.1:9090/-/healthy
docker compose exec grafana wget -qO- http://127.0.0.1:3000/api/health
docker compose exec loki wget -qO- http://127.0.0.1:3100/ready
docker compose exec alertmanager wget -qO- http://127.0.0.1:9093/-/healthy
```
### Reload Prometheus after rule/config changes
Prometheus is started with `--web.enable-lifecycle`, so a SIGHUP or HTTP call reloads config:
```bash
curl -X POST http://localhost:9090/-/reload
```
### Inspect logs
```bash
# All backend logs in Loki via Grafana Explore, or locally:
docker compose logs --tail 500 backend
# Specific service:
docker compose logs -f prometheus
```
### Storage usage
```bash
docker system df -v
docker compose exec prometheus du -sh /prometheus
docker compose exec loki du -sh /loki
docker compose exec grafana du -sh /var/lib/grafana
```
---
## Backup and Disaster Recovery
The observability data lives in named volumes:
- `prometheus_data`
- `loki_data`
- `grafana_data`
- `alertmanager_data`
### Backup volumes
```bash
# Stop the stack to ensure consistency
docker compose down
# Back up each volume to a tarball
docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \
tar czf /backups/prometheus-$(date +%F).tar.gz -C /data .
docker run --rm -v manage_loki_data:/data -v $(pwd)/backups:/backups alpine \
tar czf /backups/loki-$(date +%F).tar.gz -C /data .
docker run --rm -v manage_grafana_data:/data -v $(pwd)/backups:/backups alpine \
tar czf /backups/grafana-$(date +%F).tar.gz -C /data .
docker run --rm -v manage_alertmanager_data:/data -v $(pwd)/backups:/backups alpine \
tar czf /backups/alertmanager-$(date +%F).tar.gz -C /data .
# Start the stack again
docker compose up -d
```
> Replace `manage_` with your actual Docker Compose project name if different.
### Restore a volume
```bash
docker compose down
docker volume rm manage_prometheus_data
docker volume create manage_prometheus_data
docker run --rm -v manage_prometheus_data:/data -v $(pwd)/backups:/backups alpine \
tar xzf /backups/prometheus-YYYY-MM-DD.tar.gz -C /data
docker compose up -d
```
---
## Scaling Notes
- The current `deploy.resources` blocks are tuned for a small homelab. Raise memory limits if you monitor many machines or retain logs longer than 30 days.
- Loki is configured for single-node filesystem storage. For larger deployments, migrate to object storage (S3/GCS/MinIO) and a shared index.
- Prometheus remote-write or Thanos/Cortex can be added later for long-term metrics without changing application instrumentation.
frontend/package-lock.json
Generated
+8 -50
View File
@@ -35,7 +35,7 @@
"@tailwindcss/postcss": "^4.3.0",
"@tailwindcss/vite": "^4.3.0",
"@types/d3": "^7.4.3",
"@types/node": "^24.12.2",
"@types/node": "^24.13.2",
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^6.0.1",
@@ -3375,9 +3375,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3395,9 +3392,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3415,9 +3409,6 @@
"ppc64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3435,9 +3426,6 @@
"s390x"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3455,9 +3443,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3475,9 +3460,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3737,9 +3719,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3757,9 +3736,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3777,9 +3753,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -3797,9 +3770,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -4254,12 +4224,12 @@
"license": "MIT"
},
"node_modules/@types/node": {
"version": "24.12.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz",
"integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==",
"version": "24.13.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.2.tgz",
"integrity": "sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==",
"license": "MIT",
"dependencies": {
"undici-types": "~7.16.0"
"undici-types": "~7.18.0"
}
},
"node_modules/@types/parse-json": {
@@ -7532,9 +7502,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7556,9 +7523,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7580,9 +7544,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7604,9 +7565,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -9774,9 +9732,9 @@
}
},
"node_modules/undici-types": {
"version": "7.16.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
"version": "7.18.2",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz",
"integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
"license": "MIT"
},
"node_modules/unicorn-magic": {
frontend/package.json
+1 -1
View File
@@ -37,7 +37,7 @@
"@tailwindcss/postcss": "^4.3.0",
"@tailwindcss/vite": "^4.3.0",
"@types/d3": "^7.4.3",
"@types/node": "^24.12.2",
"@types/node": "^24.13.2",
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^6.0.1",
frontend/src/App.tsx
+11 -12
View File
@@ -5,6 +5,7 @@ import {
NavLink,
useLocation,
Outlet,
Navigate,
} from "react-router-dom";
import {
QueryClient,
@@ -14,13 +15,13 @@ import {
import { useEffect, useMemo, useState } from "react";
import { AuthProvider, useAuth } from "react-oidc-context";
import { Dashboard } from "./pages/Dashboard";
import { Monitoring } from "./pages/Monitoring";
import { Applications } from "./pages/Applications";
import { Settings } from "./pages/Settings";
import { UsersPage } from "./pages/Users";
import { FileBrowser } from "./pages/FileBrowser";
import { Actions } from "./pages/Actions";
import BackupsPage from "./components/BackupsPage";
import { ObservabilityPage } from "./components/ObservabilityPage";
import { getOidcConfig, isOidcConfigured, setAccessToken } from "./auth";
import { fetchAppVersion } from "./api/client";
import { FRONTEND_VERSION_LABEL } from "./version";
@@ -80,7 +81,7 @@ function useDarkMode() {
// Navigation items for sidebar
const navItems = [
{ path: "/", label: "Dashboard", icon: LayoutDashboard },
{ path: "/monitoring", label: "Monitoring", icon: Activity },
{ path: "/observability", label: "Observability", icon: Activity },
{ path: "/applications", label: "Media", icon: Monitor },
{ path: "/files", label: "Files", icon: FolderOpen },
{ path: "/users", label: "Users", icon: Users },
@@ -114,9 +115,7 @@ function Sidebar({
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-primary font-bold text-primary-foreground">
M
</div>
{!collapsed && (
<span className="font-semibold">Manage</span>
)}
{!collapsed && <span className="font-semibold">Manage</span>}
</div>
<Button
variant="ghost"
@@ -155,9 +154,7 @@ function Sidebar({
<Icon className="h-5 w-5" />
</NavLink>
</TooltipTrigger>
<TooltipContent side="right">
{item.label}
</TooltipContent>
<TooltipContent side="right">{item.label}</TooltipContent>
</Tooltip>
) : (
<NavLink
@@ -313,8 +310,8 @@ function ShellLayout({
onToggleDarkMode: () => void;
}) {
const [sidebarCollapsed, setSidebarCollapsed] = useState(false);
const [isMobile, setIsMobile] = useState(() =>
window.matchMedia("(max-width: 768px)").matches,
const [isMobile, setIsMobile] = useState(
() => window.matchMedia("(max-width: 768px)").matches,
);
useEffect(() => {
@@ -435,13 +432,14 @@ function AppInner() {
<Routes>
<Route element={<AuthenticatedApp />}>
<Route path="/" element={<Dashboard />} />
<Route path="/monitoring" element={<Monitoring />} />
<Route path="/monitoring" element={<Navigate to="/observability" replace />} />
<Route path="/applications" element={<Applications />} />
<Route path="/media" element={<Applications />} />
<Route path="/users" element={<UsersPage />} />
<Route path="/actions" element={<Actions />} />
<Route path="/files" element={<FileBrowser />} />
<Route path="/backups" element={<BackupsPage />} />
<Route path="/observability" element={<ObservabilityPage />} />
<Route path="/settings" element={<Settings />} />
</Route>
</Routes>
@@ -459,13 +457,14 @@ function AppInner() {
}
>
<Route path="/" element={<Dashboard />} />
<Route path="/monitoring" element={<Monitoring />} />
<Route path="/monitoring" element={<Navigate to="/observability" replace />} />
<Route path="/applications" element={<Applications />} />
<Route path="/media" element={<Applications />} />
<Route path="/users" element={<UsersPage />} />
<Route path="/actions" element={<Actions />} />
<Route path="/files" element={<FileBrowser />} />
<Route path="/backups" element={<BackupsPage />} />
<Route path="/observability" element={<ObservabilityPage />} />
<Route path="/settings" element={<Settings />} />
</Route>
</Routes>
frontend/src/api/client.ts
+13 -54
View File
@@ -10,12 +10,7 @@ import type {
UserMessageResponse,
UserMessageQueueStatus,
NowPlayingSession,
MonitoringPollerStatus,
MonitoringOverviewResponse,
AppVersionInfo,
MonitoringStatus,
MonitoringMetrics,
DiskSpace,
SSHKey,
SSHKeyInput,
SSHKeyGenerated,
@@ -24,7 +19,6 @@ import type {
SavedTaskRun,
MonitoringMachine,
MonitoringMachineInput,
MonitoringMachineAction,
MediaIndexStatus,
MediaIndexActionResponse,
MediaQueryResponse,
@@ -37,6 +31,9 @@ import type {
SSHValidationResult,
DashboardShortcut,
DashboardShortcutInput,
AlertmanagerAlertSummary,
AlertmanagerStatus,
PrometheusTarget,
} from "../types";
const BASE_URL = import.meta.env.VITE_API_URL || "/api";
@@ -165,11 +162,7 @@ export const fetchNowPlaying = fetchActivity;
// Monitoring
export const fetchMonitoringMachines = () =>
get<MonitoringMachine[]>("/api/monitoring/machines");
export const fetchMonitoringPoller = () =>
get<MonitoringPollerStatus>("/api/monitoring/poller");
export const fetchAppVersion = () => get<AppVersionInfo>("/api/version");
export const fetchMonitoringOverview = () =>
get<MonitoringOverviewResponse>("/api/dashboard/monitoring");
export const fetchDashboardShortcuts = () =>
get<DashboardShortcut[]>("/api/dashboard/shortcuts");
export const saveDashboardShortcut = (shortcut: DashboardShortcutInput) =>
@@ -194,45 +187,6 @@ export const deleteDashboardShortcut = (shortcutId: string) =>
del<{ status: string }>(
`/api/dashboard/shortcuts/${encodeURIComponent(shortcutId)}`,
);
export const fetchMonitoringStatus = (machineId?: string) =>
get<MonitoringStatus>(
"/api/monitoring/status",
machineId ? { machine_id: machineId } : undefined,
);
export const fetchMonitoringMetrics = (
lastSeconds?: number | null,
maxLines = 70_000,
machineId?: string,
) =>
get<MonitoringMetrics>("/api/monitoring/metrics", {
...(lastSeconds == null ? {} : { last_seconds: String(lastSeconds) }),
max_lines: String(maxLines),
...(machineId ? { machine_id: machineId } : {}),
});
export const fetchDiskSpace = (machineId?: string) =>
get<DiskSpace>(
"/api/monitoring/disk",
machineId ? { machine_id: machineId } : undefined,
);
export const startCollector = (machineId?: string) =>
post<{ message: string }>(
machineId
? `/api/monitoring/start?machine_id=${encodeURIComponent(machineId)}`
: "/api/monitoring/start",
);
export const stopCollector = (machineId?: string) =>
post<{ message: string }>(
machineId
? `/api/monitoring/stop?machine_id=${encodeURIComponent(machineId)}`
: "/api/monitoring/stop",
);
export const restartCollector = (machineId?: string) =>
post<{ message: string }>(
machineId
? `/api/monitoring/restart?machine_id=${encodeURIComponent(machineId)}`
: "/api/monitoring/restart",
);
export const fetchMonitoringSettings = () =>
get<MonitoringMachine[]>("/api/settings/machines");
export const fetchSSHKeys = () => get<SSHKey[]>("/api/settings/ssh-keys");
@@ -242,11 +196,6 @@ export const generateSSHKey = (payload: {
notes: string;
bits?: number;
}) => post<SSHKeyGenerated>("/api/settings/ssh-keys/generate", payload);
export const fetchMonitoringMachineActions = (machineId: string, limit = 10) =>
get<{ items: MonitoringMachineAction[]; total: number }>(
`/api/monitoring/machines/${encodeURIComponent(machineId)}/actions`,
{ limit: String(limit) },
);
export const saveSSHKey = (key: SSHKeyInput) =>
fetch(
buildUrl(
@@ -428,3 +377,13 @@ export const fetchUserMessageQueueStatus = () =>
export const sendUserMessage = (formData: FormData) =>
postForm<UserMessageResponse>("/api/users/message", formData);
// Observability summary endpoints
export const fetchAlertmanagerAlerts = () =>
get<AlertmanagerAlertSummary>("/api/monitoring/alerts");
export const fetchAlertmanagerStatus = () =>
get<AlertmanagerStatus>("/api/monitoring/alertmanager-status");
export const fetchPrometheusTargets = () =>
get<PrometheusTarget[]>("/api/monitoring/prometheus-targets");
frontend/src/components/MachineMonitoringSection.tsx
-354
View File
@@ -1,354 +0,0 @@
import { useMemo, useState } from "react";
import {
Alert,
Box,
Button,
Chip,
FormControl,
Grid,
InputLabel,
MenuItem,
Paper,
Select,
Stack,
Table,
TableBody,
TableCell,
TableContainer,
TableHead,
TableRow,
Typography,
} from "@mui/material";
import { MetricCard } from "./MetricCard";
import { MonitoringCharts } from "./MonitoringCharts";
import type { MonitoringMachine } from "../types";
import {
useCollectorControls,
useDiskSpace,
useMachineActions,
useMonitoringMetrics,
useMonitoringStatus,
} from "../hooks/useMonitoring";
function formatBytes(bytes: number): string {
if (!bytes || bytes === 0) return "0 B";
const units = ["B", "KB", "MB", "GB", "TB"];
let value = bytes;
let unitIdx = 0;
while (value >= 1000 && unitIdx < units.length - 1) {
value /= 1000;
unitIdx++;
}
return `${value.toFixed(1)} ${units[unitIdx]}`;
}
function formatRate(bytes: number): string {
return `${formatBytes(bytes)}/s`;
}
function avg(arr: number[]) {
return arr.length ? arr.reduce((a, b) => a + b, 0) / arr.length : 0;
}
function max(arr: number[]) {
return arr.length ? Math.max(...arr) : 0;
}
function formatActionTime(epochSeconds: number): string {
return new Date(epochSeconds * 1000).toLocaleString();
}
function ErrorBanner({ label, error }: { label: string; error: unknown }) {
if (!error) return null;
return (
<Alert severity="error">
{label}: {String(error)}
</Alert>
);
}
export function MachineMonitoringSection({
machine,
}: {
machine: MonitoringMachine;
}) {
const statusQuery = useMonitoringStatus(machine.id, machine.enabled);
const metricsQuery = useMonitoringMetrics(machine.id, machine.enabled);
const diskQuery = useDiskSpace(machine.id, machine.enabled);
const { start, stop, restart } = useCollectorControls(machine.id);
const actionsQuery = useMachineActions(machine.id, machine.enabled);
const [actionFilter, setActionFilter] = useState("all");
const [resultFilter, setResultFilter] = useState("all");
const status = statusQuery.data;
const metrics = metricsQuery.data;
const disk = diskQuery.data;
const samples = metrics?.samples ?? [];
const latest = samples.at(-1);
const cpuArr = samples.map((s) => s.cpu_pct);
const iowArr = samples.map((s) => s.iowait_pct ?? 0);
const memArr = samples.map((s) => s.mem_pct);
const netDownArr = samples.map((s) => s.net_rx_bytes_per_sec);
const netUpArr = samples.map((s) => s.net_tx_bytes_per_sec);
const diskReadArr = samples.map((s) => s.disk_read_bps);
const diskWriteArr = samples.map((s) => s.disk_write_bps);
const actions = actionsQuery.data?.items ?? [];
const visibleActions = useMemo(
() =>
actions.filter((action) => {
const actionMatches =
actionFilter === "all" || action.action === actionFilter;
const resultMatches =
resultFilter === "all" || action.status === resultFilter;
return actionMatches && resultMatches;
}),
[actions, actionFilter, resultFilter],
);
const hasQueryError =
statusQuery.error ||
metricsQuery.error ||
diskQuery.error ||
actionsQuery.error;
return (
<Stack spacing={2}>
<Stack
direction="row"
spacing={1.5}
sx={{ alignItems: "center", flexWrap: "wrap" }}
>
<Box>
<Typography variant="h6" sx={{ fontWeight: 700 }}>
{machine.name}
</Typography>
<Typography variant="caption" color="text.secondary">
{machine.mode === "local"
? "Local API host"
: `${machine.username || "user"}@${machine.host || "host"}:${machine.port}`}
{machine.media_root ? ` · media root ${machine.media_root}` : ""}
</Typography>
</Box>
<Chip label={machine.mode} variant="outlined" />
<Chip
label={machine.enabled ? "Enabled" : "Disabled"}
color={machine.enabled ? "success" : "default"}
variant="outlined"
/>
<Chip
label={status?.status ?? "unknown"}
color={status?.status?.includes("running") ? "success" : "primary"}
variant="outlined"
/>
<Button
size="small"
variant="outlined"
onClick={() => start.mutate()}
disabled={start.isPending || !machine.enabled}
>
Start
</Button>
<Button
size="small"
variant="outlined"
onClick={() => restart.mutate()}
disabled={restart.isPending || !machine.enabled}
>
Restart
</Button>
<Button
size="small"
variant="outlined"
onClick={() => stop.mutate()}
disabled={stop.isPending || !machine.enabled}
>
Stop
</Button>
</Stack>
{hasQueryError && (
<Stack spacing={1}>
<ErrorBanner label="Status" error={statusQuery.error} />
<ErrorBanner label="Metrics" error={metricsQuery.error} />
<ErrorBanner label="Disk" error={diskQuery.error} />
<ErrorBanner label="Recent activity" error={actionsQuery.error} />
</Stack>
)}
<Grid container spacing={1.5}>
<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
<MetricCard
label="CPU now"
value={latest ? `${latest.cpu_pct.toFixed(1)}%` : "-"}
subtext={`avg ${avg(cpuArr).toFixed(1)}%\npeak ${max(cpuArr).toFixed(1)}%`}
/>
</Grid>
<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
<MetricCard
label="IO Wait"
value={latest ? `${(latest.iowait_pct ?? 0).toFixed(1)}%` : "-"}
subtext={`avg ${avg(iowArr).toFixed(1)}%\npeak ${max(iowArr).toFixed(1)}%`}
/>
</Grid>
<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
<MetricCard
label="RAM now"
value={latest ? `${latest.mem_pct.toFixed(1)}%` : "-"}
subtext={`avg ${avg(memArr).toFixed(1)}%\npeak ${max(memArr).toFixed(1)}%`}
/>
</Grid>
<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
<MetricCard
label="Net down"
value={latest ? formatRate(latest.net_rx_bytes_per_sec) : "-"}
subtext={`avg ${formatRate(avg(netDownArr))}\npeak ${formatRate(max(netDownArr))}`}
/>
</Grid>
<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
<MetricCard
label="Net up"
value={latest ? formatRate(latest.net_tx_bytes_per_sec) : "-"}
subtext={`avg ${formatRate(avg(netUpArr))}\npeak ${formatRate(max(netUpArr))}`}
/>
</Grid>
<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
<MetricCard
label="Disk read"
value={latest ? formatRate(latest.disk_read_bps) : "-"}
subtext={`avg ${formatRate(avg(diskReadArr))}\npeak ${formatRate(max(diskReadArr))}`}
/>
</Grid>
<Grid size={{ xs: 6, md: 3, lg: 12 / 7 }}>
<MetricCard
label="Disk write"
value={latest ? formatRate(latest.disk_write_bps) : "-"}
subtext={`avg ${formatRate(avg(diskWriteArr))}\npeak ${formatRate(max(diskWriteArr))}`}
/>
</Grid>
</Grid>
{disk && (
<Grid container spacing={1.5}>
<Grid size={{ xs: 6, md: 3 }}>
<MetricCard label="Disk used" value={formatBytes(disk.used)} />
</Grid>
<Grid size={{ xs: 6, md: 3 }}>
<MetricCard
label="Disk available"
value={formatBytes(disk.available)}
/>
</Grid>
<Grid size={{ xs: 6, md: 3 }}>
<MetricCard label="Disk total" value={formatBytes(disk.size)} />
</Grid>
<Grid size={{ xs: 6, md: 3 }}>
<MetricCard label="Used %" value={disk.used_pct} />
</Grid>
</Grid>
)}
<Box>
<MonitoringCharts samples={samples} />
</Box>
<Stack spacing={1.5}>
<Stack
direction="row"
spacing={1}
sx={{ alignItems: "center", flexWrap: "wrap" }}
>
<Box>
<Typography variant="subtitle1" sx={{ fontWeight: 700 }}>
Recent activity
</Typography>
<Typography variant="caption" color="text.secondary">
Collected automatically by the backend poller.
</Typography>
</Box>
<Chip
label={`${visibleActions.length}/${actions.length || 0}`}
size="small"
variant="outlined"
/>
<FormControl size="small" sx={{ minWidth: 150 }}>
<InputLabel>Action</InputLabel>
<Select
label="Action"
value={actionFilter}
onChange={(e) => setActionFilter(e.target.value as string)}
>
<MenuItem value="all">All actions</MenuItem>
<MenuItem value="status lookup">Status</MenuItem>
<MenuItem value="metrics read">Metrics</MenuItem>
<MenuItem value="disk lookup">Disk</MenuItem>
<MenuItem value="collector start">Start</MenuItem>
<MenuItem value="collector stop">Stop</MenuItem>
<MenuItem value="collector restart">Restart</MenuItem>
<MenuItem value="collector diagnostics">Diagnostics</MenuItem>
</Select>
</FormControl>
<FormControl size="small" sx={{ minWidth: 120 }}>
<InputLabel>Result</InputLabel>
<Select
label="Result"
value={resultFilter}
onChange={(e) => setResultFilter(e.target.value as string)}
>
<MenuItem value="all">All results</MenuItem>
<MenuItem value="ok">OK</MenuItem>
<MenuItem value="error">Error</MenuItem>
</Select>
</FormControl>
</Stack>
{actionsQuery.error && (
<Alert severity="error">
Recent activity: {String(actionsQuery.error)}
</Alert>
)}
<TableContainer component={Paper} variant="outlined">
<Table size="small">
<TableHead>
<TableRow>
<TableCell>Time</TableCell>
<TableCell>Action</TableCell>
<TableCell>Status</TableCell>
<TableCell align="right">Duration</TableCell>
<TableCell>Message</TableCell>
</TableRow>
</TableHead>
<TableBody>
{visibleActions.length === 0 ? (
<TableRow>
<TableCell colSpan={5}>
<Typography variant="body2" color="text.secondary">
No activity matches the current filters.
</Typography>
</TableCell>
</TableRow>
) : (
visibleActions.map((action) => (
<TableRow
key={`${action.machine_id}-${action.created_at}-${action.action}-${action.status}`}
>
<TableCell>{formatActionTime(action.created_at)}</TableCell>
<TableCell>{action.action}</TableCell>
<TableCell>
<Chip
size="small"
label={action.status}
color={action.status === "ok" ? "success" : "error"}
variant="outlined"
/>
</TableCell>
<TableCell align="right">{action.duration_ms} ms</TableCell>
<TableCell>
{action.message || action.error || "-"}
</TableCell>
</TableRow>
))
)}
</TableBody>
</Table>
</TableContainer>
</Stack>
</Stack>
);
}
frontend/src/components/MonitoringCharts.impl.tsx
-1032
View File
File diff suppressed because it is too large Load Diff
frontend/src/components/MonitoringCharts.tsx
-1
View File
@@ -1 +0,0 @@
export { MonitoringCharts } from "./MonitoringCharts.impl";
frontend/src/components/MonitoringOverviewTable.tsx
-716
View File
@@ -1,716 +0,0 @@
import { useMemo, useState } from "react";
import {
Box,
Chip,
Paper,
Stack,
Table,
TableBody,
TableCell,
TableContainer,
TableHead,
TableRow,
TableSortLabel,
Tooltip,
Typography,
} from "@mui/material";
import type {
MonitoringMachineOverview,
MonitoringOverviewResponse,
} from "../types";
type SortKey =
| "machine"
| "mode"
| "status"
| "cpu"
| "iowait"
| "mem"
| "net_rx"
| "net_tx"
| "disk_read"
| "disk_write"
| "disk_used"
| "updated";
function formatBytes(bytes: number): string {
if (!bytes || bytes === 0) return "0 B";
const units = ["B", "KB", "MB", "GB", "TB"];
let value = bytes;
let unitIdx = 0;
while (value >= 1000 && unitIdx < units.length - 1) {
value /= 1000;
unitIdx++;
}
return `${value.toFixed(1)} ${units[unitIdx]}`;
}
function formatRate(bytes: number): string {
return `${formatBytes(bytes)}/s`;
}
function formatAge(epochSeconds: number | null): string {
if (!epochSeconds) return "-";
const diff = Date.now() / 1000 - epochSeconds;
if (diff < 60) return `${Math.max(0, Math.round(diff))}s ago`;
if (diff < 3600) return `${Math.round(diff / 60)}m ago`;
return `${Math.round(diff / 3600)}h ago`;
}
function formatReadableTime(epochSeconds: number | null): string {
if (!epochSeconds) return "-";
const date = new Date(epochSeconds * 1000);
return date.toLocaleString("en-US", {
month: "short",
day: "numeric",
hour: "2-digit",
minute: "2-digit",
hour12: false,
});
}
function formatClockTime(epochSeconds: number | null): string {
if (!epochSeconds) return "-";
const date = new Date(epochSeconds * 1000);
return date.toLocaleTimeString("en-US", {
hour: "2-digit",
minute: "2-digit",
hour12: false,
});
}
function formatUpdatedDetails(epochSeconds: number | null): [string, string] {
if (!epochSeconds) return ["-", "-"];
return [
formatReadableTime(epochSeconds),
`${formatClockTime(epochSeconds)} · ${formatAge(epochSeconds)}`,
];
}
function formatSummary(
summary: { avg: number; min: number; max: number } | null,
formatter: (value: number) => string,
) {
if (!summary) return { value: "-", min: "", max: "" };
return {
value: formatter(summary.avg),
min: formatter(summary.min),
max: formatter(summary.max),
};
}
function sortableText(value: string) {
return value.toLowerCase();
}
function getStatusChipProps(row: MonitoringMachineOverview) {
if (row.status_error) {
return { color: "error" as const, label: row.status_error };
}
if (row.status === "running") {
return { color: "success" as const, label: "running" };
}
if (row.status === "not running") {
return { color: "warning" as const, label: "not running" };
}
return { color: "default" as const, label: row.status || "-" };
}
function metricSortValue(
row: MonitoringMachineOverview,
key: SortKey,
): number | string {
switch (key) {
case "machine":
return sortableText(row.machine.name);
case "mode":
return row.machine.mode;
case "status":
return sortableText(row.status || row.status_error || "");
case "cpu":
return row.cpu_summary?.avg ?? -1;
case "iowait":
return row.iowait_summary?.avg ?? -1;
case "mem":
return row.mem_summary?.avg ?? -1;
case "net_rx":
return row.net_rx_summary?.avg ?? -1;
case "net_tx":
return row.net_tx_summary?.avg ?? -1;
case "disk_read":
return row.disk_read_summary?.avg ?? -1;
case "disk_write":
return row.disk_write_summary?.avg ?? -1;
case "disk_used":
return parseFloat((row.disk?.used_pct || "0").replace("%", "")) || -1;
case "updated":
return row.latest_sample?.ts ?? -1;
default:
return 0;
}
}
function NoDataCell() {
return (
<Stack
sx={{
width: "100%",
minWidth: 0,
height: "100%",
minHeight: 118,
textAlign: "center",
justifyContent: "center",
py: 0.5,
}}
>
<Typography
variant="caption"
color="text.secondary"
sx={{
fontSize: "0.7rem",
fontStyle: "italic",
lineHeight: 1.2,
}}
>
No data
</Typography>
</Stack>
);
}
function MetricCell({
value,
min,
max,
}: {
value: string;
min?: string;
max?: string;
}) {
return (
<Stack
sx={{
width: "100%",
minWidth: 0,
height: "100%",
minHeight: 118,
textAlign: "center",
justifyContent: "space-between",
py: 0.5,
}}
>
<Box
sx={{
flex: 1,
display: "flex",
alignItems: "center",
justifyContent: "center",
px: 1.5,
py: 1.25,
minHeight: 68,
borderRadius: 1,
}}
>
<Stack spacing={0.25} sx={{ alignItems: "center" }}>
<Typography
variant="caption"
color="text.secondary"
sx={{
fontSize: "0.6rem",
lineHeight: 1,
textTransform: "uppercase",
letterSpacing: "0.04em",
}}
>
10m avg
</Typography>
<Typography
variant="body1"
sx={{
fontWeight: 900,
fontSize: "1.12rem",
lineHeight: 1,
textAlign: "center",
whiteSpace: "nowrap",
fontVariantNumeric: "tabular-nums",
}}
>
{value}
</Typography>
</Stack>
</Box>
<Stack spacing={0.25} sx={{ width: "100%" }}>
{min ? (
<Box
sx={{
width: "100%",
border: 1,
borderColor: "divider",
borderRadius: 999,
px: 0.75,
py: 0.15,
fontSize: "0.6rem",
lineHeight: 1.2,
color: "text.secondary",
textAlign: "center",
whiteSpace: "nowrap",
overflow: "hidden",
textOverflow: "ellipsis",
}}
>
Min {min}
</Box>
) : null}
{max ? (
<Box
sx={{
width: "100%",
border: 1,
borderColor: "divider",
borderRadius: 999,
px: 0.75,
py: 0.15,
fontSize: "0.6rem",
lineHeight: 1.2,
color: "text.secondary",
textAlign: "center",
whiteSpace: "nowrap",
overflow: "hidden",
textOverflow: "ellipsis",
}}
>
Max {max}
</Box>
) : null}
</Stack>
</Stack>
);
}
export function MonitoringOverviewTable({
overview,
embedded = false,
}: {
overview?: MonitoringOverviewResponse;
embedded?: boolean;
}) {
const poller = overview?.poller;
const rows = overview?.machines ?? [];
const [sortKey, setSortKey] = useState<SortKey>("machine");
const [sortDirection, setSortDirection] = useState<"asc" | "desc">("asc");
const sortedRows = useMemo(() => {
const factor = sortDirection === "asc" ? 1 : -1;
return [...rows].sort((a, b) => {
const av = metricSortValue(a, sortKey);
const bv = metricSortValue(b, sortKey);
if (typeof av === "number" && typeof bv === "number") {
return (av - bv) * factor;
}
return String(av).localeCompare(String(bv)) * factor;
});
}, [rows, sortDirection, sortKey]);
const setSort = (key: SortKey) => {
if (sortKey === key) {
setSortDirection((current) => (current === "asc" ? "desc" : "asc"));
return;
}
setSortKey(key);
setSortDirection("asc");
};
return (
<Stack spacing={1.25}>
{!embedded ? (
<Stack
direction="row"
spacing={1}
sx={{ alignItems: "center", flexWrap: "wrap" }}
>
<Typography variant="subtitle1" sx={{ fontWeight: 700 }}>
Machine monitoring
</Typography>
<Chip
size="small"
variant="outlined"
color={poller?.worker_running ? "success" : "default"}
label={poller?.worker_running ? "running" : "stopped"}
/>
<Chip
size="small"
variant="outlined"
label={`Machines: ${overview?.enabled ?? 0}/${overview?.total ?? 0}`}
/>
<Typography variant="caption" color="text.secondary">
Last success:{" "}
{poller?.last_success_at
? new Date(poller.last_success_at * 1000).toLocaleString()
: "-"}{" "}
· last run: {formatAge(poller?.last_run_at ?? null)}
</Typography>
</Stack>
) : null}
{!embedded && poller?.last_error ? (
<Box>
<Typography variant="caption" color="error.main">
Poller error: {poller.last_error}
</Typography>
</Box>
) : null}
<TableContainer
component={embedded ? Box : Paper}
variant={embedded ? undefined : "outlined"}
sx={
embedded
? {
border: 1,
borderColor: "divider",
borderRadius: 1,
overflowX: "auto",
overflowY: "hidden",
}
: {
maxWidth: "100%",
overflowX: "auto",
}
}
>
<Table
size="small"
sx={{
minWidth: 1500,
tableLayout: "fixed",
"& .MuiTableCell-root": {
px: 1.1,
py: 0.9,
verticalAlign: "middle",
},
"& .MuiTableHead .MuiTableCell-root": {
fontSize: "0.7rem",
fontWeight: 700,
lineHeight: 1.15,
whiteSpace: "nowrap",
},
}}
>
<TableHead>
<TableRow>
<TableCell
sx={{ width: 260 }}
sortDirection={sortKey === "machine" ? sortDirection : false}
>
<TableSortLabel
active={sortKey === "machine"}
direction={sortKey === "machine" ? sortDirection : "asc"}
onClick={() => setSort("machine")}
>
Machine
</TableSortLabel>
</TableCell>
<TableCell
sx={{ width: 90 }}
sortDirection={sortKey === "mode" ? sortDirection : false}
>
<TableSortLabel
active={sortKey === "mode"}
direction={sortKey === "mode" ? sortDirection : "asc"}
onClick={() => setSort("mode")}
>
Mode
</TableSortLabel>
</TableCell>
<TableCell
sx={{ width: 120 }}
sortDirection={sortKey === "status" ? sortDirection : false}
>
<TableSortLabel
active={sortKey === "status"}
direction={sortKey === "status" ? sortDirection : "asc"}
onClick={() => setSort("status")}
>
Status
</TableSortLabel>
</TableCell>
{[
["cpu", "CPU"],
["iowait", "IO wait"],
["mem", "RAM"],
["net_rx", "Net down"],
["net_tx", "Net up"],
["disk_read", "Disk read"],
["disk_write", "Disk write"],
["disk_used", "Disk used"],
].map(([key, label]) => (
<TableCell
key={key}
align="center"
sx={{ width: 150 }}
sortDirection={sortKey === key ? sortDirection : false}
>
<TableSortLabel
active={sortKey === key}
direction={sortKey === key ? sortDirection : "asc"}
onClick={() => setSort(key as SortKey)}
>
{label}
</TableSortLabel>
</TableCell>
))}
<TableCell
sx={{ width: 180 }}
sortDirection={sortKey === "updated" ? sortDirection : false}
>
<TableSortLabel
active={sortKey === "updated"}
direction={sortKey === "updated" ? sortDirection : "asc"}
onClick={() => setSort("updated")}
>
Updated
</TableSortLabel>
</TableCell>
</TableRow>
</TableHead>
<TableBody>
{sortedRows.length === 0 ? (
<TableRow>
<TableCell colSpan={12}>
<Typography variant="body2" color="text.secondary">
No monitoring machines are configured.
</Typography>
</TableCell>
</TableRow>
) : (
sortedRows.map((row) => {
const machine = row.machine;
const hasMetrics = row.sample_count > 0;
const cpu = formatSummary(
row.cpu_summary,
(value) => `${value.toFixed(1)}%`,
);
const iowait = formatSummary(
row.iowait_summary,
(value) => `${value.toFixed(1)}%`,
);
const mem = formatSummary(
row.mem_summary,
(value) => `${value.toFixed(1)}%`,
);
const netDown = formatSummary(row.net_rx_summary, formatRate);
const netUp = formatSummary(row.net_tx_summary, formatRate);
const diskRead = formatSummary(
row.disk_read_summary,
formatRate,
);
const diskWrite = formatSummary(
row.disk_write_summary,
formatRate,
);
const disk = row.disk
? {
value: row.disk.used_pct,
min: `Used ${formatBytes(row.disk.used)}`,
max: `Avail ${formatBytes(row.disk.available)}`,
}
: null;
const updated = formatUpdatedDetails(
row.latest_sample?.ts ?? null,
);
return (
<TableRow key={machine.id}>
<TableCell sx={{ width: 260 }}>
<Stack spacing={0.25} sx={{ minWidth: 0 }}>
<Stack
direction="row"
spacing={0.75}
sx={{
alignItems: "center",
flexWrap: "nowrap",
minWidth: 0,
}}
>
<Typography
variant="body2"
sx={{
fontWeight: 700,
minWidth: 0,
overflow: "hidden",
textOverflow: "ellipsis",
whiteSpace: "nowrap",
}}
>
{machine.name}
</Typography>
{!machine.enabled && (
<Chip
size="small"
label="disabled"
variant="outlined"
/>
)}
</Stack>
<Typography
variant="caption"
color="text.secondary"
sx={{
minWidth: 0,
overflow: "hidden",
textOverflow: "ellipsis",
whiteSpace: "nowrap",
}}
>
{machine.mode === "local"
? "Local API host"
: `${machine.username || "user"}@${machine.host || "host"}:${machine.port}`}
</Typography>
</Stack>
</TableCell>
<TableCell sx={{ width: 90 }}>{machine.mode}</TableCell>
<TableCell sx={{ width: 120 }}>
{(() => {
const chip = getStatusChipProps(row);
return (
<Tooltip
title={
row.metrics_error
? `Metrics: ${row.metrics_error}`
: row.sample_count === 0
? "No metrics collected yet. Start the collector to begin monitoring."
: ""
}
arrow
>
<Chip
size="small"
variant="outlined"
color={chip.color}
label={chip.label}
/>
</Tooltip>
);
})()}
</TableCell>
<TableCell align="center">
{hasMetrics ? (
<MetricCell
value={cpu.value}
min={cpu.min}
max={cpu.max}
/>
) : (
<NoDataCell />
)}
</TableCell>
<TableCell align="center">
{hasMetrics ? (
<MetricCell
value={iowait.value}
min={iowait.min}
max={iowait.max}
/>
) : (
<NoDataCell />
)}
</TableCell>
<TableCell align="center">
{hasMetrics ? (
<MetricCell
value={mem.value}
min={mem.min}
max={mem.max}
/>
) : (
<NoDataCell />
)}
</TableCell>
<TableCell align="center">
{hasMetrics ? (
<MetricCell
value={netDown.value}
min={netDown.min}
max={netDown.max}
/>
) : (
<NoDataCell />
)}
</TableCell>
<TableCell align="center">
{hasMetrics ? (
<MetricCell
value={netUp.value}
min={netUp.min}
max={netUp.max}
/>
) : (
<NoDataCell />
)}
</TableCell>
<TableCell align="center">
{hasMetrics ? (
<MetricCell
value={diskRead.value}
min={diskRead.min}
max={diskRead.max}
/>
) : (
<NoDataCell />
)}
</TableCell>
<TableCell align="center">
{hasMetrics ? (
<MetricCell
value={diskWrite.value}
min={diskWrite.min}
max={diskWrite.max}
/>
) : (
<NoDataCell />
)}
</TableCell>
<TableCell align="center">
{disk ? (
<MetricCell
value={disk.value}
min={disk.min}
max={disk.max}
/>
) : hasMetrics ? (
"-"
) : (
<NoDataCell />
)}
</TableCell>
<TableCell sx={{ width: 180 }}>
<Stack spacing={0.1} sx={{ alignItems: "center" }}>
{hasMetrics ? (
updated.map((line) => (
<Typography
key={line}
variant="caption"
color="text.secondary"
sx={{ lineHeight: 1.1 }}
>
{line}
</Typography>
))
) : (
<Typography
variant="caption"
color="text.secondary"
sx={{ lineHeight: 1.1, fontStyle: "italic" }}
>
No data yet
</Typography>
)}
</Stack>
</TableCell>
</TableRow>
);
})
)}
</TableBody>
</Table>
</TableContainer>
</Stack>
);
}
frontend/src/components/ObservabilityPage.tsx
+698
View File
@@ -0,0 +1,698 @@
import {
Component,
useEffect,
useMemo,
useRef,
useState,
type ElementType,
type ErrorInfo,
type ReactNode,
} from "react";
import { Link } from "react-router-dom";
import {
Activity,
AlertTriangle,
Bell,
CheckCircle2,
ChevronDown,
ExternalLink,
Inbox,
PanelTop,
Radio,
RefreshCw,
Server,
ServerOff,
XCircle,
} from "lucide-react";
import {
useAlertmanagerAlerts,
useAlertmanagerStatus,
usePrometheusTargets,
useMonitoringMachines,
} from "../hooks/useObservability";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Badge } from "@/components/ui/badge";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Skeleton } from "@/components/ui/skeleton";
import {
Collapsible,
CollapsibleContent,
CollapsibleTrigger,
} from "@/components/ui/collapsible";
import type {
AlertmanagerAlert,
MonitoringMachine,
PrometheusTarget,
} from "../types";
function severityVariant(
severity: string,
): "default" | "secondary" | "destructive" | "outline" {
switch (severity.toLowerCase()) {
case "critical":
return "destructive";
case "warning":
return "default";
case "info":
return "secondary";
default:
return "outline";
}
}
function HealthCard({
title,
status,
detail,
icon: Icon,
isLoading,
}: {
title: string;
status: "ok" | "warning" | "error" | "unknown";
detail: string;
icon: ElementType;
isLoading?: boolean;
}) {
const statusIcon =
status === "ok" ? (
<CheckCircle2 className="h-5 w-5 text-green-500" />
) : status === "warning" ? (
<AlertTriangle className="h-5 w-5 text-amber-500" />
) : status === "error" ? (
<XCircle className="h-5 w-5 text-red-500" />
) : (
<Radio className="h-5 w-5 text-muted-foreground" />
);
return (
<Card>
<CardHeader className="flex flex-row items-center justify-between space-y-0 pb-2">
<CardTitle className="text-sm font-medium">{title}</CardTitle>
<Icon className="h-4 w-4 text-muted-foreground" />
</CardHeader>
<CardContent>
<div className="flex items-center gap-2">
{isLoading ? <Skeleton className="h-5 w-5" /> : statusIcon}
<span className="text-2xl font-bold capitalize">{status}</span>
</div>
<p className="mt-1 text-xs text-muted-foreground">{detail}</p>
</CardContent>
</Card>
);
}
function EmptyState({
icon: Icon,
title,
description,
action,
}: {
icon: ElementType;
title: string;
description: string;
action?: ReactNode;
}) {
return (
<div className="flex h-full min-h-[160px] flex-col items-center justify-center gap-2 rounded-md border p-6 text-center">
<Icon className="h-8 w-8 text-muted-foreground" />
<div className="font-medium">{title}</div>
<div className="max-w-md text-sm text-muted-foreground">
{description}
</div>
{action ? <div className="mt-2">{action}</div> : null}
</div>
);
}
function QueryError({
label,
error,
refetch,
}: {
label: string;
error: Error | null;
refetch: () => void;
}) {
if (!error) return null;
return (
<Alert variant="destructive">
<AlertTitle>{label} failed</AlertTitle>
<AlertDescription className="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
<span className="break-words">{error.message}</span>
<Button variant="outline" size="sm" onClick={() => refetch()}>
<RefreshCw className="mr-1 h-3 w-3" />
Retry
</Button>
</AlertDescription>
</Alert>
);
}
function AlertItem({ alert }: { alert: AlertmanagerAlert }) {
return (
<Collapsible>
<CollapsibleTrigger asChild>
<div className="group cursor-pointer rounded-lg border p-3 transition-colors hover:bg-muted/50">
<div className="flex items-start justify-between gap-2">
<div className="font-medium text-sm">{alert.name}</div>
<div className="flex items-center gap-1">
<Badge variant={severityVariant(alert.severity)}>
{alert.severity}
</Badge>
<ChevronDown className="h-4 w-4 text-muted-foreground transition-transform group-data-[state=open]:rotate-180" />
</div>
</div>
<div className="mt-1 text-xs text-muted-foreground">
{alert.summary || alert.description}
</div>
{alert.active_since && (
<div className="mt-1 text-[10px] text-muted-foreground">
Since {new Date(alert.active_since).toLocaleString()}
</div>
)}
</div>
</CollapsibleTrigger>
<CollapsibleContent className="overflow-hidden">
<div className="space-y-2 rounded-b-lg border-x border-b p-3 text-sm">
{alert.description && (
<div>
<span className="font-medium">Description:</span>{" "}
{alert.description}
</div>
)}
<div className="grid grid-cols-2 gap-2 text-xs">
{alert.job_name && (
<div>
<span className="font-medium">Job:</span> {alert.job_name}
</div>
)}
{alert.category && (
<div>
<span className="font-medium">Category:</span> {alert.category}
</div>
)}
<div>
<span className="font-medium">State:</span> {alert.state}
</div>
<div>
<span className="font-medium">Since:</span>{" "}
{alert.active_since
? new Date(alert.active_since).toLocaleString()
: "unknown"}
</div>
</div>
{alert.labels && Object.keys(alert.labels).length > 0 && (
<div className="flex flex-wrap gap-1 pt-1">
{Object.entries(alert.labels).map(([key, value]) => (
<Badge key={key} variant="secondary" className="text-[10px]">
{key}={value}
</Badge>
))}
</div>
)}
</div>
</CollapsibleContent>
</Collapsible>
);
}
function TargetsTable({ targets }: { targets: PrometheusTarget[] }) {
return (
<div className="space-y-3">
{targets.map((target, idx) => (
<div key={idx} className="rounded-lg border p-3">
<div className="font-mono text-sm">{target.targets.join(", ")}</div>
{target.labels && Object.keys(target.labels).length > 0 && (
<div className="mt-2 flex flex-wrap gap-1">
{Object.entries(target.labels).map(([key, value]) => (
<Badge key={key} variant="outline" className="text-[10px]">
{key}: {value}
</Badge>
))}
</div>
)}
</div>
))}
</div>
);
}
class GrafanaErrorBoundary extends Component<
{ children: ReactNode; fallback: ReactNode },
{ hasError: boolean }
> {
constructor(props: { children: ReactNode; fallback: ReactNode }) {
super(props);
this.state = { hasError: false };
}
static getDerivedStateFromError() {
return { hasError: true };
}
componentDidCatch(error: Error, errorInfo: ErrorInfo) {
console.error("Grafana panel error:", error, errorInfo);
}
render() {
if (this.state.hasError) {
return this.props.fallback;
}
return this.props.children;
}
}
function GrafanaPanel({ src, title }: { src: string; title: string }) {
const [loaded, setLoaded] = useState(false);
const [failed, setFailed] = useState(false);
const [iframeKey, setIframeKey] = useState(0);
const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
useEffect(() => {
timerRef.current = setTimeout(() => setFailed(true), 10_000);
return () => {
if (timerRef.current) clearTimeout(timerRef.current);
};
}, [iframeKey]);
const handleLoad = () => {
if (timerRef.current) clearTimeout(timerRef.current);
setLoaded(true);
setFailed(false);
};
const handleError = () => {
if (timerRef.current) clearTimeout(timerRef.current);
setFailed(true);
};
const reload = () => {
setLoaded(false);
setFailed(false);
setIframeKey((k) => k + 1);
};
if (!src) {
return (
<EmptyState
icon={PanelTop}
title="No Grafana URL"
description="Select a machine to load a Grafana panel."
/>
);
}
const fallback = (
<EmptyState
icon={PanelTop}
title="Grafana panel unavailable"
description="The panel did not load in time or Grafana is unreachable."
action={
<div className="flex flex-wrap justify-center gap-2">
<Button variant="outline" size="sm" onClick={reload}>
<RefreshCw className="mr-1 h-3 w-3" />
Reload
</Button>
<Button variant="outline" size="sm" asChild>
<a href={src} target="_blank" rel="noopener noreferrer">
<ExternalLink className="mr-1 h-3 w-3" />
Open in Grafana
</a>
</Button>
</div>
}
/>
);
return (
<GrafanaErrorBoundary key={iframeKey} fallback={fallback}>
<div className="relative h-full min-h-[320px] w-full overflow-hidden rounded-md border">
{!loaded && !failed && (
<div className="absolute inset-0 z-10 p-4">
<Skeleton className="h-full w-full" />
</div>
)}
{failed ? (
<div className="absolute inset-0 z-10 bg-background p-2">
{fallback}
</div>
) : (
<iframe
key={iframeKey}
title={title}
src={src}
className="h-full min-h-[320px] w-full"
allow="fullscreen"
sandbox="allow-scripts allow-same-origin allow-popups allow-forms"
onLoad={handleLoad}
onError={handleError}
/>
)}
</div>
</GrafanaErrorBoundary>
);
}
export function ObservabilityPage() {
const {
data: alertsSummary,
isLoading: alertsLoading,
error: alertsError,
refetch: refetchAlerts,
} = useAlertmanagerAlerts();
const {
data: alertmanagerStatus,
isLoading: statusLoading,
error: statusError,
refetch: refetchStatus,
} = useAlertmanagerStatus();
const {
data: prometheusTargets,
isLoading: targetsLoading,
error: targetsError,
refetch: refetchTargets,
} = usePrometheusTargets();
const {
data: machines = [],
isLoading: machinesLoading,
error: machinesError,
refetch: refetchMachines,
} = useMonitoringMachines();
const [selectedMachineId, setSelectedMachineId] = useState<string>("");
const selectedMachine = useMemo<MonitoringMachine | null>(
() =>
machines.find((m) => m.id === selectedMachineId) ?? machines[0] ?? null,
[machines, selectedMachineId],
);
const grafanaBase = "/grafana";
const nodeExporterDashboardUrl = useMemo(() => {
if (!selectedMachine) return "";
const instance = `${selectedMachine.host || "localhost"}:9100`;
return `${grafanaBase}/d/node-exporter-overview/node-exporter-overview?kiosk&var-instance=${encodeURIComponent(instance)}`;
}, [selectedMachine]);
const logsUrl = useMemo(() => {
if (!selectedMachine) return "";
const container =
selectedMachine.mode === "local" ? "backend" : selectedMachine.name;
return `${grafanaBase}/explore?orgId=1&left=${encodeURIComponent(
JSON.stringify({
datasource: "Loki",
queries: [{ refId: "A", expr: `{container="${container}"}` }],
range: { from: "now-1h", to: "now" },
}),
)}`;
}, [selectedMachine]);
const alertmanagerStatusDetail = alertmanagerStatus?.up
? alertmanagerStatus.version
? `version ${alertmanagerStatus.version}`
: "reachable"
: "unreachable";
const targetsCount = prometheusTargets?.length ?? 0;
const targetsStatus: "ok" | "warning" | "error" | "unknown" = targetsLoading
? "unknown"
: targetsError
? "error"
: targetsCount > 0
? "ok"
: "warning";
const alertStatus: "ok" | "warning" | "error" | "unknown" = alertsLoading
? "unknown"
: alertsError
? "error"
: (alertsSummary?.total ?? 0) > 0
? alertsSummary?.alerts.some((a) => a.severity === "critical")
? "error"
: "warning"
: "ok";
const machinesStatus: "ok" | "warning" | "error" | "unknown" = machinesLoading
? "unknown"
: machinesError
? "error"
: machines.length > 0
? "ok"
: "warning";
return (
<div className="space-y-6">
<div className="space-y-1">
<h1 className="text-2xl font-bold tracking-tight">Observability</h1>
<p className="text-sm text-muted-foreground">
Unified view of metrics, logs, and alerts from Prometheus, Grafana,
Loki, and Alertmanager.
</p>
</div>
<div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-4">
<HealthCard
title="Alertmanager"
status={
statusError
? "error"
: alertmanagerStatus?.up
? "ok"
: statusLoading
? "unknown"
: "error"
}
detail={alertmanagerStatusDetail}
icon={Bell}
isLoading={statusLoading}
/>
<HealthCard
title="Active Alerts"
status={alertStatus}
detail={`${alertsSummary?.total ?? 0} firing alert${(alertsSummary?.total ?? 0) === 1 ? "" : "s"}`}
icon={AlertTriangle}
isLoading={alertsLoading}
/>
<HealthCard
title="Prometheus Targets"
status={targetsStatus}
detail={`${targetsCount} remote Node Exporter target${targetsCount === 1 ? "" : "s"}`}
icon={Radio}
isLoading={targetsLoading}
/>
<HealthCard
title="Machines"
status={machinesStatus}
detail={`${machines.length} monitoring machine${machines.length === 1 ? "" : "s"}`}
icon={Server}
isLoading={machinesLoading}
/>
</div>
<div className="space-y-3">
{statusError && (
<QueryError
label="Alertmanager status"
error={statusError}
refetch={refetchStatus}
/>
)}
{alertsError && (
<QueryError
label="Active alerts"
error={alertsError}
refetch={refetchAlerts}
/>
)}
{targetsError && (
<QueryError
label="Prometheus targets"
error={targetsError}
refetch={refetchTargets}
/>
)}
{machinesError && (
<QueryError
label="Monitoring machines"
error={machinesError}
refetch={refetchMachines}
/>
)}
</div>
{alertsSummary?.error && (
<Alert variant="destructive">
<AlertTitle>Alertmanager unreachable</AlertTitle>
<AlertDescription>
The UI cannot reach Alertmanager right now. Alerts shown here may be
stale.
</AlertDescription>
</Alert>
)}
<div className="grid gap-6 lg:grid-cols-2">
<div className="space-y-6">
<Card>
<CardHeader>
<CardTitle className="flex items-center gap-2">
<Bell className="h-4 w-4" />
Recent Alerts
</CardTitle>
</CardHeader>
<CardContent className="space-y-3">
{alertsLoading ? (
<div className="space-y-2">
<Skeleton className="h-16 w-full" />
<Skeleton className="h-16 w-full" />
<Skeleton className="h-16 w-full" />
</div>
) : !alertsSummary || alertsSummary.total === 0 ? (
<EmptyState
icon={Inbox}
title="No active alerts"
description="Everything looks quiet. Alertmanager will list firing alerts here when they occur."
/>
) : (
<>
{alertsSummary.alerts.map((alert, idx) => (
<AlertItem key={`${alert.name}-${idx}`} alert={alert} />
))}
{alertsSummary.total > alertsSummary.alerts.length && (
<div className="text-center text-xs text-muted-foreground">
{alertsSummary.total - alertsSummary.alerts.length} more
alert
{alertsSummary.total - alertsSummary.alerts.length === 1
? ""
: "s"}{" "}
in Alertmanager
</div>
)}
</>
)}
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle className="flex items-center gap-2">
<Radio className="h-4 w-4" />
Prometheus Targets
</CardTitle>
</CardHeader>
<CardContent>
{targetsLoading ? (
<div className="space-y-2">
<Skeleton className="h-16 w-full" />
<Skeleton className="h-16 w-full" />
</div>
) : !prometheusTargets || prometheusTargets.length === 0 ? (
<EmptyState
icon={Radio}
title="No Node Exporter targets"
description="Enable Node Exporter on an SSH machine in Settings to populate Prometheus scrape targets."
action={
<Button variant="outline" size="sm" asChild>
<Link to="/settings">Open Settings</Link>
</Button>
}
/>
) : (
<TargetsTable targets={prometheusTargets} />
)}
</CardContent>
</Card>
</div>
<Card>
<CardHeader className="flex flex-col gap-4 sm:flex-row sm:items-center sm:justify-between">
<CardTitle className="flex items-center gap-2">
<Activity className="h-4 w-4" />
Machine Dashboard
</CardTitle>
<Select
value={selectedMachine?.id ?? ""}
onValueChange={setSelectedMachineId}
disabled={machines.length === 0}
>
<SelectTrigger className="w-full sm:w-[240px]">
<SelectValue placeholder="Select machine" />
</SelectTrigger>
<SelectContent>
{machines.map((machine) => (
<SelectItem key={machine.id} value={machine.id}>
{machine.name}
</SelectItem>
))}
</SelectContent>
</Select>
</CardHeader>
<CardContent className="space-y-4">
{selectedMachine ? (
<>
<div className="flex items-center justify-between">
<div className="text-sm font-medium">
{selectedMachine.name} metrics
</div>
<Button variant="outline" size="sm" asChild>
<a
href={nodeExporterDashboardUrl}
target="_blank"
rel="noopener noreferrer"
className="gap-1"
>
Open in Grafana
<ExternalLink className="h-3 w-3" />
</a>
</Button>
</div>
<GrafanaPanel
key={nodeExporterDashboardUrl}
src={nodeExporterDashboardUrl}
title={`${selectedMachine.name} metrics`}
/>
<div className="flex items-center justify-between pt-2">
<div className="text-sm font-medium">Recent logs</div>
<Button variant="outline" size="sm" asChild>
<a
href={logsUrl}
target="_blank"
rel="noopener noreferrer"
className="gap-1"
>
Explore in Grafana
<ExternalLink className="h-3 w-3" />
</a>
</Button>
</div>
<GrafanaPanel
key={logsUrl}
src={logsUrl}
title={`${selectedMachine.name} logs`}
/>
</>
) : (
<EmptyState
icon={ServerOff}
title="No machine selected"
description="Add monitoring machines in Settings to embed Grafana dashboards."
action={
<Button variant="outline" size="sm" asChild>
<Link to="/settings">Open Settings</Link>
</Button>
}
/>
)}
</CardContent>
</Card>
</div>
</div>
);
}
frontend/src/components/ui/alert.tsx
+76
View File
@@ -0,0 +1,76 @@
import * as React from "react"
import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const alertVariants = cva(
"group/alert relative grid w-full gap-0.5 rounded-lg border px-2.5 py-2 text-left text-sm has-data-[slot=alert-action]:relative has-data-[slot=alert-action]:pr-18 has-[>svg]:grid-cols-[auto_1fr] has-[>svg]:gap-x-2 *:[svg]:row-span-2 *:[svg]:translate-y-0.5 *:[svg]:text-current *:[svg:not([class*='size-'])]:size-4",
{
variants: {
variant: {
default: "bg-card text-card-foreground",
destructive:
"bg-card text-destructive *:data-[slot=alert-description]:text-destructive/90 *:[svg]:text-current",
},
},
defaultVariants: {
variant: "default",
},
}
)
function Alert({
className,
variant,
...props
}: React.ComponentProps<"div"> & VariantProps<typeof alertVariants>) {
return (
<div
data-slot="alert"
role="alert"
className={cn(alertVariants({ variant }), className)}
{...props}
/>
)
}
function AlertTitle({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="alert-title"
className={cn(
"font-medium group-has-[>svg]/alert:col-start-2 [&_a]:underline [&_a]:underline-offset-3 [&_a]:hover:text-foreground",
className
)}
{...props}
/>
)
}
function AlertDescription({
className,
...props
}: React.ComponentProps<"div">) {
return (
<div
data-slot="alert-description"
className={cn(
"text-sm text-balance text-muted-foreground md:text-pretty [&_a]:underline [&_a]:underline-offset-3 [&_a]:hover:text-foreground [&_p:not(:last-child)]:mb-4",
className
)}
{...props}
/>
)
}
function AlertAction({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="alert-action"
className={cn("absolute top-2 right-2", className)}
{...props}
/>
)
}
export { Alert, AlertTitle, AlertDescription, AlertAction }
frontend/src/components/ui/badge.tsx
+49
View File
@@ -0,0 +1,49 @@
import * as React from "react";
import { cva, type VariantProps } from "class-variance-authority";
import { Slot } from "radix-ui";
import { cn } from "@/lib/utils";
const badgeVariants = cva(
"group/badge inline-flex h-5 w-fit shrink-0 items-center justify-center gap-1 overflow-hidden rounded-4xl border border-transparent px-2 py-0.5 text-xs font-medium whitespace-nowrap transition-all focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 has-data-[icon=inline-end]:pr-1.5 has-data-[icon=inline-start]:pl-1.5 aria-invalid:border-destructive aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 [&>svg]:pointer-events-none [&>svg]:size-3!",
{
variants: {
variant: {
default: "bg-primary text-primary-foreground [a]:hover:bg-primary/80",
secondary:
"bg-secondary text-secondary-foreground [a]:hover:bg-secondary/80",
destructive:
"bg-destructive/10 text-destructive focus-visible:ring-destructive/20 dark:bg-destructive/20 dark:focus-visible:ring-destructive/40 [a]:hover:bg-destructive/20",
outline:
"border-border text-foreground [a]:hover:bg-muted [a]:hover:text-muted-foreground",
ghost:
"hover:bg-muted hover:text-muted-foreground dark:hover:bg-muted/50",
link: "text-primary underline-offset-4 hover:underline",
},
},
defaultVariants: {
variant: "default",
},
},
);
export function Badge({
className,
variant = "default",
asChild = false,
...props
}: React.ComponentProps<"span"> &
VariantProps<typeof badgeVariants> & { asChild?: boolean }) {
const Comp = asChild ? Slot.Root : "span";
return (
<Comp
data-slot="badge"
data-variant={variant}
className={cn(badgeVariants({ variant }), className)}
{...props}
/>
);
}
Badge.displayName = "Badge";
frontend/src/components/ui/card.tsx
+103
View File
@@ -0,0 +1,103 @@
import * as React from "react"
import { cn } from "@/lib/utils"
function Card({
className,
size = "default",
...props
}: React.ComponentProps<"div"> & { size?: "default" | "sm" }) {
return (
<div
data-slot="card"
data-size={size}
className={cn(
"group/card flex flex-col gap-(--card-spacing) overflow-hidden rounded-xl bg-card py-(--card-spacing) text-sm text-card-foreground ring-1 ring-foreground/10 [--card-spacing:--spacing(4)] has-data-[slot=card-footer]:pb-0 has-[>img:first-child]:pt-0 data-[size=sm]:[--card-spacing:--spacing(3)] data-[size=sm]:has-data-[slot=card-footer]:pb-0 *:[img:first-child]:rounded-t-xl *:[img:last-child]:rounded-b-xl",
className
)}
{...props}
/>
)
}
function CardHeader({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="card-header"
className={cn(
"group/card-header @container/card-header grid auto-rows-min items-start gap-1 rounded-t-xl px-(--card-spacing) has-data-[slot=card-action]:grid-cols-[1fr_auto] has-data-[slot=card-description]:grid-rows-[auto_auto] [.border-b]:pb-(--card-spacing)",
className
)}
{...props}
/>
)
}
function CardTitle({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="card-title"
className={cn(
"text-base leading-snug font-medium group-data-[size=sm]/card:text-sm",
className
)}
{...props}
/>
)
}
function CardDescription({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="card-description"
className={cn("text-sm text-muted-foreground", className)}
{...props}
/>
)
}
function CardAction({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="card-action"
className={cn(
"col-start-2 row-span-2 row-start-1 self-start justify-self-end",
className
)}
{...props}
/>
)
}
function CardContent({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="card-content"
className={cn("px-(--card-spacing)", className)}
{...props}
/>
)
}
function CardFooter({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="card-footer"
className={cn(
"flex items-center rounded-b-xl border-t bg-muted/50 p-(--card-spacing)",
className
)}
{...props}
/>
)
}
export {
Card,
CardHeader,
CardFooter,
CardTitle,
CardAction,
CardDescription,
CardContent,
}
frontend/src/components/ui/select.tsx
+190
View File
@@ -0,0 +1,190 @@
import * as React from "react"
import { Select as SelectPrimitive } from "radix-ui"
import { cn } from "@/lib/utils"
import { ChevronDownIcon, CheckIcon, ChevronUpIcon } from "lucide-react"
function Select({
...props
}: React.ComponentProps<typeof SelectPrimitive.Root>) {
return <SelectPrimitive.Root data-slot="select" {...props} />
}
function SelectGroup({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.Group>) {
return (
<SelectPrimitive.Group
data-slot="select-group"
className={cn("scroll-my-1 p-1", className)}
{...props}
/>
)
}
function SelectValue({
...props
}: React.ComponentProps<typeof SelectPrimitive.Value>) {
return <SelectPrimitive.Value data-slot="select-value" {...props} />
}
function SelectTrigger({
className,
size = "default",
children,
...props
}: React.ComponentProps<typeof SelectPrimitive.Trigger> & {
size?: "sm" | "default"
}) {
return (
<SelectPrimitive.Trigger
data-slot="select-trigger"
data-size={size}
className={cn(
"flex w-fit items-center justify-between gap-1.5 rounded-lg border border-input bg-transparent py-2 pr-2 pl-2.5 text-sm whitespace-nowrap transition-colors outline-none select-none focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 disabled:cursor-not-allowed disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 data-placeholder:text-muted-foreground data-[size=default]:h-8 data-[size=sm]:h-7 data-[size=sm]:rounded-[min(var(--radius-md),10px)] *:data-[slot=select-value]:line-clamp-1 *:data-[slot=select-value]:flex *:data-[slot=select-value]:items-center *:data-[slot=select-value]:gap-1.5 dark:bg-input/30 dark:hover:bg-input/50 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
className
)}
{...props}
>
{children}
<SelectPrimitive.Icon asChild>
<ChevronDownIcon className="pointer-events-none size-4 text-muted-foreground" />
</SelectPrimitive.Icon>
</SelectPrimitive.Trigger>
)
}
function SelectContent({
className,
children,
position = "item-aligned",
align = "center",
...props
}: React.ComponentProps<typeof SelectPrimitive.Content>) {
return (
<SelectPrimitive.Portal>
<SelectPrimitive.Content
data-slot="select-content"
data-align-trigger={position === "item-aligned"}
className={cn("relative z-50 max-h-(--radix-select-content-available-height) min-w-36 origin-(--radix-select-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-lg bg-popover text-popover-foreground shadow-md ring-1 ring-foreground/10 duration-100 data-[align-trigger=true]:animate-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-open:animate-in data-open:fade-in-0 data-open:zoom-in-95 data-closed:animate-out data-closed:fade-out-0 data-closed:zoom-out-95", position ==="popper"&&"data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1", className )}
position={position}
align={align}
{...props}
>
<SelectScrollUpButton />
<SelectPrimitive.Viewport
data-position={position}
className={cn(
"data-[position=popper]:h-(--radix-select-trigger-height) data-[position=popper]:w-full data-[position=popper]:min-w-(--radix-select-trigger-width)",
position === "popper" && ""
)}
>
{children}
</SelectPrimitive.Viewport>
<SelectScrollDownButton />
</SelectPrimitive.Content>
</SelectPrimitive.Portal>
)
}
function SelectLabel({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.Label>) {
return (
<SelectPrimitive.Label
data-slot="select-label"
className={cn("px-1.5 py-1 text-xs text-muted-foreground", className)}
{...props}
/>
)
}
function SelectItem({
className,
children,
...props
}: React.ComponentProps<typeof SelectPrimitive.Item>) {
return (
<SelectPrimitive.Item
data-slot="select-item"
className={cn(
"relative flex w-full cursor-default items-center gap-1.5 rounded-md py-1 pr-8 pl-1.5 text-sm outline-hidden select-none focus:bg-accent focus:text-accent-foreground not-data-[variant=destructive]:focus:**:text-accent-foreground data-disabled:pointer-events-none data-disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 *:[span]:last:flex *:[span]:last:items-center *:[span]:last:gap-2",
className
)}
{...props}
>
<span className="pointer-events-none absolute right-2 flex size-4 items-center justify-center">
<SelectPrimitive.ItemIndicator>
<CheckIcon className="pointer-events-none" />
</SelectPrimitive.ItemIndicator>
</span>
<SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
</SelectPrimitive.Item>
)
}
function SelectSeparator({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.Separator>) {
return (
<SelectPrimitive.Separator
data-slot="select-separator"
className={cn("pointer-events-none -mx-1 my-1 h-px bg-border", className)}
{...props}
/>
)
}
function SelectScrollUpButton({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.ScrollUpButton>) {
return (
<SelectPrimitive.ScrollUpButton
data-slot="select-scroll-up-button"
className={cn(
"z-10 flex cursor-default items-center justify-center bg-popover py-1 [&_svg:not([class*='size-'])]:size-4",
className
)}
{...props}
>
<ChevronUpIcon
/>
</SelectPrimitive.ScrollUpButton>
)
}
function SelectScrollDownButton({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.ScrollDownButton>) {
return (
<SelectPrimitive.ScrollDownButton
data-slot="select-scroll-down-button"
className={cn(
"z-10 flex cursor-default items-center justify-center bg-popover py-1 [&_svg:not([class*='size-'])]:size-4",
className
)}
{...props}
>
<ChevronDownIcon
/>
</SelectPrimitive.ScrollDownButton>
)
}
export {
Select,
SelectContent,
SelectGroup,
SelectItem,
SelectLabel,
SelectScrollDownButton,
SelectScrollUpButton,
SelectSeparator,
SelectTrigger,
SelectValue,
}
frontend/src/components/ui/skeleton.tsx
+13
View File
@@ -0,0 +1,13 @@
import { cn } from "@/lib/utils"
function Skeleton({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="skeleton"
className={cn("animate-pulse rounded-md bg-muted", className)}
{...props}
/>
)
}
export { Skeleton }
frontend/src/hooks/useDashboard.ts
-9
View File
@@ -5,7 +5,6 @@ import {
fetchCounts,
fetchDashboardShortcuts,
fetchLibraries,
fetchMonitoringOverview,
saveDashboardShortcut,
} from "../api/client";
import type { DashboardShortcutInput } from "../types";
@@ -34,14 +33,6 @@ export function useActivity(machineId?: string) {
});
}
export function useMonitoringOverview() {
return useQuery({
queryKey: ["dashboard", "monitoring"],
queryFn: fetchMonitoringOverview,
refetchInterval: 30_000,
});
}
// Backward-compatible alias used by older code.
export const useNowPlaying = useActivity;
frontend/src/hooks/useMonitoring.ts
-85
View File
@@ -1,85 +0,0 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import {
fetchMonitoringStatus,
fetchMonitoringMetrics,
fetchDiskSpace,
fetchMonitoringMachines,
fetchMonitoringMachineActions,
fetchMonitoringPoller,
startCollector,
stopCollector,
restartCollector,
} from "../api/client";
export function useMonitoringMachines() {
return useQuery({
queryKey: ["monitoring", "machines"],
queryFn: fetchMonitoringMachines,
refetchInterval: 30_000,
});
}
export function useMonitoringPoller() {
return useQuery({
queryKey: ["monitoring", "poller"],
queryFn: fetchMonitoringPoller,
refetchInterval: 30_000,
});
}
export function useMonitoringStatus(machineId?: string, enabled = true) {
return useQuery({
queryKey: ["monitoring", "status", machineId ?? "default"],
queryFn: () => fetchMonitoringStatus(machineId),
refetchInterval: 30_000,
enabled,
});
}
export function useMonitoringMetrics(machineId?: string, enabled = true) {
return useQuery({
queryKey: ["monitoring", "metrics", machineId ?? "default"],
queryFn: () => fetchMonitoringMetrics(undefined, 70_000, machineId),
refetchInterval: 15_000,
enabled,
});
}
export function useDiskSpace(machineId?: string, enabled = true) {
return useQuery({
queryKey: ["monitoring", "disk", machineId ?? "default"],
queryFn: () => fetchDiskSpace(machineId),
staleTime: 60_000,
enabled,
});
}
export function useMachineActions(machineId: string, enabled = true) {
return useQuery({
queryKey: ["monitoring", "actions", machineId],
queryFn: () => fetchMonitoringMachineActions(machineId),
refetchInterval: 15_000,
enabled,
});
}
export function useCollectorControls(machineId?: string) {
const queryClient = useQueryClient();
const invalidate = () =>
queryClient.invalidateQueries({ queryKey: ["monitoring"] });
const start = useMutation({
mutationFn: () => startCollector(machineId),
onSuccess: invalidate,
});
const stop = useMutation({
mutationFn: () => stopCollector(machineId),
onSuccess: invalidate,
});
const restart = useMutation({
mutationFn: () => restartCollector(machineId),
onSuccess: invalidate,
});
return { start, stop, restart };
}
frontend/src/hooks/useObservability.ts
+47
View File
@@ -0,0 +1,47 @@
import { useQuery } from "@tanstack/react-query";
import {
fetchAlertmanagerAlerts,
fetchAlertmanagerStatus,
fetchPrometheusTargets,
fetchMonitoringMachines,
} from "../api/client";
export function useAlertmanagerAlerts() {
return useQuery({
queryKey: ["observability", "alerts"],
queryFn: fetchAlertmanagerAlerts,
retry: 2,
staleTime: 10_000,
refetchInterval: 15_000,
});
}
export function useAlertmanagerStatus() {
return useQuery({
queryKey: ["observability", "alertmanager-status"],
queryFn: fetchAlertmanagerStatus,
retry: 2,
staleTime: 10_000,
refetchInterval: 30_000,
});
}
export function usePrometheusTargets() {
return useQuery({
queryKey: ["observability", "prometheus-targets"],
queryFn: fetchPrometheusTargets,
retry: 2,
staleTime: 10_000,
refetchInterval: 30_000,
});
}
export function useMonitoringMachines() {
return useQuery({
queryKey: ["monitoring", "machines"],
queryFn: fetchMonitoringMachines,
retry: 2,
staleTime: 10_000,
refetchInterval: 30_000,
});
}
frontend/src/pages/Dashboard.tsx
-10
View File
@@ -26,13 +26,11 @@ import {
useActivity,
useDashboardShortcuts,
useDeleteDashboardShortcut,
useMonitoringOverview,
useSaveDashboardShortcut,
} from "../hooks/useDashboard";
import { useMonitoringSettings } from "../hooks/useSettings";
import type { DashboardShortcut, DashboardShortcutInput } from "../types";
import { NowPlaying } from "../components/NowPlaying";
import { MonitoringOverviewTable } from "../components/MonitoringOverviewTable";
import { SectionCard } from "../components/SectionCard";
import { DialogFooter } from "../components/DialogFooter";
import BackupDashboardWidget from "../components/BackupDashboardWidget";
@@ -325,7 +323,6 @@ export function Dashboard() {
const selectedJellyfinId =
activeJellyfinMachineId || jellyfinMachines[0]?.id || "";
const { data: activity } = useActivity(selectedJellyfinId || undefined);
const { data: monitoringOverview } = useMonitoringOverview();
const { data: shortcuts = [] } = useDashboardShortcuts();
const saveShortcut = useSaveDashboardShortcut();
const deleteShortcut = useDeleteDashboardShortcut();
@@ -438,13 +435,6 @@ export function Dashboard() {
) : null}
</SectionCard>
<SectionCard
title="Monitoring overview"
description="10-minute averages and fleet status across all configured machines."
>
<MonitoringOverviewTable overview={monitoringOverview} embedded />
</SectionCard>
<BackupDashboardWidget />
<ShortcutDialog
frontend/src/pages/Monitoring.tsx
-106
View File
@@ -1,106 +0,0 @@
import { useMemo, useState } from "react";
import { Alert, Button, Stack, Tab, Typography } from "@mui/material";
import { useNavigate } from "react-router-dom";
import {
useMonitoringMachines,
useMonitoringPoller,
} from "../hooks/useMonitoring";
import { useMonitoringOverview } from "../hooks/useDashboard";
import { MachineMonitoringSection } from "../components/MachineMonitoringSection";
import { MonitoringOverviewTable } from "../components/MonitoringOverviewTable";
import { SectionCard } from "../components/SectionCard";
import { TabbedCard } from "../components/TabbedCard";
type MonitoringTab = "overview" | string;
export function Monitoring() {
const { data: machines = [], isLoading, error } = useMonitoringMachines();
const { data: poller } = useMonitoringPoller();
const { data: overview } = useMonitoringOverview();
const navigate = useNavigate();
const [tab, setTab] = useState<MonitoringTab>("overview");
const selectedMachine = useMemo(
() => machines.find((machine) => machine.id === tab) ?? null,
[machines, tab],
);
return (
<Stack spacing={2.25}>
<Stack spacing={0.5}>
<Typography variant="h5" sx={{ fontWeight: 800 }}>
Monitoring
</Typography>
<Typography variant="body2" color="text.secondary">
Review fleet health first, then switch to a machine-specific tab.
</Typography>
</Stack>
{error ? <Alert severity="error">{String(error)}</Alert> : null}
{isLoading ? (
<Alert severity="info">Loading monitoring machines...</Alert>
) : null}
<TabbedCard
value={tab}
onChange={setTab}
tabs={[
<Tab key="overview" value="overview" label="Overview" />,
...machines.map((machine) => (
<Tab key={machine.id} value={machine.id} label={machine.name} />
)),
]}
contentSx={{ p: 1.5 }}
>
{tab === "overview" ? (
<Stack spacing={1.5}>
<Stack
direction="row"
spacing={1}
sx={{ alignItems: "center", flexWrap: "wrap" }}
>
<Typography variant="subtitle1" sx={{ fontWeight: 700 }}>
Fleet overview
</Typography>
<Typography variant="caption" color="text.secondary">
{poller?.worker_running ? "running" : "stopped"}
</Typography>
</Stack>
{!isLoading && machines.length === 0 ? (
<Alert
severity="warning"
action={
<Button
color="inherit"
size="small"
onClick={() => navigate("/settings")}
>
Open Settings
</Button>
}
>
No monitoring machines are configured yet.
</Alert>
) : overview ? (
<MonitoringOverviewTable overview={overview} embedded />
) : (
<Alert severity="info">Loading fleet overview...</Alert>
)}
</Stack>
) : selectedMachine ? (
<SectionCard
title={selectedMachine.name}
description={
selectedMachine.mode === "local"
? "Local API host"
: `${selectedMachine.username || "user"}@${selectedMachine.host || "host"}:${selectedMachine.port}`
}
>
<MachineMonitoringSection machine={selectedMachine} />
</SectionCard>
) : null}
</TabbedCard>
</Stack>
);
}
frontend/src/types/index.ts
+32 -83
View File
@@ -246,28 +246,6 @@ export interface SavedTaskRun {
error: string;
}
export interface MonitoringMachineAction {
machine_id: string;
machine_name: string;
mode: "local" | "ssh";
action: string;
status: string;
created_at: number;
duration_ms: number;
request_id: string;
message: string;
error: string;
stdout_tail: string;
stderr_tail: string;
}
export interface MetricSummary {
avg: number;
min: number;
max: number;
count: number;
}
export interface ResetLocalDatabaseInput {
confirm_phrase: string;
acknowledge_settings_loss: boolean;
@@ -291,20 +269,6 @@ export interface SSHValidationResult {
known_hosts_updated: boolean;
}
export interface MonitoringPollerStatus {
worker_running: boolean;
stop_requested: boolean;
last_run_at: number | null;
last_success_at: number | null;
last_error: string;
last_cycle_ms: number | null;
poll_count: number;
error_count: number;
interval_seconds: number;
initial_delay_seconds: number;
retention_days: number;
}
export interface AppVersionInfo {
app: string;
backend_version: string;
@@ -312,53 +276,6 @@ export interface AppVersionInfo {
backend_label: string;
}
export interface MonitoringMachineOverview {
machine: MonitoringMachine;
status: string;
status_error: string;
metrics_error: string;
disk_error: string;
latest_sample: MonitoringSample | null;
sample_count: number;
cpu_summary: MetricSummary | null;
iowait_summary: MetricSummary | null;
mem_summary: MetricSummary | null;
net_rx_summary: MetricSummary | null;
net_tx_summary: MetricSummary | null;
disk_read_summary: MetricSummary | null;
disk_write_summary: MetricSummary | null;
disk: DiskSpace | null;
}
export interface MonitoringOverviewResponse {
poller: MonitoringPollerStatus;
machines: MonitoringMachineOverview[];
total: number;
enabled: number;
}
export interface MonitoringStatus {
status: string;
}
export interface MonitoringSample {
ts: number;
cpu_pct: number;
iowait_pct?: number;
mem_pct: number;
net_rx_bytes_per_sec: number;
net_tx_bytes_per_sec: number;
disk_read_bps: number;
disk_write_bps: number;
}
export interface MonitoringMetrics {
samples: MonitoringSample[];
total_samples: number;
filtered_samples: number;
cutoff_ts: number;
}
export interface DiskSpace {
filesystem: string;
size: number;
@@ -503,3 +420,35 @@ export interface DashboardShortcutInput {
user_id: string;
notes: string;
}
export interface AlertmanagerAlert {
name: string;
severity: string;
category: string;
job_name: string;
summary: string;
description: string;
active_since: string;
state: string;
labels: Record<string, string>;
}
export interface AlertmanagerAlertSummary {
total: number;
by_severity: Record<string, number>;
alerts: AlertmanagerAlert[];
error?: string;
}
export interface AlertmanagerStatus {
up: boolean;
version: string;
uptime: string;
name: string;
peers: string[];
}
export interface PrometheusTarget {
labels: Record<string, string>;
targets: string[];
}
frontend/vite.config.ts
+5
View File
@@ -30,6 +30,11 @@ export default defineConfig(({ mode }) => {
changeOrigin: true,
secure: false,
},
"/grafana": {
target: "http://localhost:3000",
changeOrigin: true,
rewrite: (path: string) => path.replace(/^\/grafana/, ""),
},
},
},
resolve: {
monitoring/alertmanager/alertmanager.yml
+44
View File
@@ -0,0 +1,44 @@
global:
smtp_smarthost: '${SMTP_HOST:-smtp.example.com}:${SMTP_PORT:-587}'
smtp_from: '${SMTP_FROM_ADDRESS:-alerts@example.com}'
smtp_auth_username: '${SMTP_USERNAME:-}'
smtp_auth_password: '${SMTP_PASSWORD}'
route:
receiver: 'default'
group_by: ['alertname', 'severity', 'category']
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
routes:
- match:
severity: critical
receiver: 'critical'
continue: true
receivers:
- name: 'default'
email_configs:
- to: '${ALERT_EMAIL_TO}'
send_resolved: true
headers:
Subject: '[Manage] {{ .GroupLabels.alertname }}'
- name: 'critical'
email_configs:
- to: '${ALERT_EMAIL_TO}'
send_resolved: true
headers:
Subject: '[Manage CRITICAL] {{ .GroupLabels.alertname }}'
- name: 'webhook'
webhook_configs:
- url: '${ALERTMANAGER_WEBHOOK_URL:-http://backend:8000/api/monitoring/alertmanager-webhook}'
send_resolved: true
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
equal: ['alertname', 'instance']
monitoring/alloy/config.alloy
+39
View File
@@ -0,0 +1,39 @@
logging {
level = "info"
}
// Promtail-compatible Docker log scraping via the local container engine.
// Requires mounting /var/run/docker.sock and /var/lib/docker/containers.
discovery.docker "containers" {
host = "unix:///var/run/docker.sock"
}
discovery.relabel "container_labels" {
targets = discovery.docker.containers.targets
rule {
source_labels = ["__meta_docker_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_docker_container_log_stream"]
target_label = "stream"
}
}
loki.source.docker "default" {
host = "unix:///var/run/docker.sock"
targets = discovery.relabel.container_labels.output
forward_to = [loki.write.local.receiver]
labels = {
job = "docker",
}
}
loki.write "local" {
endpoint {
name = "loki"
url = "http://loki:3100/loki/api/v1/push"
}
}
monitoring/grafana/grafana.ini
+32
View File
@@ -0,0 +1,32 @@
[server]
# Allow embedding Manage in iframes
root_url = %(protocol)s://%(domain)s:%(http_port)s/
serve_from_sub_path = false
[security]
allow_embedding = true
cookie_samesite = lax
[auth]
disable_login_form = false
disable_signout_menu = false
[auth.generic_oauth]
enabled = true
name = Authentik
allow_sign_up = true
client_id = ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID}
client_secret = ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET}
scopes = openid profile email
auth_url = ${GF_AUTH_GENERIC_OAUTH_AUTH_URL}
token_url = ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL}
api_url = ${GF_AUTH_GENERIC_OAUTH_API_URL}
role_attribute_path = "contains(groups[*], 'grafana-admins') && 'Admin' || contains(groups[*], 'grafana-editors') && 'Editor' || 'Viewer'"
[users]
auto_assign_org = true
auto_assign_org_role = Viewer
[dataproxy]
# Keep dashboard queries responsive in the iframe
timeout = 30
monitoring/grafana/provisioning/dashboards-json/dashboards.yml
+12
View File
@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: "Manage dashboards"
orgId: 1
folder: "Manage"
type: file
disableDeletion: false
updateIntervalSeconds: 60
allowUiUpdates: true
options:
path: /etc/grafana/provisioning/dashboards-json/dashboards
monitoring/grafana/provisioning/dashboards-json/dashboards/manage-overview.json
+312
View File
@@ -0,0 +1,312 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "sum(rate(manage_api_requests_total[5m])) by (method)",
"refId": "A"
}
],
"title": "API Request Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 500
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "sum(increase(manage_backup_runs_total[1h]))",
"refId": "A"
}
],
"title": "Backup Runs (1h)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"pieType": "donut",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "sum by (status) (manage_backup_runs_total)",
"legendFormat": "{{ status }}",
"refId": "A"
}
],
"title": "Backup Runs by Status",
"type": "piechart"
},
{
"datasource": {
"type": "loki",
"uid": "${loki_datasource}"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false,
"prettifyLogMessage": false,
"enableLogDetails": true,
"dedupStrategy": "none"
},
"pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${loki_datasource}"
},
"expr": "{container=\"/backend\"} |= \"\"",
"refId": "A"
}
],
"title": "Backend Logs",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": ["manage"],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"label": "Metrics",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {
"selected": false,
"text": "Loki",
"value": "Loki"
},
"hide": 0,
"includeAll": false,
"label": "Logs",
"multi": false,
"name": "loki_datasource",
"options": [],
"query": "loki",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Manage Overview",
"uid": "manage-overview",
"version": 1,
"weekStart": ""
}
monitoring/grafana/provisioning/dashboards-json/dashboards/node-exporter-overview.json
+400
View File
@@ -0,0 +1,400 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
"legendFormat": "{{ instance }}",
"refId": "A"
}
],
"title": "CPU Utilization",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 90
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
"legendFormat": "{{ instance }}",
"refId": "A"
}
],
"title": "Memory Utilization",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 85
},
{
"color": "red",
"value": 95
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "100 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"} * 100)",
"legendFormat": "{{ instance }}",
"refId": "A"
}
],
"title": "Root Disk Usage",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "irate(node_network_receive_bytes_total[5m])",
"legendFormat": "rx {{ instance }} {{ device }}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "irate(node_network_transmit_bytes_total[5m])",
"legendFormat": "tx {{ instance }} {{ device }}",
"refId": "B"
}
],
"title": "Network Traffic",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": ["manage", "node-exporter"],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"label": "Metrics",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Node Exporter Overview",
"uid": "node-exporter-overview",
"version": 1,
"weekStart": ""
}
monitoring/grafana/provisioning/datasources/datasources.yml
+19
View File
@@ -0,0 +1,19 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
timeInterval: 15s
- name: Loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
maxLines: 1000
monitoring/loki/loki.yml
+51
View File
@@ -0,0 +1,51 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: info
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
storage_config:
tsdb_shipper:
active_index_directory: /loki/tsdb-index
cache_location: /loki/tsdb-cache
compactor:
working_directory: /loki/compactor
retention_enabled: true
retention_delete_delay: 2h
retention_delete_worker_count: 150
compaction_interval: 10m
limits_config:
retention_period: 720h
reject_old_samples: true
reject_old_samples_max_age: 168h
monitoring/prometheus/prometheus.yml
+53
View File
@@ -0,0 +1,53 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
external_labels:
monitor: manage
alerting:
alertmanagers:
- static_configs:
- targets:
- alertmanager:9093
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- localhost:9090
- job_name: manage-backend
static_configs:
- targets:
- backend:8000
metrics_path: /metrics
scrape_interval: 15s
- job_name: node-exporter
static_configs:
- targets:
- node-exporter:9100
- job_name: loki
static_configs:
- targets:
- loki:3100
- job_name: alertmanager
static_configs:
- targets:
- alertmanager:9093
- job_name: grafana
static_configs:
- targets:
- grafana:3000
- job_name: node-exporter-remote
file_sd_configs:
- files:
- /etc/prometheus/file-sd/node_exporter_targets.json
refresh_interval: 30s
monitoring/prometheus/rules/backup_alerts.yml
+54
View File
@@ -0,0 +1,54 @@
groups:
- name: backup
rules:
- alert: BackupJobFailed
expr: increase(manage_backup_runs_total{status="failed"}[1h]) > 0
for: 0m
labels:
severity: critical
category: backup
annotations:
summary: "Backup job {{ $labels.job_name }} failed"
description: "At least one failed backup run was recorded for job {{ $labels.job_name }} within the last hour."
- alert: BackupJobStuck
expr: time() - manage_backup_runs_last_success_timestamp > 86400
for: 5m
labels:
severity: warning
category: backup
annotations:
summary: "Backup job {{ $labels.job_name }} has not succeeded in 24h"
description: "No successful backup run has been recorded for job {{ $labels.job_name }} in the last 24 hours."
- name: observability_health
rules:
- alert: PrometheusTargetMissing
expr: up == 0
for: 2m
labels:
severity: warning
category: observability
annotations:
summary: "Prometheus target {{ $labels.job }} / {{ $labels.instance }} is down"
description: "The Prometheus scrape target for {{ $labels.job }} on {{ $labels.instance }} has been unreachable for more than 2 minutes."
- alert: AlertmanagerDown
expr: up{job="alertmanager"} == 0
for: 2m
labels:
severity: critical
category: observability
annotations:
summary: "Alertmanager is down"
description: "Alertmanager has been unreachable for more than 2 minutes. Alerts may not be delivered."
- alert: GrafanaDown
expr: up{job="grafana"} == 0
for: 2m
labels:
severity: warning
category: observability
annotations:
summary: "Grafana is down"
description: "Grafana has been unreachable for more than 2 minutes. Dashboards are unavailable."