alex/manage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

Browse Source
This commit is contained in:
Developer
2026-06-16 11:08:32 +00:00
parent 615e02e970
commit e2ad731b5f
80 changed files with 5020 additions and 3665 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/media_library_viewer_api/clients/jellyfin.py
+14 -12
View File
@@ -53,7 +53,9 @@ class JellyfinClient:
{
"X-Emby-Token": api_key,
"Accept": "application/json",
"X-Emby-Authorization": 'MediaBrowser Client="MediaLibraryViewer", Device="Streamlit", DeviceId="streamlit", Version="0.1"',
"X-Emby-Authorization": (
'MediaBrowser Client="MediaLibraryViewer", Device="Streamlit", DeviceId="streamlit", Version="0.1"'
),
}
)
@@ -61,9 +63,7 @@ class JellyfinClient:
"""GET a Jellyfin endpoint and include useful response text on errors."""
clean_params = {k: v for k, v in params.items() if v is not None and v != ""}
logger.debug("Jellyfin GET %s params=%s", path, sorted(clean_params.keys()))
response = self.session.get(
f"{self.base_url}{path}", params=clean_params, timeout=self.timeout
)
response = self.session.get(f"{self.base_url}{path}", params=clean_params, timeout=self.timeout)
try:
response.raise_for_status()
except requests.HTTPError as exc:
@@ -175,14 +175,16 @@ class JellyfinClient:
series = self.item_count(user_id, "Series", parent_id=lib_id)
episodes = self.item_count(user_id, "Episode", parent_id=lib_id)
total = self.item_count(user_id, "Movie,Episode,Video,Audio,Series", parent_id=lib_id)
results.append({
"library": lib_name,
"type": lib_type,
"movies": movies,
"series": series,
"episodes": episodes,
"total": total,
})
results.append(
{
"library": lib_name,
"type": lib_type,
"movies": movies,
"series": series,
"episodes": episodes,
"total": total,
}
)
return results
def sessions(self, active_within_seconds: int | None = None) -> list[dict[str, Any]]:
backend/src/media_library_viewer_api/clients/jellyseerr.py
+5 -7
View File
@@ -40,16 +40,12 @@ class JellyseerrClient:
"""GET a Jellyseerr endpoint and include useful response text on errors."""
clean_params = {k: v for k, v in params.items() if v is not None and v != ""}
logger.debug("Jellyseerr GET %s params=%s", path, sorted(clean_params.keys()))
response = self.session.get(
f"{self.base_url}/api/v1{path}", params=clean_params, timeout=self.timeout
)
response = self.session.get(f"{self.base_url}/api/v1{path}", params=clean_params, timeout=self.timeout)
try:
response.raise_for_status()
except requests.HTTPError as exc:
detail = response.text[:500]
logger.warning(
"Jellyseerr GET %s failed status=%s url=%s", path, response.status_code, response.url
)
logger.warning("Jellyseerr GET %s failed status=%s url=%s", path, response.status_code, response.url)
raise requests.HTTPError(
f"{response.status_code} for {response.url}: {detail}",
response=response,
@@ -105,7 +101,9 @@ class JellyseerrClient:
return results
page_results = payload.get("results") or []
page_items = [item for item in page_results if isinstance(item, dict)] if isinstance(page_results, list) else []
page_items = (
[item for item in page_results if isinstance(item, dict)] if isinstance(page_results, list) else []
)
results.extend(page_items)
page_info = payload.get("pageInfo") or {}
backend/src/media_library_viewer_api/clients/resources.py
-340
View File
@@ -1,340 +0,0 @@
"""Remote resource collection helpers.
The app does not require Prometheus, Netdata, or sysstat. Instead it can install
and manage a tiny POSIX-sh collector under /tmp on the remote server. The
collector samples Linux /proc and /sys counters every 10 seconds and appends JSON
Lines. This module starts/stops the collector and reads those JSONL samples.
"""
from __future__ import annotations
import json
import logging
import shlex
from dataclasses import dataclass
from typing import Any
from media_library_viewer_api.clients.ssh import RemoteSSHClient
logger = logging.getLogger(__name__)
# POSIX shell script copied to the remote server by start_resource_collector().
# Keep this script bash-free because many NAS/media servers have minimal shells.
COLLECTOR_SCRIPT = r'''#!/bin/sh
set -u
OUT="${1:-/tmp/media_library_viewer_metrics.jsonl}"
INTERVAL="${2:-10}"
RETENTION_SECONDS="${3:-604800}"
MAX_LINES="${4:-70000}"
PRUNE_EVERY_SAMPLES="${5:-60}"
mkdir -p "$(dirname "$OUT")"
echo "collector starting at $(date -Is 2>/dev/null || date), interval=${INTERVAL}s, retention=${RETENTION_SECONDS}s, max_lines=${MAX_LINES}, out=${OUT}"
read_cpu() {
awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8+$9+$10, $5+$6, $6}' /proc/stat
}
read_mem_pct() {
awk '
/^MemTotal:/ {total=$2}
/^MemAvailable:/ {avail=$2}
END {if (total > 0) printf "%.2f", (total-avail)*100/total; else printf "0"}
' /proc/meminfo
}
read_net_bytes() {
awk '
NR > 2 {
split($0, parts, ":")
iface = parts[1]
stats = parts[2]
gsub(/^[ \t]+|[ \t]+$/, "", iface)
gsub(/^[ \t]+|[ \t]+$/, "", stats)
if (iface == "lo" || iface == "" || stats == "") next
split(stats, values, /[ \t]+/)
# /proc/net/dev after the colon:
# receive bytes are field 1, transmit bytes are field 9.
# Trim the stats block before split; otherwise leading whitespace can make
# values[1] empty in some awk implementations, resulting in zero rates.
rx += values[1] + 0
tx += values[9] + 0
}
END {printf "%.0f %.0f", rx, tx}
' /proc/net/dev
}
read_disk_bytes() {
read_sectors=0
written_sectors=0
for dev in /sys/block/*; do
[ -r "$dev/stat" ] || continue
name="$(basename "$dev")"
case "$name" in
loop*|ram*|fd*|sr*) continue ;;
esac
# Linux /sys/block/<dev>/stat fields: 3=sectors read, 7=sectors written.
# Use POSIX sh parsing instead of bash arrays so this works on minimal systems.
set -- $(cat "$dev/stat")
sectors_read="${3:-0}"
sectors_written="${7:-0}"
read_sectors=$((read_sectors + sectors_read))
written_sectors=$((written_sectors + sectors_written))
done
printf "%s %s" "$((read_sectors * 512))" "$((written_sectors * 512))"
}
set -- $(read_cpu)
prev_total="${1:-0}"
prev_idle="${2:-0}"
prev_iowait="${3:-0}"
set -- $(read_net_bytes)
prev_rx="${1:-0}"
prev_tx="${2:-0}"
set -- $(read_disk_bytes)
prev_disk_read="${1:-0}"
prev_disk_write="${2:-0}"
prev_ts="$(date +%s)"
sample_count=0
prune_metrics_file() {
[ -f "$OUT" ] || return 0
cutoff="$1"
tmp="${OUT}.$$.tmp"
awk -v cutoff="$cutoff" '
match($0, /"ts":[0-9]+/) {
ts = substr($0, RSTART + 5, RLENGTH - 5)
if (ts >= cutoff) print $0
}
' "$OUT" | tail -n "$MAX_LINES" > "$tmp" && mv "$tmp" "$OUT"
rm -f "$tmp"
}
while true; do
now_ts="$(date +%s)"
dt=$((now_ts - prev_ts))
if [ "$dt" -le 0 ]; then dt=1; fi
set -- $(read_cpu)
total="${1:-0}"
idle="${2:-0}"
iowait="${3:-0}"
set -- $(read_net_bytes)
rx="${1:-0}"
tx="${2:-0}"
set -- $(read_disk_bytes)
disk_read="${1:-0}"
disk_write="${2:-0}"
mem_pct="$(read_mem_pct)"
total_delta=$((total - prev_total))
idle_delta=$((idle - prev_idle))
iowait_delta=$((iowait - prev_iowait))
rx_delta=$((rx - prev_rx))
tx_delta=$((tx - prev_tx))
disk_read_delta=$((disk_read - prev_disk_read))
disk_write_delta=$((disk_write - prev_disk_write))
cpu_pct="$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN {if (total > 0) printf "%.2f", (total-idle)*100/total; else printf "0"}')"
iowait_pct="$(awk -v total="$total_delta" -v iow="$iowait_delta" 'BEGIN {if (total > 0) printf "%.2f", iow*100/total; else printf "0"}')"
rx_bytes_per_sec="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
tx_bytes_per_sec="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
rx_bps="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
tx_bps="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
disk_read_bps="$(awk -v bytes="$disk_read_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
disk_write_bps="$(awk -v bytes="$disk_write_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
printf '{"ts":%s,"cpu_pct":%s,"iowait_pct":%s,"mem_pct":%s,"net_rx_bytes_per_sec":%s,"net_tx_bytes_per_sec":%s,"net_rx_bps":%s,"net_tx_bps":%s,"disk_read_bps":%s,"disk_write_bps":%s}\n' \
"$now_ts" "$cpu_pct" "$iowait_pct" "$mem_pct" "$rx_bytes_per_sec" "$tx_bytes_per_sec" "$rx_bps" "$tx_bps" "$disk_read_bps" "$disk_write_bps" >> "$OUT"
sample_count=$((sample_count + 1))
if [ $((sample_count % PRUNE_EVERY_SAMPLES)) -eq 0 ]; then
prune_metrics_file "$((now_ts - RETENTION_SECONDS))"
fi
prev_total="$total"
prev_idle="$idle"
prev_iowait="$iowait"
prev_rx="$rx"
prev_tx="$tx"
prev_disk_read="$disk_read"
prev_disk_write="$disk_write"
prev_ts="$now_ts"
sleep "$INTERVAL"
done
'''
@dataclass(frozen=True)
class ResourceMonitorPaths:
"""Remote file locations used by the lightweight resource collector."""
metrics_file: str = "/tmp/media_library_viewer_metrics.jsonl"
pid_file: str = "/tmp/media_library_viewer_metrics.pid"
script_file: str = "/tmp/media_library_viewer_metrics_collector.sh"
log_file: str = "/tmp/media_library_viewer_metrics.log"
def start_resource_collector(
ssh: RemoteSSHClient,
interval_seconds: int = 10,
retention_seconds: int = 7 * 24 * 60 * 60,
max_lines: int = 70_000,
paths: ResourceMonitorPaths = ResourceMonitorPaths(),
) -> str:
"""Install and start the remote metrics collector if it is not running.
Starting a fresh collector removes old metrics/log files because schema
changes during development can otherwise leave mixed JSONL records behind.
The collector prunes its own metrics file to 7 days / max_lines.
"""
command = f"""
cat > {shlex.quote(paths.script_file)} <<'MLV_RESOURCE_COLLECTOR'
{COLLECTOR_SCRIPT}
MLV_RESOURCE_COLLECTOR
chmod +x {shlex.quote(paths.script_file)}
if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
echo "already running pid=$(cat {shlex.quote(paths.pid_file)})"
else
rm -f {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)}
nohup {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {int(interval_seconds)} {int(retention_seconds)} {int(max_lines)} >> {shlex.quote(paths.log_file)} 2>&1 &
echo $! > {shlex.quote(paths.pid_file)}
echo "started pid=$(cat {shlex.quote(paths.pid_file)})"
fi
"""
logger.info(
"Starting remote resource collector interval=%ss retention=%ss max_lines=%s", interval_seconds, retention_seconds, max_lines
)
result = ssh.run(command, timeout=20)
if result.exit_status != 0:
logger.warning("Failed to start remote resource collector: %s", result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to start resource collector")
logger.info("Remote resource collector start response: %s", result.stdout.strip())
return result.stdout.strip()
def stop_resource_collector(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
"""Stop the remote collector process if the pid file points to one."""
command = f"""
if [ -f {shlex.quote(paths.pid_file)} ]; then
pid="$(cat {shlex.quote(paths.pid_file)})"
if kill -0 "$pid" 2>/dev/null; then
kill "$pid"
echo "stopped pid=$pid"
else
echo "not running"
fi
rm -f {shlex.quote(paths.pid_file)}
else
echo "not running"
fi
"""
logger.info("Stopping remote resource collector")
result = ssh.run(command, timeout=20)
if result.exit_status != 0:
logger.warning("Failed to stop remote resource collector: %s", result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to stop resource collector")
logger.info("Remote resource collector stop response: %s", result.stdout.strip())
return result.stdout.strip()
def restart_resource_collector(
ssh: RemoteSSHClient,
interval_seconds: int = 10,
retention_seconds: int = 7 * 24 * 60 * 60,
max_lines: int = 70_000,
paths: ResourceMonitorPaths = ResourceMonitorPaths(),
) -> str:
logger.info("Restarting remote resource collector")
stop_message = stop_resource_collector(ssh, paths)
start_message = start_resource_collector(ssh, interval_seconds, retention_seconds, max_lines, paths)
return f"{stop_message}\n{start_message}"
def resource_collector_status(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
"""Return a short human-readable status string for the dashboard."""
command = f"""
if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
echo "running"
else
echo "not running"
fi
"""
result = ssh.run(command, timeout=10)
if result.exit_status != 0:
logger.warning("Failed to read collector status: %s", result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to check collector status")
status = result.stdout.strip()
logger.info("Resource collector status: %s", status)
return status
def resource_collector_debug_info(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
"""Collect remote diagnostics for troubleshooting missing metrics."""
command = f"""
echo "status:"
if [ -f {shlex.quote(paths.pid_file)} ]; then
pid="$(cat {shlex.quote(paths.pid_file)})"
echo "pid_file=$pid"
if kill -0 "$pid" 2>/dev/null; then echo "process=running"; else echo "process=not-running"; fi
else
echo "pid_file=missing"
fi
echo "files:"
ls -l {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} 2>&1 || true
echo "sample_count:"
if [ -f {shlex.quote(paths.metrics_file)} ]; then wc -l < {shlex.quote(paths.metrics_file)}; else echo 0; fi
echo "last_samples:"
if [ -f {shlex.quote(paths.metrics_file)} ]; then tail -n 5 {shlex.quote(paths.metrics_file)}; fi
echo "log_tail:"
if [ -f {shlex.quote(paths.log_file)} ]; then tail -n 40 {shlex.quote(paths.log_file)}; fi
echo "netdev_snapshot:"
cat /proc/net/dev 2>&1 || true
"""
logger.info("Collecting resource collector diagnostics")
result = ssh.run(command, timeout=20)
output = (result.stdout or "") + (result.stderr or "")
logger.debug("Resource collector diagnostics length=%s", len(output))
return output
def read_resource_metrics(ssh: RemoteSSHClient, max_lines: int = 1000, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> list[dict[str, Any]]:
"""Read recent JSONL metric samples from the remote collector file."""
command = f"test -f {shlex.quote(paths.metrics_file)} && tail -n {int(max_lines)} {shlex.quote(paths.metrics_file)} || true"
logger.debug("Reading up to %s resource metric lines", max_lines)
result = ssh.run(command, timeout=20)
if result.exit_status != 0:
logger.warning("Failed to read resource metrics: %s", result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to read resource metrics")
rows = []
for line in result.stdout.splitlines():
line = line.strip()
if not line:
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
continue
return rows
def disk_space(ssh: RemoteSSHClient, path: str = "/") -> dict[str, Any]:
"""Return df information for the filesystem containing ``path``."""
command = (
"df -P -B1 -- "
+ shlex.quote(path or "/")
+ " | awk 'NR==2 {printf \"{\\\"filesystem\\\":\\\"%s\\\",\\\"size\\\":%s,\\\"used\\\":%s,\\\"available\\\":%s,\\\"used_pct\\\":\\\"%s\\\",\\\"mount\\\":\\\"%s\\\"}\", $1,$2,$3,$4,$5,$6}'"
)
logger.debug("Reading disk space for path=%s", path)
result = ssh.run(command, timeout=20)
if result.exit_status != 0 or not result.stdout.strip():
logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
data = json.loads(result.stdout)
logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct"))
return data
backend/src/media_library_viewer_api/clients/ssh.py
+2 -6
View File
@@ -75,14 +75,10 @@ class RemoteSSHClient:
client = paramiko.SSHClient()
client.load_system_host_keys()
known_hosts_file = Path(self.known_hosts_path) if self.known_hosts_path else None
trusted_before = bool(
known_hosts_file and has_known_host(self.host, self.port, known_hosts_file)
)
trusted_before = bool(known_hosts_file and has_known_host(self.host, self.port, known_hosts_file))
if known_hosts_file and known_hosts_file.is_file():
client.load_host_keys(str(known_hosts_file))
client.set_missing_host_key_policy(
paramiko.RejectPolicy() if trusted_before else paramiko.AutoAddPolicy()
)
client.set_missing_host_key_policy(paramiko.RejectPolicy() if trusted_before else paramiko.AutoAddPolicy())
connect_kwargs: dict[str, Any] = {
"hostname": self.host,
"port": self.port,
backend/src/media_library_viewer_api/config.py
+8 -4
View File
@@ -23,6 +23,7 @@ class Settings(BaseSettings):
# Logging
log_level: str = "INFO"
log_format: str = "text" # "text" or "json"
# Auth / OIDC (Authentik-compatible JWT validation)
auth_enabled: bool = False
@@ -48,7 +49,6 @@ class Settings(BaseSettings):
ssh_port: int = 22
ssh_key_directory: str = ""
ssh_key_name: str = ""
ssh_key_file: str = "/run/secrets/ssh_private_key"
ssh_password: str = ""
ssh_known_hosts_path: str = ""
@@ -57,6 +57,12 @@ class Settings(BaseSettings):
monitoring_poll_initial_delay_seconds: int = 20
monitoring_action_retention_days: int = 30
# Observability
prometheus_enabled: bool = True
prometheus_file_sd_dir: str = "/app/backend/.cache/prometheus-file-sd"
alertmanager_url: str = "http://alertmanager:9093"
alertmanager_webhook_url: str = "" # Optional receiver for alertmanager webhook notifications
# Remote paths
remote_media_root: str = ""
remote_path_prefix: str = ""
@@ -72,8 +78,6 @@ class Settings(BaseSettings):
@property
def ssh_key_path(self) -> str:
if self.ssh_key_file:
return self.ssh_key_file
if not self.ssh_key_directory or not self.ssh_key_name:
return ""
return str(Path(self.ssh_key_directory) / self.ssh_key_name)
@@ -82,7 +86,7 @@ class Settings(BaseSettings):
def ssh_known_hosts_file(self) -> Path:
return Path(self.ssh_known_hosts_path or ".cache/media_library_viewer/known_hosts")
model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"}
model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"} # type: ignore[assignment]
def _find_env_file() -> str | None:
backend/src/media_library_viewer_api/dependencies.py
+66 -32
View File
@@ -19,12 +19,16 @@ from media_library_viewer_api.clients.jellyseerr import JellyseerrClient
from media_library_viewer_api.clients.local import LocalCommandClient
from media_library_viewer_api.clients.ssh import RemoteSSHClient
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.services.mail_queue import MailQueue, get_mail_queue as _get_mail_queue
from media_library_viewer_api.services.mail_queue import MailQueue
from media_library_viewer_api.services.mail_queue import get_mail_queue as _get_mail_queue
from media_library_viewer_api.services.monitoring_poller import (
MonitoringPoller,
)
from media_library_viewer_api.services.monitoring_poller import (
get_monitoring_poller as _get_monitoring_poller,
)
from media_library_viewer_api.services.settings_store import SettingsStore, get_settings_store as _get_settings_store
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.settings_store import get_settings_store as _get_settings_store
logger = logging.getLogger(__name__)
@@ -39,7 +43,9 @@ def _request_machine_id(request: Request | None) -> str | None:
@lru_cache(maxsize=32)
def _jellyfin_client_for(cache_key: tuple[str, str, str]) -> JellyfinClient:
machine_id, url, api_key = cache_key
logger.info("Creating Jellyfin client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>")
logger.info(
"Creating Jellyfin client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>"
)
return JellyfinClient(url, api_key)
@@ -52,13 +58,19 @@ def _jellyseerr_client_for(cache_key: tuple[str, str]) -> JellyseerrClient | Non
api_key = (settings or {}).get("jellyseerr_api_key") if settings else ""
if not api_key:
return None
logger.info("Creating Jellyseerr client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>")
logger.info(
"Creating Jellyseerr client machine_id=%s url=%s", machine_id or "<default>", url.rstrip("/") or "<unset>"
)
return JellyseerrClient(url, api_key)
@lru_cache(maxsize=32)
def _ssh_client_for(cache_key: tuple[str, str, str, int, str, str | None, str | None, str | None, str | None]) -> RemoteSSHClient:
machine_id, host, username, port, key_filename, password, private_key, private_key_passphrase, known_hosts_path = cache_key
def _ssh_client_for(
cache_key: tuple[str, str, str, int, str, str | None, str | None, str | None, str | None],
) -> RemoteSSHClient:
machine_id, host, username, port, key_filename, password, private_key, private_key_passphrase, known_hosts_path = (
cache_key
)
logger.info(
"Creating SSH client machine_id=%s host=%s user=%s port=%s key=%s password=%s private_key=%s passphrase=%s",
machine_id or "<default>",
@@ -141,7 +153,9 @@ def get_jellyfin_client(request: Request = None) -> JellyfinClient:
cache_key = (machine["id"], machine["jellyfin_url"], machine.get("jellyfin_api_key") or "")
return _jellyfin_client_for(cache_key)
raise RuntimeError("No Jellyfin machine is configured. Add a machine with jellyfin_url and jellyfin_api_key in Settings.")
raise RuntimeError(
"No Jellyfin machine is configured. Add a machine with jellyfin_url and jellyfin_api_key in Settings."
)
def get_jellyseerr_client(request: Request = None) -> JellyseerrClient | None:
@@ -160,6 +174,37 @@ def get_jellyseerr_client(request: Request = None) -> JellyseerrClient | None:
return None
def _ssh_client_from_machine_config(machine: dict[str, Any], store: SettingsStore | None = None) -> RemoteSSHClient:
"""Build a RemoteSSHClient from a machine config dict."""
store = store or get_settings_store()
known_hosts_path = get_settings().ssh_known_hosts_file
key_data = None
key_passphrase = None
ssh_key_id = str(machine.get("ssh_key_id") or "").strip()
if ssh_key_id:
ssh_key = store.get_ssh_key(ssh_key_id)
if ssh_key:
key_data = ssh_key.get("private_key") or None
key_passphrase = ssh_key.get("passphrase") or None
if not key_data and machine.get("ssh_private_key"):
key_data = machine.get("ssh_private_key") or None
key_passphrase = machine.get("ssh_private_key_passphrase") or None
cache_key = (
machine["id"],
machine["host"],
machine["username"],
int(machine.get("port") or 22),
f"{machine.get('key_directory')}/{machine.get('key_name')}"
if machine.get("key_directory") and machine.get("key_name")
else "",
machine.get("password") or None,
key_data,
key_passphrase,
str(known_hosts_path),
)
return _ssh_client_for(cache_key)
def get_ssh_client(request: Request = None):
"""Return a command client for the selected machine or legacy env fallback."""
store = get_settings_store()
@@ -172,30 +217,7 @@ def get_ssh_client(request: Request = None):
logger.info("Creating LocalCommandClient machine_id=%s", machine["id"])
return LocalCommandClient()
if machine and machine.get("host") and machine.get("username"):
known_hosts_path = get_settings().ssh_known_hosts_file
key_data = None
key_passphrase = None
ssh_key_id = str(machine.get("ssh_key_id") or "").strip()
if ssh_key_id:
ssh_key = store.get_ssh_key(ssh_key_id)
if ssh_key:
key_data = ssh_key.get("private_key") or None
key_passphrase = ssh_key.get("passphrase") or None
if not key_data and machine.get("ssh_private_key"):
key_data = machine.get("ssh_private_key") or None
key_passphrase = machine.get("ssh_private_key_passphrase") or None
cache_key = (
machine["id"],
machine["host"],
machine["username"],
int(machine.get("port") or 22),
f"{machine.get('key_directory')}/{machine.get('key_name')}" if machine.get("key_directory") and machine.get("key_name") else "",
machine.get("password") or None,
key_data,
key_passphrase,
str(known_hosts_path),
)
return _ssh_client_for(cache_key)
return _ssh_client_from_machine_config(machine, store)
settings = get_settings()
logger.info(
@@ -209,7 +231,19 @@ def get_ssh_client(request: Request = None):
)
if not settings.ssh_key_path:
raise RuntimeError("No SSH machine is configured and SSH key settings must be configured")
return _ssh_client_for(("legacy", settings.ssh_host, settings.ssh_username, settings.ssh_port, settings.ssh_key_path, settings.ssh_password or None, None, None, str(settings.ssh_known_hosts_file)))
return _ssh_client_for(
(
"legacy",
settings.ssh_host,
settings.ssh_username,
settings.ssh_port,
settings.ssh_key_path,
settings.ssh_password or None,
None,
None,
str(settings.ssh_known_hosts_file),
)
)
def get_mail_queue() -> MailQueue:
backend/src/media_library_viewer_api/domain/media.py
+5 -1
View File
@@ -32,7 +32,11 @@ def media_streams(item: dict[str, Any], stream_type: str | None = None) -> list[
streams.extend(source.get("MediaStreams") or [])
if stream_type is None:
return streams
return [stream for stream in streams if str(stream.get("Type") or stream.get("codec_type") or "").lower() == stream_type.lower()]
return [
stream
for stream in streams
if str(stream.get("Type") or stream.get("codec_type") or "").lower() == stream_type.lower()
]
def stream_value(stream: dict[str, Any], *keys: str) -> Any:
backend/src/media_library_viewer_api/jobs.py
+36
View File
@@ -52,6 +52,42 @@ JOB_TEMPLATES: dict[str, JobTemplate] = {
description="Lists empty directories under the selected path. Does not delete anything.",
command_template="find {path} -type d -empty -print",
),
"install_node_exporter": JobTemplate(
name="Install Node Exporter",
description="Downloads and installs prometheus-node-exporter via package manager (apt/dnf/yum/zypper).",
command_template=(
"set -e; "
"if command -v apt-get >/dev/null 2>&1; then "
"sudo apt-get update && sudo apt-get install -y prometheus-node-exporter; "
"elif command -v dnf >/dev/null 2>&1; then "
"sudo dnf install -y prometheus-node-exporter; "
"elif command -v yum >/dev/null 2>&1; then "
"sudo yum install -y prometheus-node-exporter; "
"elif command -v zypper >/dev/null 2>&1; then "
"sudo zypper install -y prometheus-node-exporter; "
"else echo 'No supported package manager found' >&2; exit 1; "
"fi; "
"sudo systemctl enable --now prometheus-node-exporter; "
"echo installed at {path}"
),
),
"restart_node_exporter": JobTemplate(
name="Restart Node Exporter",
description="Restarts the prometheus-node-exporter systemd service.",
command_template="sudo systemctl restart prometheus-node-exporter; echo restarted at {path}",
),
"node_exporter_status": JobTemplate(
name="Node Exporter status",
description="Checks whether prometheus-node-exporter is installed, enabled, and running.",
command_template=(
"systemctl status prometheus-node-exporter --no-pager || true; "
"echo '---'; "
"command -v node_exporter >/dev/null 2>&1 "
"&& node_exporter --version 2>&1 | head -1 "
"|| echo 'node_exporter binary not found'; "
"echo checked {path}"
),
),
}
backend/src/media_library_viewer_api/logging_utils.py
+44 -5
View File
@@ -4,19 +4,42 @@ from __future__ import annotations
import logging
import os
from typing import Any
from urllib.parse import urlsplit
from pythonjsonlogger import jsonlogger
def configure_logging(level_name: str | None = None) -> int:
def _json_formatter() -> logging.Formatter:
"""Return a JSON formatter that preserves timestamp, level, logger, and message."""
return jsonlogger.JsonFormatter(
fmt="%(timestamp)s %(level)s %(name)s %(message)s",
rename_fields={"asctime": "timestamp", "levelname": "level"},
)
def _text_formatter() -> logging.Formatter:
"""Return the legacy plain-text formatter."""
return logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
def configure_logging(level_name: str | None = None, log_format: str | None = None) -> int:
"""Configure root logging once and return the numeric log level."""
resolved_name = (level_name or os.getenv("LOG_LEVEL", "INFO")).upper()
level = getattr(logging, resolved_name, logging.INFO)
use_json = (log_format or os.getenv("LOG_FORMAT", "text")).lower() == "json"
root = logging.getLogger()
if not root.handlers:
logging.basicConfig(
level=level,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
handler = logging.StreamHandler()
handler.setFormatter(_json_formatter() if use_json else _text_formatter())
logging.basicConfig(level=level, handlers=[handler])
else:
# Reconfigure existing handler formatters if the format changed.
formatter = _json_formatter() if use_json else _text_formatter()
for handler in root.handlers:
handler.setFormatter(formatter)
root.setLevel(level)
logging.getLogger("media_library_viewer_api").setLevel(level)
logging.getLogger("uvicorn").setLevel(level)
@@ -40,6 +63,7 @@ def describe_settings(settings: object) -> dict[str, str]:
"""Return a secret-safe summary of the current backend settings."""
return {
"log_level": str(getattr(settings, "log_level", "INFO") or "INFO").upper(),
"log_format": str(getattr(settings, "log_format", "text") or "text").lower(),
"auth_enabled": str(bool(getattr(settings, "auth_enabled", True))),
"oidc_issuer_url": _sanitize_url(getattr(settings, "oidc_issuer_url", "")),
"oidc_audience": getattr(settings, "oidc_audience", "") or "<unset>",
@@ -60,3 +84,18 @@ def describe_settings(settings: object) -> dict[str, str]:
"remote_media_root": getattr(settings, "remote_media_root", "") or "<unset>",
"remote_path_prefix": getattr(settings, "remote_path_prefix", "") or "<unset>",
}
def sanitize_log_extra(extra: dict[str, Any] | None) -> dict[str, Any]:
"""Remove obvious secret keys from log extra fields before emission."""
if not extra:
return {}
blocked_suffixes = ("_password", "_secret", "_token", "_key", "private_key", "passphrase")
sanitized: dict[str, Any] = {}
for key, value in extra.items():
lower_key = key.lower()
if any(lower_key.endswith(suffix) for suffix in blocked_suffixes):
sanitized[key] = "<redacted>" if value else ""
else:
sanitized[key] = value
return sanitized
backend/src/media_library_viewer_api/main.py
+50 -10
View File
@@ -7,13 +7,20 @@ import time
from contextlib import asynccontextmanager
import uvicorn
from fastapi import FastAPI, Request
from fastapi import FastAPI, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import Response as FastAPIResponse
from media_library_viewer_api.auth import require_jwt_auth, validate_auth_settings
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
from media_library_viewer_api.dependencies import get_mail_queue, get_monitoring_poller, get_settings_store
from media_library_viewer_api.logging_utils import configure_logging, describe_settings, sanitize_log_extra
from media_library_viewer_api.observability import (
get_request_id,
metrics_payload,
record_request,
set_current_request_id,
)
from media_library_viewer_api.routers import backups as backups_router
from media_library_viewer_api.routers import dashboard, files, jobs, media, monitoring, tasks, users
from media_library_viewer_api.routers.settings import router as settings_router
@@ -28,10 +35,16 @@ logger = logging.getLogger(__name__)
async def lifespan(app: FastAPI):
"""Application lifespan — startup/shutdown."""
settings = get_settings()
configure_logging(settings.log_level)
configure_logging(settings.log_level, settings.log_format)
validate_auth_settings(settings)
logger.info("Backend startup complete: %s", describe_settings(settings))
logger.info("Managed known_hosts will be populated lazily on first successful SSH connection")
try:
from media_library_viewer_api.services.targets import write_prometheus_targets
write_prometheus_targets(get_settings_store())
except Exception:
logger.exception("Failed to write Prometheus file-SD targets during startup")
mail_queue = get_mail_queue()
monitoring_poller = get_monitoring_poller()
backup_poller = get_backup_poller()
@@ -49,8 +62,7 @@ app = FastAPI(
title="Manage API",
version=get_backend_version(),
description=(
"Manage API for Jellyfin media browsing, SSH file inspection, "
"server monitoring, and JWT-protected access."
"Manage API for Jellyfin media browsing, SSH file inspection, server monitoring, and JWT-protected access."
),
lifespan=lifespan,
)
@@ -69,30 +81,51 @@ app.add_middleware(
allow_headers=["*"],
)
@app.middleware("http")
async def enforce_jwt_auth(request: Request, call_next):
if request.url.path in {"/api/health", "/api/version"}:
if request.url.path in {"/api/health", "/api/version", "/metrics"}:
return await call_next(request)
return await require_jwt_auth(request, call_next)
@app.middleware("http")
async def log_requests(request: Request, call_next):
"""Log every API request with timing and outcome."""
"""Log every API request with timing, outcome, and request id."""
start = time.perf_counter()
request_id = get_request_id(request)
set_current_request_id(request_id)
request.state.request_id = request_id
client_host = request.client.host if request.client else "unknown"
logger.info("request start %s %s client=%s", request.method, request.url.path, client_host)
extra = sanitize_log_extra({"request_id": request_id, "client_host": client_host})
logger.info("request start %s %s", request.method, request.url.path, extra=extra)
try:
response = await call_next(request)
except Exception:
logger.exception("request error %s %s client=%s", request.method, request.url.path, client_host)
logger.exception(
"request error %s %s",
request.method,
request.url.path,
extra=sanitize_log_extra({"request_id": request_id, "client_host": client_host}),
)
raise
elapsed_ms = (time.perf_counter() - start) * 1000.0
response.headers["X-Request-Id"] = request_id
record_request(request, response, elapsed_ms / 1000.0)
logger.info(
"request end %s %s status=%s elapsed_ms=%.1f",
request.method,
request.url.path,
response.status_code,
elapsed_ms,
extra=sanitize_log_extra(
{
"request_id": request_id,
"client_host": client_host,
"status_code": response.status_code,
"elapsed_ms": round(elapsed_ms, 1),
}
),
)
return response
@@ -123,5 +156,12 @@ def version_info() -> dict[str, str]:
return get_version_info()
@app.get("/metrics")
def metrics() -> Response:
"""Expose Prometheus metrics."""
data, content_type = metrics_payload()
return FastAPIResponse(content=data, media_type=content_type)
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)
backend/src/media_library_viewer_api/observability.py
+154
View File
@@ -0,0 +1,154 @@
"""Application observability: metrics and request correlation.
This module owns Prometheus metrics and request-id generation so that the
rest of the backend can stay focused on business logic.
"""
from __future__ import annotations
import uuid
from contextvars import ContextVar
from typing import Any
from fastapi import Request, Response
from prometheus_client import (
CONTENT_TYPE_LATEST,
Counter,
Gauge,
Histogram,
generate_latest,
)
# Context-local request id for code paths that cannot receive a Request object.
_current_request_id: ContextVar[str | None] = ContextVar("request_id", default=None)
# Request metrics
REQUESTS_TOTAL = Counter(
"manage_api_requests_total",
"Total API requests",
["method", "path", "status_code"],
)
REQUEST_DURATION = Histogram(
"manage_api_request_duration_seconds",
"API request duration",
["method", "path"],
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0),
)
# Operation metrics
SSH_COMMANDS_TOTAL = Counter(
"manage_ssh_commands_total",
"Total SSH/local commands executed",
["machine_id", "action", "status"],
)
SSH_COMMAND_DURATION = Histogram(
"manage_ssh_command_duration_seconds",
"SSH/local command duration",
["machine_id", "action"],
buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0),
)
MEDIA_INDEX_BUILDS_TOTAL = Counter(
"manage_media_index_builds_total",
"Total media index build attempts",
["status"],
)
MEDIA_INDEX_BUILD_DURATION = Histogram(
"manage_media_index_build_duration_seconds",
"Media index build duration",
buckets=(1.0, 5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0),
)
BACKUP_RUNS_TOTAL = Counter(
"manage_backup_runs_total",
"Total backup runs",
["job_name", "status"],
)
BACKUP_RUNS_LAST_SUCCESS = Gauge(
"manage_backup_runs_last_success_timestamp",
"Unix timestamp of the last successful backup run per job",
["job_name"],
)
MAIL_QUEUE_SIZE = Counter(
"manage_mail_queue_messages_total",
"Total messages enqueued",
["status"],
)
def set_current_request_id(request_id: str | None) -> None:
"""Set the context-local request id."""
_current_request_id.set(request_id)
def get_current_request_id() -> str | None:
"""Return the current context-local request id or None."""
return _current_request_id.get()
def generate_request_id() -> str:
"""Return a short unique request id."""
return uuid.uuid4().hex[:16]
def get_request_id(request: Request | None = None) -> str:
"""Resolve a request id from the request header, context, or a new value."""
if request is not None:
header = request.headers.get("x-request-id") or request.headers.get("x-correlation-id")
if header:
return header.strip()
existing = _current_request_id.get()
if existing:
return existing
new_id = generate_request_id()
_current_request_id.set(new_id)
return new_id
def metrics_payload() -> tuple[bytes, str]:
"""Return the Prometheus metrics payload and content type."""
return generate_latest(), CONTENT_TYPE_LATEST
def record_request(request: Request, response: Response, duration_seconds: float) -> None:
"""Record Prometheus metrics for a completed request."""
status = str(response.status_code)
path = request.url.path
method = request.method
REQUESTS_TOTAL.labels(method=method, path=path, status_code=status).inc()
REQUEST_DURATION.labels(method=method, path=path).observe(duration_seconds)
def record_ssh_command(machine_id: str, action: str, status: str, duration_seconds: float) -> None:
"""Record metrics for an SSH/local command."""
SSH_COMMANDS_TOTAL.labels(machine_id=machine_id or "unknown", action=action, status=status).inc()
SSH_COMMAND_DURATION.labels(machine_id=machine_id or "unknown", action=action).observe(duration_seconds)
def record_media_index_build(status: str, duration_seconds: float | None = None) -> None:
"""Record metrics for a media index build."""
MEDIA_INDEX_BUILDS_TOTAL.labels(status=status).inc()
if duration_seconds is not None:
MEDIA_INDEX_BUILD_DURATION.observe(duration_seconds)
def record_backup_run(job_name: str, status: str, success: bool = False) -> None:
"""Record metrics for a backup run."""
job_name = job_name or "unknown"
BACKUP_RUNS_TOTAL.labels(job_name=job_name, status=status).inc()
if success:
BACKUP_RUNS_LAST_SUCCESS.labels(job_name=job_name).set_to_current_time()
def record_mail_queue(status: str) -> None:
"""Record metrics for a mail queue message outcome."""
MAIL_QUEUE_SIZE.labels(status=status).inc()
def log_extra(request: Request | None = None, **kwargs: Any) -> dict[str, Any]:
"""Build a standard extra dict for structured logging."""
extra: dict[str, Any] = {"request_id": get_request_id(request)}
extra.update(kwargs)
return extra
backend/src/media_library_viewer_api/path_utils.py
+1 -1
View File
@@ -57,7 +57,7 @@ def map_path_to_media_root(path: str, media_root: str) -> str:
if root_anchor in raw_parts:
anchor_index = raw_parts.index(root_anchor)
remainder_parts = raw_parts[anchor_index + 1:]
remainder_parts = raw_parts[anchor_index + 1 :]
resolved = posixpath.join(normalized_root, *remainder_parts) if remainder_parts else normalized_root
logger.debug("Mapped path to media root path=%s media_root=%s resolved=%s", path, normalized_root, resolved)
return resolved
backend/src/media_library_viewer_api/routers/backups.py
+20 -13
View File
@@ -8,6 +8,7 @@ from ..models.backups import (
BackupReportRequest,
BackupRunResponse,
)
from ..observability import record_backup_run
from ..services.backup_alert_engine import generate_alerts_for_run
from ..services.settings_store import SettingsStore, get_settings_store
@@ -17,20 +18,24 @@ router = APIRouter(prefix="/api/backups", tags=["backups"])
def _get_or_create_job(store: SettingsStore, report: BackupReportRequest) -> dict[str, Any]:
job = store.get_backup_job_by_name(report.name)
if not job:
job = store.upsert_backup_job({
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
})
job = store.upsert_backup_job(
{
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
}
)
elif report.schedule_interval_seconds:
store.upsert_backup_job({
"id": job["id"],
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
})
store.upsert_backup_job(
{
"id": job["id"],
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
}
)
job = store.get_backup_job(job["id"])
return job
@@ -61,6 +66,7 @@ def post_backup_report(
"details": report.details,
}
run = store.create_backup_run(run_data)
record_backup_run(job_name=report.name, status=report.status, success=report.status == "success")
# Generate alerts
previous_runs = store.list_backup_runs(job_id=job["id"], status="success", limit=20)
@@ -93,6 +99,7 @@ def post_backup_start(
"status": "in_progress",
}
run = store.create_backup_run(run_data)
record_backup_run(job_name=report.name, status="in_progress")
# Map details -> details_json for response model
run["details_json"] = run.pop("details", None)
return BackupRunResponse(**run)
backend/src/media_library_viewer_api/routers/dashboard.py
+14 -60
View File
@@ -11,12 +11,10 @@ from fastapi import APIRouter, Depends
from media_library_viewer_api.clients.jellyfin import JellyfinClient
from media_library_viewer_api.dependencies import (
get_jellyfin_client,
get_monitoring_poller,
get_settings_store,
get_user_id,
)
from media_library_viewer_api.models.backups import BackupDashboardSummary
from media_library_viewer_api.services.monitoring_actions import collect_machine_overview
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
@@ -46,52 +44,6 @@ def get_library_counts(
return client.library_item_counts(user_id, libraries)
@router.get("/monitoring")
def get_monitoring_overview(
store=Depends(get_settings_store),
) -> dict[str, Any]:
"""Return one lightweight monitoring row per configured machine."""
machines = [m for m in store.list_machines() if m.get("enabled")]
rows: list[dict[str, Any]] = []
for machine in machines:
try:
rows.append(collect_machine_overview(machine, store))
except Exception as exc:
logger.exception(
"Dashboard monitoring overview failed for machine_id=%s",
machine.get("id"),
)
rows.append({
"machine": {k: machine.get(k) for k in (
"id", "name", "mode", "enabled", "host", "port", "username",
"media_root", "path_prefix", "notes",
)},
"status": "",
"status_error": str(exc),
"metrics_error": str(exc),
"disk_error": str(exc),
"sample_count": 0,
"latest_sample": None,
"cpu_summary": None,
"iowait_summary": None,
"mem_summary": None,
"net_rx_summary": None,
"net_tx_summary": None,
"disk_read_summary": None,
"disk_write_summary": None,
"disk": None,
})
poller = get_monitoring_poller().snapshot()
enabled_count = sum(1 for machine in machines if machine.get("enabled"))
logger.info("Dashboard monitoring machines=%s enabled=%s", len(machines), enabled_count)
return {
"poller": poller,
"machines": rows,
"total": len(machines),
"enabled": enabled_count,
}
@router.get("/shortcuts")
def get_shortcuts(
store=Depends(get_settings_store),
@@ -144,8 +96,8 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[
has_item = bool(item)
series = item.get("SeriesName") or ""
title = (
f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")
) if has_item else "(idle)"
(f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")) if has_item else "(idle)"
)
if not has_item:
state_label = "idle"
@@ -162,16 +114,18 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[
if not transcode_type:
transcode_type.append("active")
results.append({
"user": session.get("UserName") or "Unknown",
"title": title,
"type": item.get("Type", "") if has_item else "",
"state": state_label,
"transcoding": "yes" if is_transcoding else "no",
"transcoding_type": ", ".join(transcode_type),
"device": session.get("DeviceName") or session.get("Client") or "",
"session_id": session.get("Id") or "",
})
results.append(
{
"user": session.get("UserName") or "Unknown",
"title": title,
"type": item.get("Type", "") if has_item else "",
"state": state_label,
"transcoding": "yes" if is_transcoding else "no",
"transcoding_type": ", ".join(transcode_type),
"device": session.get("DeviceName") or session.get("Client") or "",
"session_id": session.get("Id") or "",
}
)
return results
backend/src/media_library_viewer_api/routers/media.py
+3
View File
@@ -16,6 +16,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status
from media_library_viewer_api.clients.jellyfin import JellyfinClient
from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id
from media_library_viewer_api.observability import record_media_index_build
from media_library_viewer_api.services.media_index import MediaIndex
logger = logging.getLogger(__name__)
@@ -169,6 +170,7 @@ def post_build_index(
},
)
media_status = index.status()
record_media_index_build(status="started")
logger.info("Media index build started pid=%s", process.pid)
return {"status": "started", **_serialize_status(media_status)}
@@ -256,6 +258,7 @@ def force_stop_build(index: MediaIndex = Depends(get_media_index)) -> dict[str,
},
)
media_status = index.status()
record_media_index_build(status="force_stopped")
return {"status": "force_stopped", **_serialize_status(media_status)}
backend/src/media_library_viewer_api/routers/monitoring.py
+146 -100
View File
@@ -1,31 +1,70 @@
"""Monitoring router — metrics, collector controls, and per-machine status."""
"""Monitoring router — disk checks, action history, and observability stack status."""
from __future__ import annotations
import logging
import time
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Body, Depends, HTTPException, Query
from media_library_viewer_api.clients.resources import (
disk_space,
read_resource_metrics,
resource_collector_status,
)
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_settings_store
from media_library_viewer_api.services.monitoring_actions import (
poll_machine_diagnostics,
disk_space,
run_machine_operation,
start_collector,
stop_collector,
restart_collector,
)
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import build_node_exporter_targets
logger = logging.getLogger(__name__)
def _alertmanager_client() -> Any:
"""Return a simple HTTP client for the configured Alertmanager URL."""
import requests
settings = get_settings()
return requests.Session(), settings.alertmanager_url
def _webhook_client() -> Any:
"""Return a simple HTTP client for the optional webhook receiver URL."""
import requests
settings = get_settings()
return requests.Session(), settings.alertmanager_webhook_url
def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
"""Build a UI-friendly summary from Alertmanager /api/v1/alerts payload."""
by_severity: dict[str, int] = {}
open_alerts: list[dict[str, Any]] = []
for alert in alerts:
labels = alert.get("labels") or {}
annotations = alert.get("annotations") or {}
severity = labels.get("severity", "unknown")
by_severity[severity] = by_severity.get(severity, 0) + 1
open_alerts.append(
{
"name": labels.get("alertname", "unknown"),
"severity": severity,
"category": labels.get("category", ""),
"job_name": labels.get("job_name", labels.get("job", "")),
"summary": annotations.get("summary", ""),
"description": annotations.get("description", ""),
"active_since": alert.get("startsAt"),
"state": alert.get("status", "firing"),
"labels": labels,
}
)
open_alerts.sort(key=lambda a: (a["severity"] not in {"critical", "warning"}, a["severity"], a["name"]))
return {
"total": len(open_alerts),
"by_severity": by_severity,
"alerts": open_alerts[:50],
}
router = APIRouter(prefix="/api/monitoring", tags=["monitoring"])
@@ -63,7 +102,11 @@ def get_poller_status() -> dict[str, Any]:
from media_library_viewer_api.dependencies import get_monitoring_poller
poller = get_monitoring_poller().snapshot()
logger.info("Monitoring poller status requested running=%s poll_count=%s", poller.get("worker_running"), poller.get("poll_count"))
logger.info(
"Monitoring poller status requested running=%s poll_count=%s",
poller.get("worker_running"),
poller.get("poll_count"),
)
return poller
@@ -81,53 +124,6 @@ def get_machine_actions(
return {"items": actions, "total": len(actions)}
@router.get("/status")
def get_status(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Return collector running status for a given machine."""
machine = _resolve_machine(store, machine_id)
status = run_machine_operation(machine, store, "status lookup", resource_collector_status)
logger.info("Monitoring status requested machine_id=%s status=%s", machine["id"], status)
return {"status": status}
@router.get("/metrics")
def get_metrics(
max_lines: int = 70_000,
last_seconds: int | None = None,
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Return resource metric samples for a given machine."""
machine = _resolve_machine(store, machine_id)
def _read_metrics(client):
return read_resource_metrics(client, max_lines=max_lines)
rows = run_machine_operation(machine, store, "metrics read", _read_metrics)
if last_seconds is None:
filtered = rows
cutoff_ts = 0.0
else:
cutoff_ts = time.time() - last_seconds
filtered = [row for row in rows if float(row.get("ts", 0)) >= cutoff_ts]
logger.info(
"Monitoring metrics requested machine_id=%s total=%s filtered=%s last_seconds=%s",
machine["id"],
len(rows),
len(filtered),
last_seconds,
)
return {
"samples": filtered,
"total_samples": len(rows),
"filtered_samples": len(filtered),
"cutoff_ts": cutoff_ts,
}
@router.get("/disk")
def get_disk_space(
machine_id: str | None = Query(default=None),
@@ -146,49 +142,99 @@ def get_disk_space(
)
@router.post("/start")
def post_start(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Start the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = start_collector(machine, store)
logger.info("Monitoring collector start result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/prometheus-targets")
def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
"""Return Prometheus file-SD targets for remote Node Exporters.
The backend writes these targets to a JSON file that Prometheus reads via
file_sd_configs. This endpoint returns the same list live from the store so
the UI can preview which machines will be scraped.
"""
targets = build_node_exporter_targets(store)
logger.info("Prometheus targets requested count=%s", len(targets))
return targets
@router.post("/stop")
def post_stop(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Stop the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = stop_collector(machine, store)
logger.info("Monitoring collector stop result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/alerts")
def get_alertmanager_alerts() -> dict[str, Any]:
"""Return a summary of active Alertmanager alerts for the UI.
Proxies the Alertmanager `/api/v1/alerts` endpoint and reshapes the payload
into a stable, UI-friendly format. If Alertmanager is unreachable, the
endpoint returns an empty summary and logs the failure so the UI can still
render a health card instead of an error page.
"""
session, base_url = _alertmanager_client()
try:
response = session.get(f"{base_url}/api/v1/alerts", timeout=5)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager alerts from %s", base_url)
return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_unreachable"}
if data.get("status") != "success":
return {"total": 0, "by_severity": {}, "alerts": [], "error": data.get("error", "unknown")}
summary = _summary_from_alerts(data.get("data", []))
logger.info("Alertmanager alerts requested total=%s", summary["total"])
return summary
@router.post("/restart")
def post_restart(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Restart the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = restart_collector(machine, store)
logger.info("Monitoring collector restart result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/alertmanager-status")
def get_alertmanager_status() -> dict[str, Any]:
"""Return Alertmanager cluster/status for the UI health card.
Uses the Alertmanager `/api/v2/status` endpoint and exposes only the high-
level fields the UI needs: uptime, version, and whether the cluster is
healthy.
"""
session, base_url = _alertmanager_client()
try:
response = session.get(f"{base_url}/api/v2/status", timeout=5)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager status from %s", base_url)
return {"up": False, "version": "", "uptime": ""}
cluster = data.get("cluster") or {}
status = data.get("clusterStatus") or {}
return {
"up": True,
"version": data.get("versionInfo", {}).get("version", ""),
"uptime": status.get("createdAt", ""),
"name": "",
"peers": [p.get("name", "") for p in cluster.get("peers", [])],
}
@router.get("/diagnostics")
def get_diagnostics(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Return collector debug info for a given machine."""
machine = _resolve_machine(store, machine_id)
diagnostics = poll_machine_diagnostics(machine, store)["diagnostics"]
logger.info("Monitoring diagnostics requested machine_id=%s", machine["id"])
return {"diagnostics": diagnostics}
@router.post("/alertmanager-webhook")
def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
"""Receive alerts from Alertmanager and optionally forward to a webhook URL.
This endpoint is the receiver referenced by the optional `webhook_configs`
block in Alertmanager. It logs the payload for audit/debug purposes and, if
`ALERTMANAGER_WEBHOOK_URL` is configured, forwards the alert JSON verbatim.
Forwarding is best-effort: a failure to reach the downstream webhook does
not fail this endpoint, so Alertmanager sees a successful delivery.
"""
alerts = payload.get("alerts", [])
logger.info(
"Received Alertmanager webhook with %s alert(s), status=%s",
len(alerts),
payload.get("status", "unknown"),
)
session, webhook_url = _webhook_client()
if webhook_url:
try:
response = session.post(webhook_url, json=payload, timeout=10)
response.raise_for_status()
logger.info("Forwarded Alertmanager webhook to %s", webhook_url)
except Exception:
logger.exception("Failed to forward Alertmanager webhook to %s", webhook_url)
else:
logger.debug("No ALERTMANAGER_WEBHOOK_URL configured; webhook stored in logs only")
return {"status": "received"}
backend/src/media_library_viewer_api/routers/settings.py
+25 -14
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
from io import StringIO
from typing import Any
@@ -12,11 +13,13 @@ from pydantic import BaseModel, Field
from media_library_viewer_api.clients.ssh import RemoteSSHClient
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_monitoring_poller, get_settings_store
from media_library_viewer_api.services.monitoring_actions import start_collector
from media_library_viewer_api.services.db_maintenance import remove_sqlite_database
from media_library_viewer_api.services.known_hosts import has_known_host
from media_library_viewer_api.services.media_index import MediaIndex
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import write_prometheus_targets
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/settings", tags=["settings"])
@@ -95,9 +98,7 @@ def _raise_ssh_validation_error(host: str, port: int, exc: Exception) -> None:
if "protocol banner" in lowered:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail=(
f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."
),
detail=(f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."),
) from exc
if "no authentication methods available" in lowered or "authentication failed" in lowered:
raise HTTPException(
@@ -125,10 +126,12 @@ def _validate_saved_machine_ssh(machine: MonitoringMachineInput, store: Settings
client.close()
def _start_machine_collector(machine: MonitoringMachineInput, store: SettingsStore) -> None:
if "monitoring" not in {str(service).strip().lower() for service in machine.services}:
return
start_collector(machine.model_dump(exclude_none=True), store)
def _write_prometheus_targets(store: SettingsStore) -> None:
"""Regenerate Prometheus file-SD targets after machine changes."""
try:
write_prometheus_targets(store)
except Exception:
logger.exception("Failed to write Prometheus file-SD targets")
@router.post("/machines/test-ssh")
@@ -137,7 +140,9 @@ def test_machine_ssh(
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
if str(machine.mode or "").strip().lower() != "ssh":
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines"
)
client, host, port = _resolve_ssh_client(machine, store)
settings = get_settings()
@@ -174,7 +179,8 @@ def test_machine_ssh(
return {
"status": "ok",
"message": (
f"SSH connection succeeded for {host}:{port}; host key {'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
f"SSH connection succeeded for {host}:{port}; host key "
f"{'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
),
"host": host,
"port": port,
@@ -188,11 +194,11 @@ def post_machine(
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine.id)
_write_prometheus_targets(store)
poller = get_monitoring_poller()
try:
saved_machine = MonitoringMachineInput.model_validate(saved)
_validate_saved_machine_ssh(saved_machine, store)
_start_machine_collector(saved_machine, store)
finally:
poller.start()
poller.kick()
@@ -208,11 +214,11 @@ def put_machine(
if not store.get_machine(machine_id):
raise HTTPException(status_code=404, detail="Machine not found")
saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine_id)
_write_prometheus_targets(store)
poller = get_monitoring_poller()
try:
saved_machine = MonitoringMachineInput.model_validate(saved)
_validate_saved_machine_ssh(saved_machine, store)
_start_machine_collector(saved_machine, store)
finally:
poller.start()
poller.kick()
@@ -224,6 +230,7 @@ def delete_machine(machine_id: str, store: SettingsStore = Depends(get_settings_
if not store.get_machine(machine_id):
raise HTTPException(status_code=404, detail="Machine not found")
store.delete_machine(machine_id)
_write_prometheus_targets(store)
return {"status": "deleted"}
@@ -311,8 +318,12 @@ def reset_local_database(
expected = "RESET LOCAL DATABASE"
if payload.confirm_phrase.strip().upper() != expected:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Confirmation phrase does not match")
if not (payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected")
if not (
payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible
):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected"
)
settings_removed = remove_sqlite_database(store.db_path)
media_index = MediaIndex()
backend/src/media_library_viewer_api/routers/users_impl.py
+6 -4
View File
@@ -212,7 +212,8 @@ def _merge_users(
"source_summary": summary,
"name_source": name_source,
"access_source": access_source,
"jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId")) or None,
"jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId"))
or None,
"jellyseerr_username": str(
(seerr_user or {}).get("username") or (seerr_user or {}).get("jellyfinUsername") or ""
),
@@ -364,9 +365,10 @@ async def post_user_message(
text_body=text_body,
attachments=attachment_payloads,
)
from_address = str(getattr(settings, "smtp_from_address", "") or "").strip() or str(
getattr(settings, "smtp_username", "") or ""
).strip()
from_address = (
str(getattr(settings, "smtp_from_address", "") or "").strip()
or str(getattr(settings, "smtp_username", "") or "").strip()
)
logger.info(
"Users message queued request_id=%s subject=%s recipients=%s attachments=%s skipped=%s",
request_id,
backend/src/media_library_viewer_api/services/backup_alert_engine.py
+66 -54
View File
@@ -13,20 +13,21 @@ def generate_alerts_for_run(
# 1. Failed status alert
if run["status"] == "failure":
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "failed_status",
"severity": "critical",
"message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "failed_status",
"severity": "critical",
"message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
}
)
# 2. Anomaly size alert
bytes_transferred = run.get("bytes_transferred")
if bytes_transferred is not None and previous_runs:
successful_runs = [
r for r in previous_runs
if r["status"] == "success" and r.get("bytes_transferred") is not None
r for r in previous_runs if r["status"] == "success" and r.get("bytes_transferred") is not None
]
if len(successful_runs) >= 3:
sizes = [r["bytes_transferred"] for r in successful_runs[-7:]]
@@ -34,24 +35,28 @@ def generate_alerts_for_run(
if median_size > 0:
ratio = bytes_transferred / median_size
if bytes_transferred == 0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
}
)
elif ratio < 0.1 or ratio > 3.0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": (
f"Backup job '{job_name}' size anomaly: "
f"{bytes_transferred} bytes (median: {median_size})"
),
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": (
f"Backup job '{job_name}' size anomaly: "
f"{bytes_transferred} bytes (median: {median_size})"
),
}
)
# 3. Anomaly duration alert
duration_ms = run.get("duration_ms")
@@ -61,16 +66,17 @@ def generate_alerts_for_run(
durations = [r["duration_ms"] for r in successful_runs[-7:]]
median_duration = statistics.median(durations)
if median_duration > 0 and duration_ms / median_duration > 3.0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_duration",
"severity": "warning",
"message": (
f"Backup job '{job_name}' duration anomaly: "
f"{duration_ms}ms (median: {median_duration}ms)"
),
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_duration",
"severity": "warning",
"message": (
f"Backup job '{job_name}' duration anomaly: {duration_ms}ms (median: {median_duration}ms)"
),
}
)
return alerts
@@ -82,6 +88,7 @@ def check_missed_schedules(
) -> list[dict[str, Any]]:
alerts = []
import time
now = int(time.time())
for job in jobs:
@@ -93,31 +100,36 @@ def check_missed_schedules(
if not latest_run:
# No runs ever — alert if job is older than interval * 1.5
if now - job["created_at"] > interval * 1.5:
alerts.append({
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
})
alerts.append(
{
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
}
)
else:
last_run_time = latest_run["started_at"]
if now - last_run_time > interval * 1.5:
# Check if there's already an unresolved missed_schedule alert
has_open_alert = any(
a["alert_type"] == "missed_schedule" and a["resolved_at"] is None
for a in existing_alerts if a["job_id"] == job["id"]
for a in existing_alerts
if a["job_id"] == job["id"]
)
if not has_open_alert:
alerts.append({
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": (
f"Backup job '{job['name']}' missed schedule: "
f"last run at {last_run_time} (expected every {interval}s)"
),
})
alerts.append(
{
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": (
f"Backup job '{job['name']}' missed schedule: "
f"last run at {last_run_time} (expected every {interval}s)"
),
}
)
return alerts
backend/src/media_library_viewer_api/services/mail_queue.py
+3
View File
@@ -15,6 +15,7 @@ import uuid
from dataclasses import dataclass, field
from typing import Any
from media_library_viewer_api.observability import record_mail_queue
from media_library_viewer_api.services.mailer import EmailAttachment, describe_smtp_error, send_email_message
logger = logging.getLogger(__name__)
@@ -206,6 +207,7 @@ class MailQueue:
result.get("recipient_count", 0),
result.get("attachment_count", 0),
)
record_mail_queue("sent")
except Exception as exc:
friendly_error = describe_smtp_error(exc)
with self._lock:
@@ -220,6 +222,7 @@ class MailQueue:
getattr(message, "request_id", "unknown"),
friendly_error,
)
record_mail_queue("failed")
finally:
self._queue.task_done()
backend/src/media_library_viewer_api/services/mailer_impl.py
+25 -10
View File
@@ -4,8 +4,8 @@ from __future__ import annotations
import logging
import mimetypes
import socket
import smtplib
import socket
import ssl
from dataclasses import dataclass
from email.message import EmailMessage
@@ -13,7 +13,6 @@ from email.utils import formataddr
from html.parser import HTMLParser
from typing import Any, Iterable
logger = logging.getLogger(__name__)
@@ -164,7 +163,11 @@ def _smtp_sender_not_authorized(error: Exception) -> bool:
else:
raw_error_text = str(raw_error)
text = f"{code} {raw_error_text} {error}".lower()
return code in {551, 553} or "not authorised to send from this header address" in text or "not authorized to send from this header address" in text
return (
code in {551, 553}
or "not authorised to send from this header address" in text
or "not authorized to send from this header address" in text
)
def _smtp_attempt_metadata(mode: dict[str, Any]) -> dict[str, Any]:
@@ -203,10 +206,15 @@ def describe_smtp_error(error: Exception) -> str:
)
if isinstance(item, (smtplib.SMTPDataError, smtplib.SMTPResponseException)):
smtp_code = getattr(item, "smtp_code", None)
if smtp_code in {551, 553} or "not authorised to send from this header address" in lowered or "not authorized to send from this header address" in lowered:
if (
smtp_code in {551, 553}
or "not authorised to send from this header address" in lowered
or "not authorized to send from this header address" in lowered
):
return (
"SMTP server rejected the configured From address. Use an authorized alias "
"for this account or change SMTP_FROM_ADDRESS to a sender the provider allows."
"SMTP server rejected the configured From address. "
"Use an authorized alias for this account or change "
"SMTP_FROM_ADDRESS to a sender the provider allows."
)
if isinstance(item, smtplib.SMTPConnectError):
return "SMTP connection was rejected by the server. Check the host and port."
@@ -247,7 +255,9 @@ def build_email_message(
msg.set_content(plain_text or "")
for attachment in attachments:
content_type = attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream"
content_type = (
attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream"
)
maintype, subtype = content_type.split("/", 1) if "/" in content_type else ("application", "octet-stream")
msg.add_attachment(
attachment.data,
@@ -357,7 +367,9 @@ def send_email_message(
)
if _smtp_sender_not_authorized(exc) and fallback_from_address:
logger.warning(
"SMTP send sender rejected label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s; retrying with smtp_username",
"SMTP send sender rejected label=%s host=%s "
"port=%s transport=%s auth_user=%s "
"from_address=%s error=%s; retrying with smtp_username",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
@@ -393,7 +405,9 @@ def send_email_message(
}
)
logger.warning(
"SMTP send fallback failed label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s",
"SMTP send fallback failed label=%s host=%s "
"port=%s transport=%s auth_user=%s "
"from_address=%s error=%s",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
@@ -416,7 +430,8 @@ def send_email_message(
}
)
logger.info(
"SMTP send succeeded via smtp_username label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s",
"SMTP send succeeded via smtp_username label=%s host=%s "
"port=%s transport=%s auth_user=%s from_address=%s",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
backend/src/media_library_viewer_api/services/media_index_impl.py
+22 -17
View File
@@ -141,13 +141,20 @@ class MediaIndex:
key TEXT PRIMARY KEY,
value TEXT
);
CREATE INDEX IF NOT EXISTS idx_media_type ON media_items(type);
CREATE INDEX IF NOT EXISTS idx_media_library ON media_items(library_id);
CREATE INDEX IF NOT EXISTS idx_media_title ON media_items(title COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_series ON media_items(series COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_date_added ON media_items(date_added_ts);
CREATE INDEX IF NOT EXISTS idx_media_size ON media_items(size_bytes);
CREATE INDEX IF NOT EXISTS idx_media_bitrate ON media_items(bitrate_bps);
CREATE INDEX IF NOT EXISTS idx_media_type
ON media_items(type);
CREATE INDEX IF NOT EXISTS idx_media_library
ON media_items(library_id);
CREATE INDEX IF NOT EXISTS idx_media_title
ON media_items(title COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_series
ON media_items(series COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_date_added
ON media_items(date_added_ts);
CREATE INDEX IF NOT EXISTS idx_media_size
ON media_items(size_bytes);
CREATE INDEX IF NOT EXISTS idx_media_bitrate
ON media_items(bitrate_bps);
"""
)
@@ -208,10 +215,7 @@ class MediaIndex:
try:
with self.connect() as conn:
item_count = int(conn.execute("SELECT COUNT(*) FROM media_items").fetchone()[0])
meta = {
row[0]: row[1]
for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()
}
meta = {row[0]: row[1] for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()}
except sqlite3.Error:
return MediaIndexStatus(exists=False)
updated_at_raw = meta.get("updated_at", "")
@@ -310,7 +314,10 @@ class MediaIndex:
sort_sql = SORT_COLUMNS.get(sort_key, SORT_COLUMNS["title"])
direction = "DESC" if sort_order == "Descending" else "ASC"
# Always add stable tie-breakers.
order_sql = f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, season_number ASC, episode ASC, title COLLATE NOCASE ASC"
order_sql = (
f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, "
"season_number ASC, episode ASC, title COLLATE NOCASE ASC"
)
with self.connect() as conn:
total = int(conn.execute("SELECT COUNT(*) FROM media_items" + where_sql, params).fetchone()[0])
@@ -421,10 +428,7 @@ def build_media_index(
**row,
"path": resolve_remote_media_path(row.get("path", ""), media_root, fallback_prefix),
}
for row in (
normalize_media_item(item, library_id, current_library_name)
for item in items
)
for row in (normalize_media_item(item, library_id, current_library_name) for item in items)
)
processed_total += len(items)
current_library_processed += len(items)
@@ -432,7 +436,8 @@ def build_media_index(
ensure_not_cancelled()
emit(
"building",
f"{current_library_name or 'Library'}: {current_library_processed} / {current_library_total or '?'} items",
f"{current_library_name or 'Library'}: "
f"{current_library_processed} / {current_library_total or '?'} items",
)
logger.debug(
"Media index progress library=%s processed=%s/%s total_processed=%s",
backend/src/media_library_viewer_api/services/monitoring_actions.py
+43 -184
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import json
import logging
import shlex
import time
import uuid
from typing import Any, Callable
@@ -16,17 +17,9 @@ from typing import Any, Callable
from fastapi import HTTPException
from media_library_viewer_api.clients.local import LocalCommandClient
from media_library_viewer_api.clients.resources import (
disk_space,
read_resource_metrics,
resource_collector_debug_info,
resource_collector_status,
restart_resource_collector,
start_resource_collector,
stop_resource_collector,
)
from media_library_viewer_api.clients.ssh import RemoteSSHClient
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.observability import record_ssh_command
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
@@ -63,6 +56,28 @@ def build_machine_client(machine: dict[str, Any], store: SettingsStore):
)
def disk_space(client: Any, path: str = "/") -> dict[str, Any]:
"""Return df information for the filesystem containing ``path``.
Works against any client with a ``run`` method (local shell or SSH).
"""
command = (
"df -P -B1 -- " + shlex.quote(path or "/") + " | awk 'NR==2 {printf "
'"{\\"filesystem\\":\\"%s\\",\\"size\\":%s,"'
'"\\"used\\":%s,\\"available\\":%s,"'
'"\\"used_pct\\":\\"%s\\",\\"mount\\":\\"%s\\"}", "'
"$1,$2,$3,$4,$5,$6}'"
)
logger.debug("Reading disk space for path=%s", path)
result = client.run(command, timeout=20)
if result.exit_status != 0 or not result.stdout.strip():
logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
data = json.loads(result.stdout)
logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct"))
return data
def summarize_operation_result(action: str, result: Any) -> str:
"""Turn an operation result into a compact human-readable summary."""
if result is None:
@@ -73,10 +88,6 @@ def summarize_operation_result(action: str, result: Any) -> str:
if isinstance(result, list):
return f"{action}: {len(result)} item(s)"
if isinstance(result, dict):
if action == "metrics read":
samples = result.get("samples")
if isinstance(samples, list):
return f"{action}: {len(samples)} sample(s)"
if action.startswith("disk lookup"):
used_pct = result.get("used_pct")
mount = result.get("mount") or result.get("filesystem")
@@ -104,13 +115,21 @@ def run_machine_operation(
summarize: Callable[[Any], str] | None = None,
request_id: str = "",
raise_http: bool = True,
client: Any | None = None,
) -> Any:
"""Run a machine operation, record history, and optionally raise on failure."""
started = time.perf_counter()
client = build_machine_client(machine, store)
if client is None:
client = build_machine_client(machine, store)
try:
result = callback(client)
duration_ms = int((time.perf_counter() - started) * 1000)
record_ssh_command(
machine_id=machine.get("id") or "unknown",
action=action,
status="ok",
duration_seconds=duration_ms / 1000.0,
)
store.record_machine_action(
machine,
action,
@@ -124,6 +143,12 @@ def run_machine_operation(
raise
except Exception as exc: # pragma: no cover - transport/network fallback
duration_ms = int((time.perf_counter() - started) * 1000)
record_ssh_command(
machine_id=machine.get("id") or "unknown",
action=action,
status="error",
duration_seconds=duration_ms / 1000.0,
)
logger.exception(
"Monitoring %s failed machine_id=%s machine_name=%s",
action,
@@ -157,34 +182,13 @@ def poll_machine_snapshot(
) -> dict[str, Any]:
"""Collect a backend-scheduled snapshot for a machine.
This records the same action-history rows that the UI would otherwise get
from manual requests, but it runs entirely inside the backend on a schedule.
The legacy POSIX collector has been removed; this now records a lightweight
disk-space lookup on the same schedule so action history stays useful.
"""
request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}"
results: dict[str, Any] = {"request_id": request_id, "machine_id": machine.get("id"), "actions": []}
status = run_machine_operation(
machine,
store,
"status lookup",
resource_collector_status,
request_id=request_id,
raise_http=False,
)
results["status"] = status
results["actions"].append("status lookup")
metrics = run_machine_operation(
machine,
store,
"metrics read",
lambda client: read_resource_metrics(client, max_lines=metrics_limit),
request_id=request_id,
raise_http=False,
)
results["metrics_samples"] = len(metrics or [])
results["actions"].append("metrics read")
client = build_machine_client(machine, store)
settings = get_settings()
path = str(machine.get("media_root") or settings.media_root or "/")
disk = run_machine_operation(
@@ -194,6 +198,7 @@ def poll_machine_snapshot(
lambda client: disk_space(client, path),
request_id=request_id,
raise_http=False,
client=client,
)
results["disk_mount"] = (disk or {}).get("mount") if isinstance(disk, dict) else None
results["actions"].append("disk lookup")
@@ -201,149 +206,3 @@ def poll_machine_snapshot(
return results
def _summarize_metric_samples(samples: list[dict[str, Any]], field: str) -> dict[str, float] | None:
values = [float(sample.get(field, 0)) for sample in samples if sample.get(field) is not None]
if not values:
return None
return {
"avg": sum(values) / len(values),
"min": min(values),
"max": max(values),
"count": float(len(values)),
}
def collect_machine_overview(
machine: dict[str, Any],
store: SettingsStore,
*,
metrics_window_seconds: int = 600,
metrics_limit: int = 70_000,
) -> dict[str, Any]:
"""Collect a lightweight machine overview without recording history."""
overview: dict[str, Any] = {
"machine": {k: machine.get(k) for k in ("id", "name", "mode", "enabled", "host", "port", "username", "media_root", "path_prefix", "notes")},
"status": "",
"status_error": "",
"metrics_error": "",
"disk_error": "",
"sample_count": 0,
"latest_sample": None,
"cpu_summary": None,
"iowait_summary": None,
"mem_summary": None,
"net_rx_summary": None,
"net_tx_summary": None,
"disk_read_summary": None,
"disk_write_summary": None,
"disk": None,
}
try:
client: Any = build_machine_client(machine, store)
except Exception as exc:
overview["status_error"] = str(exc)
overview["metrics_error"] = str(exc)
overview["disk_error"] = str(exc)
return overview
settings = get_settings()
path = str(machine.get("media_root") or settings.media_root or "/")
try:
overview["status"] = resource_collector_status(client)
except Exception as exc:
overview["status_error"] = str(exc)
try:
samples = read_resource_metrics(client, max_lines=metrics_limit)
if samples:
latest_ts = max(float(sample.get("ts", 0)) for sample in samples)
window_start = latest_ts - metrics_window_seconds
metrics = [sample for sample in samples if float(sample.get("ts", 0)) >= window_start]
if not metrics:
metrics = samples
overview["sample_count"] = len(metrics)
overview["latest_sample"] = metrics[-1]
overview["cpu_summary"] = _summarize_metric_samples(metrics, "cpu_pct")
overview["iowait_summary"] = _summarize_metric_samples(metrics, "iowait_pct")
overview["mem_summary"] = _summarize_metric_samples(metrics, "mem_pct")
overview["net_rx_summary"] = _summarize_metric_samples(metrics, "net_rx_bytes_per_sec")
overview["net_tx_summary"] = _summarize_metric_samples(metrics, "net_tx_bytes_per_sec")
overview["disk_read_summary"] = _summarize_metric_samples(metrics, "disk_read_bps")
overview["disk_write_summary"] = _summarize_metric_samples(metrics, "disk_write_bps")
except Exception as exc:
overview["metrics_error"] = str(exc)
try:
overview["disk"] = disk_space(client, path)
except Exception as exc:
overview["disk_error"] = str(exc)
return overview
def poll_machine_diagnostics(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str | None = None,
) -> dict[str, Any]:
"""Collect a backend-scheduled diagnostics snapshot for a machine."""
request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}"
diagnostics = run_machine_operation(
machine,
store,
"collector diagnostics",
resource_collector_debug_info,
request_id=request_id,
raise_http=False,
)
return {"request_id": request_id, "diagnostics": diagnostics}
def start_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector start",
start_resource_collector,
request_id=request_id,
raise_http=True,
)
def stop_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector stop",
stop_resource_collector,
request_id=request_id,
raise_http=True,
)
def restart_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector restart",
restart_resource_collector,
request_id=request_id,
raise_http=True,
)
backend/src/media_library_viewer_api/services/monitoring_poller.py
+5 -4
View File
@@ -136,16 +136,17 @@ class MonitoringPoller:
request_id=f"poll:{machine['id']}:{int(time.time())}",
)
logger.info(
"Monitoring poll snapshot machine_id=%s request_id=%s status=%s metrics_samples=%s disk_mount=%s",
"Monitoring poll snapshot machine_id=%s request_id=%s disk_mount=%s actions=%s",
machine["id"],
snapshot.get("request_id"),
snapshot.get("status"),
snapshot.get("metrics_samples"),
snapshot.get("disk_mount"),
snapshot.get("actions"),
)
except Exception:
cycle_errors += 1
logger.exception("Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"])
logger.exception(
"Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"]
)
retention_seconds = config.retention_days * 24 * 60 * 60
cutoff_ts = int(time.time()) - retention_seconds
removed = store.prune_machine_actions(cutoff_ts)
backend/src/media_library_viewer_api/services/settings_store.py
+214 -68
View File
@@ -48,6 +48,9 @@ def _default_local_machine() -> dict[str, Any]:
"jellyfin_api_key": "",
"jellyseerr_url": "",
"jellyseerr_api_key": "",
"node_exporter_enabled": False,
"node_exporter_port": 9100,
"node_exporter_scrape_host": "",
"notes": "",
}
@@ -81,9 +84,7 @@ class SettingsStore:
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)"
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)")
conn.execute(
"""
CREATE TABLE IF NOT EXISTS monitoring_machine_actions (
@@ -118,9 +119,7 @@ class SettingsStore:
)
"""
)
existing_key_columns = {
row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()
}
existing_key_columns = {row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()}
if "public_key" not in existing_key_columns:
conn.execute("ALTER TABLE ssh_keys ADD COLUMN public_key TEXT NOT NULL DEFAULT ''")
if "fingerprint" not in existing_key_columns:
@@ -140,9 +139,7 @@ class SettingsStore:
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)"
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)")
conn.execute(
"""
CREATE TABLE IF NOT EXISTS saved_task_runs (
@@ -182,10 +179,16 @@ class SettingsStore:
"CREATE INDEX IF NOT EXISTS idx_dashboard_shortcuts_type ON dashboard_shortcuts(shortcut_type)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time ON monitoring_machine_actions(machine_id, created_at DESC)"
"""
CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time
ON monitoring_machine_actions(machine_id, created_at DESC)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status ON monitoring_machine_actions(action, status)"
"""
CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status
ON monitoring_machine_actions(action, status)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS backup_jobs (
@@ -251,7 +254,8 @@ class SettingsStore:
def _row_to_machine(self, row: sqlite3.Row) -> dict[str, Any]:
data = json.loads(row["config_json"])
services = self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [])
default_services = DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []
services = self._normalize_services(data.get("services"), default_services)
return {
"id": row["id"],
"name": row["name"],
@@ -274,6 +278,9 @@ class SettingsStore:
"jellyfin_api_key_set": bool(data.get("jellyfin_api_key")),
"jellyseerr_url": data.get("jellyseerr_url", ""),
"jellyseerr_api_key_set": bool(data.get("jellyseerr_api_key")),
"node_exporter_enabled": bool(data.get("node_exporter_enabled", False)),
"node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100),
"node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""),
"notes": data.get("notes", ""),
"created_at": row["created_at"],
"updated_at": row["updated_at"],
@@ -294,12 +301,18 @@ class SettingsStore:
"This machine" if mode == "local" else machine_id
)
services = self._normalize_services(payload.get("services"), (current or {}).get("services", []))
host = str(payload.get("host") if payload.get("host") is not None else (current or {}).get("host", "") or "").strip()
def _current_str(field: str, default: str = "") -> str:
return str(
payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default
).strip()
host = _current_str("host")
port = int(payload.get("port") or (current or {}).get("port", 22) or 22)
username = str(payload.get("username") if payload.get("username") is not None else (current or {}).get("username", "") or "").strip()
key_directory = str(payload.get("key_directory") if payload.get("key_directory") is not None else (current or {}).get("key_directory", "") or "").strip()
key_name = str(payload.get("key_name") if payload.get("key_name") is not None else (current or {}).get("key_name", "") or "").strip()
ssh_key_id = str(payload.get("ssh_key_id") if payload.get("ssh_key_id") is not None else (current or {}).get("ssh_key_id", "") or "").strip()
username = _current_str("username")
key_directory = _current_str("key_directory")
key_name = _current_str("key_name")
ssh_key_id = _current_str("ssh_key_id")
ssh_private_key = payload.get("ssh_private_key")
if ssh_private_key in (None, ""):
ssh_private_key = (current or {}).get("ssh_private_key", "")
@@ -312,20 +325,30 @@ class SettingsStore:
if password in (None, ""):
password = (current or {}).get("password", "")
password = str(password or "")
media_root = str(payload.get("media_root") if payload.get("media_root") is not None else (current or {}).get("media_root", "") or "").strip()
path_prefix = str(payload.get("path_prefix") if payload.get("path_prefix") is not None else (current or {}).get("path_prefix", "") or "").strip()
jellyfin_url = str(payload.get("jellyfin_url") if payload.get("jellyfin_url") is not None else (current or {}).get("jellyfin_url", "") or "").strip()
jellyfin_user_id = str(payload.get("jellyfin_user_id") if payload.get("jellyfin_user_id") is not None else (current or {}).get("jellyfin_user_id", "") or "").strip()
media_root = _current_str("media_root")
path_prefix = _current_str("path_prefix")
jellyfin_url = _current_str("jellyfin_url")
jellyfin_user_id = _current_str("jellyfin_user_id")
jellyfin_api_key = payload.get("jellyfin_api_key")
if jellyfin_api_key in (None, ""):
jellyfin_api_key = (current or {}).get("jellyfin_api_key", "")
jellyfin_api_key = str(jellyfin_api_key or "")
jellyseerr_url = str(payload.get("jellyseerr_url") if payload.get("jellyseerr_url") is not None else (current or {}).get("jellyseerr_url", "") or "").strip()
jellyseerr_url = _current_str("jellyseerr_url")
jellyseerr_api_key = payload.get("jellyseerr_api_key")
if jellyseerr_api_key in (None, ""):
jellyseerr_api_key = (current or {}).get("jellyseerr_api_key", "")
jellyseerr_api_key = str(jellyseerr_api_key or "")
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
node_exporter_enabled = bool(
payload.get("node_exporter_enabled")
if payload.get("node_exporter_enabled") is not None
else (current or {}).get("node_exporter_enabled", False)
)
node_exporter_port_raw = payload.get("node_exporter_port")
if node_exporter_port_raw is None:
node_exporter_port_raw = (current or {}).get("node_exporter_port", 9100)
node_exporter_port = int(node_exporter_port_raw or 9100)
node_exporter_scrape_host = _current_str("node_exporter_scrape_host")
notes = _current_str("notes")
if mode == "local":
host = host or "localhost"
username = username or ""
@@ -351,6 +374,9 @@ class SettingsStore:
"jellyfin_api_key": jellyfin_api_key,
"jellyseerr_url": jellyseerr_url,
"jellyseerr_api_key": jellyseerr_api_key,
"node_exporter_enabled": node_exporter_enabled,
"node_exporter_port": node_exporter_port,
"node_exporter_scrape_host": node_exporter_scrape_host,
"notes": notes,
}
@@ -380,6 +406,9 @@ class SettingsStore:
"jellyfin_api_key": machine["jellyfin_api_key"],
"jellyseerr_url": machine["jellyseerr_url"],
"jellyseerr_api_key": machine["jellyseerr_api_key"],
"node_exporter_enabled": machine["node_exporter_enabled"],
"node_exporter_port": machine["node_exporter_port"],
"node_exporter_scrape_host": machine["node_exporter_scrape_host"],
"notes": machine["notes"],
}
with self.connect() as conn:
@@ -431,7 +460,10 @@ class SettingsStore:
"name": row["name"],
"mode": row["mode"],
"enabled": bool(row["enabled"]),
"services": self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []),
"services": self._normalize_services(
data.get("services"),
DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [],
),
"host": data.get("host", ""),
"port": int(data.get("port", 22) or 22),
"username": data.get("username", ""),
@@ -448,11 +480,18 @@ class SettingsStore:
"jellyfin_api_key": data.get("jellyfin_api_key", ""),
"jellyseerr_url": data.get("jellyseerr_url", ""),
"jellyseerr_api_key": data.get("jellyseerr_api_key", ""),
"node_exporter_enabled": bool(data.get("node_exporter_enabled", False)),
"node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100),
"node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""),
"notes": data.get("notes", ""),
}
def list_machines_for_service(self, service: str) -> list[dict[str, Any]]:
return [machine for machine in self.list_machines() if service in machine.get("services", []) and machine.get("enabled")]
return [
machine
for machine in self.list_machines()
if service in machine.get("services", []) and machine.get("enabled")
]
def get_machine_for_service(self, service: str, machine_id: str | None = None) -> dict[str, Any] | None:
if machine_id:
@@ -485,10 +524,16 @@ class SettingsStore:
"jellyfin_api_key": machine["jellyfin_api_key"],
"jellyseerr_url": machine["jellyseerr_url"],
"jellyseerr_api_key": machine["jellyseerr_api_key"],
"node_exporter_enabled": machine["node_exporter_enabled"],
"node_exporter_port": machine["node_exporter_port"],
"node_exporter_scrape_host": machine["node_exporter_scrape_host"],
"notes": machine["notes"],
}
with self.connect() as conn:
existing = conn.execute("SELECT created_at FROM monitoring_machines WHERE id = ?", (machine["id"],)).fetchone()
existing = conn.execute(
"SELECT created_at FROM monitoring_machines WHERE id = ?",
(machine["id"],),
).fetchone()
created_at = int(existing[0]) if existing else now
conn.execute(
"""
@@ -538,10 +583,13 @@ class SettingsStore:
conn.execute(
"""
INSERT INTO monitoring_machine_actions
(id, machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail)
(
id, machine_id, machine_name, mode, action, status,
created_at, duration_ms, request_id, message, error,
stdout_tail, stderr_tail
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
,
""",
(
uuid.uuid4().hex,
str(machine.get("id") or ""),
@@ -577,8 +625,10 @@ class SettingsStore:
clauses.append("status = ?")
params.append(status)
sql = (
"SELECT machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail "
f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} ORDER BY created_at DESC LIMIT ?"
"SELECT machine_id, machine_name, mode, action, status, "
"created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail "
f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} "
"ORDER BY created_at DESC LIMIT ?"
)
params.append(max(1, min(int(limit), 200)))
with self.connect() as conn:
@@ -595,7 +645,6 @@ class SettingsStore:
)
return int(cur.rowcount or 0)
@staticmethod
def _private_key_summary(private_key: str) -> dict[str, str]:
if not private_key:
@@ -639,11 +688,29 @@ class SettingsStore:
if passphrase in (None, ""):
passphrase = (current or {}).get("passphrase", "")
passphrase = str(passphrase or "")
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
notes = str(
payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or ""
).strip()
summary = self._private_key_summary(private_key)
public_key = str(payload.get("public_key") if payload.get("public_key") is not None else (current or {}).get("public_key", "") or summary["public_key"] or "").strip()
fingerprint = str(payload.get("fingerprint") if payload.get("fingerprint") is not None else (current or {}).get("fingerprint", "") or summary["fingerprint"] or "").strip()
return {"id": key_id, "name": name, "private_key": private_key, "passphrase": passphrase, "public_key": public_key, "fingerprint": fingerprint, "notes": notes}
public_key = str(
payload.get("public_key")
if payload.get("public_key") is not None
else (current or {}).get("public_key", "") or summary["public_key"] or ""
).strip()
fingerprint = str(
payload.get("fingerprint")
if payload.get("fingerprint") is not None
else (current or {}).get("fingerprint", "") or summary["fingerprint"] or ""
).strip()
return {
"id": key_id,
"name": name,
"private_key": private_key,
"passphrase": passphrase,
"public_key": public_key,
"fingerprint": fingerprint,
"notes": notes,
}
def list_ssh_keys(self) -> list[dict[str, Any]]:
self.init_schema()
@@ -666,7 +733,15 @@ class SettingsStore:
if not row:
return None
summary = self._private_key_summary(str(row["private_key"] or ""))
return {"id": row["id"], "name": row["name"], "private_key": row["private_key"], "passphrase": row["passphrase"], "notes": row["notes"], "public_key": str(row["public_key"] or summary["public_key"] or ""), "fingerprint": str(row["fingerprint"] or summary["fingerprint"] or "")}
return {
"id": row["id"],
"name": row["name"],
"private_key": row["private_key"],
"passphrase": row["passphrase"],
"notes": row["notes"],
"public_key": str(row["public_key"] or summary["public_key"] or ""),
"fingerprint": str(row["fingerprint"] or summary["fingerprint"] or ""),
}
def upsert_ssh_key(self, payload: dict[str, Any], key_id: str | None = None) -> dict[str, Any]:
self.init_schema()
@@ -677,7 +752,10 @@ class SettingsStore:
created_at = int(existing[0]) if existing else now
conn.execute(
"""
INSERT INTO ssh_keys (id, name, private_key, passphrase, public_key, fingerprint, notes, created_at, updated_at)
INSERT INTO ssh_keys (
id, name, private_key, passphrase, public_key, fingerprint,
notes, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
name = excluded.name,
@@ -688,7 +766,17 @@ class SettingsStore:
notes = excluded.notes,
updated_at = excluded.updated_at
""",
(key["id"], key["name"], key["private_key"], key["passphrase"], key["public_key"], key["fingerprint"], key["notes"], created_at, now),
(
key["id"],
key["name"],
key["private_key"],
key["passphrase"],
key["public_key"],
key["fingerprint"],
key["notes"],
created_at,
now,
),
)
return self.get_ssh_key(key["id"]) or key
@@ -697,8 +785,6 @@ class SettingsStore:
with self.connect() as conn:
conn.execute("DELETE FROM ssh_keys WHERE id = ?", (key_id,))
def _row_to_task(self, row: sqlite3.Row) -> dict[str, Any]:
return {
"id": row["id"],
@@ -719,11 +805,27 @@ class SettingsStore:
task_type = str(payload.get("task_type") or (current or {}).get("task_type") or "shell").strip().lower()
if task_type not in {"shell", "python"}:
task_type = "shell"
content = str(payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or "")
content = str(
payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or ""
)
enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
default_machine_id = str(payload.get("default_machine_id") if payload.get("default_machine_id") is not None else (current or {}).get("default_machine_id", "") or "").strip()
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
return {"id": task_id, "name": name, "task_type": task_type, "content": content, "enabled": enabled, "default_machine_id": default_machine_id, "notes": notes}
default_machine_id = str(
payload.get("default_machine_id")
if payload.get("default_machine_id") is not None
else (current or {}).get("default_machine_id", "") or ""
).strip()
notes = str(
payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or ""
).strip()
return {
"id": task_id,
"name": name,
"task_type": task_type,
"content": content,
"enabled": enabled,
"default_machine_id": default_machine_id,
"notes": notes,
}
def list_tasks(self) -> list[dict[str, Any]]:
self.init_schema()
@@ -748,7 +850,10 @@ class SettingsStore:
created_at = int(existing[0]) if existing else now
conn.execute(
"""
INSERT INTO saved_tasks (id, name, task_type, content, enabled, default_machine_id, notes, created_at, updated_at)
INSERT INTO saved_tasks (
id, name, task_type, content, enabled, default_machine_id,
notes, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
name = excluded.name,
@@ -759,7 +864,17 @@ class SettingsStore:
notes = excluded.notes,
updated_at = excluded.updated_at
""",
(task["id"], task["name"], task["task_type"], task["content"], 1 if task["enabled"] else 0, task["default_machine_id"], task["notes"], created_at, now),
(
task["id"],
task["name"],
task["task_type"],
task["content"],
1 if task["enabled"] else 0,
task["default_machine_id"],
task["notes"],
created_at,
now,
),
)
return self.get_task(task["id"]) or task
@@ -796,8 +911,11 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO saved_task_runs
(id, task_id, task_name, machine_id, machine_name, task_type, status, created_at, duration_ms, request_id, stdout_tail, stderr_tail, error)
INSERT INTO saved_task_runs (
id, task_id, task_name, machine_id, machine_name, task_type,
status, created_at, duration_ms, request_id, stdout_tail,
stderr_tail, error
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
@@ -837,17 +955,25 @@ class SettingsStore:
def _normalize_shortcut_payload(self, payload: dict[str, Any], shortcut_id: str | None = None) -> dict[str, Any]:
current = self.get_shortcut(shortcut_id) if shortcut_id else None
shortcut_id = str(payload.get("id") or shortcut_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12]
shortcut_type = str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower()
shortcut_type = (
str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower()
)
if shortcut_type not in {"website", "action", "user"}:
shortcut_type = "website"
label = str(payload.get("label") or (current or {}).get("label") or "").strip() or shortcut_id
enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
icon = str(payload.get("icon") if payload.get("icon") is not None else (current or {}).get("icon", "") or "").strip()
url = str(payload.get("url") if payload.get("url") is not None else (current or {}).get("url", "") or "").strip()
task_id = str(payload.get("task_id") if payload.get("task_id") is not None else (current or {}).get("task_id", "") or "").strip()
machine_id = str(payload.get("machine_id") if payload.get("machine_id") is not None else (current or {}).get("machine_id", "") or "").strip()
user_id = str(payload.get("user_id") if payload.get("user_id") is not None else (current or {}).get("user_id", "") or "").strip()
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
def _field(field: str, default: str = "") -> str:
return str(
payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default
).strip()
icon = _field("icon")
url = _field("url")
task_id = _field("task_id")
machine_id = _field("machine_id")
user_id = _field("user_id")
notes = _field("notes")
return {
"id": shortcut_id,
"label": label,
@@ -882,7 +1008,10 @@ class SettingsStore:
shortcut = self._normalize_shortcut_payload(payload, shortcut_id)
now = int(time.time())
with self.connect() as conn:
existing = conn.execute("SELECT created_at FROM dashboard_shortcuts WHERE id = ?", (shortcut["id"],)).fetchone()
existing = conn.execute(
"SELECT created_at FROM dashboard_shortcuts WHERE id = ?",
(shortcut["id"],),
).fetchone()
created_at = int(existing[0]) if existing else now
conn.execute(
"""
@@ -926,14 +1055,16 @@ class SettingsStore:
"created_at": row["created_at"],
}
def _normalize_backup_job_payload(
self, payload: dict[str, Any], job_id: str | None = None
) -> dict[str, Any]:
def _normalize_backup_job_payload(self, payload: dict[str, Any], job_id: str | None = None) -> dict[str, Any]:
current = self.get_backup_job(job_id) if job_id else None
job_id = str(payload.get("id") or job_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12]
name = str(payload.get("name") or (current or {}).get("name") or job_id).strip() or job_id
source = str(payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or "").strip()
target = str(payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or "").strip()
source = str(
payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or ""
).strip()
target = str(
payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or ""
).strip()
schedule_interval_seconds = payload.get("schedule_interval_seconds")
if schedule_interval_seconds is None:
schedule_interval_seconds = (current or {}).get("schedule_interval_seconds")
@@ -1040,10 +1171,24 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO backup_runs (id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, created_at)
INSERT INTO backup_runs (
id, job_id, started_at, ended_at, status, bytes_transferred,
duration_ms, error_message, details_json, created_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(run_id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, now),
(
run_id,
job_id,
started_at,
ended_at,
status,
bytes_transferred,
duration_ms,
error_message,
details_json,
now,
),
)
return self.get_backup_run(run_id) or {
"id": run_id,
@@ -1128,7 +1273,10 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO backup_alerts (id, job_id, run_id, alert_type, severity, message, acknowledged, resolved_at, created_at)
INSERT INTO backup_alerts (
id, job_id, run_id, alert_type, severity, message,
acknowledged, resolved_at, created_at
)
VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?)
""",
(alert_id, job_id, run_id, alert_type, severity, message, now),
@@ -1238,9 +1386,7 @@ class SettingsStore:
)
"""
)
row = conn.execute(
"SELECT value FROM app_settings WHERE key = ?", (key,)
).fetchone()
row = conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone()
return row["value"] if row else default
def update_setting(self, key: str, value: str) -> None:
backend/src/media_library_viewer_api/services/targets.py
+77
View File
@@ -0,0 +1,77 @@
"""Prometheus file-based service discovery target management.
The backend owns the list of remote Node Exporter targets so that operators can
enable scraping per machine from the Manage UI. Prometheus reads the generated
JSON file via `file_sd_configs`; this keeps Prometheus config static and pushes
machine-specific changes into a file it can reload.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import Any
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
DEFAULT_NODE_EXPORTER_PORT = 9100
def _scrape_address(machine: dict[str, Any]) -> str | None:
"""Return host:port for the Node Exporter on a machine, or None if disabled."""
if not machine.get("node_exporter_enabled"):
return None
scrape_host = str(machine.get("node_exporter_scrape_host") or "").strip()
host = scrape_host or str(machine.get("host") or "").strip()
if not host or host == "localhost":
return None
port = int(machine.get("node_exporter_port") or DEFAULT_NODE_EXPORTER_PORT)
return f"{host}:{port}"
def build_node_exporter_targets(store: SettingsStore) -> list[dict[str, Any]]:
"""Build a file-SD target list for all enabled SSH machines.
Local machines are excluded because the Compose-managed node-exporter
service already covers the Docker host.
"""
targets: list[dict[str, Any]] = []
for machine in store.list_machines():
if not machine.get("enabled"):
continue
if str(machine.get("mode") or "local").strip().lower() != "ssh":
continue
address = _scrape_address(machine)
if not address:
continue
targets.append(
{
"targets": [address],
"labels": {
"job": "node-exporter-remote",
"machine_id": str(machine.get("id") or ""),
"machine_name": str(machine.get("name") or ""),
"instance": address,
},
}
)
return targets
def write_prometheus_targets(store: SettingsStore, file_sd_dir: Path | None = None) -> Path:
"""Render and persist Prometheus file-SD targets.
Returns the path written so callers can log or expose it.
"""
settings = get_settings()
file_sd_dir = file_sd_dir or Path(settings.prometheus_file_sd_dir)
file_sd_dir.mkdir(parents=True, exist_ok=True)
file_path = file_sd_dir / "node_exporter_targets.json"
targets = build_node_exporter_targets(store)
file_path.write_text(json.dumps(targets, indent=2), encoding="utf-8")
logger.info("Wrote %s node_exporter targets to %s", len(targets), file_path)
return file_path
backend/src/media_library_viewer_api/utils.py
-1
View File
@@ -11,7 +11,6 @@ from datetime import datetime
from pathlib import PurePosixPath
from typing import Any
VIDEO_FILE_EXTENSIONS = {
".3g2",
".3gp",
backend/src/media_library_viewer_api/version.py
+2 -1
View File
@@ -3,7 +3,8 @@
from __future__ import annotations
import os
from importlib.metadata import PackageNotFoundError, version as package_version
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as package_version
PACKAGE_NAME = "media-library-viewer-backend"
DEFAULT_VERSION = "0.1.0"
backend/src/media_library_viewer_api/workers/media_index_worker.py
+1 -1
View File
@@ -15,12 +15,12 @@ from typing import Any
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
from media_library_viewer_api.services.media_index import (
MediaIndex,
MediaIndexBuildCancelled,
build_media_index,
)
from media_library_viewer_api.logging_utils import configure_logging, describe_settings
logger = logging.getLogger(__name__)