alex/manage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

Browse Source
This commit is contained in:
Developer
2026-06-16 11:08:32 +00:00
parent 615e02e970
commit e2ad731b5f
80 changed files with 5020 additions and 3665 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/media_library_viewer_api/services/backup_alert_engine.py
+66 -54
View File
@@ -13,20 +13,21 @@ def generate_alerts_for_run(
# 1. Failed status alert
if run["status"] == "failure":
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "failed_status",
"severity": "critical",
"message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "failed_status",
"severity": "critical",
"message": f"Backup job '{job_name}' failed: {run.get('error_message', 'No error details')}",
}
)
# 2. Anomaly size alert
bytes_transferred = run.get("bytes_transferred")
if bytes_transferred is not None and previous_runs:
successful_runs = [
r for r in previous_runs
if r["status"] == "success" and r.get("bytes_transferred") is not None
r for r in previous_runs if r["status"] == "success" and r.get("bytes_transferred") is not None
]
if len(successful_runs) >= 3:
sizes = [r["bytes_transferred"] for r in successful_runs[-7:]]
@@ -34,24 +35,28 @@ def generate_alerts_for_run(
if median_size > 0:
ratio = bytes_transferred / median_size
if bytes_transferred == 0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": f"Backup job '{job_name}' transferred 0 bytes (median: {median_size})",
}
)
elif ratio < 0.1 or ratio > 3.0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": (
f"Backup job '{job_name}' size anomaly: "
f"{bytes_transferred} bytes (median: {median_size})"
),
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_size",
"severity": "warning",
"message": (
f"Backup job '{job_name}' size anomaly: "
f"{bytes_transferred} bytes (median: {median_size})"
),
}
)
# 3. Anomaly duration alert
duration_ms = run.get("duration_ms")
@@ -61,16 +66,17 @@ def generate_alerts_for_run(
durations = [r["duration_ms"] for r in successful_runs[-7:]]
median_duration = statistics.median(durations)
if median_duration > 0 and duration_ms / median_duration > 3.0:
alerts.append({
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_duration",
"severity": "warning",
"message": (
f"Backup job '{job_name}' duration anomaly: "
f"{duration_ms}ms (median: {median_duration}ms)"
),
})
alerts.append(
{
"job_id": job_id,
"run_id": run["id"],
"alert_type": "anomaly_duration",
"severity": "warning",
"message": (
f"Backup job '{job_name}' duration anomaly: {duration_ms}ms (median: {median_duration}ms)"
),
}
)
return alerts
@@ -82,6 +88,7 @@ def check_missed_schedules(
) -> list[dict[str, Any]]:
alerts = []
import time
now = int(time.time())
for job in jobs:
@@ -93,31 +100,36 @@ def check_missed_schedules(
if not latest_run:
# No runs ever — alert if job is older than interval * 1.5
if now - job["created_at"] > interval * 1.5:
alerts.append({
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
})
alerts.append(
{
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": f"Backup job '{job['name']}' has never run (expected every {interval}s)",
}
)
else:
last_run_time = latest_run["started_at"]
if now - last_run_time > interval * 1.5:
# Check if there's already an unresolved missed_schedule alert
has_open_alert = any(
a["alert_type"] == "missed_schedule" and a["resolved_at"] is None
for a in existing_alerts if a["job_id"] == job["id"]
for a in existing_alerts
if a["job_id"] == job["id"]
)
if not has_open_alert:
alerts.append({
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": (
f"Backup job '{job['name']}' missed schedule: "
f"last run at {last_run_time} (expected every {interval}s)"
),
})
alerts.append(
{
"job_id": job["id"],
"run_id": None,
"alert_type": "missed_schedule",
"severity": "warning",
"message": (
f"Backup job '{job['name']}' missed schedule: "
f"last run at {last_run_time} (expected every {interval}s)"
),
}
)
return alerts
backend/src/media_library_viewer_api/services/mail_queue.py
+3
View File
@@ -15,6 +15,7 @@ import uuid
from dataclasses import dataclass, field
from typing import Any
from media_library_viewer_api.observability import record_mail_queue
from media_library_viewer_api.services.mailer import EmailAttachment, describe_smtp_error, send_email_message
logger = logging.getLogger(__name__)
@@ -206,6 +207,7 @@ class MailQueue:
result.get("recipient_count", 0),
result.get("attachment_count", 0),
)
record_mail_queue("sent")
except Exception as exc:
friendly_error = describe_smtp_error(exc)
with self._lock:
@@ -220,6 +222,7 @@ class MailQueue:
getattr(message, "request_id", "unknown"),
friendly_error,
)
record_mail_queue("failed")
finally:
self._queue.task_done()
backend/src/media_library_viewer_api/services/mailer_impl.py
+25 -10
View File
@@ -4,8 +4,8 @@ from __future__ import annotations
import logging
import mimetypes
import socket
import smtplib
import socket
import ssl
from dataclasses import dataclass
from email.message import EmailMessage
@@ -13,7 +13,6 @@ from email.utils import formataddr
from html.parser import HTMLParser
from typing import Any, Iterable
logger = logging.getLogger(__name__)
@@ -164,7 +163,11 @@ def _smtp_sender_not_authorized(error: Exception) -> bool:
else:
raw_error_text = str(raw_error)
text = f"{code} {raw_error_text} {error}".lower()
return code in {551, 553} or "not authorised to send from this header address" in text or "not authorized to send from this header address" in text
return (
code in {551, 553}
or "not authorised to send from this header address" in text
or "not authorized to send from this header address" in text
)
def _smtp_attempt_metadata(mode: dict[str, Any]) -> dict[str, Any]:
@@ -203,10 +206,15 @@ def describe_smtp_error(error: Exception) -> str:
)
if isinstance(item, (smtplib.SMTPDataError, smtplib.SMTPResponseException)):
smtp_code = getattr(item, "smtp_code", None)
if smtp_code in {551, 553} or "not authorised to send from this header address" in lowered or "not authorized to send from this header address" in lowered:
if (
smtp_code in {551, 553}
or "not authorised to send from this header address" in lowered
or "not authorized to send from this header address" in lowered
):
return (
"SMTP server rejected the configured From address. Use an authorized alias "
"for this account or change SMTP_FROM_ADDRESS to a sender the provider allows."
"SMTP server rejected the configured From address. "
"Use an authorized alias for this account or change "
"SMTP_FROM_ADDRESS to a sender the provider allows."
)
if isinstance(item, smtplib.SMTPConnectError):
return "SMTP connection was rejected by the server. Check the host and port."
@@ -247,7 +255,9 @@ def build_email_message(
msg.set_content(plain_text or "")
for attachment in attachments:
content_type = attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream"
content_type = (
attachment.content_type or mimetypes.guess_type(attachment.filename)[0] or "application/octet-stream"
)
maintype, subtype = content_type.split("/", 1) if "/" in content_type else ("application", "octet-stream")
msg.add_attachment(
attachment.data,
@@ -357,7 +367,9 @@ def send_email_message(
)
if _smtp_sender_not_authorized(exc) and fallback_from_address:
logger.warning(
"SMTP send sender rejected label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s; retrying with smtp_username",
"SMTP send sender rejected label=%s host=%s "
"port=%s transport=%s auth_user=%s "
"from_address=%s error=%s; retrying with smtp_username",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
@@ -393,7 +405,9 @@ def send_email_message(
}
)
logger.warning(
"SMTP send fallback failed label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s error=%s",
"SMTP send fallback failed label=%s host=%s "
"port=%s transport=%s auth_user=%s "
"from_address=%s error=%s",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
@@ -416,7 +430,8 @@ def send_email_message(
}
)
logger.info(
"SMTP send succeeded via smtp_username label=%s host=%s port=%s transport=%s auth_user=%s from_address=%s",
"SMTP send succeeded via smtp_username label=%s host=%s "
"port=%s transport=%s auth_user=%s from_address=%s",
meta["label"],
meta["smtp_host"],
meta["smtp_port"],
backend/src/media_library_viewer_api/services/media_index_impl.py
+22 -17
View File
@@ -141,13 +141,20 @@ class MediaIndex:
key TEXT PRIMARY KEY,
value TEXT
);
CREATE INDEX IF NOT EXISTS idx_media_type ON media_items(type);
CREATE INDEX IF NOT EXISTS idx_media_library ON media_items(library_id);
CREATE INDEX IF NOT EXISTS idx_media_title ON media_items(title COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_series ON media_items(series COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_date_added ON media_items(date_added_ts);
CREATE INDEX IF NOT EXISTS idx_media_size ON media_items(size_bytes);
CREATE INDEX IF NOT EXISTS idx_media_bitrate ON media_items(bitrate_bps);
CREATE INDEX IF NOT EXISTS idx_media_type
ON media_items(type);
CREATE INDEX IF NOT EXISTS idx_media_library
ON media_items(library_id);
CREATE INDEX IF NOT EXISTS idx_media_title
ON media_items(title COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_series
ON media_items(series COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_date_added
ON media_items(date_added_ts);
CREATE INDEX IF NOT EXISTS idx_media_size
ON media_items(size_bytes);
CREATE INDEX IF NOT EXISTS idx_media_bitrate
ON media_items(bitrate_bps);
"""
)
@@ -208,10 +215,7 @@ class MediaIndex:
try:
with self.connect() as conn:
item_count = int(conn.execute("SELECT COUNT(*) FROM media_items").fetchone()[0])
meta = {
row[0]: row[1]
for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()
}
meta = {row[0]: row[1] for row in conn.execute("SELECT key, value FROM index_metadata").fetchall()}
except sqlite3.Error:
return MediaIndexStatus(exists=False)
updated_at_raw = meta.get("updated_at", "")
@@ -310,7 +314,10 @@ class MediaIndex:
sort_sql = SORT_COLUMNS.get(sort_key, SORT_COLUMNS["title"])
direction = "DESC" if sort_order == "Descending" else "ASC"
# Always add stable tie-breakers.
order_sql = f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, season_number ASC, episode ASC, title COLLATE NOCASE ASC"
order_sql = (
f" ORDER BY {sort_sql} {direction}, series COLLATE NOCASE ASC, "
"season_number ASC, episode ASC, title COLLATE NOCASE ASC"
)
with self.connect() as conn:
total = int(conn.execute("SELECT COUNT(*) FROM media_items" + where_sql, params).fetchone()[0])
@@ -421,10 +428,7 @@ def build_media_index(
**row,
"path": resolve_remote_media_path(row.get("path", ""), media_root, fallback_prefix),
}
for row in (
normalize_media_item(item, library_id, current_library_name)
for item in items
)
for row in (normalize_media_item(item, library_id, current_library_name) for item in items)
)
processed_total += len(items)
current_library_processed += len(items)
@@ -432,7 +436,8 @@ def build_media_index(
ensure_not_cancelled()
emit(
"building",
f"{current_library_name or 'Library'}: {current_library_processed} / {current_library_total or '?'} items",
f"{current_library_name or 'Library'}: "
f"{current_library_processed} / {current_library_total or '?'} items",
)
logger.debug(
"Media index progress library=%s processed=%s/%s total_processed=%s",
backend/src/media_library_viewer_api/services/monitoring_actions.py
+43 -184
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import json
import logging
import shlex
import time
import uuid
from typing import Any, Callable
@@ -16,17 +17,9 @@ from typing import Any, Callable
from fastapi import HTTPException
from media_library_viewer_api.clients.local import LocalCommandClient
from media_library_viewer_api.clients.resources import (
disk_space,
read_resource_metrics,
resource_collector_debug_info,
resource_collector_status,
restart_resource_collector,
start_resource_collector,
stop_resource_collector,
)
from media_library_viewer_api.clients.ssh import RemoteSSHClient
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.observability import record_ssh_command
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
@@ -63,6 +56,28 @@ def build_machine_client(machine: dict[str, Any], store: SettingsStore):
)
def disk_space(client: Any, path: str = "/") -> dict[str, Any]:
"""Return df information for the filesystem containing ``path``.
Works against any client with a ``run`` method (local shell or SSH).
"""
command = (
"df -P -B1 -- " + shlex.quote(path or "/") + " | awk 'NR==2 {printf "
'"{\\"filesystem\\":\\"%s\\",\\"size\\":%s,"'
'"\\"used\\":%s,\\"available\\":%s,"'
'"\\"used_pct\\":\\"%s\\",\\"mount\\":\\"%s\\"}", "'
"$1,$2,$3,$4,$5,$6}'"
)
logger.debug("Reading disk space for path=%s", path)
result = client.run(command, timeout=20)
if result.exit_status != 0 or not result.stdout.strip():
logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout)
raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
data = json.loads(result.stdout)
logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct"))
return data
def summarize_operation_result(action: str, result: Any) -> str:
"""Turn an operation result into a compact human-readable summary."""
if result is None:
@@ -73,10 +88,6 @@ def summarize_operation_result(action: str, result: Any) -> str:
if isinstance(result, list):
return f"{action}: {len(result)} item(s)"
if isinstance(result, dict):
if action == "metrics read":
samples = result.get("samples")
if isinstance(samples, list):
return f"{action}: {len(samples)} sample(s)"
if action.startswith("disk lookup"):
used_pct = result.get("used_pct")
mount = result.get("mount") or result.get("filesystem")
@@ -104,13 +115,21 @@ def run_machine_operation(
summarize: Callable[[Any], str] | None = None,
request_id: str = "",
raise_http: bool = True,
client: Any | None = None,
) -> Any:
"""Run a machine operation, record history, and optionally raise on failure."""
started = time.perf_counter()
client = build_machine_client(machine, store)
if client is None:
client = build_machine_client(machine, store)
try:
result = callback(client)
duration_ms = int((time.perf_counter() - started) * 1000)
record_ssh_command(
machine_id=machine.get("id") or "unknown",
action=action,
status="ok",
duration_seconds=duration_ms / 1000.0,
)
store.record_machine_action(
machine,
action,
@@ -124,6 +143,12 @@ def run_machine_operation(
raise
except Exception as exc: # pragma: no cover - transport/network fallback
duration_ms = int((time.perf_counter() - started) * 1000)
record_ssh_command(
machine_id=machine.get("id") or "unknown",
action=action,
status="error",
duration_seconds=duration_ms / 1000.0,
)
logger.exception(
"Monitoring %s failed machine_id=%s machine_name=%s",
action,
@@ -157,34 +182,13 @@ def poll_machine_snapshot(
) -> dict[str, Any]:
"""Collect a backend-scheduled snapshot for a machine.
This records the same action-history rows that the UI would otherwise get
from manual requests, but it runs entirely inside the backend on a schedule.
The legacy POSIX collector has been removed; this now records a lightweight
disk-space lookup on the same schedule so action history stays useful.
"""
request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}"
results: dict[str, Any] = {"request_id": request_id, "machine_id": machine.get("id"), "actions": []}
status = run_machine_operation(
machine,
store,
"status lookup",
resource_collector_status,
request_id=request_id,
raise_http=False,
)
results["status"] = status
results["actions"].append("status lookup")
metrics = run_machine_operation(
machine,
store,
"metrics read",
lambda client: read_resource_metrics(client, max_lines=metrics_limit),
request_id=request_id,
raise_http=False,
)
results["metrics_samples"] = len(metrics or [])
results["actions"].append("metrics read")
client = build_machine_client(machine, store)
settings = get_settings()
path = str(machine.get("media_root") or settings.media_root or "/")
disk = run_machine_operation(
@@ -194,6 +198,7 @@ def poll_machine_snapshot(
lambda client: disk_space(client, path),
request_id=request_id,
raise_http=False,
client=client,
)
results["disk_mount"] = (disk or {}).get("mount") if isinstance(disk, dict) else None
results["actions"].append("disk lookup")
@@ -201,149 +206,3 @@ def poll_machine_snapshot(
return results
def _summarize_metric_samples(samples: list[dict[str, Any]], field: str) -> dict[str, float] | None:
values = [float(sample.get(field, 0)) for sample in samples if sample.get(field) is not None]
if not values:
return None
return {
"avg": sum(values) / len(values),
"min": min(values),
"max": max(values),
"count": float(len(values)),
}
def collect_machine_overview(
machine: dict[str, Any],
store: SettingsStore,
*,
metrics_window_seconds: int = 600,
metrics_limit: int = 70_000,
) -> dict[str, Any]:
"""Collect a lightweight machine overview without recording history."""
overview: dict[str, Any] = {
"machine": {k: machine.get(k) for k in ("id", "name", "mode", "enabled", "host", "port", "username", "media_root", "path_prefix", "notes")},
"status": "",
"status_error": "",
"metrics_error": "",
"disk_error": "",
"sample_count": 0,
"latest_sample": None,
"cpu_summary": None,
"iowait_summary": None,
"mem_summary": None,
"net_rx_summary": None,
"net_tx_summary": None,
"disk_read_summary": None,
"disk_write_summary": None,
"disk": None,
}
try:
client: Any = build_machine_client(machine, store)
except Exception as exc:
overview["status_error"] = str(exc)
overview["metrics_error"] = str(exc)
overview["disk_error"] = str(exc)
return overview
settings = get_settings()
path = str(machine.get("media_root") or settings.media_root or "/")
try:
overview["status"] = resource_collector_status(client)
except Exception as exc:
overview["status_error"] = str(exc)
try:
samples = read_resource_metrics(client, max_lines=metrics_limit)
if samples:
latest_ts = max(float(sample.get("ts", 0)) for sample in samples)
window_start = latest_ts - metrics_window_seconds
metrics = [sample for sample in samples if float(sample.get("ts", 0)) >= window_start]
if not metrics:
metrics = samples
overview["sample_count"] = len(metrics)
overview["latest_sample"] = metrics[-1]
overview["cpu_summary"] = _summarize_metric_samples(metrics, "cpu_pct")
overview["iowait_summary"] = _summarize_metric_samples(metrics, "iowait_pct")
overview["mem_summary"] = _summarize_metric_samples(metrics, "mem_pct")
overview["net_rx_summary"] = _summarize_metric_samples(metrics, "net_rx_bytes_per_sec")
overview["net_tx_summary"] = _summarize_metric_samples(metrics, "net_tx_bytes_per_sec")
overview["disk_read_summary"] = _summarize_metric_samples(metrics, "disk_read_bps")
overview["disk_write_summary"] = _summarize_metric_samples(metrics, "disk_write_bps")
except Exception as exc:
overview["metrics_error"] = str(exc)
try:
overview["disk"] = disk_space(client, path)
except Exception as exc:
overview["disk_error"] = str(exc)
return overview
def poll_machine_diagnostics(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str | None = None,
) -> dict[str, Any]:
"""Collect a backend-scheduled diagnostics snapshot for a machine."""
request_id = request_id or f"poll:{machine.get('id') or uuid.uuid4().hex}"
diagnostics = run_machine_operation(
machine,
store,
"collector diagnostics",
resource_collector_debug_info,
request_id=request_id,
raise_http=False,
)
return {"request_id": request_id, "diagnostics": diagnostics}
def start_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector start",
start_resource_collector,
request_id=request_id,
raise_http=True,
)
def stop_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector stop",
stop_resource_collector,
request_id=request_id,
raise_http=True,
)
def restart_collector(
machine: dict[str, Any],
store: SettingsStore,
*,
request_id: str = "",
) -> Any:
return run_machine_operation(
machine,
store,
"collector restart",
restart_resource_collector,
request_id=request_id,
raise_http=True,
)
backend/src/media_library_viewer_api/services/monitoring_poller.py
+5 -4
View File
@@ -136,16 +136,17 @@ class MonitoringPoller:
request_id=f"poll:{machine['id']}:{int(time.time())}",
)
logger.info(
"Monitoring poll snapshot machine_id=%s request_id=%s status=%s metrics_samples=%s disk_mount=%s",
"Monitoring poll snapshot machine_id=%s request_id=%s disk_mount=%s actions=%s",
machine["id"],
snapshot.get("request_id"),
snapshot.get("status"),
snapshot.get("metrics_samples"),
snapshot.get("disk_mount"),
snapshot.get("actions"),
)
except Exception:
cycle_errors += 1
logger.exception("Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"])
logger.exception(
"Monitoring poll snapshot failed machine_id=%s machine_name=%s", machine["id"], machine["name"]
)
retention_seconds = config.retention_days * 24 * 60 * 60
cutoff_ts = int(time.time()) - retention_seconds
removed = store.prune_machine_actions(cutoff_ts)
backend/src/media_library_viewer_api/services/settings_store.py
+214 -68
View File
@@ -48,6 +48,9 @@ def _default_local_machine() -> dict[str, Any]:
"jellyfin_api_key": "",
"jellyseerr_url": "",
"jellyseerr_api_key": "",
"node_exporter_enabled": False,
"node_exporter_port": 9100,
"node_exporter_scrape_host": "",
"notes": "",
}
@@ -81,9 +84,7 @@ class SettingsStore:
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)"
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_monitoring_machines_mode ON monitoring_machines(mode)")
conn.execute(
"""
CREATE TABLE IF NOT EXISTS monitoring_machine_actions (
@@ -118,9 +119,7 @@ class SettingsStore:
)
"""
)
existing_key_columns = {
row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()
}
existing_key_columns = {row[1] for row in conn.execute("PRAGMA table_info(ssh_keys)").fetchall()}
if "public_key" not in existing_key_columns:
conn.execute("ALTER TABLE ssh_keys ADD COLUMN public_key TEXT NOT NULL DEFAULT ''")
if "fingerprint" not in existing_key_columns:
@@ -140,9 +139,7 @@ class SettingsStore:
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)"
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_saved_tasks_name ON saved_tasks(name)")
conn.execute(
"""
CREATE TABLE IF NOT EXISTS saved_task_runs (
@@ -182,10 +179,16 @@ class SettingsStore:
"CREATE INDEX IF NOT EXISTS idx_dashboard_shortcuts_type ON dashboard_shortcuts(shortcut_type)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time ON monitoring_machine_actions(machine_id, created_at DESC)"
"""
CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_machine_time
ON monitoring_machine_actions(machine_id, created_at DESC)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status ON monitoring_machine_actions(action, status)"
"""
CREATE INDEX IF NOT EXISTS idx_monitoring_machine_actions_action_status
ON monitoring_machine_actions(action, status)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS backup_jobs (
@@ -251,7 +254,8 @@ class SettingsStore:
def _row_to_machine(self, row: sqlite3.Row) -> dict[str, Any]:
data = json.loads(row["config_json"])
services = self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [])
default_services = DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []
services = self._normalize_services(data.get("services"), default_services)
return {
"id": row["id"],
"name": row["name"],
@@ -274,6 +278,9 @@ class SettingsStore:
"jellyfin_api_key_set": bool(data.get("jellyfin_api_key")),
"jellyseerr_url": data.get("jellyseerr_url", ""),
"jellyseerr_api_key_set": bool(data.get("jellyseerr_api_key")),
"node_exporter_enabled": bool(data.get("node_exporter_enabled", False)),
"node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100),
"node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""),
"notes": data.get("notes", ""),
"created_at": row["created_at"],
"updated_at": row["updated_at"],
@@ -294,12 +301,18 @@ class SettingsStore:
"This machine" if mode == "local" else machine_id
)
services = self._normalize_services(payload.get("services"), (current or {}).get("services", []))
host = str(payload.get("host") if payload.get("host") is not None else (current or {}).get("host", "") or "").strip()
def _current_str(field: str, default: str = "") -> str:
return str(
payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default
).strip()
host = _current_str("host")
port = int(payload.get("port") or (current or {}).get("port", 22) or 22)
username = str(payload.get("username") if payload.get("username") is not None else (current or {}).get("username", "") or "").strip()
key_directory = str(payload.get("key_directory") if payload.get("key_directory") is not None else (current or {}).get("key_directory", "") or "").strip()
key_name = str(payload.get("key_name") if payload.get("key_name") is not None else (current or {}).get("key_name", "") or "").strip()
ssh_key_id = str(payload.get("ssh_key_id") if payload.get("ssh_key_id") is not None else (current or {}).get("ssh_key_id", "") or "").strip()
username = _current_str("username")
key_directory = _current_str("key_directory")
key_name = _current_str("key_name")
ssh_key_id = _current_str("ssh_key_id")
ssh_private_key = payload.get("ssh_private_key")
if ssh_private_key in (None, ""):
ssh_private_key = (current or {}).get("ssh_private_key", "")
@@ -312,20 +325,30 @@ class SettingsStore:
if password in (None, ""):
password = (current or {}).get("password", "")
password = str(password or "")
media_root = str(payload.get("media_root") if payload.get("media_root") is not None else (current or {}).get("media_root", "") or "").strip()
path_prefix = str(payload.get("path_prefix") if payload.get("path_prefix") is not None else (current or {}).get("path_prefix", "") or "").strip()
jellyfin_url = str(payload.get("jellyfin_url") if payload.get("jellyfin_url") is not None else (current or {}).get("jellyfin_url", "") or "").strip()
jellyfin_user_id = str(payload.get("jellyfin_user_id") if payload.get("jellyfin_user_id") is not None else (current or {}).get("jellyfin_user_id", "") or "").strip()
media_root = _current_str("media_root")
path_prefix = _current_str("path_prefix")
jellyfin_url = _current_str("jellyfin_url")
jellyfin_user_id = _current_str("jellyfin_user_id")
jellyfin_api_key = payload.get("jellyfin_api_key")
if jellyfin_api_key in (None, ""):
jellyfin_api_key = (current or {}).get("jellyfin_api_key", "")
jellyfin_api_key = str(jellyfin_api_key or "")
jellyseerr_url = str(payload.get("jellyseerr_url") if payload.get("jellyseerr_url") is not None else (current or {}).get("jellyseerr_url", "") or "").strip()
jellyseerr_url = _current_str("jellyseerr_url")
jellyseerr_api_key = payload.get("jellyseerr_api_key")
if jellyseerr_api_key in (None, ""):
jellyseerr_api_key = (current or {}).get("jellyseerr_api_key", "")
jellyseerr_api_key = str(jellyseerr_api_key or "")
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
node_exporter_enabled = bool(
payload.get("node_exporter_enabled")
if payload.get("node_exporter_enabled") is not None
else (current or {}).get("node_exporter_enabled", False)
)
node_exporter_port_raw = payload.get("node_exporter_port")
if node_exporter_port_raw is None:
node_exporter_port_raw = (current or {}).get("node_exporter_port", 9100)
node_exporter_port = int(node_exporter_port_raw or 9100)
node_exporter_scrape_host = _current_str("node_exporter_scrape_host")
notes = _current_str("notes")
if mode == "local":
host = host or "localhost"
username = username or ""
@@ -351,6 +374,9 @@ class SettingsStore:
"jellyfin_api_key": jellyfin_api_key,
"jellyseerr_url": jellyseerr_url,
"jellyseerr_api_key": jellyseerr_api_key,
"node_exporter_enabled": node_exporter_enabled,
"node_exporter_port": node_exporter_port,
"node_exporter_scrape_host": node_exporter_scrape_host,
"notes": notes,
}
@@ -380,6 +406,9 @@ class SettingsStore:
"jellyfin_api_key": machine["jellyfin_api_key"],
"jellyseerr_url": machine["jellyseerr_url"],
"jellyseerr_api_key": machine["jellyseerr_api_key"],
"node_exporter_enabled": machine["node_exporter_enabled"],
"node_exporter_port": machine["node_exporter_port"],
"node_exporter_scrape_host": machine["node_exporter_scrape_host"],
"notes": machine["notes"],
}
with self.connect() as conn:
@@ -431,7 +460,10 @@ class SettingsStore:
"name": row["name"],
"mode": row["mode"],
"enabled": bool(row["enabled"]),
"services": self._normalize_services(data.get("services"), DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else []),
"services": self._normalize_services(
data.get("services"),
DEFAULT_SERVICES if row["id"] == LOCAL_MACHINE_ID else [],
),
"host": data.get("host", ""),
"port": int(data.get("port", 22) or 22),
"username": data.get("username", ""),
@@ -448,11 +480,18 @@ class SettingsStore:
"jellyfin_api_key": data.get("jellyfin_api_key", ""),
"jellyseerr_url": data.get("jellyseerr_url", ""),
"jellyseerr_api_key": data.get("jellyseerr_api_key", ""),
"node_exporter_enabled": bool(data.get("node_exporter_enabled", False)),
"node_exporter_port": int(data.get("node_exporter_port", 9100) or 9100),
"node_exporter_scrape_host": data.get("node_exporter_scrape_host", ""),
"notes": data.get("notes", ""),
}
def list_machines_for_service(self, service: str) -> list[dict[str, Any]]:
return [machine for machine in self.list_machines() if service in machine.get("services", []) and machine.get("enabled")]
return [
machine
for machine in self.list_machines()
if service in machine.get("services", []) and machine.get("enabled")
]
def get_machine_for_service(self, service: str, machine_id: str | None = None) -> dict[str, Any] | None:
if machine_id:
@@ -485,10 +524,16 @@ class SettingsStore:
"jellyfin_api_key": machine["jellyfin_api_key"],
"jellyseerr_url": machine["jellyseerr_url"],
"jellyseerr_api_key": machine["jellyseerr_api_key"],
"node_exporter_enabled": machine["node_exporter_enabled"],
"node_exporter_port": machine["node_exporter_port"],
"node_exporter_scrape_host": machine["node_exporter_scrape_host"],
"notes": machine["notes"],
}
with self.connect() as conn:
existing = conn.execute("SELECT created_at FROM monitoring_machines WHERE id = ?", (machine["id"],)).fetchone()
existing = conn.execute(
"SELECT created_at FROM monitoring_machines WHERE id = ?",
(machine["id"],),
).fetchone()
created_at = int(existing[0]) if existing else now
conn.execute(
"""
@@ -538,10 +583,13 @@ class SettingsStore:
conn.execute(
"""
INSERT INTO monitoring_machine_actions
(id, machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail)
(
id, machine_id, machine_name, mode, action, status,
created_at, duration_ms, request_id, message, error,
stdout_tail, stderr_tail
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
,
""",
(
uuid.uuid4().hex,
str(machine.get("id") or ""),
@@ -577,8 +625,10 @@ class SettingsStore:
clauses.append("status = ?")
params.append(status)
sql = (
"SELECT machine_id, machine_name, mode, action, status, created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail "
f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} ORDER BY created_at DESC LIMIT ?"
"SELECT machine_id, machine_name, mode, action, status, "
"created_at, duration_ms, request_id, message, error, stdout_tail, stderr_tail "
f"FROM monitoring_machine_actions WHERE {' AND '.join(clauses)} "
"ORDER BY created_at DESC LIMIT ?"
)
params.append(max(1, min(int(limit), 200)))
with self.connect() as conn:
@@ -595,7 +645,6 @@ class SettingsStore:
)
return int(cur.rowcount or 0)
@staticmethod
def _private_key_summary(private_key: str) -> dict[str, str]:
if not private_key:
@@ -639,11 +688,29 @@ class SettingsStore:
if passphrase in (None, ""):
passphrase = (current or {}).get("passphrase", "")
passphrase = str(passphrase or "")
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
notes = str(
payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or ""
).strip()
summary = self._private_key_summary(private_key)
public_key = str(payload.get("public_key") if payload.get("public_key") is not None else (current or {}).get("public_key", "") or summary["public_key"] or "").strip()
fingerprint = str(payload.get("fingerprint") if payload.get("fingerprint") is not None else (current or {}).get("fingerprint", "") or summary["fingerprint"] or "").strip()
return {"id": key_id, "name": name, "private_key": private_key, "passphrase": passphrase, "public_key": public_key, "fingerprint": fingerprint, "notes": notes}
public_key = str(
payload.get("public_key")
if payload.get("public_key") is not None
else (current or {}).get("public_key", "") or summary["public_key"] or ""
).strip()
fingerprint = str(
payload.get("fingerprint")
if payload.get("fingerprint") is not None
else (current or {}).get("fingerprint", "") or summary["fingerprint"] or ""
).strip()
return {
"id": key_id,
"name": name,
"private_key": private_key,
"passphrase": passphrase,
"public_key": public_key,
"fingerprint": fingerprint,
"notes": notes,
}
def list_ssh_keys(self) -> list[dict[str, Any]]:
self.init_schema()
@@ -666,7 +733,15 @@ class SettingsStore:
if not row:
return None
summary = self._private_key_summary(str(row["private_key"] or ""))
return {"id": row["id"], "name": row["name"], "private_key": row["private_key"], "passphrase": row["passphrase"], "notes": row["notes"], "public_key": str(row["public_key"] or summary["public_key"] or ""), "fingerprint": str(row["fingerprint"] or summary["fingerprint"] or "")}
return {
"id": row["id"],
"name": row["name"],
"private_key": row["private_key"],
"passphrase": row["passphrase"],
"notes": row["notes"],
"public_key": str(row["public_key"] or summary["public_key"] or ""),
"fingerprint": str(row["fingerprint"] or summary["fingerprint"] or ""),
}
def upsert_ssh_key(self, payload: dict[str, Any], key_id: str | None = None) -> dict[str, Any]:
self.init_schema()
@@ -677,7 +752,10 @@ class SettingsStore:
created_at = int(existing[0]) if existing else now
conn.execute(
"""
INSERT INTO ssh_keys (id, name, private_key, passphrase, public_key, fingerprint, notes, created_at, updated_at)
INSERT INTO ssh_keys (
id, name, private_key, passphrase, public_key, fingerprint,
notes, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
name = excluded.name,
@@ -688,7 +766,17 @@ class SettingsStore:
notes = excluded.notes,
updated_at = excluded.updated_at
""",
(key["id"], key["name"], key["private_key"], key["passphrase"], key["public_key"], key["fingerprint"], key["notes"], created_at, now),
(
key["id"],
key["name"],
key["private_key"],
key["passphrase"],
key["public_key"],
key["fingerprint"],
key["notes"],
created_at,
now,
),
)
return self.get_ssh_key(key["id"]) or key
@@ -697,8 +785,6 @@ class SettingsStore:
with self.connect() as conn:
conn.execute("DELETE FROM ssh_keys WHERE id = ?", (key_id,))
def _row_to_task(self, row: sqlite3.Row) -> dict[str, Any]:
return {
"id": row["id"],
@@ -719,11 +805,27 @@ class SettingsStore:
task_type = str(payload.get("task_type") or (current or {}).get("task_type") or "shell").strip().lower()
if task_type not in {"shell", "python"}:
task_type = "shell"
content = str(payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or "")
content = str(
payload.get("content") if payload.get("content") is not None else (current or {}).get("content", "") or ""
)
enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
default_machine_id = str(payload.get("default_machine_id") if payload.get("default_machine_id") is not None else (current or {}).get("default_machine_id", "") or "").strip()
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
return {"id": task_id, "name": name, "task_type": task_type, "content": content, "enabled": enabled, "default_machine_id": default_machine_id, "notes": notes}
default_machine_id = str(
payload.get("default_machine_id")
if payload.get("default_machine_id") is not None
else (current or {}).get("default_machine_id", "") or ""
).strip()
notes = str(
payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or ""
).strip()
return {
"id": task_id,
"name": name,
"task_type": task_type,
"content": content,
"enabled": enabled,
"default_machine_id": default_machine_id,
"notes": notes,
}
def list_tasks(self) -> list[dict[str, Any]]:
self.init_schema()
@@ -748,7 +850,10 @@ class SettingsStore:
created_at = int(existing[0]) if existing else now
conn.execute(
"""
INSERT INTO saved_tasks (id, name, task_type, content, enabled, default_machine_id, notes, created_at, updated_at)
INSERT INTO saved_tasks (
id, name, task_type, content, enabled, default_machine_id,
notes, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
name = excluded.name,
@@ -759,7 +864,17 @@ class SettingsStore:
notes = excluded.notes,
updated_at = excluded.updated_at
""",
(task["id"], task["name"], task["task_type"], task["content"], 1 if task["enabled"] else 0, task["default_machine_id"], task["notes"], created_at, now),
(
task["id"],
task["name"],
task["task_type"],
task["content"],
1 if task["enabled"] else 0,
task["default_machine_id"],
task["notes"],
created_at,
now,
),
)
return self.get_task(task["id"]) or task
@@ -796,8 +911,11 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO saved_task_runs
(id, task_id, task_name, machine_id, machine_name, task_type, status, created_at, duration_ms, request_id, stdout_tail, stderr_tail, error)
INSERT INTO saved_task_runs (
id, task_id, task_name, machine_id, machine_name, task_type,
status, created_at, duration_ms, request_id, stdout_tail,
stderr_tail, error
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
@@ -837,17 +955,25 @@ class SettingsStore:
def _normalize_shortcut_payload(self, payload: dict[str, Any], shortcut_id: str | None = None) -> dict[str, Any]:
current = self.get_shortcut(shortcut_id) if shortcut_id else None
shortcut_id = str(payload.get("id") or shortcut_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12]
shortcut_type = str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower()
shortcut_type = (
str(payload.get("shortcut_type") or (current or {}).get("shortcut_type") or "website").strip().lower()
)
if shortcut_type not in {"website", "action", "user"}:
shortcut_type = "website"
label = str(payload.get("label") or (current or {}).get("label") or "").strip() or shortcut_id
enabled = bool(payload.get("enabled", (current or {}).get("enabled", True)))
icon = str(payload.get("icon") if payload.get("icon") is not None else (current or {}).get("icon", "") or "").strip()
url = str(payload.get("url") if payload.get("url") is not None else (current or {}).get("url", "") or "").strip()
task_id = str(payload.get("task_id") if payload.get("task_id") is not None else (current or {}).get("task_id", "") or "").strip()
machine_id = str(payload.get("machine_id") if payload.get("machine_id") is not None else (current or {}).get("machine_id", "") or "").strip()
user_id = str(payload.get("user_id") if payload.get("user_id") is not None else (current or {}).get("user_id", "") or "").strip()
notes = str(payload.get("notes") if payload.get("notes") is not None else (current or {}).get("notes", "") or "").strip()
def _field(field: str, default: str = "") -> str:
return str(
payload.get(field) if payload.get(field) is not None else (current or {}).get(field, default) or default
).strip()
icon = _field("icon")
url = _field("url")
task_id = _field("task_id")
machine_id = _field("machine_id")
user_id = _field("user_id")
notes = _field("notes")
return {
"id": shortcut_id,
"label": label,
@@ -882,7 +1008,10 @@ class SettingsStore:
shortcut = self._normalize_shortcut_payload(payload, shortcut_id)
now = int(time.time())
with self.connect() as conn:
existing = conn.execute("SELECT created_at FROM dashboard_shortcuts WHERE id = ?", (shortcut["id"],)).fetchone()
existing = conn.execute(
"SELECT created_at FROM dashboard_shortcuts WHERE id = ?",
(shortcut["id"],),
).fetchone()
created_at = int(existing[0]) if existing else now
conn.execute(
"""
@@ -926,14 +1055,16 @@ class SettingsStore:
"created_at": row["created_at"],
}
def _normalize_backup_job_payload(
self, payload: dict[str, Any], job_id: str | None = None
) -> dict[str, Any]:
def _normalize_backup_job_payload(self, payload: dict[str, Any], job_id: str | None = None) -> dict[str, Any]:
current = self.get_backup_job(job_id) if job_id else None
job_id = str(payload.get("id") or job_id or uuid.uuid4().hex[:12]).strip() or uuid.uuid4().hex[:12]
name = str(payload.get("name") or (current or {}).get("name") or job_id).strip() or job_id
source = str(payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or "").strip()
target = str(payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or "").strip()
source = str(
payload.get("source") if payload.get("source") is not None else (current or {}).get("source", "") or ""
).strip()
target = str(
payload.get("target") if payload.get("target") is not None else (current or {}).get("target", "") or ""
).strip()
schedule_interval_seconds = payload.get("schedule_interval_seconds")
if schedule_interval_seconds is None:
schedule_interval_seconds = (current or {}).get("schedule_interval_seconds")
@@ -1040,10 +1171,24 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO backup_runs (id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, created_at)
INSERT INTO backup_runs (
id, job_id, started_at, ended_at, status, bytes_transferred,
duration_ms, error_message, details_json, created_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(run_id, job_id, started_at, ended_at, status, bytes_transferred, duration_ms, error_message, details_json, now),
(
run_id,
job_id,
started_at,
ended_at,
status,
bytes_transferred,
duration_ms,
error_message,
details_json,
now,
),
)
return self.get_backup_run(run_id) or {
"id": run_id,
@@ -1128,7 +1273,10 @@ class SettingsStore:
with self.connect() as conn:
conn.execute(
"""
INSERT INTO backup_alerts (id, job_id, run_id, alert_type, severity, message, acknowledged, resolved_at, created_at)
INSERT INTO backup_alerts (
id, job_id, run_id, alert_type, severity, message,
acknowledged, resolved_at, created_at
)
VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?)
""",
(alert_id, job_id, run_id, alert_type, severity, message, now),
@@ -1238,9 +1386,7 @@ class SettingsStore:
)
"""
)
row = conn.execute(
"SELECT value FROM app_settings WHERE key = ?", (key,)
).fetchone()
row = conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone()
return row["value"] if row else default
def update_setting(self, key: str, value: str) -> None:
backend/src/media_library_viewer_api/services/targets.py
+77
View File
@@ -0,0 +1,77 @@
"""Prometheus file-based service discovery target management.
The backend owns the list of remote Node Exporter targets so that operators can
enable scraping per machine from the Manage UI. Prometheus reads the generated
JSON file via `file_sd_configs`; this keeps Prometheus config static and pushes
machine-specific changes into a file it can reload.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import Any
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
DEFAULT_NODE_EXPORTER_PORT = 9100
def _scrape_address(machine: dict[str, Any]) -> str | None:
"""Return host:port for the Node Exporter on a machine, or None if disabled."""
if not machine.get("node_exporter_enabled"):
return None
scrape_host = str(machine.get("node_exporter_scrape_host") or "").strip()
host = scrape_host or str(machine.get("host") or "").strip()
if not host or host == "localhost":
return None
port = int(machine.get("node_exporter_port") or DEFAULT_NODE_EXPORTER_PORT)
return f"{host}:{port}"
def build_node_exporter_targets(store: SettingsStore) -> list[dict[str, Any]]:
"""Build a file-SD target list for all enabled SSH machines.
Local machines are excluded because the Compose-managed node-exporter
service already covers the Docker host.
"""
targets: list[dict[str, Any]] = []
for machine in store.list_machines():
if not machine.get("enabled"):
continue
if str(machine.get("mode") or "local").strip().lower() != "ssh":
continue
address = _scrape_address(machine)
if not address:
continue
targets.append(
{
"targets": [address],
"labels": {
"job": "node-exporter-remote",
"machine_id": str(machine.get("id") or ""),
"machine_name": str(machine.get("name") or ""),
"instance": address,
},
}
)
return targets
def write_prometheus_targets(store: SettingsStore, file_sd_dir: Path | None = None) -> Path:
"""Render and persist Prometheus file-SD targets.
Returns the path written so callers can log or expose it.
"""
settings = get_settings()
file_sd_dir = file_sd_dir or Path(settings.prometheus_file_sd_dir)
file_sd_dir.mkdir(parents=True, exist_ok=True)
file_path = file_sd_dir / "node_exporter_targets.json"
targets = build_node_exporter_targets(store)
file_path.write_text(json.dumps(targets, indent=2), encoding="utf-8")
logger.info("Wrote %s node_exporter targets to %s", len(targets), file_path)
return file_path