feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

This commit is contained in:
Developer
2026-06-16 11:08:32 +00:00
parent 615e02e970
commit e2ad731b5f
80 changed files with 5020 additions and 3665 deletions
@@ -8,6 +8,7 @@ from ..models.backups import (
BackupReportRequest,
BackupRunResponse,
)
from ..observability import record_backup_run
from ..services.backup_alert_engine import generate_alerts_for_run
from ..services.settings_store import SettingsStore, get_settings_store
@@ -17,20 +18,24 @@ router = APIRouter(prefix="/api/backups", tags=["backups"])
def _get_or_create_job(store: SettingsStore, report: BackupReportRequest) -> dict[str, Any]:
job = store.get_backup_job_by_name(report.name)
if not job:
job = store.upsert_backup_job({
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
})
job = store.upsert_backup_job(
{
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
}
)
elif report.schedule_interval_seconds:
store.upsert_backup_job({
"id": job["id"],
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
})
store.upsert_backup_job(
{
"id": job["id"],
"name": report.name,
"source": report.source,
"target": report.target,
"schedule_interval_seconds": report.schedule_interval_seconds,
}
)
job = store.get_backup_job(job["id"])
return job
@@ -61,6 +66,7 @@ def post_backup_report(
"details": report.details,
}
run = store.create_backup_run(run_data)
record_backup_run(job_name=report.name, status=report.status, success=report.status == "success")
# Generate alerts
previous_runs = store.list_backup_runs(job_id=job["id"], status="success", limit=20)
@@ -93,6 +99,7 @@ def post_backup_start(
"status": "in_progress",
}
run = store.create_backup_run(run_data)
record_backup_run(job_name=report.name, status="in_progress")
# Map details -> details_json for response model
run["details_json"] = run.pop("details", None)
return BackupRunResponse(**run)
@@ -11,12 +11,10 @@ from fastapi import APIRouter, Depends
from media_library_viewer_api.clients.jellyfin import JellyfinClient
from media_library_viewer_api.dependencies import (
get_jellyfin_client,
get_monitoring_poller,
get_settings_store,
get_user_id,
)
from media_library_viewer_api.models.backups import BackupDashboardSummary
from media_library_viewer_api.services.monitoring_actions import collect_machine_overview
from media_library_viewer_api.services.settings_store import SettingsStore
logger = logging.getLogger(__name__)
@@ -46,52 +44,6 @@ def get_library_counts(
return client.library_item_counts(user_id, libraries)
@router.get("/monitoring")
def get_monitoring_overview(
store=Depends(get_settings_store),
) -> dict[str, Any]:
"""Return one lightweight monitoring row per configured machine."""
machines = [m for m in store.list_machines() if m.get("enabled")]
rows: list[dict[str, Any]] = []
for machine in machines:
try:
rows.append(collect_machine_overview(machine, store))
except Exception as exc:
logger.exception(
"Dashboard monitoring overview failed for machine_id=%s",
machine.get("id"),
)
rows.append({
"machine": {k: machine.get(k) for k in (
"id", "name", "mode", "enabled", "host", "port", "username",
"media_root", "path_prefix", "notes",
)},
"status": "",
"status_error": str(exc),
"metrics_error": str(exc),
"disk_error": str(exc),
"sample_count": 0,
"latest_sample": None,
"cpu_summary": None,
"iowait_summary": None,
"mem_summary": None,
"net_rx_summary": None,
"net_tx_summary": None,
"disk_read_summary": None,
"disk_write_summary": None,
"disk": None,
})
poller = get_monitoring_poller().snapshot()
enabled_count = sum(1 for machine in machines if machine.get("enabled"))
logger.info("Dashboard monitoring machines=%s enabled=%s", len(machines), enabled_count)
return {
"poller": poller,
"machines": rows,
"total": len(machines),
"enabled": enabled_count,
}
@router.get("/shortcuts")
def get_shortcuts(
store=Depends(get_settings_store),
@@ -144,8 +96,8 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[
has_item = bool(item)
series = item.get("SeriesName") or ""
title = (
f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")
) if has_item else "(idle)"
(f"{series} - {item.get('Name', '')}" if series else item.get("Name", "Unknown")) if has_item else "(idle)"
)
if not has_item:
state_label = "idle"
@@ -162,16 +114,18 @@ def _map_sessions_to_activity_rows(sessions: list[dict[str, Any]]) -> list[dict[
if not transcode_type:
transcode_type.append("active")
results.append({
"user": session.get("UserName") or "Unknown",
"title": title,
"type": item.get("Type", "") if has_item else "",
"state": state_label,
"transcoding": "yes" if is_transcoding else "no",
"transcoding_type": ", ".join(transcode_type),
"device": session.get("DeviceName") or session.get("Client") or "",
"session_id": session.get("Id") or "",
})
results.append(
{
"user": session.get("UserName") or "Unknown",
"title": title,
"type": item.get("Type", "") if has_item else "",
"state": state_label,
"transcoding": "yes" if is_transcoding else "no",
"transcoding_type": ", ".join(transcode_type),
"device": session.get("DeviceName") or session.get("Client") or "",
"session_id": session.get("Id") or "",
}
)
return results
@@ -16,6 +16,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status
from media_library_viewer_api.clients.jellyfin import JellyfinClient
from media_library_viewer_api.dependencies import get_jellyfin_client, get_user_id
from media_library_viewer_api.observability import record_media_index_build
from media_library_viewer_api.services.media_index import MediaIndex
logger = logging.getLogger(__name__)
@@ -169,6 +170,7 @@ def post_build_index(
},
)
media_status = index.status()
record_media_index_build(status="started")
logger.info("Media index build started pid=%s", process.pid)
return {"status": "started", **_serialize_status(media_status)}
@@ -256,6 +258,7 @@ def force_stop_build(index: MediaIndex = Depends(get_media_index)) -> dict[str,
},
)
media_status = index.status()
record_media_index_build(status="force_stopped")
return {"status": "force_stopped", **_serialize_status(media_status)}
@@ -1,31 +1,70 @@
"""Monitoring router — metrics, collector controls, and per-machine status."""
"""Monitoring router — disk checks, action history, and observability stack status."""
from __future__ import annotations
import logging
import time
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Body, Depends, HTTPException, Query
from media_library_viewer_api.clients.resources import (
disk_space,
read_resource_metrics,
resource_collector_status,
)
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_settings_store
from media_library_viewer_api.services.monitoring_actions import (
poll_machine_diagnostics,
disk_space,
run_machine_operation,
start_collector,
stop_collector,
restart_collector,
)
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import build_node_exporter_targets
logger = logging.getLogger(__name__)
def _alertmanager_client() -> Any:
"""Return a simple HTTP client for the configured Alertmanager URL."""
import requests
settings = get_settings()
return requests.Session(), settings.alertmanager_url
def _webhook_client() -> Any:
"""Return a simple HTTP client for the optional webhook receiver URL."""
import requests
settings = get_settings()
return requests.Session(), settings.alertmanager_webhook_url
def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
"""Build a UI-friendly summary from Alertmanager /api/v1/alerts payload."""
by_severity: dict[str, int] = {}
open_alerts: list[dict[str, Any]] = []
for alert in alerts:
labels = alert.get("labels") or {}
annotations = alert.get("annotations") or {}
severity = labels.get("severity", "unknown")
by_severity[severity] = by_severity.get(severity, 0) + 1
open_alerts.append(
{
"name": labels.get("alertname", "unknown"),
"severity": severity,
"category": labels.get("category", ""),
"job_name": labels.get("job_name", labels.get("job", "")),
"summary": annotations.get("summary", ""),
"description": annotations.get("description", ""),
"active_since": alert.get("startsAt"),
"state": alert.get("status", "firing"),
"labels": labels,
}
)
open_alerts.sort(key=lambda a: (a["severity"] not in {"critical", "warning"}, a["severity"], a["name"]))
return {
"total": len(open_alerts),
"by_severity": by_severity,
"alerts": open_alerts[:50],
}
router = APIRouter(prefix="/api/monitoring", tags=["monitoring"])
@@ -63,7 +102,11 @@ def get_poller_status() -> dict[str, Any]:
from media_library_viewer_api.dependencies import get_monitoring_poller
poller = get_monitoring_poller().snapshot()
logger.info("Monitoring poller status requested running=%s poll_count=%s", poller.get("worker_running"), poller.get("poll_count"))
logger.info(
"Monitoring poller status requested running=%s poll_count=%s",
poller.get("worker_running"),
poller.get("poll_count"),
)
return poller
@@ -81,53 +124,6 @@ def get_machine_actions(
return {"items": actions, "total": len(actions)}
@router.get("/status")
def get_status(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Return collector running status for a given machine."""
machine = _resolve_machine(store, machine_id)
status = run_machine_operation(machine, store, "status lookup", resource_collector_status)
logger.info("Monitoring status requested machine_id=%s status=%s", machine["id"], status)
return {"status": status}
@router.get("/metrics")
def get_metrics(
max_lines: int = 70_000,
last_seconds: int | None = None,
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Return resource metric samples for a given machine."""
machine = _resolve_machine(store, machine_id)
def _read_metrics(client):
return read_resource_metrics(client, max_lines=max_lines)
rows = run_machine_operation(machine, store, "metrics read", _read_metrics)
if last_seconds is None:
filtered = rows
cutoff_ts = 0.0
else:
cutoff_ts = time.time() - last_seconds
filtered = [row for row in rows if float(row.get("ts", 0)) >= cutoff_ts]
logger.info(
"Monitoring metrics requested machine_id=%s total=%s filtered=%s last_seconds=%s",
machine["id"],
len(rows),
len(filtered),
last_seconds,
)
return {
"samples": filtered,
"total_samples": len(rows),
"filtered_samples": len(filtered),
"cutoff_ts": cutoff_ts,
}
@router.get("/disk")
def get_disk_space(
machine_id: str | None = Query(default=None),
@@ -146,49 +142,99 @@ def get_disk_space(
)
@router.post("/start")
def post_start(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Start the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = start_collector(machine, store)
logger.info("Monitoring collector start result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/prometheus-targets")
def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
"""Return Prometheus file-SD targets for remote Node Exporters.
The backend writes these targets to a JSON file that Prometheus reads via
file_sd_configs. This endpoint returns the same list live from the store so
the UI can preview which machines will be scraped.
"""
targets = build_node_exporter_targets(store)
logger.info("Prometheus targets requested count=%s", len(targets))
return targets
@router.post("/stop")
def post_stop(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Stop the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = stop_collector(machine, store)
logger.info("Monitoring collector stop result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/alerts")
def get_alertmanager_alerts() -> dict[str, Any]:
"""Return a summary of active Alertmanager alerts for the UI.
Proxies the Alertmanager `/api/v1/alerts` endpoint and reshapes the payload
into a stable, UI-friendly format. If Alertmanager is unreachable, the
endpoint returns an empty summary and logs the failure so the UI can still
render a health card instead of an error page.
"""
session, base_url = _alertmanager_client()
try:
response = session.get(f"{base_url}/api/v1/alerts", timeout=5)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager alerts from %s", base_url)
return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_unreachable"}
if data.get("status") != "success":
return {"total": 0, "by_severity": {}, "alerts": [], "error": data.get("error", "unknown")}
summary = _summary_from_alerts(data.get("data", []))
logger.info("Alertmanager alerts requested total=%s", summary["total"])
return summary
@router.post("/restart")
def post_restart(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Restart the machine's resource collector."""
machine = _resolve_machine(store, machine_id)
message = restart_collector(machine, store)
logger.info("Monitoring collector restart result machine_id=%s: %s", machine["id"], message)
return {"message": message}
@router.get("/alertmanager-status")
def get_alertmanager_status() -> dict[str, Any]:
"""Return Alertmanager cluster/status for the UI health card.
Uses the Alertmanager `/api/v2/status` endpoint and exposes only the high-
level fields the UI needs: uptime, version, and whether the cluster is
healthy.
"""
session, base_url = _alertmanager_client()
try:
response = session.get(f"{base_url}/api/v2/status", timeout=5)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager status from %s", base_url)
return {"up": False, "version": "", "uptime": ""}
cluster = data.get("cluster") or {}
status = data.get("clusterStatus") or {}
return {
"up": True,
"version": data.get("versionInfo", {}).get("version", ""),
"uptime": status.get("createdAt", ""),
"name": "",
"peers": [p.get("name", "") for p in cluster.get("peers", [])],
}
@router.get("/diagnostics")
def get_diagnostics(
machine_id: str | None = Query(default=None),
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, str]:
"""Return collector debug info for a given machine."""
machine = _resolve_machine(store, machine_id)
diagnostics = poll_machine_diagnostics(machine, store)["diagnostics"]
logger.info("Monitoring diagnostics requested machine_id=%s", machine["id"])
return {"diagnostics": diagnostics}
@router.post("/alertmanager-webhook")
def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
"""Receive alerts from Alertmanager and optionally forward to a webhook URL.
This endpoint is the receiver referenced by the optional `webhook_configs`
block in Alertmanager. It logs the payload for audit/debug purposes and, if
`ALERTMANAGER_WEBHOOK_URL` is configured, forwards the alert JSON verbatim.
Forwarding is best-effort: a failure to reach the downstream webhook does
not fail this endpoint, so Alertmanager sees a successful delivery.
"""
alerts = payload.get("alerts", [])
logger.info(
"Received Alertmanager webhook with %s alert(s), status=%s",
len(alerts),
payload.get("status", "unknown"),
)
session, webhook_url = _webhook_client()
if webhook_url:
try:
response = session.post(webhook_url, json=payload, timeout=10)
response.raise_for_status()
logger.info("Forwarded Alertmanager webhook to %s", webhook_url)
except Exception:
logger.exception("Failed to forward Alertmanager webhook to %s", webhook_url)
else:
logger.debug("No ALERTMANAGER_WEBHOOK_URL configured; webhook stored in logs only")
return {"status": "received"}
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
from io import StringIO
from typing import Any
@@ -12,11 +13,13 @@ from pydantic import BaseModel, Field
from media_library_viewer_api.clients.ssh import RemoteSSHClient
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_monitoring_poller, get_settings_store
from media_library_viewer_api.services.monitoring_actions import start_collector
from media_library_viewer_api.services.db_maintenance import remove_sqlite_database
from media_library_viewer_api.services.known_hosts import has_known_host
from media_library_viewer_api.services.media_index import MediaIndex
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import write_prometheus_targets
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/settings", tags=["settings"])
@@ -95,9 +98,7 @@ def _raise_ssh_validation_error(host: str, port: int, exc: Exception) -> None:
if "protocol banner" in lowered:
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail=(
f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."
),
detail=(f"SSH banner not received from {host}:{port}; the backend could not complete the SSH handshake."),
) from exc
if "no authentication methods available" in lowered or "authentication failed" in lowered:
raise HTTPException(
@@ -125,10 +126,12 @@ def _validate_saved_machine_ssh(machine: MonitoringMachineInput, store: Settings
client.close()
def _start_machine_collector(machine: MonitoringMachineInput, store: SettingsStore) -> None:
if "monitoring" not in {str(service).strip().lower() for service in machine.services}:
return
start_collector(machine.model_dump(exclude_none=True), store)
def _write_prometheus_targets(store: SettingsStore) -> None:
"""Regenerate Prometheus file-SD targets after machine changes."""
try:
write_prometheus_targets(store)
except Exception:
logger.exception("Failed to write Prometheus file-SD targets")
@router.post("/machines/test-ssh")
@@ -137,7 +140,9 @@ def test_machine_ssh(
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
if str(machine.mode or "").strip().lower() != "ssh":
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="SSH validation only applies to SSH machines"
)
client, host, port = _resolve_ssh_client(machine, store)
settings = get_settings()
@@ -174,7 +179,8 @@ def test_machine_ssh(
return {
"status": "ok",
"message": (
f"SSH connection succeeded for {host}:{port}; host key {'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
f"SSH connection succeeded for {host}:{port}; host key "
f"{'was recorded' if known_hosts_updated else 'was already trusted'} and authentication worked."
),
"host": host,
"port": port,
@@ -188,11 +194,11 @@ def post_machine(
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine.id)
_write_prometheus_targets(store)
poller = get_monitoring_poller()
try:
saved_machine = MonitoringMachineInput.model_validate(saved)
_validate_saved_machine_ssh(saved_machine, store)
_start_machine_collector(saved_machine, store)
finally:
poller.start()
poller.kick()
@@ -208,11 +214,11 @@ def put_machine(
if not store.get_machine(machine_id):
raise HTTPException(status_code=404, detail="Machine not found")
saved = store.upsert_machine(machine.model_dump(exclude_none=True), machine_id)
_write_prometheus_targets(store)
poller = get_monitoring_poller()
try:
saved_machine = MonitoringMachineInput.model_validate(saved)
_validate_saved_machine_ssh(saved_machine, store)
_start_machine_collector(saved_machine, store)
finally:
poller.start()
poller.kick()
@@ -224,6 +230,7 @@ def delete_machine(machine_id: str, store: SettingsStore = Depends(get_settings_
if not store.get_machine(machine_id):
raise HTTPException(status_code=404, detail="Machine not found")
store.delete_machine(machine_id)
_write_prometheus_targets(store)
return {"status": "deleted"}
@@ -311,8 +318,12 @@ def reset_local_database(
expected = "RESET LOCAL DATABASE"
if payload.confirm_phrase.strip().upper() != expected:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Confirmation phrase does not match")
if not (payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected")
if not (
payload.acknowledge_settings_loss and payload.acknowledge_media_index_loss and payload.acknowledge_irreversible
):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="All confirmation checkboxes must be selected"
)
settings_removed = remove_sqlite_database(store.db_path)
media_index = MediaIndex()
@@ -212,7 +212,8 @@ def _merge_users(
"source_summary": summary,
"name_source": name_source,
"access_source": access_source,
"jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId")) or None,
"jellyseerr_user_id": _safe_int((seerr_user or {}).get("id") or (seerr_user or {}).get("userId"))
or None,
"jellyseerr_username": str(
(seerr_user or {}).get("username") or (seerr_user or {}).get("jellyfinUsername") or ""
),
@@ -364,9 +365,10 @@ async def post_user_message(
text_body=text_body,
attachments=attachment_payloads,
)
from_address = str(getattr(settings, "smtp_from_address", "") or "").strip() or str(
getattr(settings, "smtp_username", "") or ""
).strip()
from_address = (
str(getattr(settings, "smtp_from_address", "") or "").strip()
or str(getattr(settings, "smtp_username", "") or "").strip()
)
logger.info(
"Users message queued request_id=%s subject=%s recipients=%s attachments=%s skipped=%s",
request_id,