feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI
This commit is contained in:
Developer
@@ -1,31 +1,70 @@
|
||||
"""Monitoring router — metrics, collector controls, and per-machine status."""
|
||||
"""Monitoring router — disk checks, action history, and observability stack status."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
|
||||
from media_library_viewer_api.clients.resources import (
|
||||
disk_space,
|
||||
read_resource_metrics,
|
||||
resource_collector_status,
|
||||
)
|
||||
from media_library_viewer_api.config import get_settings
|
||||
from media_library_viewer_api.dependencies import get_settings_store
|
||||
from media_library_viewer_api.services.monitoring_actions import (
|
||||
poll_machine_diagnostics,
|
||||
disk_space,
|
||||
run_machine_operation,
|
||||
start_collector,
|
||||
stop_collector,
|
||||
restart_collector,
|
||||
)
|
||||
from media_library_viewer_api.services.settings_store import SettingsStore
|
||||
from media_library_viewer_api.services.targets import build_node_exporter_targets
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _alertmanager_client() -> Any:
|
||||
"""Return a simple HTTP client for the configured Alertmanager URL."""
|
||||
import requests
|
||||
|
||||
settings = get_settings()
|
||||
return requests.Session(), settings.alertmanager_url
|
||||
|
||||
|
||||
def _webhook_client() -> Any:
|
||||
"""Return a simple HTTP client for the optional webhook receiver URL."""
|
||||
import requests
|
||||
|
||||
settings = get_settings()
|
||||
return requests.Session(), settings.alertmanager_webhook_url
|
||||
|
||||
|
||||
def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Build a UI-friendly summary from Alertmanager /api/v1/alerts payload."""
|
||||
by_severity: dict[str, int] = {}
|
||||
open_alerts: list[dict[str, Any]] = []
|
||||
for alert in alerts:
|
||||
labels = alert.get("labels") or {}
|
||||
annotations = alert.get("annotations") or {}
|
||||
severity = labels.get("severity", "unknown")
|
||||
by_severity[severity] = by_severity.get(severity, 0) + 1
|
||||
open_alerts.append(
|
||||
{
|
||||
"name": labels.get("alertname", "unknown"),
|
||||
"severity": severity,
|
||||
"category": labels.get("category", ""),
|
||||
"job_name": labels.get("job_name", labels.get("job", "")),
|
||||
"summary": annotations.get("summary", ""),
|
||||
"description": annotations.get("description", ""),
|
||||
"active_since": alert.get("startsAt"),
|
||||
"state": alert.get("status", "firing"),
|
||||
"labels": labels,
|
||||
}
|
||||
)
|
||||
open_alerts.sort(key=lambda a: (a["severity"] not in {"critical", "warning"}, a["severity"], a["name"]))
|
||||
return {
|
||||
"total": len(open_alerts),
|
||||
"by_severity": by_severity,
|
||||
"alerts": open_alerts[:50],
|
||||
}
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/monitoring", tags=["monitoring"])
|
||||
|
||||
|
||||
@@ -63,7 +102,11 @@ def get_poller_status() -> dict[str, Any]:
|
||||
from media_library_viewer_api.dependencies import get_monitoring_poller
|
||||
|
||||
poller = get_monitoring_poller().snapshot()
|
||||
logger.info("Monitoring poller status requested running=%s poll_count=%s", poller.get("worker_running"), poller.get("poll_count"))
|
||||
logger.info(
|
||||
"Monitoring poller status requested running=%s poll_count=%s",
|
||||
poller.get("worker_running"),
|
||||
poller.get("poll_count"),
|
||||
)
|
||||
return poller
|
||||
|
||||
|
||||
@@ -81,53 +124,6 @@ def get_machine_actions(
|
||||
return {"items": actions, "total": len(actions)}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def get_status(
|
||||
machine_id: str | None = Query(default=None),
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> dict[str, str]:
|
||||
"""Return collector running status for a given machine."""
|
||||
machine = _resolve_machine(store, machine_id)
|
||||
status = run_machine_operation(machine, store, "status lookup", resource_collector_status)
|
||||
logger.info("Monitoring status requested machine_id=%s status=%s", machine["id"], status)
|
||||
return {"status": status}
|
||||
|
||||
|
||||
@router.get("/metrics")
|
||||
def get_metrics(
|
||||
max_lines: int = 70_000,
|
||||
last_seconds: int | None = None,
|
||||
machine_id: str | None = Query(default=None),
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> dict[str, Any]:
|
||||
"""Return resource metric samples for a given machine."""
|
||||
machine = _resolve_machine(store, machine_id)
|
||||
|
||||
def _read_metrics(client):
|
||||
return read_resource_metrics(client, max_lines=max_lines)
|
||||
|
||||
rows = run_machine_operation(machine, store, "metrics read", _read_metrics)
|
||||
if last_seconds is None:
|
||||
filtered = rows
|
||||
cutoff_ts = 0.0
|
||||
else:
|
||||
cutoff_ts = time.time() - last_seconds
|
||||
filtered = [row for row in rows if float(row.get("ts", 0)) >= cutoff_ts]
|
||||
logger.info(
|
||||
"Monitoring metrics requested machine_id=%s total=%s filtered=%s last_seconds=%s",
|
||||
machine["id"],
|
||||
len(rows),
|
||||
len(filtered),
|
||||
last_seconds,
|
||||
)
|
||||
return {
|
||||
"samples": filtered,
|
||||
"total_samples": len(rows),
|
||||
"filtered_samples": len(filtered),
|
||||
"cutoff_ts": cutoff_ts,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/disk")
|
||||
def get_disk_space(
|
||||
machine_id: str | None = Query(default=None),
|
||||
@@ -146,49 +142,99 @@ def get_disk_space(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/start")
|
||||
def post_start(
|
||||
machine_id: str | None = Query(default=None),
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> dict[str, str]:
|
||||
"""Start the machine's resource collector."""
|
||||
machine = _resolve_machine(store, machine_id)
|
||||
message = start_collector(machine, store)
|
||||
logger.info("Monitoring collector start result machine_id=%s: %s", machine["id"], message)
|
||||
return {"message": message}
|
||||
@router.get("/prometheus-targets")
|
||||
def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
|
||||
"""Return Prometheus file-SD targets for remote Node Exporters.
|
||||
|
||||
The backend writes these targets to a JSON file that Prometheus reads via
|
||||
file_sd_configs. This endpoint returns the same list live from the store so
|
||||
the UI can preview which machines will be scraped.
|
||||
"""
|
||||
targets = build_node_exporter_targets(store)
|
||||
logger.info("Prometheus targets requested count=%s", len(targets))
|
||||
return targets
|
||||
|
||||
|
||||
@router.post("/stop")
|
||||
def post_stop(
|
||||
machine_id: str | None = Query(default=None),
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> dict[str, str]:
|
||||
"""Stop the machine's resource collector."""
|
||||
machine = _resolve_machine(store, machine_id)
|
||||
message = stop_collector(machine, store)
|
||||
logger.info("Monitoring collector stop result machine_id=%s: %s", machine["id"], message)
|
||||
return {"message": message}
|
||||
@router.get("/alerts")
|
||||
def get_alertmanager_alerts() -> dict[str, Any]:
|
||||
"""Return a summary of active Alertmanager alerts for the UI.
|
||||
|
||||
Proxies the Alertmanager `/api/v1/alerts` endpoint and reshapes the payload
|
||||
into a stable, UI-friendly format. If Alertmanager is unreachable, the
|
||||
endpoint returns an empty summary and logs the failure so the UI can still
|
||||
render a health card instead of an error page.
|
||||
"""
|
||||
session, base_url = _alertmanager_client()
|
||||
try:
|
||||
response = session.get(f"{base_url}/api/v1/alerts", timeout=5)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch Alertmanager alerts from %s", base_url)
|
||||
return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_unreachable"}
|
||||
|
||||
if data.get("status") != "success":
|
||||
return {"total": 0, "by_severity": {}, "alerts": [], "error": data.get("error", "unknown")}
|
||||
|
||||
summary = _summary_from_alerts(data.get("data", []))
|
||||
logger.info("Alertmanager alerts requested total=%s", summary["total"])
|
||||
return summary
|
||||
|
||||
|
||||
@router.post("/restart")
|
||||
def post_restart(
|
||||
machine_id: str | None = Query(default=None),
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> dict[str, str]:
|
||||
"""Restart the machine's resource collector."""
|
||||
machine = _resolve_machine(store, machine_id)
|
||||
message = restart_collector(machine, store)
|
||||
logger.info("Monitoring collector restart result machine_id=%s: %s", machine["id"], message)
|
||||
return {"message": message}
|
||||
@router.get("/alertmanager-status")
|
||||
def get_alertmanager_status() -> dict[str, Any]:
|
||||
"""Return Alertmanager cluster/status for the UI health card.
|
||||
|
||||
Uses the Alertmanager `/api/v2/status` endpoint and exposes only the high-
|
||||
level fields the UI needs: uptime, version, and whether the cluster is
|
||||
healthy.
|
||||
"""
|
||||
session, base_url = _alertmanager_client()
|
||||
try:
|
||||
response = session.get(f"{base_url}/api/v2/status", timeout=5)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch Alertmanager status from %s", base_url)
|
||||
return {"up": False, "version": "", "uptime": ""}
|
||||
|
||||
cluster = data.get("cluster") or {}
|
||||
status = data.get("clusterStatus") or {}
|
||||
return {
|
||||
"up": True,
|
||||
"version": data.get("versionInfo", {}).get("version", ""),
|
||||
"uptime": status.get("createdAt", ""),
|
||||
"name": "",
|
||||
"peers": [p.get("name", "") for p in cluster.get("peers", [])],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/diagnostics")
|
||||
def get_diagnostics(
|
||||
machine_id: str | None = Query(default=None),
|
||||
store: SettingsStore = Depends(get_settings_store),
|
||||
) -> dict[str, str]:
|
||||
"""Return collector debug info for a given machine."""
|
||||
machine = _resolve_machine(store, machine_id)
|
||||
diagnostics = poll_machine_diagnostics(machine, store)["diagnostics"]
|
||||
logger.info("Monitoring diagnostics requested machine_id=%s", machine["id"])
|
||||
return {"diagnostics": diagnostics}
|
||||
@router.post("/alertmanager-webhook")
|
||||
def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
|
||||
"""Receive alerts from Alertmanager and optionally forward to a webhook URL.
|
||||
|
||||
This endpoint is the receiver referenced by the optional `webhook_configs`
|
||||
block in Alertmanager. It logs the payload for audit/debug purposes and, if
|
||||
`ALERTMANAGER_WEBHOOK_URL` is configured, forwards the alert JSON verbatim.
|
||||
Forwarding is best-effort: a failure to reach the downstream webhook does
|
||||
not fail this endpoint, so Alertmanager sees a successful delivery.
|
||||
"""
|
||||
alerts = payload.get("alerts", [])
|
||||
logger.info(
|
||||
"Received Alertmanager webhook with %s alert(s), status=%s",
|
||||
len(alerts),
|
||||
payload.get("status", "unknown"),
|
||||
)
|
||||
|
||||
session, webhook_url = _webhook_client()
|
||||
if webhook_url:
|
||||
try:
|
||||
response = session.post(webhook_url, json=payload, timeout=10)
|
||||
response.raise_for_status()
|
||||
logger.info("Forwarded Alertmanager webhook to %s", webhook_url)
|
||||
except Exception:
|
||||
logger.exception("Failed to forward Alertmanager webhook to %s", webhook_url)
|
||||
else:
|
||||
logger.debug("No ALERTMANAGER_WEBHOOK_URL configured; webhook stored in logs only")
|
||||
|
||||
return {"status": "received"}
|
||||
|
||||
Reference in New Issue
Block a user