Files
manage/backend/src/media_library_viewer_api/routers/monitoring.py
T
Developer 0c5698c903 feat(observability): service discovery, health cards, alertmanager widget
Slice 3 of observability-service-registry (frontend). The Observability
page discovers Grafana from the service registry instead of env vars,
adds Grafana + Prometheus health cards, and ships an alertmanager
active_alerts dashboard widget.

- types: added GrafanaStatus + PrometheusStatus; added optional
  service_id/error to AlertmanagerStatus.
- api/client.ts + hooks/useObservability.ts: fetchGrafanaStatus,
  fetchPrometheusStatus, useGrafanaStatus, usePrometheusStatus.
- widgets/AlertmanagerAlertsWidget.tsx (new): presentational widget
  consuming the active_alerts summary shape (total/by_severity/alerts);
  exported from widgets/index.ts.
- integrations/registry.ts: alertmanager binding (active_alerts kind,
  30s refresh, optional severity_filter); registry.test.ts updated to
  6 service types incl alertmanager + a resolve test.
- components/ObservabilityPage.tsx: removed
  import.meta.env.VITE_GRAFANA_URL; derive GRAFANA_BASE_URL from the
  first enabled grafana service via useServiceInstances("grafana");
  added Grafana + Prometheus HealthCards (up/not-configured/unreachable)
  with QueryError retry blocks; machine dashboard shows a "No Grafana
  service configured" empty-state linking to /services when none is set.

npm run build (tsc -b + vite) clean; 0 lint errors; 72 frontend tests
pass. Reviewed fresh-context (read-only): no blockers.
2026-06-24 08:23:23 +00:00

258 lines
9.2 KiB
Python

"""Monitoring router — observability service status.
Observability components (Alertmanager, Grafana, Prometheus) are resolved from
the service registry, not environment variables. The endpoints pick the first
enabled instance of a type when no ``service_id`` is given, and return graceful
"not configured" / "unreachable" payloads so the UI always renders a health card.
"""
from __future__ import annotations
import logging
from typing import Any
import requests
from fastapi import APIRouter, Body, Depends
from media_library_viewer_api.dependencies import get_settings_store
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import build_node_exporter_targets
from media_library_viewer_api.widgets.sources import ServiceRecord, build_service_record
logger = logging.getLogger(__name__)
def _resolve_service_record(
store: SettingsStore, service_type: str, service_id: str | None = None
) -> ServiceRecord | None:
"""Return the requested service instance, else the first enabled one.
Returns ``None`` when the instance does not exist / is the wrong type, or
when no enabled instance of ``service_type`` is configured.
"""
if service_id:
row = store.get_service(service_id)
if not row or row.get("service_type") != service_type:
return None
if not row.get("enabled", True):
return None
return build_service_record(store, row)
for row in store.list_services(service_type):
if row.get("enabled", True):
return build_service_record(store, row)
return None
def _base_url(service: ServiceRecord) -> str:
return str(service.config.get("base_url") or "").rstrip("/")
def _timeout(service: ServiceRecord, default: int) -> int:
return int(service.config.get("timeout_seconds") or default)
def _auth_headers(service: ServiceRecord) -> dict[str, str]:
api_key = str(service.secrets.get("api_key") or "")
return {"Authorization": f"Bearer {api_key}"} if api_key else {}
def _status_response(service: ServiceRecord | None, *, version: str = "", error: str | None = None) -> dict[str, Any]:
return {
"up": error is None,
"version": version or "",
"service_id": service.id if service else "",
"name": service.name if service else "",
"error": error,
}
def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
"""Build a UI-friendly summary from Alertmanager /api/v1/alerts payload."""
from media_library_viewer_api.integrations.alertmanager import summarize_alerts
return summarize_alerts(alerts)
router = APIRouter(prefix="/api/monitoring", tags=["monitoring"])
@router.get("/machines")
def get_machines(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
"""Return enabled monitoring machines for the UI."""
return [m for m in store.list_machines() if m.get("enabled")]
@router.get("/prometheus-targets")
def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
"""Return Prometheus scrape targets for remote Node Exporters.
External Prometheus instances consume this list via ``http_sd_configs``.
"""
targets = build_node_exporter_targets(store)
logger.info("Prometheus targets requested count=%s", len(targets))
return targets
@router.get("/alerts")
def get_alertmanager_alerts(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Return a summary of active Alertmanager alerts for the UI.
Resolves an ``alertmanager`` service instance from the registry. When none
is configured the endpoint returns an empty summary with an
``alertmanager_not_configured`` error so the UI can render a health card.
"""
service = _resolve_service_record(store, "alertmanager", service_id)
if service is None:
return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_not_configured"}
try:
response = requests.get(
f"{_base_url(service)}/api/v1/alerts",
headers=_auth_headers(service),
timeout=_timeout(service, 5),
)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager alerts")
return {
"total": 0,
"by_severity": {},
"alerts": [],
"error": "alertmanager_unreachable",
"service_id": service.id,
"name": service.name,
}
if data.get("status") != "success":
return {
"total": 0,
"by_severity": {},
"alerts": [],
"error": data.get("error", "unknown"),
"service_id": service.id,
"name": service.name,
}
summary = _summary_from_alerts(data.get("data", []))
summary["service_id"] = service.id
summary["name"] = service.name
logger.info("Alertmanager alerts requested total=%s", summary["total"])
return summary
@router.get("/alertmanager-status")
def get_alertmanager_status(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Return Alertmanager cluster/status for the UI health card."""
service = _resolve_service_record(store, "alertmanager", service_id)
if service is None:
return {
"up": False,
"version": "",
"uptime": "",
"name": "",
"peers": [],
"error": "alertmanager_not_configured",
}
try:
response = requests.get(
f"{_base_url(service)}/api/v2/status",
headers=_auth_headers(service),
timeout=_timeout(service, 5),
)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager status")
return {
"up": False,
"version": "",
"uptime": "",
"name": service.name,
"peers": [],
"service_id": service.id,
"error": "alertmanager_unreachable",
}
cluster = data.get("cluster") or {}
status = data.get("clusterStatus") or {}
return {
"up": True,
"version": data.get("versionInfo", {}).get("version", ""),
"uptime": status.get("createdAt", ""),
"name": service.name,
"peers": [p.get("name", "") for p in cluster.get("peers", [])],
"service_id": service.id,
"error": None,
}
@router.get("/grafana-status")
def get_grafana_status(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Probe a Grafana service instance's ``/api/health`` endpoint."""
service = _resolve_service_record(store, "grafana", service_id)
if service is None:
return _status_response(None, error="no_service_configured")
try:
response = requests.get(
f"{_base_url(service)}/api/health",
headers=_auth_headers(service),
timeout=_timeout(service, 5),
)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Grafana status")
return _status_response(service, error="grafana_unreachable")
return _status_response(service, version=data.get("version", ""))
@router.get("/prometheus-status")
def get_prometheus_status(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Probe a Prometheus service instance's health and build info."""
service = _resolve_service_record(store, "prometheus", service_id)
if service is None:
return _status_response(None, error="no_service_configured")
base = _base_url(service)
timeout = _timeout(service, 10)
headers = _auth_headers(service)
try:
health = requests.get(f"{base}/-/healthy", headers=headers, timeout=timeout)
health.raise_for_status()
build_info = requests.get(f"{base}/api/v1/status/buildinfo", headers=headers, timeout=timeout)
build_info.raise_for_status()
version = build_info.json().get("data", {}).get("version", "")
except Exception:
logger.exception("Failed to fetch Prometheus status")
return _status_response(service, error="prometheus_unreachable")
return _status_response(service, version=version)
@router.post("/alertmanager-webhook")
def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
"""Receive alerts from Alertmanager and log them for audit/debug.
This endpoint is the receiver referenced by the optional ``webhook_configs``
block in Alertmanager. It is log-only: received payloads are recorded but not
forwarded anywhere. (The previous outbound relay to ``ALERTMANAGER_WEBHOOK_URL``
was removed when observability became service-registry configured.)
"""
alerts = payload.get("alerts", [])
logger.info(
"Received Alertmanager webhook with %s alert(s), status=%s",
len(alerts),
payload.get("status", "unknown"),
)
return {"status": "received"}