alex/manage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(observability): resolve services from registry, add health endpoints

Browse Source 
Slice 2 of observability-service-registry. The monitoring router resolves
observability components from the service registry instead of env vars.

- routers/monitoring.py: removed _alertmanager_client/_webhook_client env
  readers + the get_settings import. Added _resolve_service_record(store,
  service_type, service_id?) -> ServiceRecord|None (requested instance with
  type+enabled checks, else first enabled instance), plus _base_url/_timeout/
  _auth_headers (Bearer from api_key)/_status_response helpers.
- /alerts + /alertmanager-status now take service_id? + Depends(store),
  resolve an alertmanager service, return graceful not-configured/
  unreachable payloads including service_id/name; status down-branches now
  include peers:[] + error (fixes prior type drift).
- NEW /grafana-status (probes /api/health) and /prometheus-status (probes
  /-/healthy then /api/v1/status/buildinfo) returning
  {up,version,service_id,name,error}.
- Webhook receiver is now log-only (dropped the outbound
  ALERTMANAGER_WEBHOOK_URL forward).
- tests: rewrote TestAlertmanager + TestAlertmanagerWebhook to mock
  _resolve_service_record/requests.get (not-configured via empty registry);
  added TestGrafanaStatus/TestPrometheusStatus and a TestResolveServiceRecord
  unit class covering service_id match/type-mismatch/disabled and first-
  enabled/none-enabled paths.

Orphaned config fields alertmanager_url/alertmanager_webhook_url and the
env-var removal land in Slice 5. ruff clean; 240 backend tests pass.

Reviewed fresh-context (read-only): no blockers.
This commit is contained in:
Developer
2026-06-24 07:53:25 +00:00
parent 7d49df3e7d
commit 14771ae990
3 changed files with 408 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/media_library_viewer_api/routers/monitoring.py
+171 -60
View File
@@ -1,34 +1,71 @@
"""Monitoring router — observability stack status (Alertmanager + Prometheus)."""
"""Monitoring router — observability service status.
Observability components (Alertmanager, Grafana, Prometheus) are resolved from
the service registry, not environment variables. The endpoints pick the first
enabled instance of a type when no ``service_id`` is given, and return graceful
"not configured" / "unreachable" payloads so the UI always renders a health card.
"""
from __future__ import annotations
import logging
from typing import Any
import requests
from fastapi import APIRouter, Body, Depends
from media_library_viewer_api.config import get_settings
from media_library_viewer_api.dependencies import get_settings_store
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import build_node_exporter_targets
from media_library_viewer_api.widgets.sources import ServiceRecord, build_service_record
logger = logging.getLogger(__name__)
def _alertmanager_client() -> Any:
"""Return a simple HTTP client for the configured Alertmanager URL."""
import requests
def _resolve_service_record(
store: SettingsStore, service_type: str, service_id: str | None = None
) -> ServiceRecord | None:
"""Return the requested service instance, else the first enabled one.
settings = get_settings()
return requests.Session(), settings.alertmanager_url
Returns ``None`` when the instance does not exist / is the wrong type, or
when no enabled instance of ``service_type`` is configured.
"""
if service_id:
row = store.get_service(service_id)
if not row or row.get("service_type") != service_type:
return None
if not row.get("enabled", True):
return None
return build_service_record(store, row)
for row in store.list_services(service_type):
if row.get("enabled", True):
return build_service_record(store, row)
return None
def _webhook_client() -> Any:
"""Return a simple HTTP client for the optional webhook receiver URL."""
import requests
def _base_url(service: ServiceRecord) -> str:
return str(service.config.get("base_url") or "").rstrip("/")
settings = get_settings()
return requests.Session(), settings.alertmanager_webhook_url
def _timeout(service: ServiceRecord, default: int) -> int:
return int(service.config.get("timeout_seconds") or default)
def _auth_headers(service: ServiceRecord) -> dict[str, str]:
api_key = str(service.secrets.get("api_key") or "")
return {"Authorization": f"Bearer {api_key}"} if api_key else {}
def _status_response(
service: ServiceRecord | None, *, version: str = "", error: str | None = None
) -> dict[str, Any]:
return {
"up": error is None,
"version": version or "",
"service_id": service.id if service else "",
"name": service.name if service else "",
"error": error,
}
def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
@@ -49,11 +86,9 @@ def get_machines(store: SettingsStore = Depends(get_settings_store)) -> list[dic
@router.get("/prometheus-targets")
def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
"""Return Prometheus file-SD targets for remote Node Exporters.
"""Return Prometheus scrape targets for remote Node Exporters.
The backend writes these targets to a JSON file that Prometheus reads via
file_sd_configs. This endpoint returns the same list live from the store so
the UI can preview which machines will be scraped.
External Prometheus instances consume this list via ``http_sd_configs``.
"""
targets = build_node_exporter_targets(store)
logger.info("Prometheus targets requested count=%s", len(targets))
@@ -61,52 +96,90 @@ def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -
@router.get("/alerts")
def get_alertmanager_alerts() -> dict[str, Any]:
def get_alertmanager_alerts(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Return a summary of active Alertmanager alerts for the UI.
Proxies the Alertmanager `/api/v1/alerts` endpoint and reshapes the payload
into a stable, UI-friendly format. If Alertmanager is unreachable (or not
configured via ``ALERTMANAGER_URL``), the endpoint returns an empty summary
so the UI can still render a health card instead of an error page.
Resolves an ``alertmanager`` service instance from the registry. When none
is configured the endpoint returns an empty summary with an
``alertmanager_not_configured`` error so the UI can render a health card.
"""
session, base_url = _alertmanager_client()
if not base_url:
service = _resolve_service_record(store, "alertmanager", service_id)
if service is None:
return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_not_configured"}
try:
response = session.get(f"{base_url}/api/v1/alerts", timeout=5)
response = requests.get(
f"{_base_url(service)}/api/v1/alerts",
headers=_auth_headers(service),
timeout=_timeout(service, 5),
)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager alerts from %s", base_url)
return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_unreachable"}
logger.exception("Failed to fetch Alertmanager alerts")
return {
"total": 0,
"by_severity": {},
"alerts": [],
"error": "alertmanager_unreachable",
"service_id": service.id,
"name": service.name,
}
if data.get("status") != "success":
return {"total": 0, "by_severity": {}, "alerts": [], "error": data.get("error", "unknown")}
return {
"total": 0,
"by_severity": {},
"alerts": [],
"error": data.get("error", "unknown"),
"service_id": service.id,
"name": service.name,
}
summary = _summary_from_alerts(data.get("data", []))
summary["service_id"] = service.id
summary["name"] = service.name
logger.info("Alertmanager alerts requested total=%s", summary["total"])
return summary
@router.get("/alertmanager-status")
def get_alertmanager_status() -> dict[str, Any]:
"""Return Alertmanager cluster/status for the UI health card.
Uses the Alertmanager `/api/v2/status` endpoint and exposes only the high-
level fields the UI needs: uptime, version, and whether the cluster is
healthy. Returns ``up=False`` when Alertmanager is unreachable or not
configured via ``ALERTMANAGER_URL``.
"""
session, base_url = _alertmanager_client()
if not base_url:
return {"up": False, "version": "", "uptime": ""}
def get_alertmanager_status(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Return Alertmanager cluster/status for the UI health card."""
service = _resolve_service_record(store, "alertmanager", service_id)
if service is None:
return {
"up": False,
"version": "",
"uptime": "",
"name": "",
"peers": [],
"error": "alertmanager_not_configured",
}
try:
response = session.get(f"{base_url}/api/v2/status", timeout=5)
response = requests.get(
f"{_base_url(service)}/api/v2/status",
headers=_auth_headers(service),
timeout=_timeout(service, 5),
)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Alertmanager status from %s", base_url)
return {"up": False, "version": "", "uptime": ""}
logger.exception("Failed to fetch Alertmanager status")
return {
"up": False,
"version": "",
"uptime": "",
"name": service.name,
"peers": [],
"service_id": service.id,
"error": "alertmanager_unreachable",
}
cluster = data.get("cluster") or {}
status = data.get("clusterStatus") or {}
@@ -114,20 +187,70 @@ def get_alertmanager_status() -> dict[str, Any]:
"up": True,
"version": data.get("versionInfo", {}).get("version", ""),
"uptime": status.get("createdAt", ""),
"name": "",
"name": service.name,
"peers": [p.get("name", "") for p in cluster.get("peers", [])],
"service_id": service.id,
"error": None,
}
@router.get("/grafana-status")
def get_grafana_status(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Probe a Grafana service instance's ``/api/health`` endpoint."""
service = _resolve_service_record(store, "grafana", service_id)
if service is None:
return _status_response(None, error="no_service_configured")
try:
response = requests.get(
f"{_base_url(service)}/api/health",
headers=_auth_headers(service),
timeout=_timeout(service, 5),
)
response.raise_for_status()
data = response.json()
except Exception:
logger.exception("Failed to fetch Grafana status")
return _status_response(service, error="grafana_unreachable")
return _status_response(service, version=data.get("version", ""))
@router.get("/prometheus-status")
def get_prometheus_status(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Probe a Prometheus service instance's health and build info."""
service = _resolve_service_record(store, "prometheus", service_id)
if service is None:
return _status_response(None, error="no_service_configured")
base = _base_url(service)
timeout = _timeout(service, 10)
headers = _auth_headers(service)
try:
health = requests.get(f"{base}/-/healthy", headers=headers, timeout=timeout)
health.raise_for_status()
build_info = requests.get(
f"{base}/api/v1/status/buildinfo", headers=headers, timeout=timeout
)
build_info.raise_for_status()
version = build_info.json().get("data", {}).get("version", "")
except Exception:
logger.exception("Failed to fetch Prometheus status")
return _status_response(service, error="prometheus_unreachable")
return _status_response(service, version=version)
@router.post("/alertmanager-webhook")
def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
"""Receive alerts from Alertmanager and optionally forward to a webhook URL.
"""Receive alerts from Alertmanager and log them for audit/debug.
This endpoint is the receiver referenced by the optional `webhook_configs`
block in Alertmanager. It logs the payload for audit/debug purposes and, if
`ALERTMANAGER_WEBHOOK_URL` is configured, forwards the alert JSON verbatim.
Forwarding is best-effort: a failure to reach the downstream webhook does
not fail this endpoint, so Alertmanager sees a successful delivery.
This endpoint is the receiver referenced by the optional ``webhook_configs``
block in Alertmanager. It is log-only: received payloads are recorded but not
forwarded anywhere. (The previous outbound relay to ``ALERTMANAGER_WEBHOOK_URL``
was removed when observability became service-registry configured.)
"""
alerts = payload.get("alerts", [])
logger.info(
@@ -135,16 +258,4 @@ def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[st
len(alerts),
payload.get("status", "unknown"),
)
session, webhook_url = _webhook_client()
if webhook_url:
try:
response = session.post(webhook_url, json=payload, timeout=10)
response.raise_for_status()
logger.info("Forwarded Alertmanager webhook to %s", webhook_url)
except Exception:
logger.exception("Failed to forward Alertmanager webhook to %s", webhook_url)
else:
logger.debug("No ALERTMANAGER_WEBHOOK_URL configured; webhook stored in logs only")
return {"status": "received"}