Developer
231 lines
8.2 KiB
Python
231 lines
8.2 KiB
Python
"""Monitoring router — observability service status.
|
|
|
|
Observability components (Alertmanager, Prometheus) are resolved from
|
|
the service registry, not environment variables. The endpoints pick the first
|
|
enabled instance of a type when no ``service_id`` is given, and return graceful
|
|
"not configured" / "unreachable" payloads so the UI always renders a health card.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any
|
|
|
|
import requests
|
|
from fastapi import APIRouter, Body, Depends
|
|
|
|
from media_library_viewer_api.clients.http_timeout import http_timeout
|
|
from media_library_viewer_api.dependencies import get_settings_store
|
|
from media_library_viewer_api.services.service_resolution import resolve_service_record
|
|
from media_library_viewer_api.services.settings_store import SettingsStore
|
|
from media_library_viewer_api.widgets.sources import ServiceRecord
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _base_url(service: ServiceRecord) -> str:
|
|
return str(service.config.get("base_url") or "").rstrip("/")
|
|
|
|
|
|
def _timeout(service: ServiceRecord, default: int) -> tuple[float, float]:
|
|
"""Return a (connect, read) timeout tuple from the service config."""
|
|
read = int(service.config.get("timeout_seconds") or default)
|
|
return http_timeout(read)
|
|
|
|
|
|
def _auth_headers(service: ServiceRecord) -> dict[str, str]:
|
|
api_key = str(service.secrets.get("api_key") or "")
|
|
return {"Authorization": f"Bearer {api_key}"} if api_key else {}
|
|
|
|
|
|
def _status_response(service: ServiceRecord | None, *, version: str = "", error: str | None = None) -> dict[str, Any]:
|
|
return {
|
|
"up": error is None,
|
|
"version": version or "",
|
|
"service_id": service.id if service else "",
|
|
"name": service.name if service else "",
|
|
"error": error,
|
|
}
|
|
|
|
|
|
def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
|
|
"""Build a UI-friendly summary from Alertmanager /api/v1/alerts payload."""
|
|
from media_library_viewer_api.integrations.alertmanager import summarize_alerts
|
|
|
|
return summarize_alerts(alerts)
|
|
|
|
|
|
router = APIRouter(prefix="/api/monitoring", tags=["monitoring"])
|
|
|
|
|
|
|
|
|
|
@router.get("/alerts")
|
|
def get_alertmanager_alerts(
|
|
service_id: str | None = None,
|
|
store: SettingsStore = Depends(get_settings_store),
|
|
) -> dict[str, Any]:
|
|
"""Return a summary of active Alertmanager alerts for the UI.
|
|
|
|
Resolves an ``alertmanager`` service instance from the registry. When none
|
|
is configured the endpoint returns an empty summary with an
|
|
``alertmanager_not_configured`` error so the UI can render a health card.
|
|
"""
|
|
service = resolve_service_record(store, "alertmanager", service_id)
|
|
if service is None:
|
|
return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_not_configured"}
|
|
try:
|
|
response = requests.get(
|
|
f"{_base_url(service)}/api/v1/alerts",
|
|
headers=_auth_headers(service),
|
|
timeout=_timeout(service, 5),
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
except Exception:
|
|
logger.exception("Failed to fetch Alertmanager alerts")
|
|
return {
|
|
"total": 0,
|
|
"by_severity": {},
|
|
"alerts": [],
|
|
"error": "alertmanager_unreachable",
|
|
"service_id": service.id,
|
|
"name": service.name,
|
|
}
|
|
|
|
if data.get("status") != "success":
|
|
return {
|
|
"total": 0,
|
|
"by_severity": {},
|
|
"alerts": [],
|
|
"error": data.get("error", "unknown"),
|
|
"service_id": service.id,
|
|
"name": service.name,
|
|
}
|
|
|
|
summary = _summary_from_alerts(data.get("data", []))
|
|
summary["service_id"] = service.id
|
|
summary["name"] = service.name
|
|
logger.info("Alertmanager alerts requested total=%s", summary["total"])
|
|
return summary
|
|
|
|
|
|
@router.get("/alertmanager-status")
|
|
def get_alertmanager_status(
|
|
service_id: str | None = None,
|
|
store: SettingsStore = Depends(get_settings_store),
|
|
) -> dict[str, Any]:
|
|
"""Return Alertmanager cluster/status for the UI health card."""
|
|
service = resolve_service_record(store, "alertmanager", service_id)
|
|
if service is None:
|
|
return {
|
|
"up": False,
|
|
"version": "",
|
|
"uptime": "",
|
|
"name": "",
|
|
"peers": [],
|
|
"error": "alertmanager_not_configured",
|
|
}
|
|
try:
|
|
response = requests.get(
|
|
f"{_base_url(service)}/api/v2/status",
|
|
headers=_auth_headers(service),
|
|
timeout=_timeout(service, 5),
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
except Exception:
|
|
logger.exception("Failed to fetch Alertmanager status")
|
|
return {
|
|
"up": False,
|
|
"version": "",
|
|
"uptime": "",
|
|
"name": service.name,
|
|
"peers": [],
|
|
"service_id": service.id,
|
|
"error": "alertmanager_unreachable",
|
|
}
|
|
|
|
cluster = data.get("cluster") or {}
|
|
status = data.get("clusterStatus") or {}
|
|
return {
|
|
"up": True,
|
|
"version": data.get("versionInfo", {}).get("version", ""),
|
|
"uptime": status.get("createdAt", ""),
|
|
"name": service.name,
|
|
"peers": [p.get("name", "") for p in cluster.get("peers", [])],
|
|
"service_id": service.id,
|
|
"error": None,
|
|
}
|
|
|
|
|
|
@router.get("/prometheus-status")
|
|
def get_prometheus_status(
|
|
service_id: str | None = None,
|
|
store: SettingsStore = Depends(get_settings_store),
|
|
) -> dict[str, Any]:
|
|
"""Probe a Prometheus service's health via the Grafana gateway path (GM-110).
|
|
|
|
Issues a trivial ``up`` query through Grafana ``/api/ds/query``. Success
|
|
validates the full path: Grafana is reachable, the API key works, and the
|
|
Prometheus datasource responds.
|
|
"""
|
|
service = resolve_service_record(store, "prometheus", service_id)
|
|
if service is None:
|
|
return _status_response(None, error="no_service_configured")
|
|
grafana_url = str(service.config.get("grafana_url") or "").rstrip("/")
|
|
api_key = str(service.secrets.get("grafana_api_key") or "")
|
|
datasource_uid = str(service.config.get("datasource_uid") or "prometheus")
|
|
timeout = int(service.config.get("timeout_seconds") or 60)
|
|
if not grafana_url or not api_key:
|
|
return _status_response(service, error="gateway_not_configured")
|
|
body = {
|
|
"queries": [
|
|
{
|
|
"datasource": {"uid": datasource_uid, "type": "prometheus"},
|
|
"expr": "up",
|
|
"format": "time_series",
|
|
"intervalMs": 15_000,
|
|
"maxDataPoints": 1,
|
|
"refId": "A",
|
|
}
|
|
],
|
|
"from": "now-1m",
|
|
"to": "now",
|
|
}
|
|
try:
|
|
resp = requests.post(
|
|
f"{grafana_url}/api/ds/query",
|
|
json=body,
|
|
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
|
timeout=http_timeout(timeout),
|
|
)
|
|
resp.raise_for_status()
|
|
except requests.HTTPError as exc:
|
|
status_code = exc.response.status_code if exc.response else 0
|
|
if status_code in (401, 403):
|
|
return _status_response(service, error="auth_failed")
|
|
return _status_response(service, error="gateway_error")
|
|
except requests.RequestException:
|
|
logger.exception("Failed to fetch Prometheus status via gateway")
|
|
return _status_response(service, error="prometheus_unreachable")
|
|
return _status_response(service, version="ok")
|
|
|
|
|
|
@router.post("/alertmanager-webhook")
|
|
def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
|
|
"""Receive alerts from Alertmanager and log them for audit/debug.
|
|
|
|
This endpoint is the receiver referenced by the optional ``webhook_configs``
|
|
block in Alertmanager. It is log-only: received payloads are recorded but not
|
|
forwarded anywhere. (The previous outbound relay to ``ALERTMANAGER_WEBHOOK_URL``
|
|
was removed when observability became service-registry configured.)
|
|
"""
|
|
alerts = payload.get("alerts", [])
|
|
logger.info(
|
|
"Received Alertmanager webhook with %s alert(s), status=%s",
|
|
len(alerts),
|
|
payload.get("status", "unknown"),
|
|
)
|
|
return {"status": "received"}
|