feat(observability): resolve services from registry, add health endpoints

Slice 2 of observability-service-registry. The monitoring router resolves observability components from the service registry instead of env vars. - routers/monitoring.py: removed _alertmanager_client/_webhook_client env readers + the get_settings import. Added _resolve_service_record(store, service_type, service_id?) -> ServiceRecord|None (requested instance with type+enabled checks, else first enabled instance), plus _base_url/_timeout/ _auth_headers (Bearer from api_key)/_status_response helpers. - /alerts + /alertmanager-status now take service_id? + Depends(store), resolve an alertmanager service, return graceful not-configured/ unreachable payloads including service_id/name; status down-branches now include peers:[] + error (fixes prior type drift). - NEW /grafana-status (probes /api/health) and /prometheus-status (probes /-/healthy then /api/v1/status/buildinfo) returning {up,version,service_id,name,error}. - Webhook receiver is now log-only (dropped the outbound ALERTMANAGER_WEBHOOK_URL forward). - tests: rewrote TestAlertmanager + TestAlertmanagerWebhook to mock _resolve_service_record/requests.get (not-configured via empty registry); added TestGrafanaStatus/TestPrometheusStatus and a TestResolveServiceRecord unit class covering service_id match/type-mismatch/disabled and first- enabled/none-enabled paths. Orphaned config fields alertmanager_url/alertmanager_webhook_url and the env-var removal land in Slice 5. ruff clean; 240 backend tests pass. Reviewed fresh-context (read-only): no blockers.
2026-06-24 07:53:25 +00:00
parent 7d49df3e7d
commit 14771ae990
3 changed files with 408 additions and 123 deletions
@@ -1,34 +1,71 @@
-"""Monitoring router — observability stack status (Alertmanager + Prometheus)."""
+"""Monitoring router — observability service status.
+
+Observability components (Alertmanager, Grafana, Prometheus) are resolved from
+the service registry, not environment variables. The endpoints pick the first
+enabled instance of a type when no ``service_id`` is given, and return graceful
+"not configured" / "unreachable" payloads so the UI always renders a health card.
+"""

 from __future__ import annotations

 import logging
 from typing import Any

+import requests
 from fastapi import APIRouter, Body, Depends

-from media_library_viewer_api.config import get_settings
 from media_library_viewer_api.dependencies import get_settings_store
 from media_library_viewer_api.services.settings_store import SettingsStore
 from media_library_viewer_api.services.targets import build_node_exporter_targets
+from media_library_viewer_api.widgets.sources import ServiceRecord, build_service_record

 logger = logging.getLogger(__name__)


-def _alertmanager_client() -> Any:
-    """Return a simple HTTP client for the configured Alertmanager URL."""
-    import requests
+def _resolve_service_record(
+    store: SettingsStore, service_type: str, service_id: str | None = None
+) -> ServiceRecord | None:
+    """Return the requested service instance, else the first enabled one.

-    settings = get_settings()
-    return requests.Session(), settings.alertmanager_url
+    Returns ``None`` when the instance does not exist / is the wrong type, or
+    when no enabled instance of ``service_type`` is configured.
+    """
+    if service_id:
+        row = store.get_service(service_id)
+        if not row or row.get("service_type") != service_type:
+            return None
+        if not row.get("enabled", True):
+            return None
+        return build_service_record(store, row)
+    for row in store.list_services(service_type):
+        if row.get("enabled", True):
+            return build_service_record(store, row)
+    return None


-def _webhook_client() -> Any:
-    """Return a simple HTTP client for the optional webhook receiver URL."""
-    import requests
+def _base_url(service: ServiceRecord) -> str:
+    return str(service.config.get("base_url") or "").rstrip("/")

-    settings = get_settings()
-    return requests.Session(), settings.alertmanager_webhook_url
+
+def _timeout(service: ServiceRecord, default: int) -> int:
+    return int(service.config.get("timeout_seconds") or default)
+
+
+def _auth_headers(service: ServiceRecord) -> dict[str, str]:
+    api_key = str(service.secrets.get("api_key") or "")
+    return {"Authorization": f"Bearer {api_key}"} if api_key else {}
+
+
+def _status_response(
+    service: ServiceRecord | None, *, version: str = "", error: str | None = None
+) -> dict[str, Any]:
+    return {
+        "up": error is None,
+        "version": version or "",
+        "service_id": service.id if service else "",
+        "name": service.name if service else "",
+        "error": error,
+    }


 def _summary_from_alerts(alerts: list[dict[str, Any]]) -> dict[str, Any]:
@@ -49,11 +86,9 @@ def get_machines(store: SettingsStore = Depends(get_settings_store)) -> list[dic

@router.get("/prometheus-targets")
 def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -> list[dict[str, Any]]:
-    """Return Prometheus file-SD targets for remote Node Exporters.
+    """Return Prometheus scrape targets for remote Node Exporters.

-    The backend writes these targets to a JSON file that Prometheus reads via
-    file_sd_configs. This endpoint returns the same list live from the store so
-    the UI can preview which machines will be scraped.
+    External Prometheus instances consume this list via ``http_sd_configs``.
    """
    targets = build_node_exporter_targets(store)
    logger.info("Prometheus targets requested count=%s", len(targets))
@@ -61,52 +96,90 @@ def get_prometheus_targets(store: SettingsStore = Depends(get_settings_store)) -


@router.get("/alerts")
-def get_alertmanager_alerts() -> dict[str, Any]:
+def get_alertmanager_alerts(
+    service_id: str | None = None,
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, Any]:
    """Return a summary of active Alertmanager alerts for the UI.

-    Proxies the Alertmanager `/api/v1/alerts` endpoint and reshapes the payload
-    into a stable, UI-friendly format. If Alertmanager is unreachable (or not
-    configured via ``ALERTMANAGER_URL``), the endpoint returns an empty summary
-    so the UI can still render a health card instead of an error page.
+    Resolves an ``alertmanager`` service instance from the registry. When none
+    is configured the endpoint returns an empty summary with an
+    ``alertmanager_not_configured`` error so the UI can render a health card.
    """
-    session, base_url = _alertmanager_client()
-    if not base_url:
+    service = _resolve_service_record(store, "alertmanager", service_id)
+    if service is None:
        return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_not_configured"}
    try:
-        response = session.get(f"{base_url}/api/v1/alerts", timeout=5)
+        response = requests.get(
+            f"{_base_url(service)}/api/v1/alerts",
+            headers=_auth_headers(service),
+            timeout=_timeout(service, 5),
+        )
        response.raise_for_status()
        data = response.json()
    except Exception:
-        logger.exception("Failed to fetch Alertmanager alerts from %s", base_url)
-        return {"total": 0, "by_severity": {}, "alerts": [], "error": "alertmanager_unreachable"}
+        logger.exception("Failed to fetch Alertmanager alerts")
+        return {
+            "total": 0,
+            "by_severity": {},
+            "alerts": [],
+            "error": "alertmanager_unreachable",
+            "service_id": service.id,
+            "name": service.name,
+        }

    if data.get("status") != "success":
-        return {"total": 0, "by_severity": {}, "alerts": [], "error": data.get("error", "unknown")}
+        return {
+            "total": 0,
+            "by_severity": {},
+            "alerts": [],
+            "error": data.get("error", "unknown"),
+            "service_id": service.id,
+            "name": service.name,
+        }

    summary = _summary_from_alerts(data.get("data", []))
+    summary["service_id"] = service.id
+    summary["name"] = service.name
    logger.info("Alertmanager alerts requested total=%s", summary["total"])
    return summary


@router.get("/alertmanager-status")
-def get_alertmanager_status() -> dict[str, Any]:
-    """Return Alertmanager cluster/status for the UI health card.
-
-    Uses the Alertmanager `/api/v2/status` endpoint and exposes only the high-
-    level fields the UI needs: uptime, version, and whether the cluster is
-    healthy. Returns ``up=False`` when Alertmanager is unreachable or not
-    configured via ``ALERTMANAGER_URL``.
-    """
-    session, base_url = _alertmanager_client()
-    if not base_url:
-        return {"up": False, "version": "", "uptime": ""}
+def get_alertmanager_status(
+    service_id: str | None = None,
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, Any]:
+    """Return Alertmanager cluster/status for the UI health card."""
+    service = _resolve_service_record(store, "alertmanager", service_id)
+    if service is None:
+        return {
+            "up": False,
+            "version": "",
+            "uptime": "",
+            "name": "",
+            "peers": [],
+            "error": "alertmanager_not_configured",
+        }
    try:
-        response = session.get(f"{base_url}/api/v2/status", timeout=5)
+        response = requests.get(
+            f"{_base_url(service)}/api/v2/status",
+            headers=_auth_headers(service),
+            timeout=_timeout(service, 5),
+        )
        response.raise_for_status()
        data = response.json()
    except Exception:
-        logger.exception("Failed to fetch Alertmanager status from %s", base_url)
-        return {"up": False, "version": "", "uptime": ""}
+        logger.exception("Failed to fetch Alertmanager status")
+        return {
+            "up": False,
+            "version": "",
+            "uptime": "",
+            "name": service.name,
+            "peers": [],
+            "service_id": service.id,
+            "error": "alertmanager_unreachable",
+        }

    cluster = data.get("cluster") or {}
    status = data.get("clusterStatus") or {}
@@ -114,20 +187,70 @@ def get_alertmanager_status() -> dict[str, Any]:
        "up": True,
        "version": data.get("versionInfo", {}).get("version", ""),
        "uptime": status.get("createdAt", ""),
-        "name": "",
+        "name": service.name,
        "peers": [p.get("name", "") for p in cluster.get("peers", [])],
+        "service_id": service.id,
+        "error": None,
    }


+@router.get("/grafana-status")
+def get_grafana_status(
+    service_id: str | None = None,
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, Any]:
+    """Probe a Grafana service instance's ``/api/health`` endpoint."""
+    service = _resolve_service_record(store, "grafana", service_id)
+    if service is None:
+        return _status_response(None, error="no_service_configured")
+    try:
+        response = requests.get(
+            f"{_base_url(service)}/api/health",
+            headers=_auth_headers(service),
+            timeout=_timeout(service, 5),
+        )
+        response.raise_for_status()
+        data = response.json()
+    except Exception:
+        logger.exception("Failed to fetch Grafana status")
+        return _status_response(service, error="grafana_unreachable")
+    return _status_response(service, version=data.get("version", ""))
+
+
+@router.get("/prometheus-status")
+def get_prometheus_status(
+    service_id: str | None = None,
+    store: SettingsStore = Depends(get_settings_store),
+) -> dict[str, Any]:
+    """Probe a Prometheus service instance's health and build info."""
+    service = _resolve_service_record(store, "prometheus", service_id)
+    if service is None:
+        return _status_response(None, error="no_service_configured")
+    base = _base_url(service)
+    timeout = _timeout(service, 10)
+    headers = _auth_headers(service)
+    try:
+        health = requests.get(f"{base}/-/healthy", headers=headers, timeout=timeout)
+        health.raise_for_status()
+        build_info = requests.get(
+            f"{base}/api/v1/status/buildinfo", headers=headers, timeout=timeout
+        )
+        build_info.raise_for_status()
+        version = build_info.json().get("data", {}).get("version", "")
+    except Exception:
+        logger.exception("Failed to fetch Prometheus status")
+        return _status_response(service, error="prometheus_unreachable")
+    return _status_response(service, version=version)
+
+
@router.post("/alertmanager-webhook")
 def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[str, str]:
-    """Receive alerts from Alertmanager and optionally forward to a webhook URL.
+    """Receive alerts from Alertmanager and log them for audit/debug.

-    This endpoint is the receiver referenced by the optional `webhook_configs`
-    block in Alertmanager. It logs the payload for audit/debug purposes and, if
-    `ALERTMANAGER_WEBHOOK_URL` is configured, forwards the alert JSON verbatim.
-    Forwarding is best-effort: a failure to reach the downstream webhook does
-    not fail this endpoint, so Alertmanager sees a successful delivery.
+    This endpoint is the receiver referenced by the optional ``webhook_configs``
+    block in Alertmanager. It is log-only: received payloads are recorded but not
+    forwarded anywhere. (The previous outbound relay to ``ALERTMANAGER_WEBHOOK_URL``
+    was removed when observability became service-registry configured.)
    """
    alerts = payload.get("alerts", [])
    logger.info(
@@ -135,16 +258,4 @@ def receive_alertmanager_webhook(payload: dict[str, Any] = Body(...)) -> dict[st
        len(alerts),
        payload.get("status", "unknown"),
    )
-
-    session, webhook_url = _webhook_client()
-    if webhook_url:
-        try:
-            response = session.post(webhook_url, json=payload, timeout=10)
-            response.raise_for_status()
-            logger.info("Forwarded Alertmanager webhook to %s", webhook_url)
-        except Exception:
-            logger.exception("Failed to forward Alertmanager webhook to %s", webhook_url)
-    else:
-        logger.debug("No ALERTMANAGER_WEBHOOK_URL configured; webhook stored in logs only")
-
    return {"status": "received"}
@@ -24,6 +24,10 @@ from media_library_viewer_api.main import app
 from media_library_viewer_api.routers.media import get_media_index
 from media_library_viewer_api.services.media_index import MediaIndex
 from media_library_viewer_api.services.settings_store import SettingsStore
+from media_library_viewer_api.widgets.sources import ServiceRecord
+
+# Short alias for the monitoring router module under test.
+_MON = "media_library_viewer_api.routers.monitoring"

 # --- Fixtures ---

@@ -626,6 +630,76 @@ class TestMonitoring:
        assert data[0]["labels"]["job"] == "node-exporter-remote"


+class TestResolveServiceRecord:
+    """Unit tests for _resolve_service_record (service_id + first-enabled paths)."""
+
+    def _store(self, rows):
+        store = MagicMock()
+        store.get_service = lambda sid: next((r for r in rows if r["id"] == sid), None)
+        def list_filtered(service_type=None):
+            return [r for r in rows if r["service_type"] == (service_type or r["service_type"])]
+
+        store.list_services = list_filtered
+        return store
+
+    def test_service_id_match_returns_record(self):
+        from media_library_viewer_api.routers.monitoring import _resolve_service_record
+
+        row = {"id": "am1", "service_type": "alertmanager", "name": "AM", "enabled": True, "config": {}, "secrets": {}}
+        store = self._store([row])
+        with patch(f"{_MON}.build_service_record", return_value="RECORD") as mock_build:
+            result = _resolve_service_record(store, "alertmanager", "am1")
+        assert result == "RECORD"
+        mock_build.assert_called_once_with(store, row)
+
+    def test_service_id_type_mismatch_returns_none(self):
+        from media_library_viewer_api.routers.monitoring import _resolve_service_record
+
+        row = {"id": "x1", "service_type": "grafana", "name": "G", "enabled": True, "config": {}, "secrets": {}}
+        store = self._store([row])
+        assert _resolve_service_record(store, "alertmanager", "x1") is None
+
+    def test_service_id_disabled_returns_none(self):
+        from media_library_viewer_api.routers.monitoring import _resolve_service_record
+
+        row = {"id": "am1", "service_type": "alertmanager", "name": "AM", "enabled": False, "config": {}, "secrets": {}}
+        store = self._store([row])
+        assert _resolve_service_record(store, "alertmanager", "am1") is None
+
+    def test_no_service_id_returns_first_enabled(self):
+        from media_library_viewer_api.routers.monitoring import _resolve_service_record
+
+        rows = [
+            {
+                "id": "am1",
+                "service_type": "alertmanager",
+                "name": "Disabled",
+                "enabled": False,
+                "config": {},
+                "secrets": {},
+            },
+            {
+                "id": "am2",
+                "service_type": "alertmanager",
+                "name": "Active",
+                "enabled": True,
+                "config": {},
+                "secrets": {},
+            },
+        ]
+        store = self._store(rows)
+        with patch(f"{_MON}.build_service_record", return_value="RECORD") as mock_build:
+            result = _resolve_service_record(store, "alertmanager", None)
+        assert result == "RECORD"
+        mock_build.assert_called_once_with(store, rows[1])
+
+    def test_no_service_id_and_none_enabled_returns_none(self):
+        from media_library_viewer_api.routers.monitoring import _resolve_service_record
+
+        store = self._store([])
+        assert _resolve_service_record(store, "alertmanager", None) is None
+
+
 class TestSettingsMachines:
    def test_post_machine_rewrites_prometheus_targets(self, test_client):
        with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets:
@@ -665,75 +739,120 @@ class TestSettingsMachines:
        write_targets.assert_called_once()


+def _am_service(name="Alertmanager", **config):
+    cfg = {"base_url": "http://alertmanager:9093", "timeout_seconds": 5}
+    cfg.update(config)
+    return ServiceRecord(id="am1", service_type="alertmanager", name=name, config=cfg)
+
+
 class TestAlertmanager:
-    def test_alerts_endpoint_when_alertmanager_unreachable(self, test_client):
-        with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
-            session = MagicMock()
-            session.get.side_effect = Exception("connection refused")
-            mock_client.return_value = (session, "http://alertmanager:9093")
+    def test_alerts_endpoint_when_not_configured(self, test_client):
+        # No alertmanager service seeded -> registry resolves None.
+        response = test_client.get("/api/monitoring/alerts")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["total"] == 0
+        assert data["error"] == "alertmanager_not_configured"
+
+    def test_alerts_endpoint_when_unreachable(self, test_client):
+        service = _am_service()
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", side_effect=Exception("connection refused")),
+        ):
            response = test_client.get("/api/monitoring/alerts")
        assert response.status_code == 200
        data = response.json()
        assert data["total"] == 0
        assert data["error"] == "alertmanager_unreachable"
-
-    def test_alerts_endpoint_when_not_configured(self, test_client):
-        with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
-            session = MagicMock()
-            mock_client.return_value = (session, "")
-            response = test_client.get("/api/monitoring/alerts")
-        assert response.status_code == 200
-        data = response.json()
-        assert data["total"] == 0
-        assert data["error"] == "alertmanager_not_configured"
-        session.get.assert_not_called()
+        assert data["service_id"] == "am1"
+        assert data["name"] == "Alertmanager"

    def test_alerts_endpoint_returns_summary(self, test_client):
-        with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
-            session = MagicMock()
-            session.get.return_value.json.return_value = {
-                "status": "success",
-                "data": [
-                    {
-                        "labels": {"alertname": "BackupJobFailed", "severity": "critical", "job_name": "test"},
-                        "annotations": {"summary": "Backup failed", "description": "details"},
-                        "startsAt": "2026-05-11T02:00:00Z",
-                        "status": "firing",
-                    }
-                ],
-            }
-            session.get.return_value.raise_for_status = MagicMock()
-            mock_client.return_value = (session, "http://alertmanager:9093")
+        service = _am_service()
+        resp = MagicMock()
+        resp.json.return_value = {
+            "status": "success",
+            "data": [
+                {
+                    "labels": {"alertname": "BackupJobFailed", "severity": "critical", "job_name": "test"},
+                    "annotations": {"summary": "Backup failed", "description": "details"},
+                    "startsAt": "2026-05-11T02:00:00Z",
+                    "status": "firing",
+                }
+            ],
+        }
+        resp.raise_for_status = MagicMock()
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", return_value=resp),
+        ):
            response = test_client.get("/api/monitoring/alerts")
        assert response.status_code == 200
        data = response.json()
        assert data["total"] == 1
        assert data["by_severity"]["critical"] == 1
        assert data["alerts"][0]["name"] == "BackupJobFailed"
+        assert data["service_id"] == "am1"

-    def test_alertmanager_status_endpoint_when_not_configured(self, test_client):
-        with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
-            session = MagicMock()
-            mock_client.return_value = (session, "")
+    def test_alerts_endpoint_sends_bearer_token(self, test_client):
+        service = _am_service()
+        service.secrets = {"api_key": "tok"}
+        resp = MagicMock()
+        resp.json.return_value = {"status": "success", "data": []}
+        resp.raise_for_status = MagicMock()
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", return_value=resp) as mock_get,
+        ):
+            test_client.get("/api/monitoring/alerts")
+        _, kwargs = mock_get.call_args
+        assert kwargs["headers"]["Authorization"] == "Bearer tok"
+
+    def test_alertmanager_status_when_not_configured(self, test_client):
+        response = test_client.get("/api/monitoring/alertmanager-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["up"] is False
+        assert data["error"] == "alertmanager_not_configured"
+        assert data["peers"] == []
+
+    def test_alertmanager_status_when_unreachable(self, test_client):
+        service = _am_service()
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", side_effect=Exception("refused")),
+        ):
            response = test_client.get("/api/monitoring/alertmanager-status")
        assert response.status_code == 200
        data = response.json()
        assert data["up"] is False
-        session.get.assert_not_called()
+        assert data["error"] == "alertmanager_unreachable"
+        assert data["name"] == "Alertmanager"

-    def test_alertmanager_status_endpoint_when_unreachable(self, test_client):
-        with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
-            session = MagicMock()
-            session.get.side_effect = Exception("connection refused")
-            mock_client.return_value = (session, "http://alertmanager:9093")
+    def test_alertmanager_status_returns_cluster_info(self, test_client):
+        service = _am_service()
+        resp = MagicMock()
+        resp.json.return_value = {
+            "versionInfo": {"version": "0.27.0"},
+            "clusterStatus": {"createdAt": "2026-06-01T00:00:00Z"},
+            "cluster": {"peers": [{"name": "am-1"}, {"name": "am-2"}]},
+        }
+        resp.raise_for_status = MagicMock()
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", return_value=resp),
+        ):
            response = test_client.get("/api/monitoring/alertmanager-status")
        assert response.status_code == 200
        data = response.json()
-        assert data["up"] is False
+        assert data["up"] is True
+        assert data["version"] == "0.27.0"
+        assert data["peers"] == ["am-1", "am-2"]


 class TestAlertmanagerWebhook:
-    """Tests for POST /api/monitoring/alertmanager-webhook."""
+    """Tests for POST /api/monitoring/alertmanager-webhook (log-only receiver)."""

    def test_webhook_receives_payload_and_logs(self, test_client, caplog):
        payload = {
@@ -753,22 +872,79 @@ class TestAlertmanagerWebhook:
        assert response.json()["status"] == "received"
        assert "Received Alertmanager webhook with 1 alert(s)" in caplog.text

-    def test_webhook_forwards_when_configured(self, test_client, caplog, monkeypatch):
-        payload = {"status": "resolved", "alerts": []}
-        forwarded = {"captured": False}

-        class FakeResponse:
-            def raise_for_status(self):
-                pass
-
-        def fake_post(url, json, timeout):
-            forwarded["captured"] = True
-            forwarded["url"] = url
-            forwarded["payload"] = json
-            return FakeResponse()
-
-        monkeypatch.setattr("requests.Session.post", lambda _self, url, json, timeout: fake_post(url, json, timeout))
-        with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"):
-            response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload)
+class TestGrafanaStatus:
+    def test_grafana_status_when_not_configured(self, test_client):
+        response = test_client.get("/api/monitoring/grafana-status")
        assert response.status_code == 200
-        assert forwarded["captured"] is False
+        data = response.json()
+        assert data["up"] is False
+        assert data["error"] == "no_service_configured"
+
+    def test_grafana_status_when_unreachable(self, test_client):
+        service = ServiceRecord(id="g1", service_type="grafana", name="Grafana", config={"base_url": "http://grafana:3000"})
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", side_effect=Exception("refused")),
+        ):
+            response = test_client.get("/api/monitoring/grafana-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["up"] is False
+        assert data["error"] == "grafana_unreachable"
+        assert data["name"] == "Grafana"
+
+    def test_grafana_status_returns_version(self, test_client):
+        service = ServiceRecord(id="g1", service_type="grafana", name="Grafana", config={"base_url": "http://grafana:3000"})
+        resp = MagicMock()
+        resp.json.return_value = {"version": "11.3.1", "database": "ok"}
+        resp.raise_for_status = MagicMock()
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", return_value=resp),
+        ):
+            response = test_client.get("/api/monitoring/grafana-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["up"] is True
+        assert data["version"] == "11.3.1"
+        assert data["service_id"] == "g1"
+
+
+class TestPrometheusStatus:
+    def test_prometheus_status_when_not_configured(self, test_client):
+        response = test_client.get("/api/monitoring/prometheus-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["up"] is False
+        assert data["error"] == "no_service_configured"
+
+    def test_prometheus_status_when_unreachable(self, test_client):
+        service = ServiceRecord(id="p1", service_type="prometheus", name="Prometheus", config={"base_url": "http://prometheus:9090"})
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", side_effect=Exception("refused")),
+        ):
+            response = test_client.get("/api/monitoring/prometheus-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["up"] is False
+        assert data["error"] == "prometheus_unreachable"
+
+    def test_prometheus_status_returns_version(self, test_client):
+        service = ServiceRecord(id="p1", service_type="prometheus", name="Prometheus", config={"base_url": "http://prometheus:9090"})
+        health = MagicMock()
+        health.raise_for_status = MagicMock()
+        build_info = MagicMock()
+        build_info.raise_for_status = MagicMock()
+        build_info.json.return_value = {"status": "success", "data": {"version": "2.55.1"}}
+        with (
+            patch(f"{_MON}._resolve_service_record", return_value=service),
+            patch(f"{_MON}.requests.get", side_effect=[health, build_info]),
+        ):
+            response = test_client.get("/api/monitoring/prometheus-status")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["up"] is True
+        assert data["version"] == "2.55.1"
+        assert data["service_id"] == "p1"
@@ -380,9 +380,7 @@ async def test_alertmanager_adapter_sends_bearer_token():
        config={"base_url": "http://am:9093"},
        secrets={"api_key": "tok"},
    )
-    payload = SimpleNamespace(
-        raise_for_status=lambda: None, json=lambda: {"status": "success", "data": []}
-    )
+    payload = SimpleNamespace(raise_for_status=lambda: None, json=lambda: {"status": "success", "data": []})
    with patch("media_library_viewer_api.widgets.sources.requests.get", return_value=payload) as mock_get:
        result = await adapter.fetch(service, "active_alerts", {})
    assert result["total"] == 0