feat(grafana-metric-gateway): slice 1 — backend gateway transport

Route all prometheus widget queries through Grafana /api/ds/query instead of
direct Prom HTTP. PrometheusConfig: drop base_url, add grafana_url +
datasource_uid; secret grafana_api_key (required). PrometheusWidgetSource →
MetricSource with _gateway_query POST method. normalize_grafana_frames
recovered from 65bae95 + shared _dedup_label helper. Gateway-path status
check. Startup old-config validation. CHANGELOG migration note. All adapter
tests rewritten for POST /api/ds/query + Grafana frames mock. Backend: 331
pytest pass, ruff clean. Frontend: build green (unchanged in S1).
This commit is contained in:
Developer
2026-07-09 21:24:44 +00:00
parent 798196ffc7
commit df80c68f89
10 changed files with 573 additions and 250 deletions
@@ -13,9 +13,10 @@ from media_library_viewer_api.integrations.base import (
class PrometheusConfig(ServiceConfigBase):
"""Non-secret Prometheus connection config."""
"""Non-secret Prometheus-via-Grafana gateway config."""
base_url: ServiceBaseUrl
grafana_url: ServiceBaseUrl
datasource_uid: str = "prometheus"
timeout_seconds: int = 10
@@ -57,7 +58,12 @@ DEFINITION = ServiceDefinition(
description="Metrics storage and PromQL queries.",
config_model=PrometheusConfig,
secret_fields=[
SecretField(key="api_key", label="API key", helper="Optional bearer token"),
SecretField(
key="grafana_api_key",
label="Grafana API key",
required=True,
helper="Service account token or API key for the Grafana gateway",
),
],
widget_kinds=[
widget_kind(
@@ -37,6 +37,23 @@ from .version import get_backend_version, get_version_info
logger = logging.getLogger(__name__)
def _validate_prometheus_gateway_config() -> None:
"""Warn (not crash) about old-shape prometheus services needing migration (GM-113)."""
try:
store = get_settings_store()
for service in store.list_services("prometheus"):
config = service.get("config") or {}
if "base_url" in config and "grafana_url" not in config:
logger.warning(
"Prometheus service '%s' (id=%s) uses the old 'base_url' config shape. "
"Reconfigure with grafana_url + grafana_api_key (see CHANGELOG).",
service.get("name"),
service.get("id"),
)
except Exception: # pragma: no cover - startup best-effort
logger.exception("Failed to validate prometheus gateway config during startup")
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Application lifespan — startup/shutdown."""
@@ -58,6 +75,7 @@ async def lifespan(app: FastAPI):
get_service_data_harness()
except Exception:
logger.exception("Failed to initialize service data harness during startup")
_validate_prometheus_gateway_config()
mail_queue = get_mail_queue()
backup_poller = get_backup_poller()
mail_queue.start()
@@ -177,23 +177,52 @@ def get_prometheus_status(
service_id: str | None = None,
store: SettingsStore = Depends(get_settings_store),
) -> dict[str, Any]:
"""Probe a Prometheus service instance's health and build info."""
"""Probe a Prometheus service's health via the Grafana gateway path (GM-110).
Issues a trivial ``up`` query through Grafana ``/api/ds/query``. Success
validates the full path: Grafana is reachable, the API key works, and the
Prometheus datasource responds.
"""
service = resolve_service_record(store, "prometheus", service_id)
if service is None:
return _status_response(None, error="no_service_configured")
base = _base_url(service)
timeout = _timeout(service, 10)
headers = _auth_headers(service)
grafana_url = str(service.config.get("grafana_url") or "").rstrip("/")
api_key = str(service.secrets.get("grafana_api_key") or "")
datasource_uid = str(service.config.get("datasource_uid") or "prometheus")
timeout = int(service.config.get("timeout_seconds") or 10)
if not grafana_url or not api_key:
return _status_response(service, error="gateway_not_configured")
body = {
"queries": [
{
"datasource": {"uid": datasource_uid, "type": "prometheus"},
"expr": "up",
"format": "time_series",
"intervalMs": 15_000,
"maxDataPoints": 1,
"refId": "A",
}
],
"from": "now-1m",
"to": "now",
}
try:
health = requests.get(f"{base}/-/healthy", headers=headers, timeout=timeout)
health.raise_for_status()
build_info = requests.get(f"{base}/api/v1/status/buildinfo", headers=headers, timeout=timeout)
build_info.raise_for_status()
version = build_info.json().get("data", {}).get("version", "")
except Exception:
logger.exception("Failed to fetch Prometheus status")
resp = requests.post(
f"{grafana_url}/api/ds/query",
json=body,
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
timeout=timeout,
)
resp.raise_for_status()
except requests.HTTPError as exc:
status_code = exc.response.status_code if exc.response else 0
if status_code in (401, 403):
return _status_response(service, error="auth_failed")
return _status_response(service, error="gateway_error")
except requests.RequestException:
logger.exception("Failed to fetch Prometheus status via gateway")
return _status_response(service, error="prometheus_unreachable")
return _status_response(service, version=version)
return _status_response(service, version="ok")
@router.post("/alertmanager-webhook")
@@ -43,6 +43,15 @@ def step_for_window(window_seconds: int, target_points: int = 200) -> int:
return max(15, round(window_seconds / target_points))
def _dedup_label(label: str, seen: dict[str, int]) -> str:
"""Apply `` (n)`` suffix on collision. Mutates and reads from ``seen`` dict."""
if label in seen:
seen[label] += 1
return f"{label} ({seen[label]})"
seen[label] = 0
return label
def normalize_prometheus_matrix(result: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Turn a Prometheus ``/api/v1/query_range`` ``data.result`` matrix into the
``{label, points:[{t:int, v:float|None}]}`` series shape the frontend chart
@@ -62,12 +71,7 @@ def normalize_prometheus_matrix(result: list[dict[str, Any]]) -> list[dict[str,
metric = entry.get("metric") or {}
values = entry.get("values") or []
parts = [f"{k}={v}" for k, v in sorted(metric.items()) if not str(k).startswith("__")]
label = " ".join(parts) if parts else "value"
if label in seen:
seen[label] += 1
label = f"{label} ({seen[label]})"
else:
seen[label] = 0
label = _dedup_label(" ".join(parts) if parts else "value", seen)
points: list[dict[str, Any]] = []
for ts, raw in values:
t = _safe_int(ts)
@@ -79,6 +83,52 @@ def normalize_prometheus_matrix(result: list[dict[str, Any]]) -> list[dict[str,
return series
def normalize_grafana_frames(raw: dict[str, Any]) -> list[dict[str, Any]]:
"""Turn a Grafana ``/api/ds/query`` response into the ``{label, points}`` series shape.
Parses ``results.<refId>.frames[]`` where each frame has:
- ``data.values``: ``[[timestamps...], [values...]]``
- ``schema.fields``: ``[{name, labels?, config?: {displayName?}}, ...]``
Label rule (same as ``normalize_prometheus_matrix``, shared via ``_dedup_label``):
1. Prefer ``config.displayName`` (explicitly set in Grafana).
2. Else use Prometheus metric labels (sorted ``k=v``, excluding ``__``-prefixed).
3. Else fall back to the field name, or ``"value"``.
4. Dedup collisions with `` (n)`` suffix.
"""
series: list[dict[str, Any]] = []
seen: dict[str, int] = {}
results = raw.get("results", {})
for _ref_id, ref_data in results.items():
for frame in ref_data.get("frames", []):
values = frame.get("data", {}).get("values", [])
if len(values) < 2:
continue
timestamps = values[0]
vals = values[1]
# Derive a meaningful series label from the frame metadata.
fields = frame.get("schema", {}).get("fields", [])
value_field = fields[-1] if fields else {}
display_name = value_field.get("config", {}).get("displayName") or value_field.get("displayName")
frame_labels = value_field.get("labels") or {}
if display_name:
label = str(display_name)
elif frame_labels:
parts = [f"{k}={v}" for k, v in sorted(frame_labels.items()) if not str(k).startswith("__")]
label = " ".join(parts) if parts else "value"
else:
label = str(value_field.get("name", "value"))
label = _dedup_label(label, seen)
points = []
for t, v in zip(timestamps, vals):
safe_t = _safe_int(t)
if safe_t is None:
continue
points.append({"t": safe_t, "v": _safe_float(v)})
series.append({"label": label, "points": points})
return series
def _safe_float(raw: Any) -> float | None:
"""Best-effort float conversion; Prometheus sentinels and junk → ``None``."""
if raw in _NON_NUMERIC:
@@ -30,7 +30,8 @@ from media_library_viewer_api.services.settings_store import SettingsStore, get_
from media_library_viewer_api.services.task_runner import run_saved_task
from media_library_viewer_api.widgets.prometheus_range import (
WINDOW_PRESETS,
normalize_prometheus_matrix,
normalize_grafana_frames,
normalize_prometheus_matrix, # noqa: F401 — kept for future direct_url path (design decision 5)
step_for_window,
)
@@ -104,113 +105,118 @@ class StaticWidgetSource:
# ---------------------------------------------------------------------------
class PrometheusWidgetSource:
"""Run PromQL queries against a Prometheus service (instant + range)."""
class MetricSource:
"""Run PromQL queries through a Grafana gateway (``/api/ds/query``)."""
async def fetch(self, service: ServiceRecord | None, widget_kind: str, config: dict[str, Any]) -> dict[str, Any]:
try:
if service is None:
return {"error": "Prometheus widget is missing its service"}
base_url = str(service.config.get("base_url") or "").rstrip("/")
grafana_url = str(service.config.get("grafana_url") or "").rstrip("/")
api_key = str(service.secrets.get("grafana_api_key") or "")
datasource_uid = str(service.config.get("datasource_uid") or "prometheus")
timeout = int(service.config.get("timeout_seconds") or 10)
if widget_kind == "chart":
return await self._fetch_chart(base_url, timeout, config)
return await self._fetch_chart(grafana_url, api_key, datasource_uid, timeout, config)
if widget_kind == "gauge":
return await self._fetch_gauge(base_url, timeout, config)
return await self._fetch_gauge(grafana_url, api_key, datasource_uid, timeout, config)
if widget_kind == "mean":
return await self._fetch_mean(base_url, timeout, config)
# Default: instant-query metric path (unchanged).
raw = await self._instant_query(base_url, timeout, config.get("promql", ""))
return raw
return await self._fetch_mean(grafana_url, api_key, datasource_uid, timeout, config)
# Default: instant-query metric path.
return await self._fetch_metric(grafana_url, api_key, datasource_uid, timeout, config)
except Exception as exc:
logger.exception("prometheus adapter failed")
return {"error": f"Prometheus query failed: {exc}"}
async def _range_query(self, base_url: str, timeout: int, promql: str, window: int) -> dict[str, Any]:
"""Run a Prometheus ``/api/v1/query_range`` over a window (seconds).
async def _gateway_query(
self,
grafana_url: str,
api_key: str,
datasource_uid: str,
timeout: int,
promql: str,
window_seconds: int | None = None,
max_data_points: int = 200,
) -> dict[str, Any]:
"""POST ``{grafana_url}/api/ds/query``; return raw Grafana JSON or ``{error}``.
Shared by the ``chart`` (SC-101) and ``mean`` widget kinds. Returns
``{"matrix": result}`` on success or ``{"error": str}`` (never raises,
per SC-103).
- ``window_seconds=None`` → instant mapping (``from=now-1m, maxDataPoints=1``).
- ``window_seconds=<N>`` → range query (``from=now-Ns``, step derived).
"""
step = step_for_window(window)
end = int(time.time())
start = end - window
try:
response = await asyncio.wait_for(
asyncio.to_thread(
requests.get,
f"{base_url}/api/v1/query_range",
params={"query": promql, "start": start, "end": end, "step": step},
timeout=timeout,
),
if not grafana_url:
return {"error": "grafana_url is required"}
if not api_key:
return {"error": "grafana_api_key is required"}
step = step_for_window(window_seconds) if window_seconds else 15
interval_ms = step * 1000
body = {
"queries": [
{
"datasource": {"uid": datasource_uid, "type": "prometheus"},
"expr": promql,
"format": "time_series",
"intervalMs": interval_ms,
"maxDataPoints": 1 if window_seconds is None else max_data_points,
"refId": "A",
}
],
"from": f"now-{window_seconds or 60}s" if window_seconds else "now-1m",
"to": "now",
}
def _do_post() -> dict[str, Any]:
resp = requests.post(
f"{grafana_url}/api/ds/query",
json=body,
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
timeout=timeout,
)
response.raise_for_status()
payload = response.json()
except asyncio.TimeoutError:
return {"error": "Prometheus query timed out"}
except requests.RequestException as exc:
logger.exception("prometheus range query failed")
return {"error": f"Prometheus query failed: {exc}"}
result = payload.get("data", {}).get("result", [])
return {"matrix": result}
resp.raise_for_status()
return resp.json()
async def _instant_query(self, base_url: str, timeout: int, promql: str) -> dict[str, Any]:
"""Run a Prometheus ``/api/v1/query`` instant query.
Shared by the ``metric`` and ``gauge`` widget kinds. Returns
``{"result": data}`` on success or ``{"error": str}`` (never raises,
per SC-103).
"""
if not promql:
return {"error": "promql is required"}
try:
response = await asyncio.wait_for(
asyncio.to_thread(
requests.get,
f"{base_url}/api/v1/query",
params={"query": promql},
timeout=timeout,
),
timeout=timeout,
)
response.raise_for_status()
payload = response.json()
return await asyncio.wait_for(asyncio.to_thread(_do_post), timeout=timeout)
except asyncio.TimeoutError:
return {"error": "Prometheus query timed out"}
return {"error": "Grafana query timed out"}
except requests.RequestException as exc:
logger.exception("prometheus instant query failed")
return {"error": f"Prometheus query failed: {exc}"}
return {"result": payload.get("data", {})}
logger.exception("grafana gateway query failed")
return {"error": f"Grafana query failed: {exc}"}
async def _fetch_chart(self, base_url: str, timeout: int, config: dict[str, Any]) -> dict[str, Any]:
"""Range query → ``{series}`` for the chart widget (SC-101..SC-104)."""
async def _fetch_chart(
self, grafana_url: str, api_key: str, datasource_uid: str, timeout: int, config: dict[str, Any]
) -> dict[str, Any]:
"""Range query → ``{series}`` for the chart widget (GM-106)."""
promql = config.get("promql")
if not promql:
return {"error": "promql is required"}
window = WINDOW_PRESETS.get(config.get("window", "1h"), WINDOW_PRESETS["1h"])
raw = await self._range_query(base_url, timeout, promql, window)
raw = await self._gateway_query(grafana_url, api_key, datasource_uid, timeout, promql, window_seconds=window)
if "error" in raw:
return raw
return {"series": normalize_prometheus_matrix(raw["matrix"])}
return {"series": normalize_grafana_frames(raw)}
async def _fetch_gauge(self, base_url: str, timeout: int, config: dict[str, Any]) -> dict[str, Any]:
"""Instant query → scalar for the gauge widget (SC-109, SC-110, SC-111).
async def _fetch_gauge(
self, grafana_url: str, api_key: str, datasource_uid: str, timeout: int, config: dict[str, Any]
) -> dict[str, Any]:
"""Instant query → scalar for the gauge widget (GM-107).
Scalar-only: a multi-series query returns an error (SC-111). Threshold
config (``warn_at``/``crit_at``/``min``/``max``/``unit``) is passed
through for the frontend renderer.
Scalar-only: a multi-series query returns an error. Threshold config is
passed through for the frontend renderer.
"""
raw = await self._instant_query(base_url, timeout, config.get("promql") or "")
promql = config.get("promql") or ""
if not promql:
return {"error": "promql is required"}
raw = await self._gateway_query(grafana_url, api_key, datasource_uid, timeout, promql, window_seconds=None)
if "error" in raw:
return raw
result = raw["result"].get("result", [])
if len(result) != 1:
series = normalize_grafana_frames(raw)
if len(series) != 1:
return {"error": "Gauge requires a single-series query; refine your PromQL"}
try:
value = float(result[0]["value"][1])
except (KeyError, IndexError, ValueError, TypeError):
points = series[0]["points"]
if not points:
return {"error": "Gauge query returned no scalar value"}
value = points[-1]["v"]
if value is None:
return {"error": "Gauge query returned no scalar value"}
return {
"value": value,
@@ -221,36 +227,46 @@ class PrometheusWidgetSource:
"unit": config.get("unit"),
}
async def _fetch_mean(self, base_url: str, timeout: int, config: dict[str, Any]) -> dict[str, Any]:
"""Range query → client-side mean for the mean widget (SC-112..SC-114).
async def _fetch_mean(
self, grafana_url: str, api_key: str, datasource_uid: str, timeout: int, config: dict[str, Any]
) -> dict[str, Any]:
"""Range query → client-side mean for the mean widget (GM-108).
Runs ``query_range`` over the configured window preset, averages all
non-null numeric samples of the single series, and returns a scalar.
Scalar-only: a multi-series query returns an error (SC-114).
Runs a gateway range query over the configured window preset, averages
all non-null numeric samples of the single series, and returns a scalar.
Scalar-only: a multi-series query returns an error.
"""
promql = config.get("promql")
if not promql:
return {"error": "promql is required"}
window = WINDOW_PRESETS.get(config.get("window", "1h"), WINDOW_PRESETS["1h"])
raw = await self._range_query(base_url, timeout, promql, window)
raw = await self._gateway_query(grafana_url, api_key, datasource_uid, timeout, promql, window_seconds=window)
if "error" in raw:
return raw
result = raw["matrix"]
if len(result) != 1:
series = normalize_grafana_frames(raw)
if len(series) != 1:
return {"error": "Mean requires a single-series query; refine your PromQL"}
points = result[0].get("values") or []
nums: list[float] = []
for _, v in points:
if v in (None, "NaN", "+Inf", "-Inf"):
continue
try:
nums.append(float(v))
except (TypeError, ValueError):
continue
nums = [p["v"] for p in series[0]["points"] if p["v"] is not None]
if not nums:
return {"error": "Mean query returned no numeric samples in the window"}
mean = sum(nums) / len(nums)
return {"value": mean, "unit": config.get("unit")}
return {"value": sum(nums) / len(nums), "unit": config.get("unit")}
async def _fetch_metric(
self, grafana_url: str, api_key: str, datasource_uid: str, timeout: int, config: dict[str, Any]
) -> dict[str, Any]:
"""Instant query → ``{result}`` for the metric widget (GM-109).
Returns ``{result: [{label, points}]}`` — the normalized series shape.
The frontend ``PrometheusMetricWidget`` renders the last point of each
series.
"""
promql = config.get("promql") or ""
if not promql:
return {"error": "promql is required"}
raw = await self._gateway_query(grafana_url, api_key, datasource_uid, timeout, promql, window_seconds=None)
if "error" in raw:
return raw
return {"result": normalize_grafana_frames(raw)}
class AlertmanagerWidgetSource:
@@ -437,7 +453,7 @@ class QbittorrentWidgetSource:
# ---------------------------------------------------------------------------
SERVICE_ADAPTERS: dict[str, WidgetSource] = {
"prometheus": PrometheusWidgetSource(),
"prometheus": MetricSource(),
"qbittorrent": QbittorrentWidgetSource(),
"alertmanager": AlertmanagerWidgetSource(),
"jellyfin": JellyfinWidgetSource(),
+16 -11
View File
@@ -751,11 +751,15 @@ class TestPrometheusStatus:
def test_prometheus_status_when_unreachable(self, test_client):
service = ServiceRecord(
id="p1", service_type="prometheus", name="Prometheus", config={"base_url": "http://prometheus:9090"}
id="p1",
service_type="prometheus",
name="Prometheus",
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus"},
secrets={"grafana_api_key": "key"},
)
with (
patch(f"{_MON}.resolve_service_record", return_value=service),
patch(f"{_MON}.requests.get", side_effect=Exception("refused")),
patch(f"{_MON}.requests.post", side_effect=__import__("requests").ConnectionError("refused")),
):
response = test_client.get("/api/monitoring/prometheus-status")
assert response.status_code == 200
@@ -763,22 +767,23 @@ class TestPrometheusStatus:
assert data["up"] is False
assert data["error"] == "prometheus_unreachable"
def test_prometheus_status_returns_version(self, test_client):
def test_prometheus_status_returns_ok(self, test_client):
service = ServiceRecord(
id="p1", service_type="prometheus", name="Prometheus", config={"base_url": "http://prometheus:9090"}
id="p1",
service_type="prometheus",
name="Prometheus",
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus"},
secrets={"grafana_api_key": "key"},
)
health = MagicMock()
health.raise_for_status = MagicMock()
build_info = MagicMock()
build_info.raise_for_status = MagicMock()
build_info.json.return_value = {"status": "success", "data": {"version": "2.55.1"}}
gateway_resp = MagicMock()
gateway_resp.raise_for_status = MagicMock()
with (
patch(f"{_MON}.resolve_service_record", return_value=service),
patch(f"{_MON}.requests.get", side_effect=[health, build_info]),
patch(f"{_MON}.requests.post", return_value=gateway_resp),
):
response = test_client.get("/api/monitoring/prometheus-status")
assert response.status_code == 200
data = response.json()
assert data["up"] is True
assert data["version"] == "2.55.1"
assert data["version"] == "ok"
assert data["service_id"] == "p1"
+151
View File
@@ -6,6 +6,8 @@ import pytest
from media_library_viewer_api.widgets.prometheus_range import (
WINDOW_PRESETS,
_dedup_label,
normalize_grafana_frames,
normalize_prometheus_matrix,
step_for_window,
)
@@ -104,3 +106,152 @@ class TestNormalizePrometheusMatrix:
{"t": 1, "v": 3.5},
{"t": 3, "v": None},
]
class TestDedupLabel:
"""The shared label-dedup helper used by both normalizers (GM-104)."""
def test_first_use_returns_label_unchanged(self) -> None:
seen: dict[str, int] = {}
assert _dedup_label("value", seen) == "value"
assert seen == {"value": 0}
def test_collision_appends_suffix(self) -> None:
seen: dict[str, int] = {}
assert _dedup_label("job=x", seen) == "job=x"
assert _dedup_label("job=x", seen) == "job=x (1)"
assert _dedup_label("job=x", seen) == "job=x (2)"
def test_different_labels_dont_collide(self) -> None:
seen: dict[str, int] = {}
assert _dedup_label("a", seen) == "a"
assert _dedup_label("b", seen) == "b"
class TestNormalizeGrafanaFrames:
"""GM-104: frames normalizer recovered from 65bae95 + shared dedup."""
def test_empty_response(self) -> None:
assert normalize_grafana_frames({"results": {}}) == []
assert normalize_grafana_frames({}) == []
def test_single_frame_with_values(self) -> None:
raw = {
"results": {
"A": {
"frames": [
{
"data": {"values": [[1000, 2000], [1.5, 2.5]]},
"schema": {"fields": [{"name": "Time"}, {"name": "Value"}]},
}
]
}
}
}
out = normalize_grafana_frames(raw)
assert len(out) == 1
assert out[0]["label"] == "Value"
assert out[0]["points"] == [
{"t": 1000, "v": 1.5},
{"t": 2000, "v": 2.5},
]
def test_display_name_takes_priority(self) -> None:
raw = {
"results": {
"A": {
"frames": [
{
"data": {"values": [[100, 200], [0.75, 0.80]]},
"schema": {
"fields": [
{"name": "Time"},
{
"name": "Value",
"labels": {"instance": "host:9100"},
"config": {"displayName": "CPU Usage"},
},
]
},
}
]
}
}
}
out = normalize_grafana_frames(raw)
assert out[0]["label"] == "CPU Usage"
def test_labels_fallback_when_no_display_name(self) -> None:
raw = {
"results": {
"A": {
"frames": [
{
"data": {"values": [[100], [1.0]]},
"schema": {
"fields": [
{"name": "Time"},
{
"name": "Value",
"labels": {"__name__": "up", "instance": "h:9100"},
},
]
},
}
]
}
}
}
out = normalize_grafana_frames(raw)
assert out[0]["label"] == "instance=h:9100"
def test_falls_back_to_value_when_no_metadata(self) -> None:
raw = {
"results": {
"A": {
"frames": [
{
"data": {"values": [[100], [1.0]]},
"schema": {"fields": [{"name": "Time"}, {}]},
}
]
}
}
}
out = normalize_grafana_frames(raw)
assert out[0]["label"] == "value"
def test_dedup_collisions(self) -> None:
raw = {
"results": {
"A": {
"frames": [
{
"data": {"values": [[100], [1.0]]},
"schema": {"fields": [{}, {"name": "Value"}]},
},
{
"data": {"values": [[100], [2.0]]},
"schema": {"fields": [{}, {"name": "Value"}]},
},
]
}
}
}
out = normalize_grafana_frames(raw)
labels = [s["label"] for s in out]
assert labels == ["Value", "Value (1)"]
def test_skips_frames_with_insufficient_values(self) -> None:
raw = {
"results": {
"A": {
"frames": [
{"data": {"values": [[100]]}, "schema": {"fields": []}},
{"data": {"values": [[100], [1.0]]}, "schema": {"fields": [{}, {}]}},
]
}
}
}
out = normalize_grafana_frames(raw)
assert len(out) == 1
+17 -17
View File
@@ -117,7 +117,7 @@ def test_widget_kind_lookup():
def test_service_config_schema_is_json_schema():
schema = get_service_definition("prometheus").config_schema
assert schema["type"] == "object"
assert "base_url" in schema["properties"]
assert "grafana_url" in schema["properties"]
# ---------------------------------------------------------------------------
@@ -182,7 +182,7 @@ def test_list_service_types(client):
def test_service_type_includes_secret_and_widget_metadata(client):
response = client.get("/api/services/types")
prom = next(item for item in response.json() if item["service_type"] == "prometheus")
assert [sf["key"] for sf in prom["secret_fields"]] == ["api_key"]
assert [sf["key"] for sf in prom["secret_fields"]] == ["grafana_api_key"]
assert set(wk["kind"] for wk in prom["widget_kinds"]) == {"metric", "chart", "gauge", "mean"}
@@ -195,8 +195,8 @@ def _prometheus_payload(**overrides):
payload = {
"service_type": "prometheus",
"name": "Production Prometheus",
"config": {"base_url": "https://prometheus.example.com"},
"secrets": {"api_key": "secret-token"},
"config": {"grafana_url": "https://grafana.example.com", "datasource_uid": "prometheus"},
"secrets": {"grafana_api_key": "secret-token"},
"enabled": True,
}
payload.update(overrides)
@@ -208,10 +208,10 @@ def test_create_and_list_service(client):
assert response.status_code == 201
created = response.json()
assert created["service_type"] == "prometheus"
assert created["config"]["base_url"] == "https://prometheus.example.com"
assert created["config"]["grafana_url"] == "https://grafana.example.com"
# Plaintext secrets are never returned.
assert "secrets" not in created
assert created["secrets_set"] == {"api_key": True}
assert created["secrets_set"] == {"grafana_api_key": True}
response = client.get("/api/services/instances")
assert response.status_code == 200
@@ -242,11 +242,11 @@ def test_update_service_preserves_unsent_secrets(client):
json={
"service_type": "prometheus",
"name": "Renamed Prometheus",
"config": {"base_url": "https://prometheus.example.com", "timeout_seconds": 10},
"config": {"grafana_url": "https://grafana.example.com", "timeout_seconds": 10},
},
).json()
assert updated["name"] == "Renamed Prometheus"
assert updated["secrets_set"] == {"api_key": True}
assert updated["secrets_set"] == {"grafana_api_key": True}
def test_update_service_can_clear_secret(client):
@@ -256,11 +256,11 @@ def test_update_service_can_clear_secret(client):
json={
"service_type": "prometheus",
"name": "Production Prometheus",
"config": {"base_url": "https://prometheus.example.com"},
"secrets": {"api_key": ""},
"config": {"grafana_url": "https://grafana.example.com"},
"secrets": {"grafana_api_key": ""},
},
).json()
assert updated["secrets_set"] == {"api_key": False}
assert updated["secrets_set"] == {"grafana_api_key": False}
def test_unknown_service_type_rejected(client):
@@ -274,7 +274,7 @@ def test_unknown_service_type_rejected(client):
def test_invalid_config_rejected(client):
response = client.post(
"/api/services/instances",
json={"service_type": "prometheus", "name": "x", "config": {"base_url": ""}},
json={"service_type": "prometheus", "name": "x", "config": {"grafana_url": ""}},
)
assert response.status_code == 422
# Force a real validation error via bad type.
@@ -292,10 +292,10 @@ def test_service_base_url_requires_http_schema(bad_url):
"""Every service base_url must include an http:// or https:// schema."""
model = get_service_definition("prometheus").config_model
with pytest.raises(ValidationError):
model.model_validate({"base_url": bad_url, "timeout_seconds": 5})
model.model_validate({"grafana_url": bad_url, "timeout_seconds": 5})
@pytest.mark.parametrize("service_type", ["prometheus", "alertmanager", "jellyfin", "authentik", "nextcloud"])
@pytest.mark.parametrize("service_type", ["alertmanager", "jellyfin", "authentik", "nextcloud"])
def test_service_base_url_accepts_absolute_urls(service_type):
model = get_service_definition(service_type).config_model
instance = model.model_validate({"base_url": "https://example.com"})
@@ -308,7 +308,7 @@ def test_unknown_secret_field_rejected(client):
json={
"service_type": "prometheus",
"name": "x",
"config": {"base_url": "https://prometheus.example.com"},
"config": {"grafana_url": "https://grafana.example.com"},
"secrets": {"password": "leak"},
},
)
@@ -321,7 +321,7 @@ def test_credential_key_in_config_rejected(client):
json={
"service_type": "prometheus",
"name": "x",
"config": {"base_url": "https://prometheus.example.com", "api_key": "leak"},
"config": {"grafana_url": "https://grafana.example.com", "api_key": "leak"},
},
)
assert response.status_code == 422
@@ -369,7 +369,7 @@ def test_delete_service_cascades_to_widgets(client, tmp_path):
"""
store = app.dependency_overrides[get_settings_store]()
service = store.upsert_service(
{"service_type": "prometheus", "name": "Prometheus", "config": {"base_url": "u"}, "enabled": True}
{"service_type": "prometheus", "name": "Prometheus", "config": {"grafana_url": "u"}, "enabled": True}
)
# Ensure the service_id column exists and seed a referencing widget.
+142 -107
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
import pytest
@@ -42,7 +43,7 @@ def client(tmp_path):
def _make_prometheus_service(client, name="Production Prometheus", **config_overrides):
config = {"base_url": "https://prometheus.example.com"}
config = {"grafana_url": "https://grafana.example.com", "datasource_uid": "prometheus"}
config.update(config_overrides)
return client.post(
"/api/services/instances",
@@ -302,7 +303,7 @@ def test_fetch_widget_service_disabled(client):
json={
"service_type": "prometheus",
"name": service["name"],
"config": {"base_url": "https://prometheus.example.com"},
"config": {"grafana_url": "https://grafana.example.com", "datasource_uid": "prometheus"},
"enabled": False,
},
)
@@ -469,39 +470,45 @@ def test_jellyfin_definition_has_now_playing_widget():
@pytest.mark.asyncio
async def test_prometheus_chart_adapter_runs_range_query():
"""SC-101: chart kind hits /api/v1/query_range and returns {series}."""
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
"""GM-106: chart kind hits /api/ds/query and returns {series}."""
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"base_url": "http://p:9090", "timeout_seconds": 5},
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus", "timeout_seconds": 5},
secrets={"grafana_api_key": "key"},
)
payload = SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {
"data": {
"result": [
{
"metric": {"__name__": "up", "instance": "h:9100"},
"values": [[100, "1"], [130, "1"]],
}
]
"results": {
"A": {
"frames": [
{
"data": {"values": [[100, 130], [1.0, 1.0]]},
"schema": {
"fields": [
{"name": "Time"},
{"name": "Value", "labels": {"__name__": "up", "instance": "h:9100"}},
]
},
}
]
}
}
},
)
with patch("media_library_viewer_api.widgets.sources.requests.get", return_value=payload) as mock_get:
with patch("media_library_viewer_api.widgets.sources.requests.post", return_value=payload) as mock_post:
result = await adapter.fetch(service, "chart", {"promql": "up", "window": "1h"})
# query_range endpoint + window-derived start/end/step params.
call = mock_get.call_args
assert call.args[0].endswith("/api/v1/query_range")
params = call.kwargs["params"]
assert params["query"] == "up"
assert {"start", "end", "step"}.issubset(params)
# {series} shape with the shared normalization (label drops __name__).
call = mock_post.call_args
assert call.args[0].endswith("/api/ds/query")
body = call.kwargs["json"]
assert body["queries"][0]["expr"] == "up"
assert body["queries"][0]["datasource"]["uid"] == "prometheus"
assert "series" in result
assert result["series"][0]["label"] == "instance=h:9100"
assert result["series"][0]["points"] == [{"t": 100, "v": 1.0}, {"t": 130, "v": 1.0}]
@@ -509,26 +516,43 @@ async def test_prometheus_chart_adapter_runs_range_query():
@pytest.mark.asyncio
async def test_prometheus_chart_adapter_requires_promql():
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
service = ServiceRecord(id="s", service_type="prometheus", name="p", config={"base_url": "http://p:9090"})
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus"},
secrets={"grafana_api_key": "key"},
)
result = await adapter.fetch(service, "chart", {"promql": ""})
assert result == {"error": "promql is required"}
@pytest.mark.asyncio
async def test_prometheus_chart_adapter_degrades_on_http_error():
"""SC-103: a connection error returns {error} rather than raising."""
"""GM-103: a connection error returns {error} rather than raising."""
import requests as req_mod
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s", service_type="prometheus", name="p", config={"base_url": "http://p:9090", "timeout_seconds": 2}
id="s",
service_type="prometheus",
name="p",
config={
"grafana_url": "http://grafana:3000",
"datasource_uid": "prometheus",
"timeout_seconds": 2,
},
secrets={"grafana_api_key": "key"},
)
with patch("media_library_viewer_api.widgets.sources.requests.get", side_effect=req_mod.ConnectionError("refused")):
with patch(
"media_library_viewer_api.widgets.sources.requests.post",
side_effect=req_mod.ConnectionError("refused"),
):
result = await adapter.fetch(service, "chart", {"promql": "up", "window": "1h"})
assert "error" in result
assert "failed" in result["error"].lower()
@@ -632,7 +656,7 @@ def test_widget_reference_lifecycle(widget_ref_client):
{
"service_type": "prometheus",
"name": "Prometheus",
"config": {"base_url": "https://prometheus.example.com"},
"config": {"grafana_url": "https://grafana.example.com", "datasource_uid": "prometheus"},
"secrets": {"api_key": "tok"},
"enabled": True,
},
@@ -690,7 +714,7 @@ def test_widget_reference_detach(widget_ref_client):
{
"service_type": "prometheus",
"name": "Prometheus",
"config": {"base_url": "https://prometheus.example.com"},
"config": {"grafana_url": "https://grafana.example.com", "datasource_uid": "prometheus"},
"secrets": {"api_key": "tok"},
"enabled": True,
},
@@ -794,46 +818,57 @@ def test_widget_reference_update_sort_order(widget_ref_client):
# ---------------------------------------------------------------------------
# Prometheus gauge + mean adapter tests (SC-109..SC-114)
# Prometheus gauge + mean adapter tests (GM-107..GM-108)
# ---------------------------------------------------------------------------
def _grafana_single_frame(values, labels=None, display_name=None):
"""Build a Grafana /api/ds/query frames response for a single series."""
field: dict[str, Any] = {"name": "Value"}
if labels:
field["labels"] = labels
if display_name:
field["config"] = {"displayName": display_name}
return {
"results": {
"A": {
"frames": [
{
"data": {"values": values},
"schema": {"fields": [{"name": "Time"}, field]},
}
]
}
}
}
@pytest.mark.asyncio
async def test_prometheus_gauge_adapter_returns_scalar():
"""SC-109: gauge kind hits /api/v1/query and returns {value, thresholds}."""
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
"""GM-107: gauge kind hits /api/ds/query and returns {value, thresholds}."""
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"base_url": "http://p:9090", "timeout_seconds": 5},
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus", "timeout_seconds": 5},
secrets={"grafana_api_key": "key"},
)
payload = SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {
"data": {
"result": [
{"metric": {"__name__": "cpu"}, "value": [100, "0.75"]},
]
}
},
json=lambda: _grafana_single_frame([[100], [0.75]]),
)
with patch("media_library_viewer_api.widgets.sources.requests.get", return_value=payload) as mock_get:
with patch("media_library_viewer_api.widgets.sources.requests.post", return_value=payload) as mock_post:
result = await adapter.fetch(
service,
"gauge",
{
"promql": "cpu_usage",
"warn_at": 0.8,
"crit_at": 0.95,
"unit": "%",
},
{"promql": "cpu_usage", "warn_at": 0.8, "crit_at": 0.95, "unit": "%"},
)
call = mock_get.call_args
assert call.args[0].endswith("/api/v1/query")
assert call.kwargs["params"]["query"] == "cpu_usage"
call = mock_post.call_args
assert call.args[0].endswith("/api/ds/query")
assert call.kwargs["json"]["queries"][0]["expr"] == "cpu_usage"
assert result["value"] == 0.75
assert result["warn_at"] == 0.8
assert result["crit_at"] == 0.95
@@ -842,28 +877,31 @@ async def test_prometheus_gauge_adapter_returns_scalar():
@pytest.mark.asyncio
async def test_prometheus_gauge_adapter_rejects_multi_series():
"""SC-111: gauge must be scalar-only; multi-series returns error."""
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
"""GM-107: gauge must be scalar-only; multi-series returns error."""
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"base_url": "http://p:9090"},
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus"},
secrets={"grafana_api_key": "key"},
)
payload = SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {
"data": {
"result": [
{"metric": {"instance": "a"}, "value": [100, "1"]},
{"metric": {"instance": "b"}, "value": [100, "2"]},
]
"results": {
"A": {
"frames": [
{"data": {"values": [[100], [1.0]]}, "schema": {"fields": [{}, {"name": "A"}]}},
{"data": {"values": [[100], [2.0]]}, "schema": {"fields": [{}, {"name": "A"}]}},
]
}
}
},
)
with patch("media_library_viewer_api.widgets.sources.requests.get", return_value=payload):
with patch("media_library_viewer_api.widgets.sources.requests.post", return_value=payload):
result = await adapter.fetch(service, "gauge", {"promql": "up"})
assert "error" in result
assert "single-series" in result["error"].lower()
@@ -871,14 +909,15 @@ async def test_prometheus_gauge_adapter_rejects_multi_series():
@pytest.mark.asyncio
async def test_prometheus_gauge_adapter_requires_promql():
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"base_url": "http://p:9090"},
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus"},
secrets={"grafana_api_key": "key"},
)
result = await adapter.fetch(service, "gauge", {"promql": ""})
assert result == {"error": "promql is required"}
@@ -886,30 +925,22 @@ async def test_prometheus_gauge_adapter_requires_promql():
@pytest.mark.asyncio
async def test_prometheus_mean_adapter_computes_average():
"""SC-112: mean kind averages non-null values over the window."""
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
"""GM-108: mean kind averages non-null values over the window."""
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"base_url": "http://p:9090", "timeout_seconds": 5},
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus", "timeout_seconds": 5},
secrets={"grafana_api_key": "key"},
)
payload = SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {
"data": {
"result": [
{
"metric": {"__name__": "cpu"},
"values": [[100, "1.0"], [130, "2.0"], [160, "3.0"]],
}
]
}
},
json=lambda: _grafana_single_frame([[100, 130, 160], [1.0, 2.0, 3.0]]),
)
with patch("media_library_viewer_api.widgets.sources.requests.get", return_value=payload):
with patch("media_library_viewer_api.widgets.sources.requests.post", return_value=payload):
result = await adapter.fetch(service, "mean", {"promql": "cpu", "window": "1h"})
assert result["value"] == 2.0
assert result["unit"] is None
@@ -917,28 +948,31 @@ async def test_prometheus_mean_adapter_computes_average():
@pytest.mark.asyncio
async def test_prometheus_mean_adapter_rejects_multi_series():
"""SC-114: mean must be scalar-only; multi-series returns error."""
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
"""GM-108: mean must be scalar-only; multi-series returns error."""
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"base_url": "http://p:9090"},
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus"},
secrets={"grafana_api_key": "key"},
)
payload = SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {
"data": {
"result": [
{"metric": {"instance": "a"}, "values": [[100, "1"]]},
{"metric": {"instance": "b"}, "values": [[100, "2"]]},
]
"results": {
"A": {
"frames": [
{"data": {"values": [[100], [1.0]]}, "schema": {"fields": [{}, {"name": "A"}]}},
{"data": {"values": [[100], [2.0]]}, "schema": {"fields": [{}, {"name": "A"}]}},
]
}
}
},
)
with patch("media_library_viewer_api.widgets.sources.requests.get", return_value=payload):
with patch("media_library_viewer_api.widgets.sources.requests.post", return_value=payload):
result = await adapter.fetch(service, "mean", {"promql": "up", "window": "1h"})
assert "error" in result
assert "single-series" in result["error"].lower()
@@ -946,45 +980,46 @@ async def test_prometheus_mean_adapter_rejects_multi_series():
@pytest.mark.asyncio
async def test_prometheus_mean_adapter_skips_nan_values():
"""SC-112: NaN / Inf values are excluded from the mean computation."""
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
"""GM-108: NaN values are excluded from the mean computation."""
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"base_url": "http://p:9090"},
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus"},
secrets={"grafana_api_key": "key"},
)
# Grafana frames shape with NaN — normalize_grafana_frames converts string "NaN" to None
payload = SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {
"data": {
"result": [
{
"metric": {},
"values": [[100, "2.0"], [130, "NaN"], [160, "4.0"]],
}
]
"results": {
"A": {
"frames": [
{"data": {"values": [[100, 130, 160], [2.0, "NaN", 4.0]]}, "schema": {"fields": [{}, {}]}}
]
}
}
},
)
with patch("media_library_viewer_api.widgets.sources.requests.get", return_value=payload):
with patch("media_library_viewer_api.widgets.sources.requests.post", return_value=payload):
result = await adapter.fetch(service, "mean", {"promql": "up", "window": "1h"})
# (2.0 + 4.0) / 2 = 3.0 (NaN excluded)
assert result["value"] == 3.0
@pytest.mark.asyncio
async def test_prometheus_mean_adapter_requires_promql():
from media_library_viewer_api.widgets.sources import PrometheusWidgetSource
from media_library_viewer_api.widgets.sources import MetricSource
adapter = PrometheusWidgetSource()
adapter = MetricSource()
service = ServiceRecord(
id="s",
service_type="prometheus",
name="p",
config={"base_url": "http://p:9090"},
config={"grafana_url": "http://grafana:3000", "datasource_uid": "prometheus"},
secrets={"grafana_api_key": "key"},
)
result = await adapter.fetch(service, "mean", {"promql": ""})
assert result == {"error": "promql is required"}