alex/manage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(observability): add Prometheus/Grafana/Loki/Alertmanager/Alloy stack and remove legacy Monitoring UI

Browse Source
This commit is contained in:
Developer
2026-06-16 11:08:32 +00:00
parent 615e02e970
commit e2ad731b5f
80 changed files with 5020 additions and 3665 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_api.py
+190 -125
View File
@@ -11,23 +11,23 @@ from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from media_library_viewer_api.main import app
from media_library_viewer_api.clients.ssh import CommandResult
from media_library_viewer_api.dependencies import (
get_ssh_client,
get_jellyfin_client,
get_jellyseerr_client,
get_mail_queue,
get_settings_store,
get_ssh_client,
get_user_id,
)
from media_library_viewer_api.clients.ssh import CommandResult
from media_library_viewer_api.main import app
from media_library_viewer_api.routers.media import get_media_index
from media_library_viewer_api.services.media_index import MediaIndex
from media_library_viewer_api.services.settings_store import SettingsStore
# --- Fixtures ---
@pytest.fixture
def mock_jellyfin():
"""Mock Jellyfin client."""
@@ -145,15 +145,29 @@ def test_client(mock_jellyfin, mock_jellyseerr, mock_ssh, tmp_path):
# --- Health ---
class TestHealth:
def test_health(self, test_client):
response = test_client.get("/api/health")
assert response.status_code == 200
assert response.json() == {"status": "ok"}
def test_metrics_endpoint(self, test_client):
response = test_client.get("/metrics")
assert response.status_code == 200
assert "manage_api_requests_total" in response.text
assert response.headers["content-type"].startswith("text/plain")
def test_request_id_header(self, test_client):
response = test_client.get("/api/health")
assert response.status_code == 200
assert "x-request-id" in response.headers
assert len(response.headers["x-request-id"]) > 0
# --- Dashboard ---
class TestDashboard:
def test_counts(self, test_client):
response = test_client.get("/api/dashboard/counts")
@@ -163,78 +177,6 @@ class TestDashboard:
assert data["series"] == 20
assert data["episodes"] == 500
def test_monitoring_overview(self, test_client):
store = MagicMock()
store.list_machines.return_value = [
{
"id": "local",
"name": "This machine",
"mode": "local",
"enabled": True,
"host": "localhost",
"port": 22,
"username": "",
"media_root": "/srv/media",
"path_prefix": "",
"notes": "",
},
{
"id": "remote1",
"name": "Remote",
"mode": "ssh",
"enabled": True,
"host": "server.example.com",
"port": 22,
"username": "alex",
"media_root": "/srv/media",
"path_prefix": "",
"notes": "",
},
]
app.dependency_overrides[get_settings_store] = lambda: store
try:
with (
patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=object()),
patch("media_library_viewer_api.services.monitoring_actions.resource_collector_status", return_value="running pid=123"),
patch(
"media_library_viewer_api.services.monitoring_actions.read_resource_metrics",
return_value=[
{
"ts": 123.0,
"cpu_pct": 10.0,
"iowait_pct": 1.0,
"mem_pct": 20.0,
"net_rx_bytes_per_sec": 100.0,
"net_tx_bytes_per_sec": 50.0,
"disk_read_bps": 1.0,
"disk_write_bps": 2.0,
}
],
),
patch(
"media_library_viewer_api.services.monitoring_actions.disk_space",
return_value={
"filesystem": "/dev/sda1",
"size": 1000,
"used": 200,
"available": 800,
"used_pct": "20.0%",
"mount": "/srv/media",
},
),
):
response = test_client.get("/api/dashboard/monitoring")
finally:
app.dependency_overrides.pop(get_settings_store, None)
assert response.status_code == 200
data = response.json()
assert data["total"] == 2
assert data["enabled"] == 2
assert len(data["machines"]) == 2
assert data["machines"][0]["latest_sample"]["cpu_pct"] == 10.0
assert data["machines"][0]["disk"]["used_pct"] == "20.0%"
def test_libraries(self, test_client):
response = test_client.get("/api/dashboard/libraries")
assert response.status_code == 200
@@ -268,6 +210,7 @@ class TestDashboard:
# --- Settings reset ---
class TestSettingsReset:
def test_reset_local_database_requires_full_confirmation(self, test_client, tmp_path):
store = SettingsStore(tmp_path / "settings.sqlite")
@@ -329,8 +272,10 @@ class TestSettingsReset:
assert store.get_machine("local") is None
assert len(store.list_machines()) == 0
# --- Users ---
class TestUsers:
def test_users_list_enriched(self, test_client):
response = test_client.get("/api/users")
@@ -418,7 +363,7 @@ class TestUsers:
)
try:
with patch("media_library_viewer_api.routers.users.get_settings", return_value=settings):
with patch("media_library_viewer_api.routers.users_impl.get_settings", return_value=settings):
response = test_client.post(
"/api/users/message",
data={
@@ -443,8 +388,10 @@ class TestUsers:
assert kwargs["subject"] == "Hello team"
assert kwargs["settings"] is settings
# --- Files ---
class TestFiles:
def test_list_directory(self, test_client):
response = test_client.get("/api/files/list", params={"path": "/media"})
@@ -486,6 +433,7 @@ class TestFiles:
# --- Media index build ---
class TestMediaIndexApi:
def test_status_includes_build_progress(self, test_client, tmp_path):
index = MediaIndex(tmp_path / "index.sqlite")
@@ -586,9 +534,11 @@ class TestMediaIndexApi:
alive_calls["count"] += 1
return alive_calls["count"] <= 2
with patch("media_library_viewer_api.routers.media._pid_is_alive", side_effect=fake_pid_is_alive), patch(
"media_library_viewer_api.routers.media.os.killpg"
) as killpg, patch("media_library_viewer_api.routers.media.time.sleep", return_value=None):
with (
patch("media_library_viewer_api.routers.media._pid_is_alive", side_effect=fake_pid_is_alive),
patch("media_library_viewer_api.routers.media.os.killpg") as killpg,
patch("media_library_viewer_api.routers.media.time.sleep", return_value=None),
):
try:
response = test_client.post("/api/media/force-stop")
assert response.status_code == 202
@@ -615,6 +565,7 @@ class TestMediaIndexApi:
# --- Jobs ---
class TestJobs:
def test_list_templates(self, test_client):
response = test_client.get("/api/jobs/templates")
@@ -645,37 +596,21 @@ class TestJobs:
# --- Monitoring ---
class TestMonitoring:
def _ensure_machine(self):
store = app.dependency_overrides[get_settings_store]()
if not store.list_machines():
store.upsert_machine({
"name": "Test Machine",
"mode": "ssh",
"enabled": True,
"services": ["monitoring", "files", "jellyfin"],
"host": "test-host",
"username": "test-user",
})
def test_status(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="running pid=1234\n", stderr=""
)
response = test_client.get("/api/monitoring/status")
assert response.status_code == 200
assert "running" in response.json()["status"]
def test_metrics_empty(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="", stderr=""
)
response = test_client.get("/api/monitoring/metrics")
assert response.status_code == 200
data = response.json()
assert data["samples"] == []
store.upsert_machine(
{
"name": "Test Machine",
"mode": "ssh",
"enabled": True,
"services": ["monitoring", "files", "jellyfin"],
"host": "test-host",
"username": "test-user",
}
)
def test_disk(self, test_client, mock_ssh):
self._ensure_machine()
@@ -685,32 +620,162 @@ class TestMonitoring:
stdout='{"filesystem":"/dev/sda1","size":1000000000,"used":500000000,"available":500000000,"used_pct":"50%","mount":"/"}',
stderr="",
)
response = test_client.get("/api/monitoring/disk")
with patch("media_library_viewer_api.services.monitoring_actions.build_machine_client", return_value=mock_ssh):
response = test_client.get("/api/monitoring/disk")
assert response.status_code == 200
data = response.json()
assert data["used_pct"] == "50%"
def test_start(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="started pid=5678\n", stderr=""
)
response = test_client.post("/api/monitoring/start")
def test_prometheus_targets_empty(self, test_client):
response = test_client.get("/api/monitoring/prometheus-targets")
assert response.status_code == 200
assert "started" in response.json()["message"]
assert response.json() == []
def test_stop(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="stopped pid=5678\n", stderr=""
def test_prometheus_targets_returns_enabled_ssh_node_exporter(self, test_client):
store = app.dependency_overrides[get_settings_store]()
store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"enabled": True,
"services": ["monitoring"],
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9200,
"node_exporter_scrape_host": "1.2.3.4",
}
)
response = test_client.post("/api/monitoring/stop")
response = test_client.get("/api/monitoring/prometheus-targets")
assert response.status_code == 200
data = response.json()
assert len(data) == 1
assert data[0]["targets"] == ["1.2.3.4:9200"]
assert data[0]["labels"]["job"] == "node-exporter-remote"
def test_restart(self, test_client, mock_ssh):
self._ensure_machine()
mock_ssh.run.return_value = CommandResult(
command="...", exit_status=0, stdout="stopped pid=5678\nstarted pid=9999\n", stderr=""
class TestSettingsMachines:
def test_post_machine_rewrites_prometheus_targets(self, test_client):
with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets:
with patch("media_library_viewer_api.routers.settings._validate_saved_machine_ssh"):
response = test_client.post(
"/api/settings/machines",
json={
"name": "remote1",
"mode": "ssh",
"enabled": True,
"services": ["monitoring"],
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9200,
"node_exporter_scrape_host": "1.2.3.4",
},
)
assert response.status_code == 201
write_targets.assert_called_once()
def test_delete_machine_rewrites_prometheus_targets(self, test_client):
store = app.dependency_overrides[get_settings_store]()
machine = store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"enabled": True,
"services": ["monitoring"],
"host": "10.0.0.5",
"username": "u",
}
)
response = test_client.post("/api/monitoring/restart")
with patch("media_library_viewer_api.routers.settings.write_prometheus_targets") as write_targets:
response = test_client.delete(f"/api/settings/machines/{machine['id']}")
assert response.status_code == 200
write_targets.assert_called_once()
class TestAlertmanager:
def test_alerts_endpoint_when_alertmanager_unreachable(self, test_client):
with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
session = MagicMock()
session.get.side_effect = Exception("connection refused")
mock_client.return_value = (session, "http://alertmanager:9093")
response = test_client.get("/api/monitoring/alerts")
assert response.status_code == 200
data = response.json()
assert data["total"] == 0
assert data["error"] == "alertmanager_unreachable"
def test_alerts_endpoint_returns_summary(self, test_client):
with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
session = MagicMock()
session.get.return_value.json.return_value = {
"status": "success",
"data": [
{
"labels": {"alertname": "BackupJobFailed", "severity": "critical", "job_name": "test"},
"annotations": {"summary": "Backup failed", "description": "details"},
"startsAt": "2026-05-11T02:00:00Z",
"status": "firing",
}
],
}
session.get.return_value.raise_for_status = MagicMock()
mock_client.return_value = (session, "http://alertmanager:9093")
response = test_client.get("/api/monitoring/alerts")
assert response.status_code == 200
data = response.json()
assert data["total"] == 1
assert data["by_severity"]["critical"] == 1
assert data["alerts"][0]["name"] == "BackupJobFailed"
def test_alertmanager_status_endpoint_when_unreachable(self, test_client):
with patch("media_library_viewer_api.routers.monitoring._alertmanager_client") as mock_client:
session = MagicMock()
session.get.side_effect = Exception("connection refused")
mock_client.return_value = (session, "http://alertmanager:9093")
response = test_client.get("/api/monitoring/alertmanager-status")
assert response.status_code == 200
data = response.json()
assert data["up"] is False
class TestAlertmanagerWebhook:
"""Tests for POST /api/monitoring/alertmanager-webhook."""
def test_webhook_receives_payload_and_logs(self, test_client, caplog):
payload = {
"status": "firing",
"alerts": [
{
"status": "firing",
"labels": {"alertname": "BackupJobFailed", "severity": "critical"},
"annotations": {"summary": "Backup failed"},
"startsAt": "2026-06-13T10:00:00Z",
}
],
}
with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"):
response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload)
assert response.status_code == 200
assert response.json()["status"] == "received"
assert "Received Alertmanager webhook with 1 alert(s)" in caplog.text
def test_webhook_forwards_when_configured(self, test_client, caplog, monkeypatch):
payload = {"status": "resolved", "alerts": []}
forwarded = {"captured": False}
class FakeResponse:
def raise_for_status(self):
pass
def fake_post(url, json, timeout):
forwarded["captured"] = True
forwarded["url"] = url
forwarded["payload"] = json
return FakeResponse()
monkeypatch.setattr("requests.Session.post", lambda _self, url, json, timeout: fake_post(url, json, timeout))
with caplog.at_level("INFO", logger="media_library_viewer_api.routers.monitoring"):
response = test_client.post("/api/monitoring/alertmanager-webhook", json=payload)
assert response.status_code == 200
assert forwarded["captured"] is False
backend/tests/test_config.py
+1
View File
@@ -2,6 +2,7 @@
import os
from unittest.mock import patch
from media_library_viewer_api.config import Settings
backend/tests/test_domain_media.py
+8 -9
View File
@@ -1,16 +1,15 @@
"""Unit tests for domain/media.py normalization helpers."""
import pytest
from media_library_viewer_api.domain.media import (
first_media_source,
media_streams,
stream_value,
is_hdr_item,
format_date_added,
timestamp_date_added,
format_rate_bits_decimal,
normalize_media_item,
display_media_row,
first_media_source,
format_date_added,
format_rate_bits_decimal,
is_hdr_item,
media_streams,
normalize_media_item,
stream_value,
timestamp_date_added,
)
backend/tests/test_jobs.py
+1 -2
View File
@@ -1,7 +1,6 @@
"""Unit tests for jobs.py template rendering and safety."""
import pytest
from media_library_viewer_api.jobs import JOB_TEMPLATES, JobTemplate, run_job
from media_library_viewer_api.jobs import JOB_TEMPLATES, JobTemplate
class TestJobTemplate:
backend/tests/test_media_index.py
+3 -3
View File
@@ -1,9 +1,9 @@
"""Unit tests for the SQLite media index service."""
import pytest
import tempfile
from pathlib import Path
from typing import Any
import pytest
from media_library_viewer_api.services.media_index import (
MediaIndex,
MediaIndexBuildCancelled,
backend/tests/test_monitoring_actions.py
+8 -18
View File
@@ -3,7 +3,7 @@ from unittest.mock import MagicMock, patch
from media_library_viewer_api.services.monitoring_actions import poll_machine_snapshot
def test_poll_machine_snapshot_records_history_entries():
def test_poll_machine_snapshot_records_disk_lookup():
store = MagicMock()
machine = {
"id": "local",
@@ -17,14 +17,6 @@ def test_poll_machine_snapshot_records_history_entries():
"media_library_viewer_api.services.monitoring_actions.build_machine_client",
return_value=object(),
) as build_client,
patch(
"media_library_viewer_api.services.monitoring_actions.resource_collector_status",
return_value="running pid=123",
) as status_fn,
patch(
"media_library_viewer_api.services.monitoring_actions.read_resource_metrics",
return_value=[{"ts": 1.0}, {"ts": 2.0}],
) as metrics_fn,
patch(
"media_library_viewer_api.services.monitoring_actions.disk_space",
return_value={"mount": "/srv/media", "used_pct": "12.5%"},
@@ -33,14 +25,12 @@ def test_poll_machine_snapshot_records_history_entries():
result = poll_machine_snapshot(machine, store, metrics_limit=123, request_id="poll:test")
assert result["request_id"] == "poll:test"
assert result["metrics_samples"] == 2
assert result["disk_mount"] == "/srv/media"
build_client.assert_called_once_with(machine)
status_fn.assert_called_once()
metrics_fn.assert_called_once_with(build_client.return_value, max_lines=123)
assert result["actions"] == ["disk lookup"]
build_client.assert_called_once_with(machine, store)
disk_fn.assert_called_once_with(build_client.return_value, "/srv/media")
assert store.record_machine_action.call_count == 3
recorded_actions = [call.args[1] for call in store.record_machine_action.call_args_list]
assert recorded_actions == ["status lookup", "metrics read", "disk lookup for /srv/media"]
assert all(call.kwargs["request_id"] == "poll:test" for call in store.record_machine_action.call_args_list)
assert all(call.args[2] == "ok" for call in store.record_machine_action.call_args_list)
assert store.record_machine_action.call_count == 1
recorded_action = store.record_machine_action.call_args
assert recorded_action.args[1] == "disk lookup for /srv/media"
assert recorded_action.kwargs["request_id"] == "poll:test"
assert recorded_action.args[2] == "ok"
backend/tests/test_observability.py
+37
View File
@@ -0,0 +1,37 @@
"""Tests for observability metrics helpers."""
import time
from prometheus_client import REGISTRY
from media_library_viewer_api.observability import record_backup_run
def _metric_samples(metric_name):
return {
tuple(s.labels.values()): s.value
for family in REGISTRY.collect()
for s in family.samples
if s.name == metric_name
}
def test_record_backup_run_increments_counter():
record_backup_run("job-a", "success", success=True)
record_backup_run("job-a", "failure")
record_backup_run("job-b", "success", success=True)
samples = _metric_samples("manage_backup_runs_total")
assert samples[("job-a", "success")] >= 1.0
assert samples[("job-a", "failure")] >= 1.0
assert samples[("job-b", "success")] >= 1.0
def test_record_backup_run_sets_last_success_timestamp():
before = time.time()
record_backup_run("job-ts", "success", success=True)
after = time.time()
samples = _metric_samples("manage_backup_runs_last_success_timestamp")
value = samples[("job-ts",)]
assert before <= value <= after
backend/tests/test_path_utils.py
+3 -2
View File
@@ -1,6 +1,5 @@
"""Unit tests for path_utils.py — path resolution logic."""
import pytest
from media_library_viewer_api.path_utils import (
apply_remote_path_prefix,
map_path_to_media_root,
@@ -32,7 +31,9 @@ class TestApplyRemotePathPrefix:
assert apply_remote_path_prefix("/media/file.mkv", "/srv/") == "/srv/media/file.mkv"
def test_path_with_spaces(self):
assert apply_remote_path_prefix("/media/My Movie (2024)/file.mkv", "/srv") == "/srv/media/My Movie (2024)/file.mkv"
assert (
apply_remote_path_prefix("/media/My Movie (2024)/file.mkv", "/srv") == "/srv/media/My Movie (2024)/file.mkv"
)
class TestMapPathToMediaRoot:
backend/tests/test_ssh_client.py
+3 -1
View File
@@ -7,7 +7,9 @@ from media_library_viewer_api.clients.ssh import RemoteSSHClient
def test_connect_uses_existing_known_hosts_without_reprobing(tmp_path):
known_hosts_path = tmp_path / "known_hosts"
known_hosts_path.write_text("example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAFAKE\n")
known_hosts_path.write_text(
"example.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID8tdNkXNS1EWbkjb2XxFHQXRrpYzl0MXaVLRmVrbmDZ\n"
)
ssh_client = MagicMock()
ssh_client.connect.return_value = None
backend/tests/test_targets.py
+109
View File
@@ -0,0 +1,109 @@
"""Tests for Prometheus file-based service discovery target generation."""
from pathlib import Path
import pytest
from media_library_viewer_api.services.settings_store import SettingsStore
from media_library_viewer_api.services.targets import (
build_node_exporter_targets,
write_prometheus_targets,
)
@pytest.fixture
def store(tmp_path: Path) -> SettingsStore:
db = SettingsStore(tmp_path / "settings.sqlite")
db.init_schema()
return db
class TestBuildNodeExporterTargets:
def test_disabled_machine_excluded(self, store: SettingsStore):
store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": False,
"node_exporter_port": 9200,
}
)
assert build_node_exporter_targets(store) == []
def test_ssh_enabled_machine_included(self, store: SettingsStore):
machine = store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9200,
"node_exporter_scrape_host": "1.2.3.4",
}
)
targets = build_node_exporter_targets(store)
assert len(targets) == 1
assert targets[0]["targets"] == ["1.2.3.4:9200"]
assert targets[0]["labels"]["machine_id"] == machine["id"]
assert targets[0]["labels"]["machine_name"] == "remote1"
assert targets[0]["labels"]["job"] == "node-exporter-remote"
def test_scrape_host_defaults_to_machine_host(self, store: SettingsStore):
store.upsert_machine(
{
"name": "remote2",
"mode": "ssh",
"host": "remote2.example.com",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9100,
}
)
targets = build_node_exporter_targets(store)
assert targets[0]["targets"] == ["remote2.example.com:9100"]
def test_local_machine_excluded(self, store: SettingsStore):
store.upsert_machine(
{
"name": "This machine",
"mode": "local",
"host": "localhost",
"username": "",
"node_exporter_enabled": True,
}
)
assert build_node_exporter_targets(store) == []
def test_missing_host_excluded(self, store: SettingsStore):
store.upsert_machine(
{
"name": "remote3",
"mode": "ssh",
"host": "",
"username": "u",
"node_exporter_enabled": True,
}
)
assert build_node_exporter_targets(store) == []
class TestWritePrometheusTargets:
def test_writes_valid_json(self, store: SettingsStore, tmp_path: Path):
store.upsert_machine(
{
"name": "remote1",
"mode": "ssh",
"host": "10.0.0.5",
"username": "u",
"node_exporter_enabled": True,
"node_exporter_port": 9200,
}
)
file_path = write_prometheus_targets(store, tmp_path)
assert file_path.exists()
assert file_path.name == "node_exporter_targets.json"
targets = build_node_exporter_targets(store)
assert len(targets) == 1
backend/tests/test_utils.py
+6 -7
View File
@@ -1,17 +1,16 @@
"""Unit tests for utils.py formatting helpers."""
import pytest
from media_library_viewer_api.utils import (
ticks_to_minutes,
ffprobe_format_summary,
format_bitrate,
format_duration,
human_size,
is_known_video_file,
format_duration,
format_bitrate,
ffprobe_format_summary,
summarize_video_streams,
summarize_audio_streams,
summarize_subtitle_streams,
summarize_streams,
summarize_subtitle_streams,
summarize_video_streams,
ticks_to_minutes,
timestamp_to_local,
)