103 lines
4.2 KiB
Python
103 lines
4.2 KiB
Python
"""Small dependency-free Prometheus exposition for the single-node appliance."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import threading
|
|
from collections import defaultdict
|
|
from collections.abc import Iterable
|
|
from datetime import UTC, datetime
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from backup_tool.config import Settings
|
|
from backup_tool.db.models import Backup, Execution, Repository, Schedule
|
|
from backup_tool.execution import ACTIVE_STATES
|
|
|
|
|
|
class Metrics:
|
|
def __init__(self) -> None:
|
|
self._lock = threading.Lock()
|
|
self._requests: dict[tuple[str, str, int], int] = defaultdict(int)
|
|
self._durations: dict[tuple[str, str], tuple[int, float]] = {}
|
|
|
|
def observe_request(self, method: str, path: str, status: int, duration_seconds: float) -> None:
|
|
route = path if path in {"/livez", "/readyz", "/metrics"} else "/api"
|
|
with self._lock:
|
|
self._requests[(method, route, status)] += 1
|
|
count, total = self._durations.get((method, route), (0, 0.0))
|
|
self._durations[(method, route)] = (count + 1, total + duration_seconds)
|
|
|
|
def render(self, operational: Iterable[tuple[str, float]]) -> str:
|
|
lines = [
|
|
"# HELP backup_tool_http_requests_total HTTP requests handled by the web role.",
|
|
"# TYPE backup_tool_http_requests_total counter",
|
|
]
|
|
with self._lock:
|
|
for (method, path, status), value in sorted(self._requests.items()):
|
|
labels = f'method="{method}",path="{path}",status="{status}"'
|
|
lines.append(f"backup_tool_http_requests_total{{{labels}}} {value}")
|
|
lines.extend(
|
|
[
|
|
"# HELP backup_tool_http_request_duration_seconds HTTP request duration.",
|
|
"# TYPE backup_tool_http_request_duration_seconds summary",
|
|
]
|
|
)
|
|
for (method, path), (count, total) in sorted(self._durations.items()):
|
|
labels = f'method="{method}",path="{path}"'
|
|
lines.append(f"backup_tool_http_request_duration_seconds_count{{{labels}}} {count}")
|
|
lines.append(
|
|
f"backup_tool_http_request_duration_seconds_sum{{{labels}}} {total:.6f}"
|
|
)
|
|
lines.extend(f"{name} {value}" for name, value in operational)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
async def collect_operational_metrics(
|
|
settings: Settings, db: AsyncSession
|
|
) -> list[tuple[str, float]]:
|
|
now = datetime.now(UTC)
|
|
active = await db.scalar(
|
|
select(func.count()).select_from(Execution).where(Execution.state.in_(ACTIVE_STATES))
|
|
)
|
|
stale = await db.scalar(
|
|
select(func.count())
|
|
.select_from(Execution)
|
|
.where(Execution.lease_expires_at.is_not(None), Execution.lease_expires_at < now)
|
|
)
|
|
failed = await db.scalar(
|
|
select(func.count()).select_from(Execution).where(Execution.state == "failed")
|
|
)
|
|
corrupt = await db.scalar(
|
|
select(func.count()).select_from(Backup).where(Backup.integrity == "corrupt")
|
|
)
|
|
schedule_lag = await db.scalar(
|
|
select(func.min(Schedule.next_nominal_at)).where(
|
|
Schedule.enabled, Schedule.next_nominal_at.is_not(None)
|
|
)
|
|
)
|
|
values = [
|
|
("backup_tool_active_executions", active or 0),
|
|
("backup_tool_stale_execution_leases", stale or 0),
|
|
("backup_tool_failed_executions", failed or 0),
|
|
("backup_tool_corrupt_backups", corrupt or 0),
|
|
(
|
|
"backup_tool_schedule_lag_seconds",
|
|
max(0.0, (now - schedule_lag).total_seconds()) if schedule_lag is not None else 0.0,
|
|
),
|
|
]
|
|
roots = list(settings.repository_roots) + list(settings.restore_roots)
|
|
for index, root in enumerate(roots):
|
|
try:
|
|
stats = os.statvfs(root)
|
|
except OSError:
|
|
continue
|
|
name = f'backup_tool_filesystem_free_bytes{{root="{index}"}}'
|
|
values.append((name, stats.f_bavail * stats.f_frsize))
|
|
unavailable = await db.scalar(
|
|
select(func.count()).select_from(Repository).where(Repository.state == "unavailable")
|
|
)
|
|
values.append(("backup_tool_unavailable_repositories", unavailable or 0))
|
|
return values
|