"""Small dependency-free Prometheus exposition for the single-node appliance.""" from __future__ import annotations import os import threading from collections import defaultdict from collections.abc import Iterable from datetime import UTC, datetime from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from backup_tool.config import Settings from backup_tool.db.models import Backup, Execution, Repository, Schedule from backup_tool.execution import ACTIVE_STATES class Metrics: def __init__(self) -> None: self._lock = threading.Lock() self._requests: dict[tuple[str, str, int], int] = defaultdict(int) self._durations: dict[tuple[str, str], tuple[int, float]] = {} def observe_request(self, method: str, path: str, status: int, duration_seconds: float) -> None: route = path if path in {"/livez", "/readyz", "/metrics"} else "/api" with self._lock: self._requests[(method, route, status)] += 1 count, total = self._durations.get((method, route), (0, 0.0)) self._durations[(method, route)] = (count + 1, total + duration_seconds) def render(self, operational: Iterable[tuple[str, float]]) -> str: lines = [ "# HELP backup_tool_http_requests_total HTTP requests handled by the web role.", "# TYPE backup_tool_http_requests_total counter", ] with self._lock: for (method, path, status), value in sorted(self._requests.items()): labels = f'method="{method}",path="{path}",status="{status}"' lines.append(f"backup_tool_http_requests_total{{{labels}}} {value}") lines.extend( [ "# HELP backup_tool_http_request_duration_seconds HTTP request duration.", "# TYPE backup_tool_http_request_duration_seconds summary", ] ) for (method, path), (count, total) in sorted(self._durations.items()): labels = f'method="{method}",path="{path}"' lines.append(f"backup_tool_http_request_duration_seconds_count{{{labels}}} {count}") lines.append( f"backup_tool_http_request_duration_seconds_sum{{{labels}}} {total:.6f}" ) lines.extend(f"{name} {value}" for name, value in operational) return "\n".join(lines) + "\n" async def collect_operational_metrics( settings: Settings, db: AsyncSession ) -> list[tuple[str, float]]: now = datetime.now(UTC) active = await db.scalar( select(func.count()).select_from(Execution).where(Execution.state.in_(ACTIVE_STATES)) ) stale = await db.scalar( select(func.count()) .select_from(Execution) .where(Execution.lease_expires_at.is_not(None), Execution.lease_expires_at < now) ) failed = await db.scalar( select(func.count()).select_from(Execution).where(Execution.state == "failed") ) corrupt = await db.scalar( select(func.count()).select_from(Backup).where(Backup.integrity == "corrupt") ) schedule_lag = await db.scalar( select(func.min(Schedule.next_nominal_at)).where( Schedule.enabled, Schedule.next_nominal_at.is_not(None) ) ) values = [ ("backup_tool_active_executions", active or 0), ("backup_tool_stale_execution_leases", stale or 0), ("backup_tool_failed_executions", failed or 0), ("backup_tool_corrupt_backups", corrupt or 0), ( "backup_tool_schedule_lag_seconds", max(0.0, (now - schedule_lag).total_seconds()) if schedule_lag is not None else 0.0, ), ] roots = list(settings.repository_roots) + list(settings.restore_roots) for index, root in enumerate(roots): try: stats = os.statvfs(root) except OSError: continue name = f'backup_tool_filesystem_free_bytes{{root="{index}"}}' values.append((name, stats.f_bavail * stats.f_frsize)) unavailable = await db.scalar( select(func.count()).select_from(Repository).where(Repository.state == "unavailable") ) values.append(("backup_tool_unavailable_repositories", unavailable or 0)) return values