manage/backend/clients/resources.py

"""Remote resource collection helpers.

The app does not require Prometheus, Netdata, or sysstat. Instead it can install
and manage a tiny POSIX-sh collector under /tmp on the remote server. The
collector samples Linux /proc and /sys counters every 10 seconds and appends JSON
Lines. This module starts/stops the collector and reads those JSONL samples.
"""

from __future__ import annotations

import json
import shlex
from dataclasses import dataclass
from typing import Any

from clients.ssh import RemoteSSHClient

# POSIX shell script copied to the remote server by start_resource_collector().
# Keep this script bash-free because many NAS/media servers have minimal shells.
COLLECTOR_SCRIPT = r'''#!/bin/sh
set -u
OUT="${1:-/tmp/media_library_viewer_metrics.jsonl}"
INTERVAL="${2:-10}"
RETENTION_SECONDS="${3:-604800}"
MAX_LINES="${4:-70000}"
PRUNE_EVERY_SAMPLES="${5:-60}"
mkdir -p "$(dirname "$OUT")"

echo "collector starting at $(date -Is 2>/dev/null || date), interval=${INTERVAL}s, retention=${RETENTION_SECONDS}s, max_lines=${MAX_LINES}, out=${OUT}"

read_cpu() {
  awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8+$9+$10, $5+$6, $6}' /proc/stat
}

read_mem_pct() {
  awk '
    /^MemTotal:/ {total=$2}
    /^MemAvailable:/ {avail=$2}
    END {if (total > 0) printf "%.2f", (total-avail)*100/total; else printf "0"}
  ' /proc/meminfo
}

read_net_bytes() {
  awk '
    NR > 2 {
      split($0, parts, ":")
      iface = parts[1]
      stats = parts[2]
      gsub(/^[ \t]+|[ \t]+$/, "", iface)
      gsub(/^[ \t]+|[ \t]+$/, "", stats)
      if (iface == "lo" || iface == "" || stats == "") next
      split(stats, values, /[ \t]+/)
      # /proc/net/dev after the colon:
      # receive bytes are field 1, transmit bytes are field 9.
      # Trim the stats block before split; otherwise leading whitespace can make
      # values[1] empty in some awk implementations, resulting in zero rates.
      rx += values[1] + 0
      tx += values[9] + 0
    }
    END {printf "%.0f %.0f", rx, tx}
  ' /proc/net/dev
}

read_disk_bytes() {
  read_sectors=0
  written_sectors=0
  for dev in /sys/block/*; do
    [ -r "$dev/stat" ] || continue
    name="$(basename "$dev")"
    case "$name" in
      loop*|ram*|fd*|sr*) continue ;;
    esac
    # Linux /sys/block/<dev>/stat fields: 3=sectors read, 7=sectors written.
    # Use POSIX sh parsing instead of bash arrays so this works on minimal systems.
    set -- $(cat "$dev/stat")
    sectors_read="${3:-0}"
    sectors_written="${7:-0}"
    read_sectors=$((read_sectors + sectors_read))
    written_sectors=$((written_sectors + sectors_written))
  done
  printf "%s %s" "$((read_sectors * 512))" "$((written_sectors * 512))"
}

set -- $(read_cpu)
prev_total="${1:-0}"
prev_idle="${2:-0}"
prev_iowait="${3:-0}"
set -- $(read_net_bytes)
prev_rx="${1:-0}"
prev_tx="${2:-0}"
set -- $(read_disk_bytes)
prev_disk_read="${1:-0}"
prev_disk_write="${2:-0}"
prev_ts="$(date +%s)"
sample_count=0

prune_metrics_file() {
  [ -f "$OUT" ] || return 0
  cutoff="$1"
  tmp="${OUT}.$$.tmp"
  awk -v cutoff="$cutoff" '
    match($0, /"ts":[0-9]+/) {
      ts = substr($0, RSTART + 5, RLENGTH - 5)
      if (ts >= cutoff) print $0
    }
  ' "$OUT" | tail -n "$MAX_LINES" > "$tmp" && mv "$tmp" "$OUT"
  rm -f "$tmp"
}

while true; do
  sleep "$INTERVAL"
  now_ts="$(date +%s)"
  dt=$((now_ts - prev_ts))
  if [ "$dt" -le 0 ]; then dt=1; fi

  set -- $(read_cpu)
  total="${1:-0}"
  idle="${2:-0}"
  iowait="${3:-0}"
  set -- $(read_net_bytes)
  rx="${1:-0}"
  tx="${2:-0}"
  set -- $(read_disk_bytes)
  disk_read="${1:-0}"
  disk_write="${2:-0}"
  mem_pct="$(read_mem_pct)"

  total_delta=$((total - prev_total))
  idle_delta=$((idle - prev_idle))
  iowait_delta=$((iowait - prev_iowait))
  rx_delta=$((rx - prev_rx))
  tx_delta=$((tx - prev_tx))
  disk_read_delta=$((disk_read - prev_disk_read))
  disk_write_delta=$((disk_write - prev_disk_write))

  cpu_pct="$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN {if (total > 0) printf "%.2f", (total-idle)*100/total; else printf "0"}')"
  iowait_pct="$(awk -v total="$total_delta" -v iow="$iowait_delta" 'BEGIN {if (total > 0) printf "%.2f", iow*100/total; else printf "0"}')"
  rx_bytes_per_sec="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
  tx_bytes_per_sec="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
  rx_bps="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
  tx_bps="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
  disk_read_bps="$(awk -v bytes="$disk_read_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
  disk_write_bps="$(awk -v bytes="$disk_write_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"

  printf '{"ts":%s,"cpu_pct":%s,"iowait_pct":%s,"mem_pct":%s,"net_rx_bytes_per_sec":%s,"net_tx_bytes_per_sec":%s,"net_rx_bps":%s,"net_tx_bps":%s,"disk_read_bps":%s,"disk_write_bps":%s}\n' \
    "$now_ts" "$cpu_pct" "$iowait_pct" "$mem_pct" "$rx_bytes_per_sec" "$tx_bytes_per_sec" "$rx_bps" "$tx_bps" "$disk_read_bps" "$disk_write_bps" >> "$OUT"

  sample_count=$((sample_count + 1))
  if [ $((sample_count % PRUNE_EVERY_SAMPLES)) -eq 0 ]; then
    prune_metrics_file "$((now_ts - RETENTION_SECONDS))"
  fi

  prev_total="$total"
  prev_idle="$idle"
  prev_iowait="$iowait"
  prev_rx="$rx"
  prev_tx="$tx"
  prev_disk_read="$disk_read"
  prev_disk_write="$disk_write"
  prev_ts="$now_ts"
done
'''


@dataclass(frozen=True)
class ResourceMonitorPaths:
    """Remote file locations used by the lightweight resource collector."""

    metrics_file: str = "/tmp/media_library_viewer_metrics.jsonl"
    pid_file: str = "/tmp/media_library_viewer_metrics.pid"
    script_file: str = "/tmp/media_library_viewer_metrics_collector.sh"
    log_file: str = "/tmp/media_library_viewer_metrics.log"


def start_resource_collector(
    ssh: RemoteSSHClient,
    interval_seconds: int = 10,
    retention_seconds: int = 7 * 24 * 60 * 60,
    max_lines: int = 70_000,
    paths: ResourceMonitorPaths = ResourceMonitorPaths(),
) -> str:
    """Install and start the remote metrics collector if it is not running.

    Starting a fresh collector removes old metrics/log files because schema
    changes during development can otherwise leave mixed JSONL records behind.
    The collector prunes its own metrics file to 7 days / max_lines.
    """
    command = f"""
cat > {shlex.quote(paths.script_file)} <<'MLV_RESOURCE_COLLECTOR'
{COLLECTOR_SCRIPT}
MLV_RESOURCE_COLLECTOR
chmod +x {shlex.quote(paths.script_file)}
if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
  echo "already running pid=$(cat {shlex.quote(paths.pid_file)})"
else
  rm -f {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)}
  nohup {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {int(interval_seconds)} {int(retention_seconds)} {int(max_lines)} >> {shlex.quote(paths.log_file)} 2>&1 &
  echo $! > {shlex.quote(paths.pid_file)}
  echo "started pid=$(cat {shlex.quote(paths.pid_file)})"
fi
"""
    result = ssh.run(command, timeout=20)
    if result.exit_status != 0:
        raise RuntimeError(result.stderr or result.stdout or "failed to start resource collector")
    return result.stdout.strip()


def stop_resource_collector(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
    """Stop the remote collector process if the pid file points to one."""
    command = f"""
if [ -f {shlex.quote(paths.pid_file)} ]; then
  pid="$(cat {shlex.quote(paths.pid_file)})"
  if kill -0 "$pid" 2>/dev/null; then
    kill "$pid"
    echo "stopped pid=$pid"
  else
    echo "not running"
  fi
  rm -f {shlex.quote(paths.pid_file)}
else
  echo "not running"
fi
"""
    result = ssh.run(command, timeout=20)
    if result.exit_status != 0:
        raise RuntimeError(result.stderr or result.stdout or "failed to stop resource collector")
    return result.stdout.strip()


def restart_resource_collector(
    ssh: RemoteSSHClient,
    interval_seconds: int = 10,
    retention_seconds: int = 7 * 24 * 60 * 60,
    max_lines: int = 70_000,
    paths: ResourceMonitorPaths = ResourceMonitorPaths(),
) -> str:
    stop_message = stop_resource_collector(ssh, paths)
    start_message = start_resource_collector(ssh, interval_seconds, retention_seconds, max_lines, paths)
    return f"{stop_message}\n{start_message}"


def resource_collector_status(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
    """Return a short human-readable status string for the dashboard."""
    command = f"""
if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
  echo "running pid=$(cat {shlex.quote(paths.pid_file)})"
else
  echo "not running"
fi
"""
    result = ssh.run(command, timeout=10)
    if result.exit_status != 0:
        raise RuntimeError(result.stderr or result.stdout or "failed to check collector status")
    return result.stdout.strip()


def resource_collector_debug_info(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
    """Collect remote diagnostics for troubleshooting missing metrics."""
    command = f"""
echo "status:"
if [ -f {shlex.quote(paths.pid_file)} ]; then
  pid="$(cat {shlex.quote(paths.pid_file)})"
  echo "pid_file=$pid"
  if kill -0 "$pid" 2>/dev/null; then echo "process=running"; else echo "process=not-running"; fi
else
  echo "pid_file=missing"
fi

echo "files:"
ls -l {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} 2>&1 || true

echo "sample_count:"
if [ -f {shlex.quote(paths.metrics_file)} ]; then wc -l < {shlex.quote(paths.metrics_file)}; else echo 0; fi

echo "last_samples:"
if [ -f {shlex.quote(paths.metrics_file)} ]; then tail -n 5 {shlex.quote(paths.metrics_file)}; fi

echo "log_tail:"
if [ -f {shlex.quote(paths.log_file)} ]; then tail -n 40 {shlex.quote(paths.log_file)}; fi

echo "netdev_snapshot:"
cat /proc/net/dev 2>&1 || true
"""
    result = ssh.run(command, timeout=20)
    return (result.stdout or "") + (result.stderr or "")


def read_resource_metrics(ssh: RemoteSSHClient, max_lines: int = 1000, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> list[dict[str, Any]]:
    """Read recent JSONL metric samples from the remote collector file."""
    command = f"test -f {shlex.quote(paths.metrics_file)} && tail -n {int(max_lines)} {shlex.quote(paths.metrics_file)} || true"
    result = ssh.run(command, timeout=20)
    if result.exit_status != 0:
        raise RuntimeError(result.stderr or result.stdout or "failed to read resource metrics")
    rows = []
    for line in result.stdout.splitlines():
        line = line.strip()
        if not line:
            continue
        try:
            rows.append(json.loads(line))
        except json.JSONDecodeError:
            continue
    return rows


def disk_space(ssh: RemoteSSHClient, path: str = "/") -> dict[str, Any]:
    """Return df information for the filesystem containing ``path``."""
    command = (
        "df -P -B1 -- "
        + shlex.quote(path or "/")
        + " | awk 'NR==2 {printf \"{\\\"filesystem\\\":\\\"%s\\\",\\\"size\\\":%s,\\\"used\\\":%s,\\\"available\\\":%s,\\\"used_pct\\\":\\\"%s\\\",\\\"mount\\\":\\\"%s\\\"}\", $1,$2,$3,$4,$5,$6}'"
    )
    result = ssh.run(command, timeout=20)
    if result.exit_status != 0 or not result.stdout.strip():
        raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
    return json.loads(result.stdout)