"""Remote resource collection helpers. The app does not require Prometheus, Netdata, or sysstat. Instead it can install and manage a tiny POSIX-sh collector under /tmp on the remote server. The collector samples Linux /proc and /sys counters every 10 seconds and appends JSON Lines. This module starts/stops the collector and reads those JSONL samples. """ from __future__ import annotations import json import shlex from dataclasses import dataclass from typing import Any from clients.ssh import RemoteSSHClient # POSIX shell script copied to the remote server by start_resource_collector(). # Keep this script bash-free because many NAS/media servers have minimal shells. COLLECTOR_SCRIPT = r'''#!/bin/sh set -u OUT="${1:-/tmp/media_library_viewer_metrics.jsonl}" INTERVAL="${2:-10}" RETENTION_SECONDS="${3:-604800}" MAX_LINES="${4:-70000}" PRUNE_EVERY_SAMPLES="${5:-60}" mkdir -p "$(dirname "$OUT")" echo "collector starting at $(date -Is 2>/dev/null || date), interval=${INTERVAL}s, retention=${RETENTION_SECONDS}s, max_lines=${MAX_LINES}, out=${OUT}" read_cpu() { awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8+$9+$10, $5+$6, $6}' /proc/stat } read_mem_pct() { awk ' /^MemTotal:/ {total=$2} /^MemAvailable:/ {avail=$2} END {if (total > 0) printf "%.2f", (total-avail)*100/total; else printf "0"} ' /proc/meminfo } read_net_bytes() { awk ' NR > 2 { split($0, parts, ":") iface = parts[1] stats = parts[2] gsub(/^[ \t]+|[ \t]+$/, "", iface) gsub(/^[ \t]+|[ \t]+$/, "", stats) if (iface == "lo" || iface == "" || stats == "") next split(stats, values, /[ \t]+/) # /proc/net/dev after the colon: # receive bytes are field 1, transmit bytes are field 9. # Trim the stats block before split; otherwise leading whitespace can make # values[1] empty in some awk implementations, resulting in zero rates. rx += values[1] + 0 tx += values[9] + 0 } END {printf "%.0f %.0f", rx, tx} ' /proc/net/dev } read_disk_bytes() { read_sectors=0 written_sectors=0 for dev in /sys/block/*; do [ -r "$dev/stat" ] || continue name="$(basename "$dev")" case "$name" in loop*|ram*|fd*|sr*) continue ;; esac # Linux /sys/block//stat fields: 3=sectors read, 7=sectors written. # Use POSIX sh parsing instead of bash arrays so this works on minimal systems. set -- $(cat "$dev/stat") sectors_read="${3:-0}" sectors_written="${7:-0}" read_sectors=$((read_sectors + sectors_read)) written_sectors=$((written_sectors + sectors_written)) done printf "%s %s" "$((read_sectors * 512))" "$((written_sectors * 512))" } set -- $(read_cpu) prev_total="${1:-0}" prev_idle="${2:-0}" prev_iowait="${3:-0}" set -- $(read_net_bytes) prev_rx="${1:-0}" prev_tx="${2:-0}" set -- $(read_disk_bytes) prev_disk_read="${1:-0}" prev_disk_write="${2:-0}" prev_ts="$(date +%s)" sample_count=0 prune_metrics_file() { [ -f "$OUT" ] || return 0 cutoff="$1" tmp="${OUT}.$$.tmp" awk -v cutoff="$cutoff" ' match($0, /"ts":[0-9]+/) { ts = substr($0, RSTART + 5, RLENGTH - 5) if (ts >= cutoff) print $0 } ' "$OUT" | tail -n "$MAX_LINES" > "$tmp" && mv "$tmp" "$OUT" rm -f "$tmp" } while true; do sleep "$INTERVAL" now_ts="$(date +%s)" dt=$((now_ts - prev_ts)) if [ "$dt" -le 0 ]; then dt=1; fi set -- $(read_cpu) total="${1:-0}" idle="${2:-0}" iowait="${3:-0}" set -- $(read_net_bytes) rx="${1:-0}" tx="${2:-0}" set -- $(read_disk_bytes) disk_read="${1:-0}" disk_write="${2:-0}" mem_pct="$(read_mem_pct)" total_delta=$((total - prev_total)) idle_delta=$((idle - prev_idle)) iowait_delta=$((iowait - prev_iowait)) rx_delta=$((rx - prev_rx)) tx_delta=$((tx - prev_tx)) disk_read_delta=$((disk_read - prev_disk_read)) disk_write_delta=$((disk_write - prev_disk_write)) cpu_pct="$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN {if (total > 0) printf "%.2f", (total-idle)*100/total; else printf "0"}')" iowait_pct="$(awk -v total="$total_delta" -v iow="$iowait_delta" 'BEGIN {if (total > 0) printf "%.2f", iow*100/total; else printf "0"}')" rx_bytes_per_sec="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')" tx_bytes_per_sec="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')" rx_bps="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')" tx_bps="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')" disk_read_bps="$(awk -v bytes="$disk_read_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')" disk_write_bps="$(awk -v bytes="$disk_write_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')" printf '{"ts":%s,"cpu_pct":%s,"iowait_pct":%s,"mem_pct":%s,"net_rx_bytes_per_sec":%s,"net_tx_bytes_per_sec":%s,"net_rx_bps":%s,"net_tx_bps":%s,"disk_read_bps":%s,"disk_write_bps":%s}\n' \ "$now_ts" "$cpu_pct" "$iowait_pct" "$mem_pct" "$rx_bytes_per_sec" "$tx_bytes_per_sec" "$rx_bps" "$tx_bps" "$disk_read_bps" "$disk_write_bps" >> "$OUT" sample_count=$((sample_count + 1)) if [ $((sample_count % PRUNE_EVERY_SAMPLES)) -eq 0 ]; then prune_metrics_file "$((now_ts - RETENTION_SECONDS))" fi prev_total="$total" prev_idle="$idle" prev_iowait="$iowait" prev_rx="$rx" prev_tx="$tx" prev_disk_read="$disk_read" prev_disk_write="$disk_write" prev_ts="$now_ts" done ''' @dataclass(frozen=True) class ResourceMonitorPaths: """Remote file locations used by the lightweight resource collector.""" metrics_file: str = "/tmp/media_library_viewer_metrics.jsonl" pid_file: str = "/tmp/media_library_viewer_metrics.pid" script_file: str = "/tmp/media_library_viewer_metrics_collector.sh" log_file: str = "/tmp/media_library_viewer_metrics.log" def start_resource_collector( ssh: RemoteSSHClient, interval_seconds: int = 10, retention_seconds: int = 7 * 24 * 60 * 60, max_lines: int = 70_000, paths: ResourceMonitorPaths = ResourceMonitorPaths(), ) -> str: """Install and start the remote metrics collector if it is not running. Starting a fresh collector removes old metrics/log files because schema changes during development can otherwise leave mixed JSONL records behind. The collector prunes its own metrics file to 7 days / max_lines. """ command = f""" cat > {shlex.quote(paths.script_file)} <<'MLV_RESOURCE_COLLECTOR' {COLLECTOR_SCRIPT} MLV_RESOURCE_COLLECTOR chmod +x {shlex.quote(paths.script_file)} if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then echo "already running pid=$(cat {shlex.quote(paths.pid_file)})" else rm -f {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} nohup {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {int(interval_seconds)} {int(retention_seconds)} {int(max_lines)} >> {shlex.quote(paths.log_file)} 2>&1 & echo $! > {shlex.quote(paths.pid_file)} echo "started pid=$(cat {shlex.quote(paths.pid_file)})" fi """ result = ssh.run(command, timeout=20) if result.exit_status != 0: raise RuntimeError(result.stderr or result.stdout or "failed to start resource collector") return result.stdout.strip() def stop_resource_collector(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str: """Stop the remote collector process if the pid file points to one.""" command = f""" if [ -f {shlex.quote(paths.pid_file)} ]; then pid="$(cat {shlex.quote(paths.pid_file)})" if kill -0 "$pid" 2>/dev/null; then kill "$pid" echo "stopped pid=$pid" else echo "not running" fi rm -f {shlex.quote(paths.pid_file)} else echo "not running" fi """ result = ssh.run(command, timeout=20) if result.exit_status != 0: raise RuntimeError(result.stderr or result.stdout or "failed to stop resource collector") return result.stdout.strip() def restart_resource_collector( ssh: RemoteSSHClient, interval_seconds: int = 10, retention_seconds: int = 7 * 24 * 60 * 60, max_lines: int = 70_000, paths: ResourceMonitorPaths = ResourceMonitorPaths(), ) -> str: stop_message = stop_resource_collector(ssh, paths) start_message = start_resource_collector(ssh, interval_seconds, retention_seconds, max_lines, paths) return f"{stop_message}\n{start_message}" def resource_collector_status(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str: """Return a short human-readable status string for the dashboard.""" command = f""" if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then echo "running pid=$(cat {shlex.quote(paths.pid_file)})" else echo "not running" fi """ result = ssh.run(command, timeout=10) if result.exit_status != 0: raise RuntimeError(result.stderr or result.stdout or "failed to check collector status") return result.stdout.strip() def resource_collector_debug_info(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str: """Collect remote diagnostics for troubleshooting missing metrics.""" command = f""" echo "status:" if [ -f {shlex.quote(paths.pid_file)} ]; then pid="$(cat {shlex.quote(paths.pid_file)})" echo "pid_file=$pid" if kill -0 "$pid" 2>/dev/null; then echo "process=running"; else echo "process=not-running"; fi else echo "pid_file=missing" fi echo "files:" ls -l {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} 2>&1 || true echo "sample_count:" if [ -f {shlex.quote(paths.metrics_file)} ]; then wc -l < {shlex.quote(paths.metrics_file)}; else echo 0; fi echo "last_samples:" if [ -f {shlex.quote(paths.metrics_file)} ]; then tail -n 5 {shlex.quote(paths.metrics_file)}; fi echo "log_tail:" if [ -f {shlex.quote(paths.log_file)} ]; then tail -n 40 {shlex.quote(paths.log_file)}; fi echo "netdev_snapshot:" cat /proc/net/dev 2>&1 || true """ result = ssh.run(command, timeout=20) return (result.stdout or "") + (result.stderr or "") def read_resource_metrics(ssh: RemoteSSHClient, max_lines: int = 1000, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> list[dict[str, Any]]: """Read recent JSONL metric samples from the remote collector file.""" command = f"test -f {shlex.quote(paths.metrics_file)} && tail -n {int(max_lines)} {shlex.quote(paths.metrics_file)} || true" result = ssh.run(command, timeout=20) if result.exit_status != 0: raise RuntimeError(result.stderr or result.stdout or "failed to read resource metrics") rows = [] for line in result.stdout.splitlines(): line = line.strip() if not line: continue try: rows.append(json.loads(line)) except json.JSONDecodeError: continue return rows def disk_space(ssh: RemoteSSHClient, path: str = "/") -> dict[str, Any]: """Return df information for the filesystem containing ``path``.""" command = ( "df -P -B1 -- " + shlex.quote(path or "/") + " | awk 'NR==2 {printf \"{\\\"filesystem\\\":\\\"%s\\\",\\\"size\\\":%s,\\\"used\\\":%s,\\\"available\\\":%s,\\\"used_pct\\\":\\\"%s\\\",\\\"mount\\\":\\\"%s\\\"}\", $1,$2,$3,$4,$5,$6}'" ) result = ssh.run(command, timeout=20) if result.exit_status != 0 or not result.stdout.strip(): raise RuntimeError(result.stderr or result.stdout or "failed to read disk space") return json.loads(result.stdout)