|
|
|
@@ -1,340 +0,0 @@
|
|
|
|
|
"""Remote resource collection helpers.
|
|
|
|
|
|
|
|
|
|
The app does not require Prometheus, Netdata, or sysstat. Instead it can install
|
|
|
|
|
and manage a tiny POSIX-sh collector under /tmp on the remote server. The
|
|
|
|
|
collector samples Linux /proc and /sys counters every 10 seconds and appends JSON
|
|
|
|
|
Lines. This module starts/stops the collector and reads those JSONL samples.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import json
|
|
|
|
|
import logging
|
|
|
|
|
import shlex
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from typing import Any
|
|
|
|
|
|
|
|
|
|
from media_library_viewer_api.clients.ssh import RemoteSSHClient
|
|
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
# POSIX shell script copied to the remote server by start_resource_collector().
|
|
|
|
|
# Keep this script bash-free because many NAS/media servers have minimal shells.
|
|
|
|
|
COLLECTOR_SCRIPT = r'''#!/bin/sh
|
|
|
|
|
set -u
|
|
|
|
|
OUT="${1:-/tmp/media_library_viewer_metrics.jsonl}"
|
|
|
|
|
INTERVAL="${2:-10}"
|
|
|
|
|
RETENTION_SECONDS="${3:-604800}"
|
|
|
|
|
MAX_LINES="${4:-70000}"
|
|
|
|
|
PRUNE_EVERY_SAMPLES="${5:-60}"
|
|
|
|
|
mkdir -p "$(dirname "$OUT")"
|
|
|
|
|
|
|
|
|
|
echo "collector starting at $(date -Is 2>/dev/null || date), interval=${INTERVAL}s, retention=${RETENTION_SECONDS}s, max_lines=${MAX_LINES}, out=${OUT}"
|
|
|
|
|
|
|
|
|
|
read_cpu() {
|
|
|
|
|
awk '/^cpu / {print $2+$3+$4+$5+$6+$7+$8+$9+$10, $5+$6, $6}' /proc/stat
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
read_mem_pct() {
|
|
|
|
|
awk '
|
|
|
|
|
/^MemTotal:/ {total=$2}
|
|
|
|
|
/^MemAvailable:/ {avail=$2}
|
|
|
|
|
END {if (total > 0) printf "%.2f", (total-avail)*100/total; else printf "0"}
|
|
|
|
|
' /proc/meminfo
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
read_net_bytes() {
|
|
|
|
|
awk '
|
|
|
|
|
NR > 2 {
|
|
|
|
|
split($0, parts, ":")
|
|
|
|
|
iface = parts[1]
|
|
|
|
|
stats = parts[2]
|
|
|
|
|
gsub(/^[ \t]+|[ \t]+$/, "", iface)
|
|
|
|
|
gsub(/^[ \t]+|[ \t]+$/, "", stats)
|
|
|
|
|
if (iface == "lo" || iface == "" || stats == "") next
|
|
|
|
|
split(stats, values, /[ \t]+/)
|
|
|
|
|
# /proc/net/dev after the colon:
|
|
|
|
|
# receive bytes are field 1, transmit bytes are field 9.
|
|
|
|
|
# Trim the stats block before split; otherwise leading whitespace can make
|
|
|
|
|
# values[1] empty in some awk implementations, resulting in zero rates.
|
|
|
|
|
rx += values[1] + 0
|
|
|
|
|
tx += values[9] + 0
|
|
|
|
|
}
|
|
|
|
|
END {printf "%.0f %.0f", rx, tx}
|
|
|
|
|
' /proc/net/dev
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
read_disk_bytes() {
|
|
|
|
|
read_sectors=0
|
|
|
|
|
written_sectors=0
|
|
|
|
|
for dev in /sys/block/*; do
|
|
|
|
|
[ -r "$dev/stat" ] || continue
|
|
|
|
|
name="$(basename "$dev")"
|
|
|
|
|
case "$name" in
|
|
|
|
|
loop*|ram*|fd*|sr*) continue ;;
|
|
|
|
|
esac
|
|
|
|
|
# Linux /sys/block/<dev>/stat fields: 3=sectors read, 7=sectors written.
|
|
|
|
|
# Use POSIX sh parsing instead of bash arrays so this works on minimal systems.
|
|
|
|
|
set -- $(cat "$dev/stat")
|
|
|
|
|
sectors_read="${3:-0}"
|
|
|
|
|
sectors_written="${7:-0}"
|
|
|
|
|
read_sectors=$((read_sectors + sectors_read))
|
|
|
|
|
written_sectors=$((written_sectors + sectors_written))
|
|
|
|
|
done
|
|
|
|
|
printf "%s %s" "$((read_sectors * 512))" "$((written_sectors * 512))"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
set -- $(read_cpu)
|
|
|
|
|
prev_total="${1:-0}"
|
|
|
|
|
prev_idle="${2:-0}"
|
|
|
|
|
prev_iowait="${3:-0}"
|
|
|
|
|
set -- $(read_net_bytes)
|
|
|
|
|
prev_rx="${1:-0}"
|
|
|
|
|
prev_tx="${2:-0}"
|
|
|
|
|
set -- $(read_disk_bytes)
|
|
|
|
|
prev_disk_read="${1:-0}"
|
|
|
|
|
prev_disk_write="${2:-0}"
|
|
|
|
|
prev_ts="$(date +%s)"
|
|
|
|
|
sample_count=0
|
|
|
|
|
|
|
|
|
|
prune_metrics_file() {
|
|
|
|
|
[ -f "$OUT" ] || return 0
|
|
|
|
|
cutoff="$1"
|
|
|
|
|
tmp="${OUT}.$$.tmp"
|
|
|
|
|
awk -v cutoff="$cutoff" '
|
|
|
|
|
match($0, /"ts":[0-9]+/) {
|
|
|
|
|
ts = substr($0, RSTART + 5, RLENGTH - 5)
|
|
|
|
|
if (ts >= cutoff) print $0
|
|
|
|
|
}
|
|
|
|
|
' "$OUT" | tail -n "$MAX_LINES" > "$tmp" && mv "$tmp" "$OUT"
|
|
|
|
|
rm -f "$tmp"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
while true; do
|
|
|
|
|
now_ts="$(date +%s)"
|
|
|
|
|
dt=$((now_ts - prev_ts))
|
|
|
|
|
if [ "$dt" -le 0 ]; then dt=1; fi
|
|
|
|
|
|
|
|
|
|
set -- $(read_cpu)
|
|
|
|
|
total="${1:-0}"
|
|
|
|
|
idle="${2:-0}"
|
|
|
|
|
iowait="${3:-0}"
|
|
|
|
|
set -- $(read_net_bytes)
|
|
|
|
|
rx="${1:-0}"
|
|
|
|
|
tx="${2:-0}"
|
|
|
|
|
set -- $(read_disk_bytes)
|
|
|
|
|
disk_read="${1:-0}"
|
|
|
|
|
disk_write="${2:-0}"
|
|
|
|
|
mem_pct="$(read_mem_pct)"
|
|
|
|
|
|
|
|
|
|
total_delta=$((total - prev_total))
|
|
|
|
|
idle_delta=$((idle - prev_idle))
|
|
|
|
|
iowait_delta=$((iowait - prev_iowait))
|
|
|
|
|
rx_delta=$((rx - prev_rx))
|
|
|
|
|
tx_delta=$((tx - prev_tx))
|
|
|
|
|
disk_read_delta=$((disk_read - prev_disk_read))
|
|
|
|
|
disk_write_delta=$((disk_write - prev_disk_write))
|
|
|
|
|
|
|
|
|
|
cpu_pct="$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN {if (total > 0) printf "%.2f", (total-idle)*100/total; else printf "0"}')"
|
|
|
|
|
iowait_pct="$(awk -v total="$total_delta" -v iow="$iowait_delta" 'BEGIN {if (total > 0) printf "%.2f", iow*100/total; else printf "0"}')"
|
|
|
|
|
rx_bytes_per_sec="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
|
|
|
|
|
tx_bytes_per_sec="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
|
|
|
|
|
rx_bps="$(awk -v bytes="$rx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
|
|
|
|
|
tx_bps="$(awk -v bytes="$tx_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes*8/dt; else printf "0"}')"
|
|
|
|
|
disk_read_bps="$(awk -v bytes="$disk_read_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
|
|
|
|
|
disk_write_bps="$(awk -v bytes="$disk_write_delta" -v dt="$dt" 'BEGIN {if (bytes >= 0) printf "%.0f", bytes/dt; else printf "0"}')"
|
|
|
|
|
|
|
|
|
|
printf '{"ts":%s,"cpu_pct":%s,"iowait_pct":%s,"mem_pct":%s,"net_rx_bytes_per_sec":%s,"net_tx_bytes_per_sec":%s,"net_rx_bps":%s,"net_tx_bps":%s,"disk_read_bps":%s,"disk_write_bps":%s}\n' \
|
|
|
|
|
"$now_ts" "$cpu_pct" "$iowait_pct" "$mem_pct" "$rx_bytes_per_sec" "$tx_bytes_per_sec" "$rx_bps" "$tx_bps" "$disk_read_bps" "$disk_write_bps" >> "$OUT"
|
|
|
|
|
|
|
|
|
|
sample_count=$((sample_count + 1))
|
|
|
|
|
if [ $((sample_count % PRUNE_EVERY_SAMPLES)) -eq 0 ]; then
|
|
|
|
|
prune_metrics_file "$((now_ts - RETENTION_SECONDS))"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
prev_total="$total"
|
|
|
|
|
prev_idle="$idle"
|
|
|
|
|
prev_iowait="$iowait"
|
|
|
|
|
prev_rx="$rx"
|
|
|
|
|
prev_tx="$tx"
|
|
|
|
|
prev_disk_read="$disk_read"
|
|
|
|
|
prev_disk_write="$disk_write"
|
|
|
|
|
prev_ts="$now_ts"
|
|
|
|
|
sleep "$INTERVAL"
|
|
|
|
|
done
|
|
|
|
|
'''
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class ResourceMonitorPaths:
|
|
|
|
|
"""Remote file locations used by the lightweight resource collector."""
|
|
|
|
|
|
|
|
|
|
metrics_file: str = "/tmp/media_library_viewer_metrics.jsonl"
|
|
|
|
|
pid_file: str = "/tmp/media_library_viewer_metrics.pid"
|
|
|
|
|
script_file: str = "/tmp/media_library_viewer_metrics_collector.sh"
|
|
|
|
|
log_file: str = "/tmp/media_library_viewer_metrics.log"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def start_resource_collector(
|
|
|
|
|
ssh: RemoteSSHClient,
|
|
|
|
|
interval_seconds: int = 10,
|
|
|
|
|
retention_seconds: int = 7 * 24 * 60 * 60,
|
|
|
|
|
max_lines: int = 70_000,
|
|
|
|
|
paths: ResourceMonitorPaths = ResourceMonitorPaths(),
|
|
|
|
|
) -> str:
|
|
|
|
|
"""Install and start the remote metrics collector if it is not running.
|
|
|
|
|
|
|
|
|
|
Starting a fresh collector removes old metrics/log files because schema
|
|
|
|
|
changes during development can otherwise leave mixed JSONL records behind.
|
|
|
|
|
The collector prunes its own metrics file to 7 days / max_lines.
|
|
|
|
|
"""
|
|
|
|
|
command = f"""
|
|
|
|
|
cat > {shlex.quote(paths.script_file)} <<'MLV_RESOURCE_COLLECTOR'
|
|
|
|
|
{COLLECTOR_SCRIPT}
|
|
|
|
|
MLV_RESOURCE_COLLECTOR
|
|
|
|
|
chmod +x {shlex.quote(paths.script_file)}
|
|
|
|
|
if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
|
|
|
|
|
echo "already running pid=$(cat {shlex.quote(paths.pid_file)})"
|
|
|
|
|
else
|
|
|
|
|
rm -f {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)}
|
|
|
|
|
nohup {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {int(interval_seconds)} {int(retention_seconds)} {int(max_lines)} >> {shlex.quote(paths.log_file)} 2>&1 &
|
|
|
|
|
echo $! > {shlex.quote(paths.pid_file)}
|
|
|
|
|
echo "started pid=$(cat {shlex.quote(paths.pid_file)})"
|
|
|
|
|
fi
|
|
|
|
|
"""
|
|
|
|
|
logger.info(
|
|
|
|
|
"Starting remote resource collector interval=%ss retention=%ss max_lines=%s", interval_seconds, retention_seconds, max_lines
|
|
|
|
|
)
|
|
|
|
|
result = ssh.run(command, timeout=20)
|
|
|
|
|
if result.exit_status != 0:
|
|
|
|
|
logger.warning("Failed to start remote resource collector: %s", result.stderr or result.stdout)
|
|
|
|
|
raise RuntimeError(result.stderr or result.stdout or "failed to start resource collector")
|
|
|
|
|
logger.info("Remote resource collector start response: %s", result.stdout.strip())
|
|
|
|
|
return result.stdout.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def stop_resource_collector(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
|
|
|
|
|
"""Stop the remote collector process if the pid file points to one."""
|
|
|
|
|
command = f"""
|
|
|
|
|
if [ -f {shlex.quote(paths.pid_file)} ]; then
|
|
|
|
|
pid="$(cat {shlex.quote(paths.pid_file)})"
|
|
|
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
|
|
|
kill "$pid"
|
|
|
|
|
echo "stopped pid=$pid"
|
|
|
|
|
else
|
|
|
|
|
echo "not running"
|
|
|
|
|
fi
|
|
|
|
|
rm -f {shlex.quote(paths.pid_file)}
|
|
|
|
|
else
|
|
|
|
|
echo "not running"
|
|
|
|
|
fi
|
|
|
|
|
"""
|
|
|
|
|
logger.info("Stopping remote resource collector")
|
|
|
|
|
result = ssh.run(command, timeout=20)
|
|
|
|
|
if result.exit_status != 0:
|
|
|
|
|
logger.warning("Failed to stop remote resource collector: %s", result.stderr or result.stdout)
|
|
|
|
|
raise RuntimeError(result.stderr or result.stdout or "failed to stop resource collector")
|
|
|
|
|
logger.info("Remote resource collector stop response: %s", result.stdout.strip())
|
|
|
|
|
return result.stdout.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def restart_resource_collector(
|
|
|
|
|
ssh: RemoteSSHClient,
|
|
|
|
|
interval_seconds: int = 10,
|
|
|
|
|
retention_seconds: int = 7 * 24 * 60 * 60,
|
|
|
|
|
max_lines: int = 70_000,
|
|
|
|
|
paths: ResourceMonitorPaths = ResourceMonitorPaths(),
|
|
|
|
|
) -> str:
|
|
|
|
|
logger.info("Restarting remote resource collector")
|
|
|
|
|
stop_message = stop_resource_collector(ssh, paths)
|
|
|
|
|
start_message = start_resource_collector(ssh, interval_seconds, retention_seconds, max_lines, paths)
|
|
|
|
|
return f"{stop_message}\n{start_message}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def resource_collector_status(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
|
|
|
|
|
"""Return a short human-readable status string for the dashboard."""
|
|
|
|
|
command = f"""
|
|
|
|
|
if [ -f {shlex.quote(paths.pid_file)} ] && kill -0 "$(cat {shlex.quote(paths.pid_file)})" 2>/dev/null; then
|
|
|
|
|
echo "running"
|
|
|
|
|
else
|
|
|
|
|
echo "not running"
|
|
|
|
|
fi
|
|
|
|
|
"""
|
|
|
|
|
result = ssh.run(command, timeout=10)
|
|
|
|
|
if result.exit_status != 0:
|
|
|
|
|
logger.warning("Failed to read collector status: %s", result.stderr or result.stdout)
|
|
|
|
|
raise RuntimeError(result.stderr or result.stdout or "failed to check collector status")
|
|
|
|
|
status = result.stdout.strip()
|
|
|
|
|
logger.info("Resource collector status: %s", status)
|
|
|
|
|
return status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def resource_collector_debug_info(ssh: RemoteSSHClient, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> str:
|
|
|
|
|
"""Collect remote diagnostics for troubleshooting missing metrics."""
|
|
|
|
|
command = f"""
|
|
|
|
|
echo "status:"
|
|
|
|
|
if [ -f {shlex.quote(paths.pid_file)} ]; then
|
|
|
|
|
pid="$(cat {shlex.quote(paths.pid_file)})"
|
|
|
|
|
echo "pid_file=$pid"
|
|
|
|
|
if kill -0 "$pid" 2>/dev/null; then echo "process=running"; else echo "process=not-running"; fi
|
|
|
|
|
else
|
|
|
|
|
echo "pid_file=missing"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
echo "files:"
|
|
|
|
|
ls -l {shlex.quote(paths.script_file)} {shlex.quote(paths.metrics_file)} {shlex.quote(paths.log_file)} 2>&1 || true
|
|
|
|
|
|
|
|
|
|
echo "sample_count:"
|
|
|
|
|
if [ -f {shlex.quote(paths.metrics_file)} ]; then wc -l < {shlex.quote(paths.metrics_file)}; else echo 0; fi
|
|
|
|
|
|
|
|
|
|
echo "last_samples:"
|
|
|
|
|
if [ -f {shlex.quote(paths.metrics_file)} ]; then tail -n 5 {shlex.quote(paths.metrics_file)}; fi
|
|
|
|
|
|
|
|
|
|
echo "log_tail:"
|
|
|
|
|
if [ -f {shlex.quote(paths.log_file)} ]; then tail -n 40 {shlex.quote(paths.log_file)}; fi
|
|
|
|
|
|
|
|
|
|
echo "netdev_snapshot:"
|
|
|
|
|
cat /proc/net/dev 2>&1 || true
|
|
|
|
|
"""
|
|
|
|
|
logger.info("Collecting resource collector diagnostics")
|
|
|
|
|
result = ssh.run(command, timeout=20)
|
|
|
|
|
output = (result.stdout or "") + (result.stderr or "")
|
|
|
|
|
logger.debug("Resource collector diagnostics length=%s", len(output))
|
|
|
|
|
return output
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def read_resource_metrics(ssh: RemoteSSHClient, max_lines: int = 1000, paths: ResourceMonitorPaths = ResourceMonitorPaths()) -> list[dict[str, Any]]:
|
|
|
|
|
"""Read recent JSONL metric samples from the remote collector file."""
|
|
|
|
|
command = f"test -f {shlex.quote(paths.metrics_file)} && tail -n {int(max_lines)} {shlex.quote(paths.metrics_file)} || true"
|
|
|
|
|
logger.debug("Reading up to %s resource metric lines", max_lines)
|
|
|
|
|
result = ssh.run(command, timeout=20)
|
|
|
|
|
if result.exit_status != 0:
|
|
|
|
|
logger.warning("Failed to read resource metrics: %s", result.stderr or result.stdout)
|
|
|
|
|
raise RuntimeError(result.stderr or result.stdout or "failed to read resource metrics")
|
|
|
|
|
rows = []
|
|
|
|
|
for line in result.stdout.splitlines():
|
|
|
|
|
line = line.strip()
|
|
|
|
|
if not line:
|
|
|
|
|
continue
|
|
|
|
|
try:
|
|
|
|
|
rows.append(json.loads(line))
|
|
|
|
|
except json.JSONDecodeError:
|
|
|
|
|
continue
|
|
|
|
|
return rows
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def disk_space(ssh: RemoteSSHClient, path: str = "/") -> dict[str, Any]:
|
|
|
|
|
"""Return df information for the filesystem containing ``path``."""
|
|
|
|
|
command = (
|
|
|
|
|
"df -P -B1 -- "
|
|
|
|
|
+ shlex.quote(path or "/")
|
|
|
|
|
+ " | awk 'NR==2 {printf \"{\\\"filesystem\\\":\\\"%s\\\",\\\"size\\\":%s,\\\"used\\\":%s,\\\"available\\\":%s,\\\"used_pct\\\":\\\"%s\\\",\\\"mount\\\":\\\"%s\\\"}\", $1,$2,$3,$4,$5,$6}'"
|
|
|
|
|
)
|
|
|
|
|
logger.debug("Reading disk space for path=%s", path)
|
|
|
|
|
result = ssh.run(command, timeout=20)
|
|
|
|
|
if result.exit_status != 0 or not result.stdout.strip():
|
|
|
|
|
logger.warning("Failed to read disk space for %s: %s", path, result.stderr or result.stdout)
|
|
|
|
|
raise RuntimeError(result.stderr or result.stdout or "failed to read disk space")
|
|
|
|
|
data = json.loads(result.stdout)
|
|
|
|
|
logger.info("Disk space path=%s mount=%s used_pct=%s", path, data.get("mount"), data.get("used_pct"))
|
|
|
|
|
return data
|
Developer