From 7f9b40cd4fb38d622f72a4452da54561727219f8 Mon Sep 17 00:00:00 2001 From: Developer Date: Tue, 16 Jun 2026 14:55:25 +0000 Subject: [PATCH] feat(observability): add standalone observability compose stack Add docker-compose.observability.yml to run Grafana, Prometheus, Loki, Alertmanager, Alloy and Node Exporter independently of Manage. Includes a standalone Prometheus config, empty file-SD placeholder, and runbook documentation. --- docker-compose.observability.yml | 222 ++++++++++++++++++ docs/observability-runbooks.md | 24 ++ .../file-sd/node_exporter_targets.json | 1 + .../prometheus/prometheus.standalone.yml | 46 ++++ 4 files changed, 293 insertions(+) create mode 100644 docker-compose.observability.yml create mode 100644 monitoring/prometheus/file-sd/node_exporter_targets.json create mode 100644 monitoring/prometheus/prometheus.standalone.yml diff --git a/docker-compose.observability.yml b/docker-compose.observability.yml new file mode 100644 index 0000000..0252342 --- /dev/null +++ b/docker-compose.observability.yml @@ -0,0 +1,222 @@ +# Standalone observability stack. +# +# Run this on a host where you want Grafana, Prometheus, Loki, Alertmanager, +# Alloy and Node Exporter without the rest of Manage. It exposes the usual +# ports directly; no Traefik or OIDC integration is required. +# +# docker compose -f docker-compose.observability.yml up -d +# +# Then open Grafana at http://localhost:3000 (admin / admin by default). +# +# To scrape a Manage backend as well, edit monitoring/prometheus/prometheus.standalone.yml +# and add a static target for it, or drop a file-SD JSON file into +# monitoring/prometheus/file-sd/. + +services: + prometheus: + image: prom/prometheus:v2.55.1 + container_name: prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" + - "--web.console.libraries=/usr/share/prometheus/console_libraries" + - "--web.console.templates=/usr/share/prometheus/consoles" + - "--web.enable-lifecycle" + volumes: + - ./monitoring/prometheus/prometheus.standalone.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/rules:/etc/prometheus/rules:ro + - ./monitoring/prometheus/file-sd:/etc/prometheus/file-sd:ro + - prometheus_data:/prometheus + ports: + - "9090:9090" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 1G + reservations: + cpus: "0.25" + memory: 256M + + loki: + image: grafana/loki:3.1.1 + container_name: loki + command: -config.file=/etc/loki/loki.yml + volumes: + - ./monitoring/loki/loki.yml:/etc/loki/loki.yml:ro + - loki_data:/loki + ports: + - "3100:3100" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 1G + reservations: + cpus: "0.25" + memory: 256M + + alloy: + image: grafana/alloy:v1.5.0 + container_name: alloy + command: + - run + - /etc/alloy/config.alloy + - --storage.path=/var/lib/alloy + volumes: + - ./monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + networks: + - monitoring + depends_on: + loki: + condition: service_healthy + restart: unless-stopped + user: root + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:12345/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + reservations: + cpus: "0.10" + memory: 128M + + grafana: + image: grafana/grafana:11.3.1 + container_name: grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID:-} + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:-} + GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${GF_AUTH_GENERIC_OAUTH_AUTH_URL:-} + GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${GF_AUTH_GENERIC_OAUTH_TOKEN_URL:-} + GF_AUTH_GENERIC_OAUTH_API_URL: ${GF_AUTH_GENERIC_OAUTH_API_URL:-} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + volumes: + - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana_data:/var/lib/grafana + ports: + - "3000:3000" + networks: + - monitoring + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + + alertmanager: + image: prom/alertmanager:v0.27.0 + container_name: alertmanager + command: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + environment: + SMTP_HOST: ${SMTP_HOST:-smtp.example.com} + SMTP_PORT: ${SMTP_PORT:-587} + SMTP_USERNAME: ${SMTP_USERNAME:-} + SMTP_PASSWORD: ${SMTP_PASSWORD:-} + SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-no-reply@example.com} + ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-admin@example.com} + volumes: + - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager_data:/alertmanager + ports: + - "9093:9093" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9093/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + reservations: + cpus: "0.10" + memory: 64M + + node-exporter: + image: prom/node-exporter:v1.8.2 + container_name: node-exporter + command: + - "--path.rootfs=/host" + volumes: + - /:/host:ro,rslave + ports: + - "9100:9100" + networks: + - monitoring + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9100/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + reservations: + cpus: "0.05" + memory: 32M + +networks: + monitoring: + +volumes: + prometheus_data: + loki_data: + grafana_data: + alertmanager_data: diff --git a/docs/observability-runbooks.md b/docs/observability-runbooks.md index 41bf1c1..1b288ef 100644 --- a/docs/observability-runbooks.md +++ b/docs/observability-runbooks.md @@ -183,6 +183,30 @@ docker compose up -d --- +## Standalone Observability Stack + +You can run the observability services on their own without the Manage backend or frontend: + +```bash +cd /path/to/manage +docker compose -f docker-compose.observability.yml up -d +``` + +This starts Prometheus, Grafana, Loki, Alertmanager, Alloy and Node Exporter. Exposed ports: + +| Service | URL | +|---------|-----| +| Grafana | | +| Prometheus | | +| Alertmanager | | +| Loki | | +| Node Exporter | | +| Alloy | | + +Grafana defaults to `admin` / `admin`. Datasources and dashboards are provisioned automatically. + +To scrape a Manage backend from this standalone stack, edit `monitoring/prometheus/prometheus.standalone.yml` and add a static target for the backend's `/metrics` endpoint, or drop a file-SD JSON file into `monitoring/prometheus/file-sd/`. + ## Scaling Notes - The current `deploy.resources` blocks are tuned for a small homelab. Raise memory limits if you monitor many machines or retain logs longer than 30 days. diff --git a/monitoring/prometheus/file-sd/node_exporter_targets.json b/monitoring/prometheus/file-sd/node_exporter_targets.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/monitoring/prometheus/file-sd/node_exporter_targets.json @@ -0,0 +1 @@ +[] diff --git a/monitoring/prometheus/prometheus.standalone.yml b/monitoring/prometheus/prometheus.standalone.yml new file mode 100644 index 0000000..ed755af --- /dev/null +++ b/monitoring/prometheus/prometheus.standalone.yml @@ -0,0 +1,46 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: manage-observability + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + + - job_name: node-exporter + static_configs: + - targets: + - node-exporter:9100 + + - job_name: loki + static_configs: + - targets: + - loki:3100 + + - job_name: alertmanager + static_configs: + - targets: + - alertmanager:9093 + + - job_name: grafana + static_configs: + - targets: + - grafana:3000 + + - job_name: node-exporter-remote + file_sd_configs: + - files: + - /etc/prometheus/file-sd/node_exporter_targets.json + refresh_interval: 30s