diff --git a/scripts/proxmox-monitor.sh b/scripts/proxmox-monitor.sh new file mode 100755 index 0000000..bb3e80a --- /dev/null +++ b/scripts/proxmox-monitor.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# proxmox-monitor.sh — Proxmox Cluster + Docker Monitoring Health Check +# Implements proxmox-monitor.prose.md (check-health section) +# +# Legs: Prometheus, Grafana, Docker Stats Exporter, PVE Exporter, PBS GC +# All legs must return 200 for healthy status. +# +# Run: bash scripts/proxmox-monitor.sh +# Exits 0 if all probes pass, 1 if any fails. + +set -uo pipefail + +CT116_HOST="192.168.68.116" +FAILED=() +TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC') + +echo "=== Proxmox Monitor — $TIMESTAMP ===" +echo "Executed from: $(pwd -P)" +echo "" + +# 1. Prometheus health (bound to 0.0.0.0:9090 on .116) +PROM_CODE=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://${CT116_HOST}:9090/-/healthy 2>/dev/null) +PROM_CODE=$(printf '%s' "$PROM_CODE" | tr -d '[:space:]') +[ -n "$PROM_CODE" ] || PROM_CODE="000" + +if [ "$PROM_CODE" = "200" ]; then + echo " ✅ Prometheus: alive" +else + echo " 🔴 Prometheus: probe-failed: ${CT116_HOST}:9090 (expected 200, got ${PROM_CODE})" + FAILED+=("prometheus") +fi + +# 2. Grafana health (bound to 0.0.0.0:3001 on .116) +GRAF_CODE=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://${CT116_HOST}:3001/api/health 2>/dev/null) +GRAF_CODE=$(printf '%s' "$GRAF_CODE" | tr -d '[:space:]') +[ -n "$GRAF_CODE" ] || GRAF_CODE="000" + +if [ "$GRAF_CODE" = "200" ]; then + echo " ✅ Grafana: alive" +else + echo " 🔴 Grafana: probe-failed: ${CT116_HOST}:3001 (expected 200, got ${GRAF_CODE})" + FAILED+=("grafana") +fi + +# 3. Docker Stats exporter (bound to 127.0.0.1:9324 on .116 — probe from .116 localhost) +DOCKER_CODE=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@${CT116_HOST} \ + "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9324/metrics" 2>/dev/null) +DOCKER_CODE=$(printf '%s' "$DOCKER_CODE" | tr -d '[:space:]') +[ -n "$DOCKER_CODE" ] || DOCKER_CODE="000" + +if [ "$DOCKER_CODE" = "200" ]; then + echo " ✅ Docker Stats: alive" +else + echo " 🔴 Docker Stats: probe-failed: CT116:127.0.0.1:9324 (expected 200, got ${DOCKER_CODE})" + FAILED+=("docker-stats") +fi + +# 4. PVE Exporter (bound to 127.0.0.1:9221 on .116 — probe from .116 localhost) +PVE_CODE=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@${CT116_HOST} \ + "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9221/metrics" 2>/dev/null) +PVE_CODE=$(printf '%s' "$PVE_CODE" | tr -d '[:space:]') +[ -n "$PVE_CODE" ] || PVE_CODE="000" + +if [ "$PVE_CODE" = "200" ]; then + echo " ✅ PVE Exporter: alive" +else + echo " 🔴 PVE Exporter: probe-failed: CT116:127.0.0.1:9221 (expected 200, got ${PVE_CODE})" + FAILED+=("pve-exporter") +fi + +# 5. PBS GC liveness (storepve-datastore GC must have run within 48h) +PBS_GC_OUTPUT=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@192.168.68.6 \ + "pct exec 107 -- proxmox-backup-manager garbage-collection list --output-format json" 2>/dev/null) +PBS_GC_OUTPUT=$(printf '%s' "$PBS_GC_OUTPUT" | tr -d '[:space:]') +[ -n "$PBS_GC_OUTPUT" ] || PBS_GC_OUTPUT="000" + +if [ "$PBS_GC_OUTPUT" = "000" ]; then + echo " 🔴 PBS GC: probe-failed: storepve:192.168.68.6 (expected JSON, got 000)" + FAILED+=("pbs-gc") +else + # Parse the JSON to get storepve-datastore's last-run-endtime + LAST_RUN_ENDTIME=$(echo "$PBS_GC_OUTPUT" | python3 -c " +import sys, json +data = json.load(sys.stdin) +for store in data: + if store['store'] == 'storepve-datastore': + print(store.get('last-run-endtime', 0)) + break +" 2>/dev/null) + + if [ -z "$LAST_RUN_ENDTIME" ] || [ "$LAST_RUN_ENDTIME" = "0" ]; then + echo " 🔴 PBS GC: never-run (storepve-datastore has no last-run-endtime)" + FAILED+=("pbs-gc") + else + # Convert epoch to age in hours + NOW_EPOCH=$(date -u +%s) + AGE_HOURS=$(( (NOW_EPOCH - LAST_RUN_ENDTIME) / 3600 )) + AGE_DAYS=$(( AGE_HOURS / 24 )) + + if [ $AGE_HOURS -gt 48 ]; then + echo " 🔴 PBS GC: stale (last run ${AGE_DAYS}d ago, pending-bytes unknown)" + FAILED+=("pbs-gc") + else + echo " ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago)" + fi + fi +fi + +# ── Summary ───────────────────────────────────────────────────────────────── +echo "" +if [ ${#FAILED[@]} -eq 0 ]; then + echo " ✅ All legs OK" + exit 0 +else + for f in "${FAILED[@]}"; do + echo " 🔴 FAILED: $f" + done + exit 1 +fi \ No newline at end of file