2026-09-19 02:51:35 +00:00
3 changed files with 305 additions and 0 deletions
@@ -89,6 +89,48 @@ agent: abiba
 | minipve | 192.168.68.12 | PVE |
 | amdpve | 192.168.68.15 | PVE + Strix Halo LLM (strix-moe) |

+## PBS GC (Proxmox Backup Server)
+
+### Schedule
+Cron `0 20 * * *` on the **storepve HOST** (192.168.68.6) = 20:00 America/New_York local = **00:00 UTC**.
+
+**NOTE**: The closed PR #116 said "20:00 UTC" — this is WRONG by four hours. Do not copy it.
+
+### What Actually Runs
+Host script `/usr/local/bin/pbs-gc.sh` runs `pct exec 107 -- proxmox-backup-manager garbage-collection start storepve-datastore`.
+
+**IMPORTANT**: The tool `proxmox-backup-manager` exists only inside CT 107 (where the PBS server runs). The storepve host has only `proxmox-backup-client`. This is why the job had never worked before 2026-09-19 00:00 UTC.
+
+### Datastore Location
+- **Datastore**: CT 107's `/mnt/pbs-backup` on the storepve ZFS dataset `/tank/pbs-backup` (pool `tank`, ~11T free)
+- **NOT** `/media/easystore2` (media library, 3.7T, 96% used — separate volume)
+
+### Liveness Check
+The new `proxmox-monitor.sh` leg checks storepve-datastore GC health:
+- Reads GC state from CT 107: `pct exec 107 -- proxmox-backup-manager garbage-collection list --output-format json`
+- **FAILS** if `last-run-endtime` is older than 48 hours
+- Reports age in hours and pending-bytes
+
+**All six verdict shapes** (exactly as emitted by the script):
+
+1. **Healthy** (fresh GC, 0 B pending):
+   `✅ PBS GC: healthy (last run 1h ago, pending-bytes: 0 B)`
+
+2. **Stale** (GC ran >48h ago):
+   `🔴 PBS GC: stale (last run 49h ago, pending-bytes: 1048576 B)`
+
+3. **Probe-failed: empty read** (000/timeout/unreadable):
+   `🔴 PBS GC: probe-failed: storepve:192.168.68.6 (expected JSON, got 000)`
+
+4. **Probe-failed: unparseable** (non-empty but invalid JSON — the "command not found" case):
+   `🔴 PBS GC: probe-failed: storepve:192.168.68.6 (unparseable JSON)`
+
+5. **Never-run: datastore absent** (valid JSON but storepve-datastore not in list):
+   `🔴 PBS GC: never-run (storepve-datastore not found in GC list)`
+
+6. **Never-run: no endtime** (valid JSON with datastore present but last-run-endtime is null/0):
+   `🔴 PBS GC: never-run (storepve-datastore has no last-run-endtime)`
+
 ## Operations

 ### view-dashboards
@@ -0,0 +1,138 @@
+#!/bin/bash
+# proxmox-monitor.sh — Proxmox Cluster + Docker Monitoring Health Check
+# Implements proxmox-monitor.prose.md (check-health section)
+#
+# Legs: Prometheus, Grafana, Docker Stats Exporter, PVE Exporter, PBS GC
+# All legs must return 200 for healthy status.
+#
+# Run: bash scripts/proxmox-monitor.sh
+# Exits 0 if all probes pass, 1 if any fails.
+
+set -uo pipefail
+
+CT116_HOST="192.168.68.116"
+FAILED=()
+TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
+
+echo "=== Proxmox Monitor — $TIMESTAMP ==="
+echo "Executed from: $(pwd -P)"
+echo ""
+
+# 1. Prometheus health (bound to 0.0.0.0:9090 on .116)
+PROM_CODE=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://${CT116_HOST}:9090/-/healthy 2>/dev/null)
+PROM_CODE=$(printf '%s' "$PROM_CODE" | tr -d '[:space:]')
+[ -n "$PROM_CODE" ] || PROM_CODE="000"
+
+if [ "$PROM_CODE" = "200" ]; then
+    echo "  ✅ Prometheus: alive"
+else
+    echo "  🔴 Prometheus: probe-failed: ${CT116_HOST}:9090 (expected 200, got ${PROM_CODE})"
+    FAILED+=("prometheus")
+fi
+
+# 2. Grafana health (bound to 0.0.0.0:3001 on .116)
+GRAF_CODE=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://${CT116_HOST}:3001/api/health 2>/dev/null)
+GRAF_CODE=$(printf '%s' "$GRAF_CODE" | tr -d '[:space:]')
+[ -n "$GRAF_CODE" ] || GRAF_CODE="000"
+
+if [ "$GRAF_CODE" = "200" ]; then
+    echo "  ✅ Grafana: alive"
+else
+    echo "  🔴 Grafana: probe-failed: ${CT116_HOST}:3001 (expected 200, got ${GRAF_CODE})"
+    FAILED+=("grafana")
+fi
+
+# 3. Docker Stats exporter (bound to 127.0.0.1:9324 on .116 — probe from .116 localhost)
+DOCKER_CODE=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@${CT116_HOST} \
+    "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9324/metrics" 2>/dev/null)
+DOCKER_CODE=$(printf '%s' "$DOCKER_CODE" | tr -d '[:space:]')
+[ -n "$DOCKER_CODE" ] || DOCKER_CODE="000"
+
+if [ "$DOCKER_CODE" = "200" ]; then
+    echo "  ✅ Docker Stats: alive"
+else
+    echo "  🔴 Docker Stats: probe-failed: CT116:127.0.0.1:9324 (expected 200, got ${DOCKER_CODE})"
+    FAILED+=("docker-stats")
+fi
+
+# 4. PVE Exporter (bound to 127.0.0.1:9221 on .116 — probe from .116 localhost)
+PVE_CODE=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@${CT116_HOST} \
+    "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9221/metrics" 2>/dev/null)
+PVE_CODE=$(printf '%s' "$PVE_CODE" | tr -d '[:space:]')
+[ -n "$PVE_CODE" ] || PVE_CODE="000"
+
+if [ "$PVE_CODE" = "200" ]; then
+    echo "  ✅ PVE Exporter: alive"
+else
+    echo "  🔴 PVE Exporter: probe-failed: CT116:127.0.0.1:9221 (expected 200, got ${PVE_CODE})"
+    FAILED+=("pve-exporter")
+fi
+
+# 5. PBS GC liveness (storepve-datastore GC must have run within 48h)
+PBS_GC_OUTPUT=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@192.168.68.6 \
+    "pct exec 107 -- proxmox-backup-manager garbage-collection list --output-format json" 2>/dev/null)
+PBS_GC_OUTPUT=$(printf '%s' "$PBS_GC_OUTPUT" | tr -d '[:space:]')
+[ -n "$PBS_GC_OUTPUT" ] || PBS_GC_OUTPUT="000"
+
+if [ "$PBS_GC_OUTPUT" = "000" ]; then
+    echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (expected JSON, got 000)"
+    FAILED+=("pbs-gc")
+else
+    # Parse the JSON to get storepve-datastore's last-run-endtime and pending-bytes
+    PBS_GC_RESULT=$(echo "$PBS_GC_OUTPUT" | python3 -c "
+import sys, json
+try:
+    data = json.load(sys.stdin)
+    for store in data:
+        if store['store'] == 'storepve-datastore':
+            endtime = store.get('last-run-endtime')
+            pending = store.get('pending-bytes', 0)
+            if endtime is None or endtime == 0:
+                print('never-run')
+            else:
+                print(f'{endtime}|{pending}')
+            break
+    else:
+        print('absent')
+except json.JSONDecodeError:
+    print('unparseable')
+" 2>/dev/null)
+    
+    if [ -z "$PBS_GC_RESULT" ] || [ "$PBS_GC_RESULT" = "unparseable" ]; then
+        echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (unparseable JSON)"
+        FAILED+=("pbs-gc")
+    elif [ "$PBS_GC_RESULT" = "absent" ]; then
+        echo "  🔴 PBS GC: never-run (storepve-datastore not found in GC list)"
+        FAILED+=("pbs-gc")
+    elif [ "$PBS_GC_RESULT" = "never-run" ]; then
+        echo "  🔴 PBS GC: never-run (storepve-datastore has no last-run-endtime)"
+        FAILED+=("pbs-gc")
+    else
+        # Parse the endtime|pending format
+        LAST_RUN_ENDTIME=$(echo "$PBS_GC_RESULT" | cut -d'|' -f1)
+        PENDING_BYTES=$(echo "$PBS_GC_RESULT" | cut -d'|' -f2)
+        
+        # Convert epoch to age in hours
+        NOW_EPOCH=$(date -u +%s)
+        AGE_HOURS=$(( (NOW_EPOCH - LAST_RUN_ENDTIME) / 3600 ))
+        
+        if [ $AGE_HOURS -gt 48 ]; then
+            echo "  🔴 PBS GC: stale (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)"
+            FAILED+=("pbs-gc")
+        else
+            echo "  ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)"
+        fi
+    fi
+fi
+
+# ── Summary ─────────────────────────────────────────────────────────────────
+echo ""
+if [ ${#FAILED[@]} -eq 0 ]; then
+    echo "  ✅ All legs OK"
+    exit 0
+else
+    for f in "${FAILED[@]}"; do
+        echo "  🔴 FAILED: $f"
+    done
+    exit 1
+fi
@@ -0,0 +1,125 @@
+#!/bin/bash
+# test_proxmox_monitor.sh — Stub-driven tests for PBS GC liveness leg
+
+set -uo pipefail
+
+SCRIPT="$(cd "$(dirname "$0")" && pwd)/proxmox-monitor.sh"
+PASS=0
+FAIL=0
+
+# ── Helpers ────────────────────────────────────────────────────────────────
+assert() {
+    local desc="$1" cond="$2"
+    if eval "$cond" 2>/dev/null; then
+        echo "  ✅ $desc"
+        PASS=$((PASS+1))
+    else
+        echo "  🔴 $desc"
+        FAIL=$((FAIL+1))
+    fi
+}
+
+# ── 1. Healthy: fresh GC, 0 B pending ─────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+FRESH_ENDTIME=$(( $(date -u +%s) - (1 * 3600) ))  # 1 hour ago
+cat > "$TMP_DIR/ssh" << EOF
+#!/bin/bash
+# Stub: return valid JSON with fresh endtime
+echo '[{"store":"storepve-datastore","last-run-endtime":$FRESH_ENDTIME,"pending-bytes":0}]'
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Healthy: PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Healthy: shows healthy verdict" '[[ "$PBS_LINE" == *"healthy"* ]]'
+assert "Healthy: shows pending-bytes 0 B" '[[ "$PBS_LINE" == *"pending-bytes: 0 B"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 2. Stale: GC >48h old ─────────────────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+OLD_ENDTIME=$(( $(date -u +%s) - (49 * 3600) ))  # 49 hours ago
+cat > "$TMP_DIR/ssh" << EOF
+#!/bin/bash
+echo '[{"store":"storepve-datastore","last-run-endtime":$OLD_ENDTIME,"pending-bytes":1048576}]'
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Stale: PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Stale: shows stale verdict" '[[ "$PBS_LINE" == *"stale"* ]]'
+assert "Stale: shows pending-bytes 1048576 B" '[[ "$PBS_LINE" == *"pending-bytes: 1048576 B"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 3. Probe-failed: empty output ─────────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Probe-failed (empty): PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Probe-failed (empty): shows probe-failed" '[[ "$PBS_LINE" == *"probe-failed"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 4. Probe-failed: unparseable output ───────────────────────────────────
+TMP_DIR=$(mktemp -d)
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+echo "proxmox-backup-manager: command not found"
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Probe-failed (unparseable): PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Probe-failed (unparseable): shows probe-failed" '[[ "$PBS_LINE" == *"probe-failed"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 5. Null endtime: never-run ────────────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+echo '[{"store":"storepve-datastore","last-run-endtime":null,"pending-bytes":0}]'
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Null endtime: PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Null endtime: shows never-run" '[[ "$PBS_LINE" == *"never-run"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 6. Datastore absent ───────────────────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+echo '[{"store":"s3-archive","last-run-endtime":null,"pending-bytes":0}]'
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Datastore absent: PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Datastore absent: shows never-run" '[[ "$PBS_LINE" == *"never-run"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── Summary ────────────────────────────────────────────────────────────────
+echo ""
+echo "Results: ${PASS} passed, ${FAIL} failed"
+if [ $FAIL -gt 0 ]; then
+    echo "  🔴 TESTS FAILED"
+    exit 1
+else
+    echo "  ✅ ALL TESTS PASSED"
+    exit 0
+fi