prose-contracts/scripts/proxmox-monitor.sh

#!/bin/bash
# proxmox-monitor.sh — Proxmox Cluster + Docker Monitoring Health Check
# Implements proxmox-monitor.prose.md (check-health section)
#
# Legs: Prometheus, Grafana, Docker Stats Exporter, PVE Exporter, PBS GC
# All legs must return 200 for healthy status.
#
# Run: bash scripts/proxmox-monitor.sh
# Exits 0 if all probes pass, 1 if any fails.

set -uo pipefail

CT116_HOST="192.168.68.116"
FAILED=()
TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')

echo "=== Proxmox Monitor — $TIMESTAMP ==="
echo "Executed from: $(pwd -P)"
echo ""

# 1. Prometheus health (bound to 0.0.0.0:9090 on .116)
PROM_CODE=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://${CT116_HOST}:9090/-/healthy 2>/dev/null)
PROM_CODE=$(printf '%s' "$PROM_CODE" | tr -d '[:space:]')
[ -n "$PROM_CODE" ] || PROM_CODE="000"

if [ "$PROM_CODE" = "200" ]; then
    echo "  ✅ Prometheus: alive"
else
    echo "  🔴 Prometheus: probe-failed: ${CT116_HOST}:9090 (expected 200, got ${PROM_CODE})"
    FAILED+=("prometheus")
fi

# 2. Grafana health (bound to 0.0.0.0:3001 on .116)
GRAF_CODE=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://${CT116_HOST}:3001/api/health 2>/dev/null)
GRAF_CODE=$(printf '%s' "$GRAF_CODE" | tr -d '[:space:]')
[ -n "$GRAF_CODE" ] || GRAF_CODE="000"

if [ "$GRAF_CODE" = "200" ]; then
    echo "  ✅ Grafana: alive"
else
    echo "  🔴 Grafana: probe-failed: ${CT116_HOST}:3001 (expected 200, got ${GRAF_CODE})"
    FAILED+=("grafana")
fi

# 3. Docker Stats exporter (bound to 127.0.0.1:9324 on .116 — probe from .116 localhost)
DOCKER_CODE=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@${CT116_HOST} \
    "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9324/metrics" 2>/dev/null)
DOCKER_CODE=$(printf '%s' "$DOCKER_CODE" | tr -d '[:space:]')
[ -n "$DOCKER_CODE" ] || DOCKER_CODE="000"

if [ "$DOCKER_CODE" = "200" ]; then
    echo "  ✅ Docker Stats: alive"
else
    echo "  🔴 Docker Stats: probe-failed: CT116:127.0.0.1:9324 (expected 200, got ${DOCKER_CODE})"
    FAILED+=("docker-stats")
fi

# 4. PVE Exporter (bound to 127.0.0.1:9221 on .116 — probe from .116 localhost)
PVE_CODE=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@${CT116_HOST} \
    "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9221/metrics" 2>/dev/null)
PVE_CODE=$(printf '%s' "$PVE_CODE" | tr -d '[:space:]')
[ -n "$PVE_CODE" ] || PVE_CODE="000"

if [ "$PVE_CODE" = "200" ]; then
    echo "  ✅ PVE Exporter: alive"
else
    echo "  🔴 PVE Exporter: probe-failed: CT116:127.0.0.1:9221 (expected 200, got ${PVE_CODE})"
    FAILED+=("pve-exporter")
fi

# 5. PBS GC liveness (storepve-datastore GC must have run within 48h)
PBS_GC_OUTPUT=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@192.168.68.6 \
    "pct exec 107 -- proxmox-backup-manager garbage-collection list --output-format json" 2>/dev/null)
PBS_GC_OUTPUT=$(printf '%s' "$PBS_GC_OUTPUT" | tr -d '[:space:]')
[ -n "$PBS_GC_OUTPUT" ] || PBS_GC_OUTPUT="000"

if [ "$PBS_GC_OUTPUT" = "000" ]; then
    echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (expected JSON, got 000)"
    FAILED+=("pbs-gc")
else
    # Parse the JSON to get storepve-datastore's last-run-endtime and pending-bytes
    PBS_GC_RESULT=$(echo "$PBS_GC_OUTPUT" | python3 -c "
import sys, json
try:
    data = json.load(sys.stdin)
    for store in data:
        if store['store'] == 'storepve-datastore':
            endtime = store.get('last-run-endtime')
            pending = store.get('pending-bytes', 0)
            if endtime is None or endtime == 0:
                print('never-run')
            else:
                print(f'{endtime}|{pending}')
            break
    else:
        print('absent')
except json.JSONDecodeError:
    print('unparseable')
" 2>/dev/null)
    
    if [ -z "$PBS_GC_RESULT" ] || [ "$PBS_GC_RESULT" = "unparseable" ]; then
        echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (unparseable JSON)"
        FAILED+=("pbs-gc")
    elif [ "$PBS_GC_RESULT" = "absent" ]; then
        echo "  🔴 PBS GC: never-run (storepve-datastore not found in GC list)"
        FAILED+=("pbs-gc")
    elif [ "$PBS_GC_RESULT" = "never-run" ]; then
        echo "  🔴 PBS GC: never-run (storepve-datastore has no last-run-endtime)"
        FAILED+=("pbs-gc")
    else
        # Parse the endtime|pending format
        LAST_RUN_ENDTIME=$(echo "$PBS_GC_RESULT" | cut -d'|' -f1)
        PENDING_BYTES=$(echo "$PBS_GC_RESULT" | cut -d'|' -f2)
        
        # Convert epoch to age in hours
        NOW_EPOCH=$(date -u +%s)
        AGE_HOURS=$(( (NOW_EPOCH - LAST_RUN_ENDTIME) / 3600 ))
        
        if [ $AGE_HOURS -gt 48 ]; then
            echo "  🔴 PBS GC: stale (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)"
            FAILED+=("pbs-gc")
        else
            echo "  ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)"
        fi
    fi
fi

# ── Summary ─────────────────────────────────────────────────────────────────
echo ""
if [ ${#FAILED[@]} -eq 0 ]; then
    echo "  ✅ All legs OK"
    exit 0
else
    for f in "${FAILED[@]}"; do
        echo "  🔴 FAILED: $f"
    done
    exit 1
fi