#!/bin/bash
# proxmox-monitor.sh — Proxmox Cluster + Docker Monitoring Health Check
# Implements proxmox-monitor.prose.md (check-health section)
#
# Legs: Prometheus, Grafana, Docker Stats Exporter, PVE Exporter, PBS GC
# All legs must return 200 for healthy status.
#
# Run: bash scripts/proxmox-monitor.sh
# Exits 0 if all probes pass, 1 if any fails.

set -uo pipefail

CT116_HOST="192.168.68.116"
FAILED=()
TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')

echo "=== Proxmox Monitor — $TIMESTAMP ==="
echo "Executed from: $(pwd -P)"
echo ""

# 1. Prometheus health (bound to 0.0.0.0:9090 on .116)
PROM_CODE=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://${CT116_HOST}:9090/-/healthy 2>/dev/null)
PROM_CODE=$(printf '%s' "$PROM_CODE" | tr -d '[:space:]')
[ -n "$PROM_CODE" ] || PROM_CODE="000"

if [ "$PROM_CODE" = "200" ]; then
    echo "  ✅ Prometheus: alive"
else
    echo "  🔴 Prometheus: probe-failed: ${CT116_HOST}:9090 (expected 200, got ${PROM_CODE})"
    FAILED+=("prometheus")
fi

# 2. Grafana health (bound to 0.0.0.0:3001 on .116)
GRAF_CODE=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://${CT116_HOST}:3001/api/health 2>/dev/null)
GRAF_CODE=$(printf '%s' "$GRAF_CODE" | tr -d '[:space:]')
[ -n "$GRAF_CODE" ] || GRAF_CODE="000"

if [ "$GRAF_CODE" = "200" ]; then
    echo "  ✅ Grafana: alive"
else
    echo "  🔴 Grafana: probe-failed: ${CT116_HOST}:3001 (expected 200, got ${GRAF_CODE})"
    FAILED+=("grafana")
fi

# 3. Docker Stats exporter (bound to 127.0.0.1:9324 on .116 — probe from .116 localhost)
DOCKER_CODE=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@${CT116_HOST} \
    "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9324/metrics" 2>/dev/null)
DOCKER_CODE=$(printf '%s' "$DOCKER_CODE" | tr -d '[:space:]')
[ -n "$DOCKER_CODE" ] || DOCKER_CODE="000"

if [ "$DOCKER_CODE" = "200" ]; then
    echo "  ✅ Docker Stats: alive"
else
    echo "  🔴 Docker Stats: probe-failed: CT116:127.0.0.1:9324 (expected 200, got ${DOCKER_CODE})"
    FAILED+=("docker-stats")
fi

# 4. PVE Exporter (bound to 127.0.0.1:9221 on .116 — probe from .116 localhost)
PVE_CODE=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@${CT116_HOST} \
    "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9221/metrics" 2>/dev/null)
PVE_CODE=$(printf '%s' "$PVE_CODE" | tr -d '[:space:]')
[ -n "$PVE_CODE" ] || PVE_CODE="000"

if [ "$PVE_CODE" = "200" ]; then
    echo "  ✅ PVE Exporter: alive"
else
    echo "  🔴 PVE Exporter: probe-failed: CT116:127.0.0.1:9221 (expected 200, got ${PVE_CODE})"
    FAILED+=("pve-exporter")
fi

# 5. PBS GC liveness (storepve-datastore GC must have run within 48h)
# Use absolute path for pct to avoid PATH issues in non-interactive ssh
PBS_GC_OUTPUT=$(ssh -o ConnectTimeout=5 -o BatchMode=yes root@192.168.68.6 \
    "/sbin/pct exec 107 -- /sbin/proxmox-backup-manager garbage-collection list --output-format json" 2>/dev/null)
PBS_GC_OUTPUT=$(printf '%s' "$PBS_GC_OUTPUT" | tr -d '[:space:]')
[ -n "$PBS_GC_OUTPUT" ] || PBS_GC_OUTPUT="000"

if [ "$PBS_GC_OUTPUT" = "000" ]; then
    echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (expected JSON, got 000)"
    FAILED+=("pbs-gc")
else
    # Parse the JSON to get storepve-datastore's state with four distinct outcomes:
    # 1. probe-failed: non-zero ssh status / empty / unparseable JSON
    # 2. running: collection in progress (last-run-endtime absent or 0, but upid present)
    # 3. stale: no completed run within 48h
    # 4. healthy: completed within 48h
    PBS_GC_RESULT=$(echo "$PBS_GC_OUTPUT" | python3 -c "
import sys, json
try:
    data = json.load(sys.stdin)
    for store in data:
        if store['store'] == 'storepve-datastore':
            endtime = store.get('last-run-endtime')
            upid = store.get('upid')
            pending = store.get('pending-bytes', 0)

            # State 2: Running (collection in progress) — last-run-endtime absent while run is in progress
            if (endtime is None or endtime == 0) and upid is not None:
                print(f'running|{pending}')
                break

            # State 3: No completed run (never-run or stale)
            if endtime is None or endtime == 0:
                print(f'no-completed-run|{pending}')
                break

            # States 3 & 4: Completed (has endtime)
            print(f'completed|{endtime}|{pending}')
            break
    else:
        print(f'absent|0')
except json.JSONDecodeError:
    print(f'unparseable|0')
" 2>/dev/null)
    
    # Parse the state|endtime|pending format
    PBS_GC_STATE=$(echo "$PBS_GC_RESULT" | cut -d'|' -f1)
    
    if [ "$PBS_GC_STATE" = "unparseable" ]; then
        # State 1: probe-failed (unparseable JSON)
        echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (unparseable JSON)"
        FAILED+=("pbs-gc")
    elif [ "$PBS_GC_STATE" = "absent" ]; then
        # State 1: probe-failed (store not found)
        echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (storepve-datastore not found)"
        FAILED+=("pbs-gc")
    elif [ "$PBS_GC_STATE" = "running" ]; then
        # State 2: collection in progress — do NOT fail
        PENDING_BYTES=$(echo "$PBS_GC_RESULT" | cut -d'|' -f2)
        echo "  ⏳ PBS GC: running (started: in-progress, pending-bytes: ${PENDING_BYTES} B)"
    elif [ "$PBS_GC_STATE" = "no-completed-run" ]; then
        # State 3: no completed run within 48h
        PENDING_BYTES=$(echo "$PBS_GC_RESULT" | cut -d'|' -f2)
        echo "  🔴 PBS GC: no completed run within 48h (pending-bytes: ${PENDING_BYTES} B)"
        FAILED+=("pbs-gc")
    else
        # States 3 & 4: completed (has endtime)
        LAST_RUN_ENDTIME=$(echo "$PBS_GC_RESULT" | cut -d'|' -f2)
        PENDING_BYTES=$(echo "$PBS_GC_RESULT" | cut -d'|' -f3)
        
        # Convert epoch to age in hours
        NOW_EPOCH=$(date -u +%s)
        AGE_HOURS=$(( (NOW_EPOCH - LAST_RUN_ENDTIME) / 3600 ))
        
        if [ $AGE_HOURS -gt 48 ]; then
            # State 3: stale (no completed run within 48h)
            echo "  🔴 PBS GC: stale — last completed run was ${AGE_HOURS}h ago (pending-bytes: ${PENDING_BYTES} B)"
            FAILED+=("pbs-gc")
        else
            # State 4: healthy (completed within 48h)
            echo "  ✅ PBS GC: healthy — last completed run ${AGE_HOURS}h ago (pending-bytes: ${PENDING_BYTES} B)"
        fi
    fi
fi

# ── Summary ─────────────────────────────────────────────────────────────────
echo ""
if [ ${#FAILED[@]} -eq 0 ]; then
    echo "  ✅ All legs OK"
    exit 0
else
    for f in "${FAILED[@]}"; do
        echo "  🔴 FAILED: $f"
    done
    exit 1
fi