fix(proxmox): Fix PBS GC liveness leg + add comprehensive test suite

(a) Probe-failure detection: now treats empty OR unparseable JSON as
     probe-failed, not never-run. This prevents 'command not found'
     outputs from being rendered as service verdicts.

(b) pending-bytes: now extracted from JSON and reported in stale verdict.

(c) Null endtime: use .get() with explicit None check, not 0 fallback.
     null values now correctly trigger never-run verdict instead of
     arithmetic crash (set -u).

(d) Tests: Added 14-assertion stub-driven suite covering: healthy,
     stale (>48h), probe-failed (empty and unparseable), null endtime,
     datastore absent. Each test stubs ssh/curl to verify exact
     behavior against the pre-fix head.

Branch: fix/pbs-gc-liveness-signal-20260919

scripts/proxmox-monitor.sh

@@ -78,30 +78,49 @@ if [ "$PBS_GC_OUTPUT" = "000" ]; then
     echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (expected JSON, got 000)"
     FAILED+=("pbs-gc")
 else
-    # Parse the JSON to get storepve-datastore's last-run-endtime
-    LAST_RUN_ENDTIME=$(echo "$PBS_GC_OUTPUT" | python3 -c "
+    # Parse the JSON to get storepve-datastore's last-run-endtime and pending-bytes
+    PBS_GC_RESULT=$(echo "$PBS_GC_OUTPUT" | python3 -c "
 import sys, json
-data = json.load(sys.stdin)
-for store in data:
-    if store['store'] == 'storepve-datastore':
-        print(store.get('last-run-endtime', 0))
-        break
+try:
+    data = json.load(sys.stdin)
+    for store in data:
+        if store['store'] == 'storepve-datastore':
+            endtime = store.get('last-run-endtime')
+            pending = store.get('pending-bytes', 0)
+            if endtime is None or endtime == 0:
+                print('never-run')
+            else:
+                print(f'{endtime}|{pending}')
+            break
+    else:
+        print('absent')
+except json.JSONDecodeError:
+    print('unparseable')
 " 2>/dev/null)
     
-    if [ -z "$LAST_RUN_ENDTIME" ] || [ "$LAST_RUN_ENDTIME" = "0" ]; then
+    if [ -z "$PBS_GC_RESULT" ] || [ "$PBS_GC_RESULT" = "unparseable" ]; then
+        echo "  🔴 PBS GC: probe-failed: storepve:192.168.68.6 (unparseable JSON)"
+        FAILED+=("pbs-gc")
+    elif [ "$PBS_GC_RESULT" = "absent" ]; then
+        echo "  🔴 PBS GC: never-run (storepve-datastore not found in GC list)"
+        FAILED+=("pbs-gc")
+    elif [ "$PBS_GC_RESULT" = "never-run" ]; then
         echo "  🔴 PBS GC: never-run (storepve-datastore has no last-run-endtime)"
         FAILED+=("pbs-gc")
     else
+        # Parse the endtime|pending format
+        LAST_RUN_ENDTIME=$(echo "$PBS_GC_RESULT" | cut -d'|' -f1)
+        PENDING_BYTES=$(echo "$PBS_GC_RESULT" | cut -d'|' -f2)
+        
         # Convert epoch to age in hours
         NOW_EPOCH=$(date -u +%s)
         AGE_HOURS=$(( (NOW_EPOCH - LAST_RUN_ENDTIME) / 3600 ))
-        AGE_DAYS=$(( AGE_HOURS / 24 ))
         
         if [ $AGE_HOURS -gt 48 ]; then
-            echo "  🔴 PBS GC: stale (last run ${AGE_DAYS}d ago, pending-bytes unknown)"
+            echo "  🔴 PBS GC: stale (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)"
             FAILED+=("pbs-gc")
         else
-            echo "  ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago)"
+            echo "  ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)"
         fi
     fi
 fi

scripts/test_proxmox_monitor.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+# test_proxmox_monitor.sh — Stub-driven tests for PBS GC liveness leg
+
+set -uo pipefail
+
+SCRIPT="$(cd "$(dirname "$0")" && pwd)/proxmox-monitor.sh"
+PASS=0
+FAIL=0
+
+# ── Helpers ────────────────────────────────────────────────────────────────
+assert() {
+    local desc="$1" cond="$2"
+    if eval "$cond" 2>/dev/null; then
+        echo "  ✅ $desc"
+        PASS=$((PASS+1))
+    else
+        echo "  🔴 $desc"
+        FAIL=$((FAIL+1))
+    fi
+}
+
+# ── 1. Healthy: fresh GC, 0 B pending ─────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+FRESH_ENDTIME=$(( $(date -u +%s) - (1 * 3600) ))  # 1 hour ago
+cat > "$TMP_DIR/ssh" << EOF
+#!/bin/bash
+# Stub: return valid JSON with fresh endtime
+echo '[{"store":"storepve-datastore","last-run-endtime":$FRESH_ENDTIME,"pending-bytes":0}]'
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Healthy: PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Healthy: shows healthy verdict" '[[ "$PBS_LINE" == *"healthy"* ]]'
+assert "Healthy: shows pending-bytes 0 B" '[[ "$PBS_LINE" == *"pending-bytes: 0 B"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 2. Stale: GC >48h old ─────────────────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+OLD_ENDTIME=$(( $(date -u +%s) - (49 * 3600) ))  # 49 hours ago
+cat > "$TMP_DIR/ssh" << EOF
+#!/bin/bash
+echo '[{"store":"storepve-datastore","last-run-endtime":$OLD_ENDTIME,"pending-bytes":1048576}]'
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Stale: PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Stale: shows stale verdict" '[[ "$PBS_LINE" == *"stale"* ]]'
+assert "Stale: shows pending-bytes 1048576 B" '[[ "$PBS_LINE" == *"pending-bytes: 1048576 B"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 3. Probe-failed: empty output ─────────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Probe-failed (empty): PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Probe-failed (empty): shows probe-failed" '[[ "$PBS_LINE" == *"probe-failed"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 4. Probe-failed: unparseable output ───────────────────────────────────
+TMP_DIR=$(mktemp -d)
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+echo "proxmox-backup-manager: command not found"
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Probe-failed (unparseable): PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Probe-failed (unparseable): shows probe-failed" '[[ "$PBS_LINE" == *"probe-failed"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 5. Null endtime: never-run ────────────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+echo '[{"store":"storepve-datastore","last-run-endtime":null,"pending-bytes":0}]'
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Null endtime: PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Null endtime: shows never-run" '[[ "$PBS_LINE" == *"never-run"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── 6. Datastore absent ───────────────────────────────────────────────────
+TMP_DIR=$(mktemp -d)
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+echo '[{"store":"s3-archive","last-run-endtime":null,"pending-bytes":0}]'
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+
+OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
+PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
+assert "Datastore absent: PBS line exists" '[[ -n "$PBS_LINE" ]]'
+assert "Datastore absent: shows never-run" '[[ "$PBS_LINE" == *"never-run"* ]]'
+rm -rf "$TMP_DIR"
+
+# ── Summary ────────────────────────────────────────────────────────────────
+echo ""
+echo "Results: ${PASS} passed, ${FAIL} failed"
+if [ $FAIL -gt 0 ]; then
+    echo "  🔴 TESTS FAILED"
+    exit 1
+else
+    echo "  ✅ ALL TESTS PASSED"
+    exit 0
+fi