From f59d1a21592e6c8c869404a430c2ab3245af6566 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 19 Sep 2026 02:28:39 +0000 Subject: [PATCH] fix(proxmox): Fix PBS GC liveness leg + add comprehensive test suite (a) Probe-failure detection: now treats empty OR unparseable JSON as probe-failed, not never-run. This prevents 'command not found' outputs from being rendered as service verdicts. (b) pending-bytes: now extracted from JSON and reported in stale verdict. (c) Null endtime: use .get() with explicit None check, not 0 fallback. null values now correctly trigger never-run verdict instead of arithmetic crash (set -u). (d) Tests: Added 14-assertion stub-driven suite covering: healthy, stale (>48h), probe-failed (empty and unparseable), null endtime, datastore absent. Each test stubs ssh/curl to verify exact behavior against the pre-fix head. Branch: fix/pbs-gc-liveness-signal-20260919 --- scripts/proxmox-monitor.sh | 41 ++++++++--- scripts/test_proxmox_monitor.sh | 125 ++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 11 deletions(-) create mode 100755 scripts/test_proxmox_monitor.sh diff --git a/scripts/proxmox-monitor.sh b/scripts/proxmox-monitor.sh index bb3e80a..689f973 100755 --- a/scripts/proxmox-monitor.sh +++ b/scripts/proxmox-monitor.sh @@ -78,30 +78,49 @@ if [ "$PBS_GC_OUTPUT" = "000" ]; then echo " 🔴 PBS GC: probe-failed: storepve:192.168.68.6 (expected JSON, got 000)" FAILED+=("pbs-gc") else - # Parse the JSON to get storepve-datastore's last-run-endtime - LAST_RUN_ENDTIME=$(echo "$PBS_GC_OUTPUT" | python3 -c " + # Parse the JSON to get storepve-datastore's last-run-endtime and pending-bytes + PBS_GC_RESULT=$(echo "$PBS_GC_OUTPUT" | python3 -c " import sys, json -data = json.load(sys.stdin) -for store in data: - if store['store'] == 'storepve-datastore': - print(store.get('last-run-endtime', 0)) - break +try: + data = json.load(sys.stdin) + for store in data: + if store['store'] == 'storepve-datastore': + endtime = store.get('last-run-endtime') + pending = store.get('pending-bytes', 0) + if endtime is None or endtime == 0: + print('never-run') + else: + print(f'{endtime}|{pending}') + break + else: + print('absent') +except json.JSONDecodeError: + print('unparseable') " 2>/dev/null) - if [ -z "$LAST_RUN_ENDTIME" ] || [ "$LAST_RUN_ENDTIME" = "0" ]; then + if [ -z "$PBS_GC_RESULT" ] || [ "$PBS_GC_RESULT" = "unparseable" ]; then + echo " 🔴 PBS GC: probe-failed: storepve:192.168.68.6 (unparseable JSON)" + FAILED+=("pbs-gc") + elif [ "$PBS_GC_RESULT" = "absent" ]; then + echo " 🔴 PBS GC: never-run (storepve-datastore not found in GC list)" + FAILED+=("pbs-gc") + elif [ "$PBS_GC_RESULT" = "never-run" ]; then echo " 🔴 PBS GC: never-run (storepve-datastore has no last-run-endtime)" FAILED+=("pbs-gc") else + # Parse the endtime|pending format + LAST_RUN_ENDTIME=$(echo "$PBS_GC_RESULT" | cut -d'|' -f1) + PENDING_BYTES=$(echo "$PBS_GC_RESULT" | cut -d'|' -f2) + # Convert epoch to age in hours NOW_EPOCH=$(date -u +%s) AGE_HOURS=$(( (NOW_EPOCH - LAST_RUN_ENDTIME) / 3600 )) - AGE_DAYS=$(( AGE_HOURS / 24 )) if [ $AGE_HOURS -gt 48 ]; then - echo " 🔴 PBS GC: stale (last run ${AGE_DAYS}d ago, pending-bytes unknown)" + echo " 🔴 PBS GC: stale (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)" FAILED+=("pbs-gc") else - echo " ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago)" + echo " ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)" fi fi fi diff --git a/scripts/test_proxmox_monitor.sh b/scripts/test_proxmox_monitor.sh new file mode 100755 index 0000000..ee18d1b --- /dev/null +++ b/scripts/test_proxmox_monitor.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# test_proxmox_monitor.sh — Stub-driven tests for PBS GC liveness leg + +set -uo pipefail + +SCRIPT="$(cd "$(dirname "$0")" && pwd)/proxmox-monitor.sh" +PASS=0 +FAIL=0 + +# ── Helpers ──────────────────────────────────────────────────────────────── +assert() { + local desc="$1" cond="$2" + if eval "$cond" 2>/dev/null; then + echo " ✅ $desc" + PASS=$((PASS+1)) + else + echo " 🔴 $desc" + FAIL=$((FAIL+1)) + fi +} + +# ── 1. Healthy: fresh GC, 0 B pending ───────────────────────────────────── +TMP_DIR=$(mktemp -d) +FRESH_ENDTIME=$(( $(date -u +%s) - (1 * 3600) )) # 1 hour ago +cat > "$TMP_DIR/ssh" << EOF +#!/bin/bash +# Stub: return valid JSON with fresh endtime +echo '[{"store":"storepve-datastore","last-run-endtime":$FRESH_ENDTIME,"pending-bytes":0}]' +exit 0 +EOF +chmod +x "$TMP_DIR/ssh" + +OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1) +PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "") +assert "Healthy: PBS line exists" '[[ -n "$PBS_LINE" ]]' +assert "Healthy: shows healthy verdict" '[[ "$PBS_LINE" == *"healthy"* ]]' +assert "Healthy: shows pending-bytes 0 B" '[[ "$PBS_LINE" == *"pending-bytes: 0 B"* ]]' +rm -rf "$TMP_DIR" + +# ── 2. Stale: GC >48h old ───────────────────────────────────────────────── +TMP_DIR=$(mktemp -d) +OLD_ENDTIME=$(( $(date -u +%s) - (49 * 3600) )) # 49 hours ago +cat > "$TMP_DIR/ssh" << EOF +#!/bin/bash +echo '[{"store":"storepve-datastore","last-run-endtime":$OLD_ENDTIME,"pending-bytes":1048576}]' +exit 0 +EOF +chmod +x "$TMP_DIR/ssh" + +OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1) +PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "") +assert "Stale: PBS line exists" '[[ -n "$PBS_LINE" ]]' +assert "Stale: shows stale verdict" '[[ "$PBS_LINE" == *"stale"* ]]' +assert "Stale: shows pending-bytes 1048576 B" '[[ "$PBS_LINE" == *"pending-bytes: 1048576 B"* ]]' +rm -rf "$TMP_DIR" + +# ── 3. Probe-failed: empty output ───────────────────────────────────────── +TMP_DIR=$(mktemp -d) +cat > "$TMP_DIR/ssh" << 'EOF' +#!/bin/bash +exit 0 +EOF +chmod +x "$TMP_DIR/ssh" + +OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1) +PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "") +assert "Probe-failed (empty): PBS line exists" '[[ -n "$PBS_LINE" ]]' +assert "Probe-failed (empty): shows probe-failed" '[[ "$PBS_LINE" == *"probe-failed"* ]]' +rm -rf "$TMP_DIR" + +# ── 4. Probe-failed: unparseable output ─────────────────────────────────── +TMP_DIR=$(mktemp -d) +cat > "$TMP_DIR/ssh" << 'EOF' +#!/bin/bash +echo "proxmox-backup-manager: command not found" +exit 0 +EOF +chmod +x "$TMP_DIR/ssh" + +OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1) +PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "") +assert "Probe-failed (unparseable): PBS line exists" '[[ -n "$PBS_LINE" ]]' +assert "Probe-failed (unparseable): shows probe-failed" '[[ "$PBS_LINE" == *"probe-failed"* ]]' +rm -rf "$TMP_DIR" + +# ── 5. Null endtime: never-run ──────────────────────────────────────────── +TMP_DIR=$(mktemp -d) +cat > "$TMP_DIR/ssh" << 'EOF' +#!/bin/bash +echo '[{"store":"storepve-datastore","last-run-endtime":null,"pending-bytes":0}]' +exit 0 +EOF +chmod +x "$TMP_DIR/ssh" + +OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1) +PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "") +assert "Null endtime: PBS line exists" '[[ -n "$PBS_LINE" ]]' +assert "Null endtime: shows never-run" '[[ "$PBS_LINE" == *"never-run"* ]]' +rm -rf "$TMP_DIR" + +# ── 6. Datastore absent ─────────────────────────────────────────────────── +TMP_DIR=$(mktemp -d) +cat > "$TMP_DIR/ssh" << 'EOF' +#!/bin/bash +echo '[{"store":"s3-archive","last-run-endtime":null,"pending-bytes":0}]' +exit 0 +EOF +chmod +x "$TMP_DIR/ssh" + +OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1) +PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "") +assert "Datastore absent: PBS line exists" '[[ -n "$PBS_LINE" ]]' +assert "Datastore absent: shows never-run" '[[ "$PBS_LINE" == *"never-run"* ]]' +rm -rf "$TMP_DIR" + +# ── Summary ──────────────────────────────────────────────────────────────── +echo "" +echo "Results: ${PASS} passed, ${FAIL} failed" +if [ $FAIL -gt 0 ]; then + echo " 🔴 TESTS FAILED" + exit 1 +else + echo " ✅ ALL TESTS PASSED" + exit 0 +fi