feat(proxmox-monitor): add PBS GC liveness signal #119
+30
-11
@@ -78,30 +78,49 @@ if [ "$PBS_GC_OUTPUT" = "000" ]; then
|
||||
echo " 🔴 PBS GC: probe-failed: storepve:192.168.68.6 (expected JSON, got 000)"
|
||||
FAILED+=("pbs-gc")
|
||||
else
|
||||
# Parse the JSON to get storepve-datastore's last-run-endtime
|
||||
LAST_RUN_ENDTIME=$(echo "$PBS_GC_OUTPUT" | python3 -c "
|
||||
# Parse the JSON to get storepve-datastore's last-run-endtime and pending-bytes
|
||||
PBS_GC_RESULT=$(echo "$PBS_GC_OUTPUT" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
for store in data:
|
||||
if store['store'] == 'storepve-datastore':
|
||||
print(store.get('last-run-endtime', 0))
|
||||
break
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
for store in data:
|
||||
if store['store'] == 'storepve-datastore':
|
||||
endtime = store.get('last-run-endtime')
|
||||
pending = store.get('pending-bytes', 0)
|
||||
if endtime is None or endtime == 0:
|
||||
print('never-run')
|
||||
else:
|
||||
print(f'{endtime}|{pending}')
|
||||
break
|
||||
else:
|
||||
print('absent')
|
||||
except json.JSONDecodeError:
|
||||
print('unparseable')
|
||||
" 2>/dev/null)
|
||||
|
||||
if [ -z "$LAST_RUN_ENDTIME" ] || [ "$LAST_RUN_ENDTIME" = "0" ]; then
|
||||
if [ -z "$PBS_GC_RESULT" ] || [ "$PBS_GC_RESULT" = "unparseable" ]; then
|
||||
echo " 🔴 PBS GC: probe-failed: storepve:192.168.68.6 (unparseable JSON)"
|
||||
FAILED+=("pbs-gc")
|
||||
elif [ "$PBS_GC_RESULT" = "absent" ]; then
|
||||
echo " 🔴 PBS GC: never-run (storepve-datastore not found in GC list)"
|
||||
FAILED+=("pbs-gc")
|
||||
elif [ "$PBS_GC_RESULT" = "never-run" ]; then
|
||||
echo " 🔴 PBS GC: never-run (storepve-datastore has no last-run-endtime)"
|
||||
FAILED+=("pbs-gc")
|
||||
else
|
||||
# Parse the endtime|pending format
|
||||
LAST_RUN_ENDTIME=$(echo "$PBS_GC_RESULT" | cut -d'|' -f1)
|
||||
PENDING_BYTES=$(echo "$PBS_GC_RESULT" | cut -d'|' -f2)
|
||||
|
||||
# Convert epoch to age in hours
|
||||
NOW_EPOCH=$(date -u +%s)
|
||||
AGE_HOURS=$(( (NOW_EPOCH - LAST_RUN_ENDTIME) / 3600 ))
|
||||
AGE_DAYS=$(( AGE_HOURS / 24 ))
|
||||
|
||||
if [ $AGE_HOURS -gt 48 ]; then
|
||||
echo " 🔴 PBS GC: stale (last run ${AGE_DAYS}d ago, pending-bytes unknown)"
|
||||
echo " 🔴 PBS GC: stale (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)"
|
||||
FAILED+=("pbs-gc")
|
||||
else
|
||||
echo " ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago)"
|
||||
echo " ✅ PBS GC: healthy (last run ${AGE_HOURS}h ago, pending-bytes: ${PENDING_BYTES} B)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
Executable
+125
@@ -0,0 +1,125 @@
|
||||
#!/bin/bash
|
||||
# test_proxmox_monitor.sh — Stub-driven tests for PBS GC liveness leg
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT="$(cd "$(dirname "$0")" && pwd)/proxmox-monitor.sh"
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
assert() {
|
||||
local desc="$1" cond="$2"
|
||||
if eval "$cond" 2>/dev/null; then
|
||||
echo " ✅ $desc"
|
||||
PASS=$((PASS+1))
|
||||
else
|
||||
echo " 🔴 $desc"
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
}
|
||||
|
||||
# ── 1. Healthy: fresh GC, 0 B pending ─────────────────────────────────────
|
||||
TMP_DIR=$(mktemp -d)
|
||||
FRESH_ENDTIME=$(( $(date -u +%s) - (1 * 3600) )) # 1 hour ago
|
||||
cat > "$TMP_DIR/ssh" << EOF
|
||||
#!/bin/bash
|
||||
# Stub: return valid JSON with fresh endtime
|
||||
echo '[{"store":"storepve-datastore","last-run-endtime":$FRESH_ENDTIME,"pending-bytes":0}]'
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$TMP_DIR/ssh"
|
||||
|
||||
OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
|
||||
PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
|
||||
assert "Healthy: PBS line exists" '[[ -n "$PBS_LINE" ]]'
|
||||
assert "Healthy: shows healthy verdict" '[[ "$PBS_LINE" == *"healthy"* ]]'
|
||||
assert "Healthy: shows pending-bytes 0 B" '[[ "$PBS_LINE" == *"pending-bytes: 0 B"* ]]'
|
||||
rm -rf "$TMP_DIR"
|
||||
|
||||
# ── 2. Stale: GC >48h old ─────────────────────────────────────────────────
|
||||
TMP_DIR=$(mktemp -d)
|
||||
OLD_ENDTIME=$(( $(date -u +%s) - (49 * 3600) )) # 49 hours ago
|
||||
cat > "$TMP_DIR/ssh" << EOF
|
||||
#!/bin/bash
|
||||
echo '[{"store":"storepve-datastore","last-run-endtime":$OLD_ENDTIME,"pending-bytes":1048576}]'
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$TMP_DIR/ssh"
|
||||
|
||||
OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
|
||||
PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
|
||||
assert "Stale: PBS line exists" '[[ -n "$PBS_LINE" ]]'
|
||||
assert "Stale: shows stale verdict" '[[ "$PBS_LINE" == *"stale"* ]]'
|
||||
assert "Stale: shows pending-bytes 1048576 B" '[[ "$PBS_LINE" == *"pending-bytes: 1048576 B"* ]]'
|
||||
rm -rf "$TMP_DIR"
|
||||
|
||||
# ── 3. Probe-failed: empty output ─────────────────────────────────────────
|
||||
TMP_DIR=$(mktemp -d)
|
||||
cat > "$TMP_DIR/ssh" << 'EOF'
|
||||
#!/bin/bash
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$TMP_DIR/ssh"
|
||||
|
||||
OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
|
||||
PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
|
||||
assert "Probe-failed (empty): PBS line exists" '[[ -n "$PBS_LINE" ]]'
|
||||
assert "Probe-failed (empty): shows probe-failed" '[[ "$PBS_LINE" == *"probe-failed"* ]]'
|
||||
rm -rf "$TMP_DIR"
|
||||
|
||||
# ── 4. Probe-failed: unparseable output ───────────────────────────────────
|
||||
TMP_DIR=$(mktemp -d)
|
||||
cat > "$TMP_DIR/ssh" << 'EOF'
|
||||
#!/bin/bash
|
||||
echo "proxmox-backup-manager: command not found"
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$TMP_DIR/ssh"
|
||||
|
||||
OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
|
||||
PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
|
||||
assert "Probe-failed (unparseable): PBS line exists" '[[ -n "$PBS_LINE" ]]'
|
||||
assert "Probe-failed (unparseable): shows probe-failed" '[[ "$PBS_LINE" == *"probe-failed"* ]]'
|
||||
rm -rf "$TMP_DIR"
|
||||
|
||||
# ── 5. Null endtime: never-run ────────────────────────────────────────────
|
||||
TMP_DIR=$(mktemp -d)
|
||||
cat > "$TMP_DIR/ssh" << 'EOF'
|
||||
#!/bin/bash
|
||||
echo '[{"store":"storepve-datastore","last-run-endtime":null,"pending-bytes":0}]'
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$TMP_DIR/ssh"
|
||||
|
||||
OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
|
||||
PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
|
||||
assert "Null endtime: PBS line exists" '[[ -n "$PBS_LINE" ]]'
|
||||
assert "Null endtime: shows never-run" '[[ "$PBS_LINE" == *"never-run"* ]]'
|
||||
rm -rf "$TMP_DIR"
|
||||
|
||||
# ── 6. Datastore absent ───────────────────────────────────────────────────
|
||||
TMP_DIR=$(mktemp -d)
|
||||
cat > "$TMP_DIR/ssh" << 'EOF'
|
||||
#!/bin/bash
|
||||
echo '[{"store":"s3-archive","last-run-endtime":null,"pending-bytes":0}]'
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$TMP_DIR/ssh"
|
||||
|
||||
OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
|
||||
PBS_LINE=$(echo "$OUT" | grep "PBS GC:" || echo "")
|
||||
assert "Datastore absent: PBS line exists" '[[ -n "$PBS_LINE" ]]'
|
||||
assert "Datastore absent: shows never-run" '[[ "$PBS_LINE" == *"never-run"* ]]'
|
||||
rm -rf "$TMP_DIR"
|
||||
|
||||
# ── Summary ────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "Results: ${PASS} passed, ${FAIL} failed"
|
||||
if [ $FAIL -gt 0 ]; then
|
||||
echo " 🔴 TESTS FAILED"
|
||||
exit 1
|
||||
else
|
||||
echo " ✅ ALL TESTS PASSED"
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user