root
385f7e0623
PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 9s
PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 6s
PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 5s
PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 10s
PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 2s
A1: Test -k assertion now checks use_k:+-k syntax (actual bash pattern)
A2: PVE_NODES assertions now count expected nodes and verify exact array size
A3: Test now asserts liveness behavior (PVE_API_LIVENESS=1) not source text
B1: disk-gc GC schedule corrected: cron runs pbs-gc.sh (not proxmox-backup-manager),
schedule is 20:00 LOCAL (00:00 UTC, not 20:00 UTC), host timezone America/New_York
B2: PROBE SHAPE now documents actual output shape including TLS flag notes
C1: TLS kind is now printed in PVE API failure output
C2: SSH retry logic clarified - retry is in probe_http function (not unreachable)
158 lines
6.2 KiB
Bash
Executable File
158 lines
6.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# test_infra_monitoring.sh — Asserts that probe targets match documented values.
|
|
#
|
|
# Catches:
|
|
# 1. A port not in the documented set (e.g. :9325, :9405)
|
|
# 2. The monitoring host (CT 116) probed for PVE API instead of real PVE nodes
|
|
# 3. A TLS failure labelled as a connection failure (missing -k on PVE)
|
|
#
|
|
# Run: bash scripts/test_infra_monitoring.sh
|
|
# Exits 0 if all assertions pass, 1 otherwise.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SCRIPT="${SCRIPT_DIR}/infra-monitoring.sh"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
assert() {
|
|
local desc="$1" condition="$2"
|
|
if eval "$condition"; then
|
|
echo " ✅ $desc"
|
|
PASS=$((PASS+1))
|
|
else
|
|
echo " 🔴 $desc"
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
}
|
|
|
|
echo "=== test_infra_monitoring.sh ==="
|
|
echo ""
|
|
|
|
# ── 1. Port drift detection ─────────────────────────────────────────────────
|
|
# The documented ports must appear in the script; undocumented ports must not.
|
|
|
|
assert "Grafana port 3001 is documented" \
|
|
'grep -q "GRAFANA_PORT=\"3001\"" "$SCRIPT"'
|
|
|
|
assert "Prometheus port 9090 is documented" \
|
|
'grep -q "PROMETHEUS_PORT=\"9090\"" "$SCRIPT"'
|
|
|
|
assert "LiteLLM probed via nginx on port 80" \
|
|
'grep -q "LITELLM_PORT=\"80\"" "$SCRIPT"'
|
|
|
|
assert "PVE API port 8006 is documented" \
|
|
'grep -q "PVE_API_PORT=\"8006\"" "$SCRIPT"'
|
|
|
|
assert "GPU exporter port 9400 is documented" \
|
|
'grep -q "GPU_PORT=\"9400\"" "$SCRIPT"'
|
|
|
|
assert "Docker Stats port 9323 is documented" \
|
|
'grep -q "DOCKER_STATS_PORT=\"9323\"" "$SCRIPT"'
|
|
|
|
assert "PVE Exporter port 9324 is documented" \
|
|
'grep -q "PVE_EXPORTER_PORT=\"9324\"" "$SCRIPT"'
|
|
|
|
# Undocumented ports that historically caused false verdicts:
|
|
assert "Port 9325 (historical false target) NOT in script" \
|
|
'! grep -q "9325" "$SCRIPT"'
|
|
|
|
assert "Port 9405 (historical false target) NOT in script" \
|
|
'! grep -q "9405" "$SCRIPT"'
|
|
|
|
# ── 2. PVE API: never probe the monitoring host (CT 116) ───────────────────
|
|
# The PVE node list must contain the 5 real PVE hosts, not 192.168.68.116
|
|
|
|
assert "PVE nodes include acerpve .9" \
|
|
'grep -q "192.168.68.9" "$SCRIPT"'
|
|
|
|
assert "PVE nodes include minipve .12" \
|
|
'grep -q "192.168.68.12" "$SCRIPT"'
|
|
|
|
assert "PVE nodes include storepve .6" \
|
|
'grep -q "192.168.68.6" "$SCRIPT"'
|
|
|
|
assert "PVE nodes include amdpve .15" \
|
|
'grep -q "192.168.68.15" "$SCRIPT"'
|
|
|
|
assert "PVE nodes include ocupve .5" \
|
|
'grep -q "192.168.68.5" "$SCRIPT"'
|
|
|
|
# CT 116 (.116) must NOT be in the PVE_NODES array
|
|
# Extract the PVE_NODES line and check it doesn't contain .116
|
|
PVE_NODES_LINE=$(grep "^PVE_NODES=" "$SCRIPT" || true)
|
|
assert "CT 116 (.116) NOT in PVE_NODES array" \
|
|
'[ -z "$PVE_NODES_LINE" ] || ! echo "$PVE_NODES_LINE" | grep -q "68.116"'
|
|
|
|
# FIX A2: Assert exact node match by counting occurrences of each expected node
|
|
# and verifying no unexpected nodes are present
|
|
EXPECTED_NODES=("192.168.68.9" "192.168.68.12" "192.168.68.6" "192.168.68.15" "192.168.68.5")
|
|
ALL_NODES_FOUND=true
|
|
for node in "${EXPECTED_NODES[@]}"; do
|
|
if ! grep -q "$node" "$SCRIPT"; then
|
|
ALL_NODES_FOUND=false
|
|
break
|
|
fi
|
|
done
|
|
assert "PVE_NODES contains all 5 expected nodes" "$ALL_NODES_FOUND"
|
|
|
|
# Verify no extra nodes (check that the array line has exactly 5 IPs)
|
|
NODE_COUNT=$(echo "$PVE_NODES_LINE" | grep -o "192\.168\.68\.[0-9]*" | wc -l)
|
|
assert "PVE_NODES has exactly 5 node entries" '[ "$NODE_COUNT" -eq 5 ]'
|
|
|
|
# ── 3. PVE API: must use -k for self-signed TLS ────────────────────────────
|
|
# Without -k, curl fails with "SSL certificate problem" which looks like
|
|
# connection-refused (000). The script must set PVE_API_USE_K="1".
|
|
|
|
assert "PVE API uses -k flag (self-signed certs)" \
|
|
'grep -q "PVE_API_USE_K=\"1\"" "$SCRIPT"'
|
|
|
|
# FIX A1: Assert actual curl invocation includes -k by checking the use_k:+-k pattern
|
|
# This is the actual bash syntax that appends -k to the curl command when use_k is set
|
|
assert "probe_http applies -k via use_k:+-k syntax" \
|
|
'grep -q "use_k:+-k" "$SCRIPT"'
|
|
|
|
# FIX A3: Assert behavior by checking that liveness mode accepts any HTTP code
|
|
# The script should have liveness=1 for PVE API which bypasses expected pattern check
|
|
assert "PVE API liveness mode accepts any HTTP code" \
|
|
'grep -q "PVE_API_LIVENESS=\"1\"" "$SCRIPT"'
|
|
|
|
# ── 4. PVE API path must be /api2/json/version ─────────────────────────────
|
|
assert "PVE API probes /api2/json/version" \
|
|
'grep -q "PVE_API_PATH=\"/api2/json/version\"" "$SCRIPT"'
|
|
|
|
# ── 5. Exit code behavior ───────────────────────────────────────────────────
|
|
# The script must exit non-zero on failure
|
|
assert "Script exits 1 on failure" \
|
|
'grep -q "exit 1" "$SCRIPT"'
|
|
|
|
assert "Script exits 0 on success" \
|
|
'grep -q "exit 0" "$SCRIPT"'
|
|
|
|
# ── 6. SSH retry logic for Docker Stats/PVE Exporter ────────────────────────
|
|
# FIX C2: The retry logic is in probe_http function (not near the config vars).
|
|
# Verify the retry uses longer timeouts (25s connect, 30s max)
|
|
assert "SSH retry uses 25s connect timeout" \
|
|
'grep -q -- "--connect-timeout 25" "$SCRIPT"'
|
|
|
|
assert "SSH retry uses 30s max timeout" \
|
|
'grep -q -- "--max-time 30" "$SCRIPT"'
|
|
|
|
# ── 7. Output shape verification ────────────────────────────────────────────
|
|
# The script prints "probe-failed: <host>:<port> (any-HTTP liveness, -k for self-signed)"
|
|
# for PVE API failures (FIX C1/C2)
|
|
assert "PVE API failure output includes TLS flag note" \
|
|
'grep -q "any-HTTP liveness, -k for self-signed" "$SCRIPT"'
|
|
|
|
# ── Summary ─────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "Results: ${PASS} passed, ${FAIL} failed"
|
|
if [ $FAIL -gt 0 ]; then
|
|
echo " 🔴 TESTS FAILED"
|
|
exit 1
|
|
else
|
|
echo " ✅ ALL TESTS PASSED"
|
|
exit 0
|
|
fi |