root
c295322c85
PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 6s
PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 17s
PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 7s
PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 11s
PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 2s
Rewrote test to run the monitor with stubbed curl/ssh on PATH that
capture the exact argv of each probe call. The test now asserts the
URL+port of every leg actually requested, not source text or config
constants.
A2: PVE node assertions now check the exact URL in the curl log
(https://192.168.68.9:8006/... must appear), so a wrong IP
(e.g. .99) fails the test.
A3: Grafana/Prometheus/LiteLLM assertions check the URL the call
actually builds, so a hardcoded wrong port in the CALL (while the
config variable stays correct) fails the test.
Mutation evidence:
A2: sed s/192.168.68.9/192.168.68.99/ in PVE_NODES -> suite FAILS
A3: sed s/"$GRAFANA_PORT"/"9999"/ in probe call -> suite FAILS
Results: 18 passed, 0 failed (baseline); 17/18 on each mutation
146 lines
5.0 KiB
Bash
Executable File
146 lines
5.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# test_infra_monitoring.sh — Asserts that probe calls use the documented targets.
|
|
#
|
|
# Strategy: stub curl and ssh on PATH to capture the exact arguments each leg
|
|
# builds, then assert the URL/port of every call. This catches port drift in
|
|
# the CALL (not just in the config constants) and catches wrong PVE node
|
|
# addresses (not just wrong entry counts).
|
|
#
|
|
# Run: bash scripts/test_infra_monitoring.sh
|
|
# Exits 0 if all assertions pass, 1 otherwise.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SCRIPT="${SCRIPT_DIR}/infra-monitoring.sh"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
assert() {
|
|
local desc="$1" condition="$2"
|
|
if eval "$condition"; then
|
|
echo " ✅ $desc"
|
|
PASS=$((PASS+1))
|
|
else
|
|
echo " 🔴 $desc"
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
}
|
|
|
|
echo "=== test_infra_monitoring.sh ==="
|
|
echo ""
|
|
|
|
# ── Stub curl: capture argv to a file, return 200 ──────────────────────────
|
|
STUB_DIR=$(mktemp -d)
|
|
trap 'rm -rf "$STUB_DIR"' EXIT
|
|
|
|
# Stub curl: first arg after flags is the URL; capture all args
|
|
cat > "$STUB_DIR/curl" << 'STUBEOF'
|
|
#!/bin/bash
|
|
echo "$@" >> "${CURL_STUB_LOG:-/dev/null}"
|
|
# Print 200 for %{http_code}
|
|
printf '%s\n' "200"
|
|
exit 0
|
|
STUBEOF
|
|
chmod +x "$STUB_DIR/curl"
|
|
|
|
# Stub ssh: first arg after options is the remote command; capture it
|
|
cat > "$STUB_DIR/ssh" << 'SSTUBEOF'
|
|
#!/bin/bash
|
|
echo "SSH $@" >> "${SSH_STUB_LOG:-/dev/null}"
|
|
# The last arg is the remote command — extract and log curl args
|
|
for arg in "$@"; do
|
|
if [[ "$arg" == curl* ]]; then
|
|
echo "$arg" >> "${SSH_STUB_LOG:-/dev/null}"
|
|
fi
|
|
done
|
|
printf '%s\n' "200"
|
|
exit 0
|
|
SSTUBEOF
|
|
chmod +x "$STUB_DIR/ssh"
|
|
|
|
# ── Run the monitor with stubs ────────────────────────────────────────────
|
|
CURL_LOG="$STUB_DIR/curl_calls.log"
|
|
SSH_LOG="$STUB_DIR/ssh_calls.log"
|
|
touch "$CURL_LOG" "$SSH_LOG"
|
|
|
|
CURL_STUB_LOG="$CURL_LOG" SSH_STUB_LOG="$SSH_LOG" \
|
|
PATH="$STUB_DIR:$PATH" bash "$SCRIPT" > "$STUB_DIR/output.txt" 2>&1
|
|
|
|
# ── 1. Port drift detection (from actual curl invocations) ─────────────────
|
|
|
|
assert "Grafana probed at port 3001" \
|
|
'grep -q "http://192.168.68.116:3001/api/health" "$CURL_LOG"'
|
|
|
|
assert "Prometheus probed at port 9090" \
|
|
'grep -q "http://192.168.68.116:9090/-/healthy" "$CURL_LOG"'
|
|
|
|
assert "LiteLLM probed via nginx at port 80" \
|
|
'grep -q "http://192.168.68.116:80/litellm/health" "$CURL_LOG"'
|
|
|
|
assert "PVE API probed at port 8006" \
|
|
'grep -q ":8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "GPU exporter probed at port 9400" \
|
|
'grep -q ":9400/metrics" "$CURL_LOG"'
|
|
|
|
# ── 2. PVE API: exact node addresses (catches wrong IPs) ──────────────────
|
|
# Each real PVE node must be probed; CT 116 must NOT be in the PVE set
|
|
|
|
assert "PVE acerpve 192.168.68.9 probed" \
|
|
'grep -q "https://192.168.68.9:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "PVE minipve 192.168.68.12 probed" \
|
|
'grep -q "https://192.168.68.12:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "PVE storepve 192.168.68.6 probed" \
|
|
'grep -q "https://192.168.68.6:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "PVE amdpve 192.168.68.15 probed" \
|
|
'grep -q "https://192.168.68.15:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "PVE ocupve 192.168.68.5 probed" \
|
|
'grep -q "https://192.168.68.5:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
# CT 116 (.116) must NOT appear as a PVE API target
|
|
assert "CT 116 (.116) NOT probed as PVE API node" \
|
|
'! grep -q "https://192.168.68.116:8006" "$CURL_LOG"'
|
|
|
|
# ── 3. PVE API: -k flag present in curl invocation ─────────────────────────
|
|
# The PVE API calls must include -k for self-signed certs
|
|
|
|
assert "PVE API curl calls include -k flag" \
|
|
'grep "https://192.168.68.9:8006" "$CURL_LOG" | grep -q -- "-k"'
|
|
|
|
# ── 4. Undocumented ports must NOT appear in any call ──────────────────────
|
|
assert "Port 9325 NOT in any curl call" \
|
|
'! grep -q ":9325" "$CURL_LOG"'
|
|
|
|
assert "Port 9405 NOT in any curl call" \
|
|
'! grep -q ":9405" "$CURL_LOG"'
|
|
|
|
# ── 5. Docker Stats / PVE Exporter: SSH-probed at correct ports ────────────
|
|
assert "Docker Stats probed at port 9323 via SSH" \
|
|
'grep -q "9323" "$SSH_LOG"'
|
|
|
|
assert "PVE Exporter probed at port 9324 via SSH" \
|
|
'grep -q "9324" "$SSH_LOG"'
|
|
|
|
# ── 6. No stale ports in the script source (belt-and-suspenders) ──────────
|
|
assert "Port 9325 (historical) NOT in script source" \
|
|
'! grep -q "9325" "$SCRIPT"'
|
|
|
|
assert "Port 9405 (historical) NOT in script source" \
|
|
'! grep -q "9405" "$SCRIPT"'
|
|
|
|
# ── Summary ─────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "Results: ${PASS} passed, ${FAIL} failed"
|
|
if [ $FAIL -gt 0 ]; then
|
|
echo " 🔴 TESTS FAILED"
|
|
exit 1
|
|
else
|
|
echo " ✅ ALL TESTS PASSED"
|
|
exit 0
|
|
fi |