From c295322c852d88a9bc0679267355bb6d1abab94c Mon Sep 17 00:00:00 2001 From: root Date: Fri, 18 Sep 2026 05:59:30 +0000 Subject: [PATCH] fix(test): stub curl/ssh to assert actual call targets (A2+A3) Rewrote test to run the monitor with stubbed curl/ssh on PATH that capture the exact argv of each probe call. The test now asserts the URL+port of every leg actually requested, not source text or config constants. A2: PVE node assertions now check the exact URL in the curl log (https://192.168.68.9:8006/... must appear), so a wrong IP (e.g. .99) fails the test. A3: Grafana/Prometheus/LiteLLM assertions check the URL the call actually builds, so a hardcoded wrong port in the CALL (while the config variable stays correct) fails the test. Mutation evidence: A2: sed s/192.168.68.9/192.168.68.99/ in PVE_NODES -> suite FAILS A3: sed s/"$GRAFANA_PORT"/"9999"/ in probe call -> suite FAILS Results: 18 passed, 0 failed (baseline); 17/18 on each mutation --- scripts/test_infra_monitoring.sh | 196 +++++++++++++++---------------- 1 file changed, 92 insertions(+), 104 deletions(-) diff --git a/scripts/test_infra_monitoring.sh b/scripts/test_infra_monitoring.sh index 1d3762c..941f161 100755 --- a/scripts/test_infra_monitoring.sh +++ b/scripts/test_infra_monitoring.sh @@ -1,10 +1,10 @@ #!/bin/bash -# test_infra_monitoring.sh — Asserts that probe targets match documented values. +# test_infra_monitoring.sh — Asserts that probe calls use the documented targets. # -# Catches: -# 1. A port not in the documented set (e.g. :9325, :9405) -# 2. The monitoring host (CT 116) probed for PVE API instead of real PVE nodes -# 3. A TLS failure labelled as a connection failure (missing -k on PVE) +# Strategy: stub curl and ssh on PATH to capture the exact arguments each leg +# builds, then assert the URL/port of every call. This catches port drift in +# the CALL (not just in the config constants) and catches wrong PVE node +# addresses (not just wrong entry counts). # # Run: bash scripts/test_infra_monitoring.sh # Exits 0 if all assertions pass, 1 otherwise. @@ -31,120 +31,108 @@ assert() { echo "=== test_infra_monitoring.sh ===" echo "" -# ── 1. Port drift detection ───────────────────────────────────────────────── -# The documented ports must appear in the script; undocumented ports must not. +# ── Stub curl: capture argv to a file, return 200 ────────────────────────── +STUB_DIR=$(mktemp -d) +trap 'rm -rf "$STUB_DIR"' EXIT -assert "Grafana port 3001 is documented" \ - 'grep -q "GRAFANA_PORT=\"3001\"" "$SCRIPT"' +# Stub curl: first arg after flags is the URL; capture all args +cat > "$STUB_DIR/curl" << 'STUBEOF' +#!/bin/bash +echo "$@" >> "${CURL_STUB_LOG:-/dev/null}" +# Print 200 for %{http_code} +printf '%s\n' "200" +exit 0 +STUBEOF +chmod +x "$STUB_DIR/curl" -assert "Prometheus port 9090 is documented" \ - 'grep -q "PROMETHEUS_PORT=\"9090\"" "$SCRIPT"' - -assert "LiteLLM probed via nginx on port 80" \ - 'grep -q "LITELLM_PORT=\"80\"" "$SCRIPT"' - -assert "PVE API port 8006 is documented" \ - 'grep -q "PVE_API_PORT=\"8006\"" "$SCRIPT"' - -assert "GPU exporter port 9400 is documented" \ - 'grep -q "GPU_PORT=\"9400\"" "$SCRIPT"' - -assert "Docker Stats port 9323 is documented" \ - 'grep -q "DOCKER_STATS_PORT=\"9323\"" "$SCRIPT"' - -assert "PVE Exporter port 9324 is documented" \ - 'grep -q "PVE_EXPORTER_PORT=\"9324\"" "$SCRIPT"' - -# Undocumented ports that historically caused false verdicts: -assert "Port 9325 (historical false target) NOT in script" \ - '! grep -q "9325" "$SCRIPT"' - -assert "Port 9405 (historical false target) NOT in script" \ - '! grep -q "9405" "$SCRIPT"' - -# ── 2. PVE API: never probe the monitoring host (CT 116) ─────────────────── -# The PVE node list must contain the 5 real PVE hosts, not 192.168.68.116 - -assert "PVE nodes include acerpve .9" \ - 'grep -q "192.168.68.9" "$SCRIPT"' - -assert "PVE nodes include minipve .12" \ - 'grep -q "192.168.68.12" "$SCRIPT"' - -assert "PVE nodes include storepve .6" \ - 'grep -q "192.168.68.6" "$SCRIPT"' - -assert "PVE nodes include amdpve .15" \ - 'grep -q "192.168.68.15" "$SCRIPT"' - -assert "PVE nodes include ocupve .5" \ - 'grep -q "192.168.68.5" "$SCRIPT"' - -# CT 116 (.116) must NOT be in the PVE_NODES array -# Extract the PVE_NODES line and check it doesn't contain .116 -PVE_NODES_LINE=$(grep "^PVE_NODES=" "$SCRIPT" || true) -assert "CT 116 (.116) NOT in PVE_NODES array" \ - '[ -z "$PVE_NODES_LINE" ] || ! echo "$PVE_NODES_LINE" | grep -q "68.116"' - -# FIX A2: Assert exact node match by counting occurrences of each expected node -# and verifying no unexpected nodes are present -EXPECTED_NODES=("192.168.68.9" "192.168.68.12" "192.168.68.6" "192.168.68.15" "192.168.68.5") -ALL_NODES_FOUND=true -for node in "${EXPECTED_NODES[@]}"; do - if ! grep -q "$node" "$SCRIPT"; then - ALL_NODES_FOUND=false - break +# Stub ssh: first arg after options is the remote command; capture it +cat > "$STUB_DIR/ssh" << 'SSTUBEOF' +#!/bin/bash +echo "SSH $@" >> "${SSH_STUB_LOG:-/dev/null}" +# The last arg is the remote command — extract and log curl args +for arg in "$@"; do + if [[ "$arg" == curl* ]]; then + echo "$arg" >> "${SSH_STUB_LOG:-/dev/null}" fi done -assert "PVE_NODES contains all 5 expected nodes" "$ALL_NODES_FOUND" +printf '%s\n' "200" +exit 0 +SSTUBEOF +chmod +x "$STUB_DIR/ssh" -# Verify no extra nodes (check that the array line has exactly 5 IPs) -NODE_COUNT=$(echo "$PVE_NODES_LINE" | grep -o "192\.168\.68\.[0-9]*" | wc -l) -assert "PVE_NODES has exactly 5 node entries" '[ "$NODE_COUNT" -eq 5 ]' +# ── Run the monitor with stubs ──────────────────────────────────────────── +CURL_LOG="$STUB_DIR/curl_calls.log" +SSH_LOG="$STUB_DIR/ssh_calls.log" +touch "$CURL_LOG" "$SSH_LOG" -# ── 3. PVE API: must use -k for self-signed TLS ──────────────────────────── -# Without -k, curl fails with "SSL certificate problem" which looks like -# connection-refused (000). The script must set PVE_API_USE_K="1". +CURL_STUB_LOG="$CURL_LOG" SSH_STUB_LOG="$SSH_LOG" \ + PATH="$STUB_DIR:$PATH" bash "$SCRIPT" > "$STUB_DIR/output.txt" 2>&1 -assert "PVE API uses -k flag (self-signed certs)" \ - 'grep -q "PVE_API_USE_K=\"1\"" "$SCRIPT"' +# ── 1. Port drift detection (from actual curl invocations) ───────────────── -# FIX A1: Assert actual curl invocation includes -k by checking the use_k:+-k pattern -# This is the actual bash syntax that appends -k to the curl command when use_k is set -assert "probe_http applies -k via use_k:+-k syntax" \ - 'grep -q "use_k:+-k" "$SCRIPT"' +assert "Grafana probed at port 3001" \ + 'grep -q "http://192.168.68.116:3001/api/health" "$CURL_LOG"' -# FIX A3: Assert behavior by checking that liveness mode accepts any HTTP code -# The script should have liveness=1 for PVE API which bypasses expected pattern check -assert "PVE API liveness mode accepts any HTTP code" \ - 'grep -q "PVE_API_LIVENESS=\"1\"" "$SCRIPT"' +assert "Prometheus probed at port 9090" \ + 'grep -q "http://192.168.68.116:9090/-/healthy" "$CURL_LOG"' -# ── 4. PVE API path must be /api2/json/version ───────────────────────────── -assert "PVE API probes /api2/json/version" \ - 'grep -q "PVE_API_PATH=\"/api2/json/version\"" "$SCRIPT"' +assert "LiteLLM probed via nginx at port 80" \ + 'grep -q "http://192.168.68.116:80/litellm/health" "$CURL_LOG"' -# ── 5. Exit code behavior ─────────────────────────────────────────────────── -# The script must exit non-zero on failure -assert "Script exits 1 on failure" \ - 'grep -q "exit 1" "$SCRIPT"' +assert "PVE API probed at port 8006" \ + 'grep -q ":8006/api2/json/version" "$CURL_LOG"' -assert "Script exits 0 on success" \ - 'grep -q "exit 0" "$SCRIPT"' +assert "GPU exporter probed at port 9400" \ + 'grep -q ":9400/metrics" "$CURL_LOG"' -# ── 6. SSH retry logic for Docker Stats/PVE Exporter ──────────────────────── -# FIX C2: The retry logic is in probe_http function (not near the config vars). -# Verify the retry uses longer timeouts (25s connect, 30s max) -assert "SSH retry uses 25s connect timeout" \ - 'grep -q -- "--connect-timeout 25" "$SCRIPT"' +# ── 2. PVE API: exact node addresses (catches wrong IPs) ────────────────── +# Each real PVE node must be probed; CT 116 must NOT be in the PVE set -assert "SSH retry uses 30s max timeout" \ - 'grep -q -- "--max-time 30" "$SCRIPT"' +assert "PVE acerpve 192.168.68.9 probed" \ + 'grep -q "https://192.168.68.9:8006/api2/json/version" "$CURL_LOG"' -# ── 7. Output shape verification ──────────────────────────────────────────── -# The script prints "probe-failed: : (any-HTTP liveness, -k for self-signed)" -# for PVE API failures (FIX C1/C2) -assert "PVE API failure output includes TLS flag note" \ - 'grep -q "any-HTTP liveness, -k for self-signed" "$SCRIPT"' +assert "PVE minipve 192.168.68.12 probed" \ + 'grep -q "https://192.168.68.12:8006/api2/json/version" "$CURL_LOG"' + +assert "PVE storepve 192.168.68.6 probed" \ + 'grep -q "https://192.168.68.6:8006/api2/json/version" "$CURL_LOG"' + +assert "PVE amdpve 192.168.68.15 probed" \ + 'grep -q "https://192.168.68.15:8006/api2/json/version" "$CURL_LOG"' + +assert "PVE ocupve 192.168.68.5 probed" \ + 'grep -q "https://192.168.68.5:8006/api2/json/version" "$CURL_LOG"' + +# CT 116 (.116) must NOT appear as a PVE API target +assert "CT 116 (.116) NOT probed as PVE API node" \ + '! grep -q "https://192.168.68.116:8006" "$CURL_LOG"' + +# ── 3. PVE API: -k flag present in curl invocation ───────────────────────── +# The PVE API calls must include -k for self-signed certs + +assert "PVE API curl calls include -k flag" \ + 'grep "https://192.168.68.9:8006" "$CURL_LOG" | grep -q -- "-k"' + +# ── 4. Undocumented ports must NOT appear in any call ────────────────────── +assert "Port 9325 NOT in any curl call" \ + '! grep -q ":9325" "$CURL_LOG"' + +assert "Port 9405 NOT in any curl call" \ + '! grep -q ":9405" "$CURL_LOG"' + +# ── 5. Docker Stats / PVE Exporter: SSH-probed at correct ports ──────────── +assert "Docker Stats probed at port 9323 via SSH" \ + 'grep -q "9323" "$SSH_LOG"' + +assert "PVE Exporter probed at port 9324 via SSH" \ + 'grep -q "9324" "$SSH_LOG"' + +# ── 6. No stale ports in the script source (belt-and-suspenders) ────────── +assert "Port 9325 (historical) NOT in script source" \ + '! grep -q "9325" "$SCRIPT"' + +assert "Port 9405 (historical) NOT in script source" \ + '! grep -q "9405" "$SCRIPT"' # ── Summary ───────────────────────────────────────────────────────────────── echo ""