fix(test): stub curl/ssh to assert actual call targets (A2+A3)

Rewrote test to run the monitor with stubbed curl/ssh on PATH that
capture the exact argv of each probe call. The test now asserts the
URL+port of every leg actually requested, not source text or config
constants.

A2: PVE node assertions now check the exact URL in the curl log
    (https://192.168.68.9:8006/... must appear), so a wrong IP
    (e.g. .99) fails the test.

A3: Grafana/Prometheus/LiteLLM assertions check the URL the call
    actually builds, so a hardcoded wrong port in the CALL (while the
    config variable stays correct) fails the test.

Mutation evidence:
  A2: sed s/192.168.68.9/192.168.68.99/ in PVE_NODES -> suite FAILS
  A3: sed s/"$GRAFANA_PORT"/"9999"/ in probe call -> suite FAILS

Results: 18 passed, 0 failed (baseline); 17/18 on each mutation

scripts/test_infra_monitoring.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
-# test_infra_monitoring.sh — Asserts that probe targets match documented values.
+# test_infra_monitoring.sh — Asserts that probe calls use the documented targets.
 #
-# Catches:
-#   1. A port not in the documented set (e.g. :9325, :9405)
-#   2. The monitoring host (CT 116) probed for PVE API instead of real PVE nodes
-#   3. A TLS failure labelled as a connection failure (missing -k on PVE)
+# Strategy: stub curl and ssh on PATH to capture the exact arguments each leg
+# builds, then assert the URL/port of every call. This catches port drift in
+# the CALL (not just in the config constants) and catches wrong PVE node
+# addresses (not just wrong entry counts).
 #
 # Run: bash scripts/test_infra_monitoring.sh
 # Exits 0 if all assertions pass, 1 otherwise.
@@ -31,120 +31,108 @@ assert() {
 echo "=== test_infra_monitoring.sh ==="
 echo ""
 
-# ── 1. Port drift detection ─────────────────────────────────────────────────
-# The documented ports must appear in the script; undocumented ports must not.
+# ── Stub curl: capture argv to a file, return 200 ──────────────────────────
+STUB_DIR=$(mktemp -d)
+trap 'rm -rf "$STUB_DIR"' EXIT
 
-assert "Grafana port 3001 is documented" \
-    'grep -q "GRAFANA_PORT=\"3001\"" "$SCRIPT"'
+# Stub curl: first arg after flags is the URL; capture all args
+cat > "$STUB_DIR/curl" << 'STUBEOF'
+#!/bin/bash
+echo "$@" >> "${CURL_STUB_LOG:-/dev/null}"
+# Print 200 for %{http_code}
+printf '%s\n' "200"
+exit 0
+STUBEOF
+chmod +x "$STUB_DIR/curl"
 
-assert "Prometheus port 9090 is documented" \
-    'grep -q "PROMETHEUS_PORT=\"9090\"" "$SCRIPT"'
-
-assert "LiteLLM probed via nginx on port 80" \
-    'grep -q "LITELLM_PORT=\"80\"" "$SCRIPT"'
-
-assert "PVE API port 8006 is documented" \
-    'grep -q "PVE_API_PORT=\"8006\"" "$SCRIPT"'
-
-assert "GPU exporter port 9400 is documented" \
-    'grep -q "GPU_PORT=\"9400\"" "$SCRIPT"'
-
-assert "Docker Stats port 9323 is documented" \
-    'grep -q "DOCKER_STATS_PORT=\"9323\"" "$SCRIPT"'
-
-assert "PVE Exporter port 9324 is documented" \
-    'grep -q "PVE_EXPORTER_PORT=\"9324\"" "$SCRIPT"'
-
-# Undocumented ports that historically caused false verdicts:
-assert "Port 9325 (historical false target) NOT in script" \
-    '! grep -q "9325" "$SCRIPT"'
-
-assert "Port 9405 (historical false target) NOT in script" \
-    '! grep -q "9405" "$SCRIPT"'
-
-# ── 2. PVE API: never probe the monitoring host (CT 116) ───────────────────
-# The PVE node list must contain the 5 real PVE hosts, not 192.168.68.116
-
-assert "PVE nodes include acerpve .9" \
-    'grep -q "192.168.68.9" "$SCRIPT"'
-
-assert "PVE nodes include minipve .12" \
-    'grep -q "192.168.68.12" "$SCRIPT"'
-
-assert "PVE nodes include storepve .6" \
-    'grep -q "192.168.68.6" "$SCRIPT"'
-
-assert "PVE nodes include amdpve .15" \
-    'grep -q "192.168.68.15" "$SCRIPT"'
-
-assert "PVE nodes include ocupve .5" \
-    'grep -q "192.168.68.5" "$SCRIPT"'
-
-# CT 116 (.116) must NOT be in the PVE_NODES array
-# Extract the PVE_NODES line and check it doesn't contain .116
-PVE_NODES_LINE=$(grep "^PVE_NODES=" "$SCRIPT" || true)
-assert "CT 116 (.116) NOT in PVE_NODES array" \
-    '[ -z "$PVE_NODES_LINE" ] || ! echo "$PVE_NODES_LINE" | grep -q "68.116"'
-
-# FIX A2: Assert exact node match by counting occurrences of each expected node
-# and verifying no unexpected nodes are present
-EXPECTED_NODES=("192.168.68.9" "192.168.68.12" "192.168.68.6" "192.168.68.15" "192.168.68.5")
-ALL_NODES_FOUND=true
-for node in "${EXPECTED_NODES[@]}"; do
-    if ! grep -q "$node" "$SCRIPT"; then
-        ALL_NODES_FOUND=false
-        break
+# Stub ssh: first arg after options is the remote command; capture it
+cat > "$STUB_DIR/ssh" << 'SSTUBEOF'
+#!/bin/bash
+echo "SSH $@" >> "${SSH_STUB_LOG:-/dev/null}"
+# The last arg is the remote command — extract and log curl args
+for arg in "$@"; do
+    if [[ "$arg" == curl* ]]; then
+        echo "$arg" >> "${SSH_STUB_LOG:-/dev/null}"
     fi
 done
-assert "PVE_NODES contains all 5 expected nodes" "$ALL_NODES_FOUND"
+printf '%s\n' "200"
+exit 0
+SSTUBEOF
+chmod +x "$STUB_DIR/ssh"
 
-# Verify no extra nodes (check that the array line has exactly 5 IPs)
-NODE_COUNT=$(echo "$PVE_NODES_LINE" | grep -o "192\.168\.68\.[0-9]*" | wc -l)
-assert "PVE_NODES has exactly 5 node entries" '[ "$NODE_COUNT" -eq 5 ]'
+# ── Run the monitor with stubs ────────────────────────────────────────────
+CURL_LOG="$STUB_DIR/curl_calls.log"
+SSH_LOG="$STUB_DIR/ssh_calls.log"
+touch "$CURL_LOG" "$SSH_LOG"
 
-# ── 3. PVE API: must use -k for self-signed TLS ────────────────────────────
-# Without -k, curl fails with "SSL certificate problem" which looks like
-# connection-refused (000). The script must set PVE_API_USE_K="1".
+CURL_STUB_LOG="$CURL_LOG" SSH_STUB_LOG="$SSH_LOG" \
+    PATH="$STUB_DIR:$PATH" bash "$SCRIPT" > "$STUB_DIR/output.txt" 2>&1
 
-assert "PVE API uses -k flag (self-signed certs)" \
-    'grep -q "PVE_API_USE_K=\"1\"" "$SCRIPT"'
+# ── 1. Port drift detection (from actual curl invocations) ─────────────────
 
-# FIX A1: Assert actual curl invocation includes -k by checking the use_k:+-k pattern
-# This is the actual bash syntax that appends -k to the curl command when use_k is set
-assert "probe_http applies -k via use_k:+-k syntax" \
-    'grep -q "use_k:+-k" "$SCRIPT"'
+assert "Grafana probed at port 3001" \
+    'grep -q "http://192.168.68.116:3001/api/health" "$CURL_LOG"'
 
-# FIX A3: Assert behavior by checking that liveness mode accepts any HTTP code
-# The script should have liveness=1 for PVE API which bypasses expected pattern check
-assert "PVE API liveness mode accepts any HTTP code" \
-    'grep -q "PVE_API_LIVENESS=\"1\"" "$SCRIPT"'
+assert "Prometheus probed at port 9090" \
+    'grep -q "http://192.168.68.116:9090/-/healthy" "$CURL_LOG"'
 
-# ── 4. PVE API path must be /api2/json/version ─────────────────────────────
-assert "PVE API probes /api2/json/version" \
-    'grep -q "PVE_API_PATH=\"/api2/json/version\"" "$SCRIPT"'
+assert "LiteLLM probed via nginx at port 80" \
+    'grep -q "http://192.168.68.116:80/litellm/health" "$CURL_LOG"'
 
-# ── 5. Exit code behavior ───────────────────────────────────────────────────
-# The script must exit non-zero on failure
-assert "Script exits 1 on failure" \
-    'grep -q "exit 1" "$SCRIPT"'
+assert "PVE API probed at port 8006" \
+    'grep -q ":8006/api2/json/version" "$CURL_LOG"'
 
-assert "Script exits 0 on success" \
-    'grep -q "exit 0" "$SCRIPT"'
+assert "GPU exporter probed at port 9400" \
+    'grep -q ":9400/metrics" "$CURL_LOG"'
 
-# ── 6. SSH retry logic for Docker Stats/PVE Exporter ────────────────────────
-# FIX C2: The retry logic is in probe_http function (not near the config vars).
-# Verify the retry uses longer timeouts (25s connect, 30s max)
-assert "SSH retry uses 25s connect timeout" \
-    'grep -q -- "--connect-timeout 25" "$SCRIPT"'
+# ── 2. PVE API: exact node addresses (catches wrong IPs) ──────────────────
+# Each real PVE node must be probed; CT 116 must NOT be in the PVE set
 
-assert "SSH retry uses 30s max timeout" \
-    'grep -q -- "--max-time 30" "$SCRIPT"'
+assert "PVE acerpve 192.168.68.9 probed" \
+    'grep -q "https://192.168.68.9:8006/api2/json/version" "$CURL_LOG"'
 
-# ── 7. Output shape verification ────────────────────────────────────────────
-# The script prints "probe-failed: <host>:<port> (any-HTTP liveness, -k for self-signed)"
-# for PVE API failures (FIX C1/C2)
-assert "PVE API failure output includes TLS flag note" \
-    'grep -q "any-HTTP liveness, -k for self-signed" "$SCRIPT"'
+assert "PVE minipve 192.168.68.12 probed" \
+    'grep -q "https://192.168.68.12:8006/api2/json/version" "$CURL_LOG"'
+
+assert "PVE storepve 192.168.68.6 probed" \
+    'grep -q "https://192.168.68.6:8006/api2/json/version" "$CURL_LOG"'
+
+assert "PVE amdpve 192.168.68.15 probed" \
+    'grep -q "https://192.168.68.15:8006/api2/json/version" "$CURL_LOG"'
+
+assert "PVE ocupve 192.168.68.5 probed" \
+    'grep -q "https://192.168.68.5:8006/api2/json/version" "$CURL_LOG"'
+
+# CT 116 (.116) must NOT appear as a PVE API target
+assert "CT 116 (.116) NOT probed as PVE API node" \
+    '! grep -q "https://192.168.68.116:8006" "$CURL_LOG"'
+
+# ── 3. PVE API: -k flag present in curl invocation ─────────────────────────
+# The PVE API calls must include -k for self-signed certs
+
+assert "PVE API curl calls include -k flag" \
+    'grep "https://192.168.68.9:8006" "$CURL_LOG" | grep -q -- "-k"'
+
+# ── 4. Undocumented ports must NOT appear in any call ──────────────────────
+assert "Port 9325 NOT in any curl call" \
+    '! grep -q ":9325" "$CURL_LOG"'
+
+assert "Port 9405 NOT in any curl call" \
+    '! grep -q ":9405" "$CURL_LOG"'
+
+# ── 5. Docker Stats / PVE Exporter: SSH-probed at correct ports ────────────
+assert "Docker Stats probed at port 9323 via SSH" \
+    'grep -q "9323" "$SSH_LOG"'
+
+assert "PVE Exporter probed at port 9324 via SSH" \
+    'grep -q "9324" "$SSH_LOG"'
+
+# ── 6. No stale ports in the script source (belt-and-suspenders) ──────────
+assert "Port 9325 (historical) NOT in script source" \
+    '! grep -q "9325" "$SCRIPT"'
+
+assert "Port 9405 (historical) NOT in script source" \
+    '! grep -q "9405" "$SCRIPT"'
 
 # ── Summary ─────────────────────────────────────────────────────────────────
 echo ""