#!/bin/bash # test_infra_monitoring.sh — Asserts that probe targets match documented values. # # Catches: # 1. A port not in the documented set (e.g. :9325, :9405) # 2. The monitoring host (CT 116) probed for PVE API instead of real PVE nodes # 3. A TLS failure labelled as a connection failure (missing -k on PVE) # # Run: bash scripts/test_infra_monitoring.sh # Exits 0 if all assertions pass, 1 otherwise. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SCRIPT="${SCRIPT_DIR}/infra-monitoring.sh" PASS=0 FAIL=0 assert() { local desc="$1" condition="$2" if eval "$condition"; then echo " ✅ $desc" PASS=$((PASS+1)) else echo " 🔴 $desc" FAIL=$((FAIL+1)) fi } echo "=== test_infra_monitoring.sh ===" echo "" # ── 1. Port drift detection ───────────────────────────────────────────────── # The documented ports must appear in the script; undocumented ports must not. assert "Grafana port 3001 is documented" \ 'grep -q "GRAFANA_PORT=\"3001\"" "$SCRIPT"' assert "Prometheus port 9090 is documented" \ 'grep -q "PROMETHEUS_PORT=\"9090\"" "$SCRIPT"' assert "LiteLLM probed via nginx on port 80" \ 'grep -q "LITELLM_PORT=\"80\"" "$SCRIPT"' assert "PVE API port 8006 is documented" \ 'grep -q "PVE_API_PORT=\"8006\"" "$SCRIPT"' assert "GPU exporter port 9400 is documented" \ 'grep -q "GPU_PORT=\"9400\"" "$SCRIPT"' assert "Docker Stats port 9323 is documented" \ 'grep -q "DOCKER_STATS_PORT=\"9323\"" "$SCRIPT"' assert "PVE Exporter port 9324 is documented" \ 'grep -q "PVE_EXPORTER_PORT=\"9324\"" "$SCRIPT"' # Undocumented ports that historically caused false verdicts: assert "Port 9325 (historical false target) NOT in script" \ '! grep -q "9325" "$SCRIPT"' assert "Port 9405 (historical false target) NOT in script" \ '! grep -q "9405" "$SCRIPT"' # ── 2. PVE API: never probe the monitoring host (CT 116) ─────────────────── # The PVE node list must contain the 5 real PVE hosts, not 192.168.68.116 assert "PVE nodes include acerpve .9" \ 'grep -q "192.168.68.9" "$SCRIPT"' assert "PVE nodes include minipve .12" \ 'grep -q "192.168.68.12" "$SCRIPT"' assert "PVE nodes include storepve .6" \ 'grep -q "192.168.68.6" "$SCRIPT"' assert "PVE nodes include amdpve .15" \ 'grep -q "192.168.68.15" "$SCRIPT"' assert "PVE nodes include ocupve .5" \ 'grep -q "192.168.68.5" "$SCRIPT"' # CT 116 (.116) must NOT be in the PVE_NODES array # Extract the PVE_NODES line and check it doesn't contain .116 PVE_NODES_LINE=$(grep "^PVE_NODES=" "$SCRIPT" || true) assert "CT 116 (.116) NOT in PVE_NODES array" \ '[ -z "$PVE_NODES_LINE" ] || ! echo "$PVE_NODES_LINE" | grep -q "68.116"' # ── 3. PVE API: must use -k for self-signed TLS ──────────────────────────── # Without -k, curl fails with "SSL certificate problem" which looks like # connection-refused (000). The script must set PVE_API_USE_K="1". assert "PVE API uses -k flag (self-signed certs)" \ 'grep -q "PVE_API_USE_K=\"1\"" "$SCRIPT"' # The probe_http function must apply use_k to the curl command assert "probe_http applies -k to curl when use_k is set" \ 'grep -q "use_k" "$SCRIPT"' # ── 4. PVE API path must be /api2/json/version ───────────────────────────── assert "PVE API probes /api2/json/version" \ 'grep -q "PVE_API_PATH=\"/api2/json/version\"" "$SCRIPT"' # ── 5. Exit code behavior ─────────────────────────────────────────────────── # The script must exit non-zero on failure assert "Script exits 1 on failure" \ 'grep -q "exit 1" "$SCRIPT"' assert "Script exits 0 on success" \ 'grep -q "exit 0" "$SCRIPT"' # ── Summary ───────────────────────────────────────────────────────────────── echo "" echo "Results: ${PASS} passed, ${FAIL} failed" if [ $FAIL -gt 0 ]; then echo " 🔴 TESTS FAILED" exit 1 else echo " ✅ ALL TESTS PASSED" exit 0 fi