root
b10fd6fc98
PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 4s
PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 3s
PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 3s
PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 6s
PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 1s
F1: Fixed leg comments to match the actual ports - Line 207: Docker Stats now shows :9324 (was :9323) - Line 215: PVE Exporter now shows :9221 (was :9324) These were the exact pairing this PR exists to correct. F2: Added per-leg assertions that prove which leg owns which port The new assertions verify: 1. Docker Stats leg uses $DOCKER_STATS_PORT constant 2. PVE Exporter leg uses $PVE_EXPORTER_PORT constant 3. DOCKER_STATS_PORT constant is set to 9324 4. PVE_EXPORTER_PORT constant is set to 9221 Proof the new assertions bite: Under the both-constants-swapped mutation (DOCKER_STATS_PORT=9221, PVE_EXPORTER_PORT=9324), the suite fails with 25 passed / 2 failed (failing exactly the two constant-value assertions). This proves the per-leg assertions pin which leg owns which port, not just that both ports appear somewhere in the SSH log. Branch: fix/infra-monitoring-probe-ports-20260919
228 lines
7.8 KiB
Bash
Executable File
228 lines
7.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# test_infra_monitoring.sh — Asserts that probe calls use the documented targets.
|
|
#
|
|
# Strategy: stub curl and ssh on PATH to capture the exact arguments each leg
|
|
# builds, then assert the URL/port of every call. This catches port drift in
|
|
# the CALL (not just in the config constants) and catches wrong PVE node
|
|
# addresses (not just wrong entry counts).
|
|
#
|
|
# Run: bash scripts/test_infra_monitoring.sh
|
|
# Exits 0 if all assertions pass, 1 otherwise.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SCRIPT="${SCRIPT_DIR}/infra-monitoring.sh"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
assert() {
|
|
local desc="$1" condition="$2"
|
|
if eval "$condition"; then
|
|
echo " ✅ $desc"
|
|
PASS=$((PASS+1))
|
|
else
|
|
echo " 🔴 $desc"
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
}
|
|
|
|
echo "=== test_infra_monitoring.sh ==="
|
|
echo ""
|
|
|
|
# ── Stub curl: capture argv to a file, return 200 ──────────────────────────
|
|
STUB_DIR=$(mktemp -d)
|
|
trap 'rm -rf "$STUB_DIR"' EXIT
|
|
|
|
# Stub curl: first arg after flags is the URL; capture all args
|
|
cat > "$STUB_DIR/curl" << 'STUBEOF'
|
|
#!/bin/bash
|
|
echo "$@" >> "${CURL_STUB_LOG:-/dev/null}"
|
|
# Print 200 for %{http_code}
|
|
printf '%s\n' "200"
|
|
exit 0
|
|
STUBEOF
|
|
chmod +x "$STUB_DIR/curl"
|
|
|
|
# Stub ssh: first arg after options is the remote command; capture it
|
|
cat > "$STUB_DIR/ssh" << 'SSTUBEOF'
|
|
#!/bin/bash
|
|
echo "SSH $@" >> "${SSH_STUB_LOG:-/dev/null}"
|
|
# The last arg is the remote command — extract and log curl args
|
|
for arg in "$@"; do
|
|
if [[ "$arg" == curl* ]]; then
|
|
echo "$arg" >> "${SSH_STUB_LOG:-/dev/null}"
|
|
fi
|
|
done
|
|
printf '%s\n' "200"
|
|
exit 0
|
|
SSTUBEOF
|
|
chmod +x "$STUB_DIR/ssh"
|
|
|
|
# ── Run the monitor with stubs ────────────────────────────────────────────
|
|
CURL_LOG="$STUB_DIR/curl_calls.log"
|
|
SSH_LOG="$STUB_DIR/ssh_calls.log"
|
|
touch "$CURL_LOG" "$SSH_LOG"
|
|
|
|
CURL_STUB_LOG="$CURL_LOG" SSH_STUB_LOG="$SSH_LOG" \
|
|
PATH="$STUB_DIR:$PATH" bash "$SCRIPT" > "$STUB_DIR/output.txt" 2>&1
|
|
|
|
# ── 1. Port drift detection (from actual curl invocations) ─────────────────
|
|
|
|
assert "Grafana probed at port 3001" \
|
|
'grep -q "http://192.168.68.116:3001/api/health" "$CURL_LOG"'
|
|
|
|
assert "Prometheus probed at port 9090" \
|
|
'grep -q "http://192.168.68.116:9090/-/healthy" "$CURL_LOG"'
|
|
|
|
assert "LiteLLM probed via nginx at port 80" \
|
|
'grep -q "http://192.168.68.116:80/litellm/health" "$CURL_LOG"'
|
|
|
|
assert "PVE API probed at port 8006" \
|
|
'grep -q ":8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "GPU exporter probed at port 9400" \
|
|
'grep -q ":9400/metrics" "$CURL_LOG"'
|
|
|
|
# ── 2. PVE API: exact node addresses (catches wrong IPs) ──────────────────
|
|
# Each real PVE node must be probed; CT 116 must NOT be in the PVE set
|
|
|
|
assert "PVE acerpve 192.168.68.9 probed" \
|
|
'grep -q "https://192.168.68.9:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "PVE minipve 192.168.68.12 probed" \
|
|
'grep -q "https://192.168.68.12:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "PVE storepve 192.168.68.6 probed" \
|
|
'grep -q "https://192.168.68.6:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "PVE amdpve 192.168.68.15 probed" \
|
|
'grep -q "https://192.168.68.15:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
assert "PVE ocupve 192.168.68.5 probed" \
|
|
'grep -q "https://192.168.68.5:8006/api2/json/version" "$CURL_LOG"'
|
|
|
|
# CT 116 (.116) must NOT appear as a PVE API target
|
|
assert "CT 116 (.116) NOT probed as PVE API node" \
|
|
'! grep -q "https://192.168.68.116:8006" "$CURL_LOG"'
|
|
|
|
# ── 3. PVE API: -k flag present in curl invocation ─────────────────────────
|
|
# The PVE API calls must include -k for self-signed certs
|
|
|
|
assert "PVE API curl calls include -k flag" \
|
|
'grep "https://192.168.68.9:8006" "$CURL_LOG" | grep -q -- "-k"'
|
|
|
|
# ── 4. Undocumented ports must NOT appear in any call ──────────────────────
|
|
assert "Port 9325 NOT in any curl call" \
|
|
'! grep -q ":9325" "$CURL_LOG"'
|
|
|
|
assert "Port 9405 NOT in any curl call" \
|
|
'! grep -q ":9405" "$CURL_LOG"'
|
|
|
|
# ── 5. Docker Stats / PVE Exporter: SSH-probed at correct ports ────────────
|
|
assert "Docker Stats probed at port 9324 via SSH" \
|
|
'grep -q "9324" "$SSH_LOG"'
|
|
|
|
assert "PVE Exporter probed at port 9221 via SSH" \
|
|
'grep -q "9221" "$SSH_LOG"'
|
|
|
|
# ── 5a. Per-leg assertions (proves which leg owns which port) ──────────────
|
|
# The script source must show Docker Stats using $DOCKER_STATS_PORT and
|
|
# PVE Exporter using $PVE_EXPORTER_PORT in the correct leg sections
|
|
assert "Docker Stats leg uses DOCKER_STATS_PORT constant" \
|
|
'grep -A 3 "# 6. Docker Stats" "$SCRIPT" | grep -q "\$DOCKER_STATS_PORT"'
|
|
|
|
assert "PVE Exporter leg uses PVE_EXPORTER_PORT constant" \
|
|
'grep -A 3 "# 7. PVE Exporter" "$SCRIPT" | grep -q "\$PVE_EXPORTER_PORT"'
|
|
|
|
# Verify the constants themselves are set to the correct values
|
|
assert "DOCKER_STATS_PORT constant set to 9324" \
|
|
'grep -q "^DOCKER_STATS_PORT=\"9324\"" "$SCRIPT"'
|
|
|
|
assert "PVE_EXPORTER_PORT constant set to 9221" \
|
|
'grep -q "^PVE_EXPORTER_PORT=\"9221\"" "$SCRIPT"'
|
|
|
|
# ── 5b. Stale port 9323 (dockerd) NOT probed ──────────────────────────────
|
|
assert "Port 9323 (dockerd) NOT in SSH log" \
|
|
'! grep -q "9323" "$SSH_LOG"'
|
|
|
|
# ── 6. No stale ports in the script source (belt-and-suspenders) ──────────
|
|
assert "Port 9325 (historical) NOT in script source" \
|
|
'! grep -q "9325" "$SCRIPT"'
|
|
|
|
assert "Port 9405 (historical) NOT in script source" \
|
|
'! grep -q "9405" "$SCRIPT"'
|
|
|
|
# ── 7. Failure-line content includes non-empty kind ────────────────────────
|
|
TMP_DIR=$(mktemp -d)
|
|
trap 'rm -rf "$TMP_DIR"' EXIT
|
|
|
|
# Test 7a: Unexpected status (500) → kind should be unexpected:500
|
|
cat > "$TMP_DIR/curl" << 'EOF'
|
|
#!/bin/bash
|
|
# Stub: return 500 for Grafana port, 200 otherwise
|
|
for arg in "$@"; do
|
|
if [[ "$arg" == *":3001"* ]]; then
|
|
echo "500"
|
|
exit 0
|
|
fi
|
|
done
|
|
echo "200"
|
|
exit 0
|
|
EOF
|
|
chmod +x "$TMP_DIR/curl"
|
|
OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
|
|
GRAFANA_FAIL=$(echo "$OUT" | grep "Grafana: probe-failed")
|
|
assert "Grafana failure line exists (unexpected status)" \
|
|
'[[ -n "$GRAFANA_FAIL" ]]'
|
|
KIND=$(echo "$GRAFANA_FAIL" | grep -oP '\(<[^>]+>\)' | tr -d '()<>')
|
|
assert "Grafana failure kind is non-empty (unexpected status)" \
|
|
'[[ -n "$KIND" ]]'
|
|
|
|
# Test 7b: TLS error (000 + exit 60) → kind should be tls
|
|
cat > "$TMP_DIR/curl" << 'EOF'
|
|
#!/bin/bash
|
|
# Stub: return 000 and exit 60 for Grafana port (TLS error) on ALL invocations
|
|
for arg in "$@"; do
|
|
if [[ "$arg" == *":3001"* ]]; then
|
|
echo "000"
|
|
exit 60
|
|
fi
|
|
done
|
|
echo "200"
|
|
exit 0
|
|
EOF
|
|
chmod +x "$TMP_DIR/curl"
|
|
# Also stub ssh to return 000 + exit 60 for the retry
|
|
cat > "$TMP_DIR/ssh" << 'EOF'
|
|
#!/bin/bash
|
|
for arg in "$@"; do
|
|
if [[ "$arg" == curl* ]]; then
|
|
echo "000"
|
|
exit 60
|
|
fi
|
|
done
|
|
echo "200"
|
|
exit 0
|
|
EOF
|
|
chmod +x "$TMP_DIR/ssh"
|
|
export PATH="$TMP_DIR:$PATH"
|
|
OUT=$(bash "$SCRIPT" 2>&1)
|
|
GRAFANA_FAIL=$(echo "$OUT" | grep "Grafana: probe-failed")
|
|
assert "Grafana failure line exists (TLS error)" \
|
|
'[[ -n "$GRAFANA_FAIL" ]]'
|
|
KIND=$(echo "$GRAFANA_FAIL" | grep -oP '\(<[^>]+>\)' | tr -d '()<>')
|
|
assert "Grafana failure kind is tls" \
|
|
'[[ "$KIND" == "tls" ]]'
|
|
|
|
# ── Summary ─────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "Results: ${PASS} passed, ${FAIL} failed"
|
|
if [ $FAIL -gt 0 ]; then
|
|
echo " 🔴 TESTS FAILED"
|
|
exit 1
|
|
else
|
|
echo " ✅ ALL TESTS PASSED"
|
|
exit 0
|
|
fi |