root
93f15709d1
PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 3s
PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 5s
PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 2s
PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 1s
PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 1s
The 2026-09-17 false-verdict incident (third recurrence) showed that prose policy is not a control: the agent probed :9325/:9405 (nonexistent ports), CT 116 for PVE API (should be real PVE nodes), and rendered TLS failures as connection-refused. This moves the canonical probe set into scripts/infra-monitoring.sh (executed verbatim by the contract) and adds scripts/test_infra_monitoring.sh which asserts every probed port matches the documented value. - scripts/infra-monitoring.sh: one script per contract pattern; all targets, ports, paths, and expected-status rules in code; -k for PVE self-signed certs; non-zero exit naming every failed target; no OK summary on failure - scripts/test_infra_monitoring.sh: 20 assertions covering port drift, monitoring-host-as-PVE-node, and missing -k flag - infrastructure-monitoring.prose.md: check-health section now references the script as executable owner; paste its raw output verbatim Proof: all 13 legs pass (exit 0); deliberately broken Grafana port (9325) produces 'probe-failed: 192.168.68.116:9325 (expected 200)' and exit 1.
122 lines
4.3 KiB
Bash
Executable File
122 lines
4.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# test_infra_monitoring.sh — Asserts that probe targets match documented values.
|
|
#
|
|
# Catches:
|
|
# 1. A port not in the documented set (e.g. :9325, :9405)
|
|
# 2. The monitoring host (CT 116) probed for PVE API instead of real PVE nodes
|
|
# 3. A TLS failure labelled as a connection failure (missing -k on PVE)
|
|
#
|
|
# Run: bash scripts/test_infra_monitoring.sh
|
|
# Exits 0 if all assertions pass, 1 otherwise.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SCRIPT="${SCRIPT_DIR}/infra-monitoring.sh"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
assert() {
|
|
local desc="$1" condition="$2"
|
|
if eval "$condition"; then
|
|
echo " ✅ $desc"
|
|
PASS=$((PASS+1))
|
|
else
|
|
echo " 🔴 $desc"
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
}
|
|
|
|
echo "=== test_infra_monitoring.sh ==="
|
|
echo ""
|
|
|
|
# ── 1. Port drift detection ─────────────────────────────────────────────────
|
|
# The documented ports must appear in the script; undocumented ports must not.
|
|
|
|
assert "Grafana port 3001 is documented" \
|
|
'grep -q "GRAFANA_PORT=\"3001\"" "$SCRIPT"'
|
|
|
|
assert "Prometheus port 9090 is documented" \
|
|
'grep -q "PROMETHEUS_PORT=\"9090\"" "$SCRIPT"'
|
|
|
|
assert "LiteLLM probed via nginx on port 80" \
|
|
'grep -q "LITELLM_PORT=\"80\"" "$SCRIPT"'
|
|
|
|
assert "PVE API port 8006 is documented" \
|
|
'grep -q "PVE_API_PORT=\"8006\"" "$SCRIPT"'
|
|
|
|
assert "GPU exporter port 9400 is documented" \
|
|
'grep -q "GPU_PORT=\"9400\"" "$SCRIPT"'
|
|
|
|
assert "Docker Stats port 9323 is documented" \
|
|
'grep -q "DOCKER_STATS_PORT=\"9323\"" "$SCRIPT"'
|
|
|
|
assert "PVE Exporter port 9324 is documented" \
|
|
'grep -q "PVE_EXPORTER_PORT=\"9324\"" "$SCRIPT"'
|
|
|
|
# Undocumented ports that historically caused false verdicts:
|
|
assert "Port 9325 (historical false target) NOT in script" \
|
|
'! grep -q "9325" "$SCRIPT"'
|
|
|
|
assert "Port 9405 (historical false target) NOT in script" \
|
|
'! grep -q "9405" "$SCRIPT"'
|
|
|
|
# ── 2. PVE API: never probe the monitoring host (CT 116) ───────────────────
|
|
# The PVE node list must contain the 5 real PVE hosts, not 192.168.68.116
|
|
|
|
assert "PVE nodes include acerpve .9" \
|
|
'grep -q "192.168.68.9" "$SCRIPT"'
|
|
|
|
assert "PVE nodes include minipve .12" \
|
|
'grep -q "192.168.68.12" "$SCRIPT"'
|
|
|
|
assert "PVE nodes include storepve .6" \
|
|
'grep -q "192.168.68.6" "$SCRIPT"'
|
|
|
|
assert "PVE nodes include amdpve .15" \
|
|
'grep -q "192.168.68.15" "$SCRIPT"'
|
|
|
|
assert "PVE nodes include ocupve .5" \
|
|
'grep -q "192.168.68.5" "$SCRIPT"'
|
|
|
|
# CT 116 (.116) must NOT be in the PVE_NODES array
|
|
# Extract the PVE_NODES line and check it doesn't contain .116
|
|
PVE_NODES_LINE=$(grep "^PVE_NODES=" "$SCRIPT" || true)
|
|
assert "CT 116 (.116) NOT in PVE_NODES array" \
|
|
'[ -z "$PVE_NODES_LINE" ] || ! echo "$PVE_NODES_LINE" | grep -q "68.116"'
|
|
|
|
# ── 3. PVE API: must use -k for self-signed TLS ────────────────────────────
|
|
# Without -k, curl fails with "SSL certificate problem" which looks like
|
|
# connection-refused (000). The script must set PVE_API_USE_K="1".
|
|
|
|
assert "PVE API uses -k flag (self-signed certs)" \
|
|
'grep -q "PVE_API_USE_K=\"1\"" "$SCRIPT"'
|
|
|
|
# The probe_http function must apply use_k to the curl command
|
|
assert "probe_http applies -k to curl when use_k is set" \
|
|
'grep -q "use_k" "$SCRIPT"'
|
|
|
|
# ── 4. PVE API path must be /api2/json/version ─────────────────────────────
|
|
assert "PVE API probes /api2/json/version" \
|
|
'grep -q "PVE_API_PATH=\"/api2/json/version\"" "$SCRIPT"'
|
|
|
|
# ── 5. Exit code behavior ───────────────────────────────────────────────────
|
|
# The script must exit non-zero on failure
|
|
assert "Script exits 1 on failure" \
|
|
'grep -q "exit 1" "$SCRIPT"'
|
|
|
|
assert "Script exits 0 on success" \
|
|
'grep -q "exit 0" "$SCRIPT"'
|
|
|
|
# ── Summary ─────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "Results: ${PASS} passed, ${FAIL} failed"
|
|
if [ $FAIL -gt 0 ]; then
|
|
echo " 🔴 TESTS FAILED"
|
|
exit 1
|
|
else
|
|
echo " ✅ ALL TESTS PASSED"
|
|
exit 0
|
|
fi
|