diff --git a/infrastructure-monitoring.prose.md b/infrastructure-monitoring.prose.md index 04e62ce..52b1ac6 100644 --- a/infrastructure-monitoring.prose.md +++ b/infrastructure-monitoring.prose.md @@ -277,6 +277,27 @@ code (or failure kind with retry details). Apply the standing probe rules: any HTTP status = ALIVE; only 000/timeout/refused = probe-failed. A redirect is not a failure. +### Docker Stats and PVE Exporter Ports + +These two exporters bind to 127.0.0.1 on CT 116 (localhost-only) and must be probed via SSH: + +| Exporter | Port | Container | Metrics | +|----------|------|-----------|---------| +| **Docker Stats** | **9324** | harness-docker-stats | `docker_container_*` (per-container CPU/mem/network) | +| **PVE Exporter** | **9221** | harness-pve-exporter | `pve_*` (5 cluster-level metrics) | + +**IMPORTANT**: Do not confuse with port 9323, which is owned by dockerd and serves the Docker Engine's own metrics (`builder_builds_*`, `containerd_build_info_*`). + +```bash +# Docker Stats (harness-docker-stats) +ssh root@192.168.68.116 "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9324/metrics" +# Expected: 200 or 404 (any HTTP status = ALIVE) + +# PVE Exporter (harness-pve-exporter) +ssh root@192.168.68.116 "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9221/metrics" +# Expected: 200 or 404 (any HTTP status = ALIVE) +``` + ### Phase 1: GPU Exporters **NVIDIA (.8 and .110)**: diff --git a/scripts/infra-monitoring.sh b/scripts/infra-monitoring.sh index bd20718..472a812 100755 --- a/scripts/infra-monitoring.sh +++ b/scripts/infra-monitoring.sh @@ -60,8 +60,8 @@ GPU_PATH="/metrics" GPU_EXPECTED="200" # Docker Stats and PVE Exporter bind to 127.0.0.1 on CT 116 -DOCKER_STATS_PORT="9323" -PVE_EXPORTER_PORT="9324" +DOCKER_STATS_PORT="9324" # harness-docker-stats (docker_container_* metrics) +PVE_EXPORTER_PORT="9221" # harness-pve-exporter (5 pve_* metrics) CT116_SSH_HOST="192.168.68.116" # Both are bare-200: 404 = container not yet started DOCKER_STATS_EXPECTED="200|404" @@ -204,7 +204,7 @@ if [ ${#GPU_FAILED[@]} -gt 0 ]; then FAILED+=("gpu-exporters: ${GPU_FAILED[*]}") fi -# 6. Docker Stats (CT 116 :9323, 127.0.0.1 via SSH) — 200|404 +# 6. Docker Stats (CT 116 :9324, 127.0.0.1 via SSH) — 200|404 if probe_http "127.0.0.1" "$DOCKER_STATS_PORT" "/" "$DOCKER_STATS_EXPECTED" "" "$CT116_SSH_HOST"; then echo " ✅ Docker Stats: alive" else @@ -212,7 +212,7 @@ else FAILED+=("docker-stats") fi -# 7. PVE Exporter (CT 116 :9324, 127.0.0.1 via SSH) — 200|404 +# 7. PVE Exporter (CT 116 :9221, 127.0.0.1 via SSH) — 200|404 if probe_http "127.0.0.1" "$PVE_EXPORTER_PORT" "/" "$PVE_EXPORTER_EXPECTED" "" "$CT116_SSH_HOST"; then echo " ✅ PVE Exporter: alive" else diff --git a/scripts/test_infra_monitoring.sh b/scripts/test_infra_monitoring.sh index c3d5567..636dc99 100755 --- a/scripts/test_infra_monitoring.sh +++ b/scripts/test_infra_monitoring.sh @@ -121,12 +121,32 @@ assert "Port 9405 NOT in any curl call" \ '! grep -q ":9405" "$CURL_LOG"' # ── 5. Docker Stats / PVE Exporter: SSH-probed at correct ports ──────────── -assert "Docker Stats probed at port 9323 via SSH" \ - 'grep -q "9323" "$SSH_LOG"' - -assert "PVE Exporter probed at port 9324 via SSH" \ +assert "Docker Stats probed at port 9324 via SSH" \ 'grep -q "9324" "$SSH_LOG"' +assert "PVE Exporter probed at port 9221 via SSH" \ + 'grep -q "9221" "$SSH_LOG"' + +# ── 5a. Per-leg assertions (proves which leg owns which port) ────────────── +# The script source must show Docker Stats using $DOCKER_STATS_PORT and +# PVE Exporter using $PVE_EXPORTER_PORT in the correct leg sections +assert "Docker Stats leg uses DOCKER_STATS_PORT constant" \ + 'grep -A 3 "# 6. Docker Stats" "$SCRIPT" | grep -q "\$DOCKER_STATS_PORT"' + +assert "PVE Exporter leg uses PVE_EXPORTER_PORT constant" \ + 'grep -A 3 "# 7. PVE Exporter" "$SCRIPT" | grep -q "\$PVE_EXPORTER_PORT"' + +# Verify the constants themselves are set to the correct values +assert "DOCKER_STATS_PORT constant set to 9324" \ + 'grep -q "^DOCKER_STATS_PORT=\"9324\"" "$SCRIPT"' + +assert "PVE_EXPORTER_PORT constant set to 9221" \ + 'grep -q "^PVE_EXPORTER_PORT=\"9221\"" "$SCRIPT"' + +# ── 5b. Stale port 9323 (dockerd) NOT probed ────────────────────────────── +assert "Port 9323 (dockerd) NOT in SSH log" \ + '! grep -q "9323" "$SSH_LOG"' + # ── 6. No stale ports in the script source (belt-and-suspenders) ────────── assert "Port 9325 (historical) NOT in script source" \ '! grep -q "9325" "$SCRIPT"'