infrastructure-monitoring.prose.md

@@ -277,6 +277,27 @@ code (or failure kind with retry details). Apply the standing probe rules: any
 HTTP status = ALIVE; only 000/timeout/refused = probe-failed. A redirect is not
 a failure.
 
+### Docker Stats and PVE Exporter Ports
+
+These two exporters bind to 127.0.0.1 on CT 116 (localhost-only) and must be probed via SSH:
+
+| Exporter | Port | Container | Metrics |
+|----------|------|-----------|---------|
+| **Docker Stats** | **9324** | harness-docker-stats | `docker_container_*` (per-container CPU/mem/network) |
+| **PVE Exporter** | **9221** | harness-pve-exporter | `pve_*` (5 cluster-level metrics) |
+
+**IMPORTANT**: Do not confuse with port 9323, which is owned by dockerd and serves the Docker Engine's own metrics (`builder_builds_*`, `containerd_build_info_*`).
+
+```bash
+# Docker Stats (harness-docker-stats)
+ssh root@192.168.68.116 "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9324/metrics"
+# Expected: 200 or 404 (any HTTP status = ALIVE)
+
+# PVE Exporter (harness-pve-exporter)
+ssh root@192.168.68.116 "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9221/metrics"
+# Expected: 200 or 404 (any HTTP status = ALIVE)
+```
+
 ### Phase 1: GPU Exporters
 
 **NVIDIA (.8 and .110)**:

scripts/infra-monitoring.sh

@@ -60,8 +60,8 @@ GPU_PATH="/metrics"
 GPU_EXPECTED="200"
 
 # Docker Stats and PVE Exporter bind to 127.0.0.1 on CT 116
-DOCKER_STATS_PORT="9323"
-PVE_EXPORTER_PORT="9324"
+DOCKER_STATS_PORT="9324"  # harness-docker-stats (docker_container_* metrics)
+PVE_EXPORTER_PORT="9221"  # harness-pve-exporter (5 pve_* metrics)
 CT116_SSH_HOST="192.168.68.116"
 # Both are bare-200: 404 = container not yet started
 DOCKER_STATS_EXPECTED="200|404"
@@ -204,7 +204,7 @@ if [ ${#GPU_FAILED[@]} -gt 0 ]; then
     FAILED+=("gpu-exporters: ${GPU_FAILED[*]}")
 fi
 
-# 6. Docker Stats (CT 116 :9323, 127.0.0.1 via SSH) — 200|404
+# 6. Docker Stats (CT 116 :9324, 127.0.0.1 via SSH) — 200|404
 if probe_http "127.0.0.1" "$DOCKER_STATS_PORT" "/" "$DOCKER_STATS_EXPECTED" "" "$CT116_SSH_HOST"; then
     echo "  ✅ Docker Stats: alive"
 else
@@ -212,7 +212,7 @@ else
     FAILED+=("docker-stats")
 fi
 
-# 7. PVE Exporter (CT 116 :9324, 127.0.0.1 via SSH) — 200|404
+# 7. PVE Exporter (CT 116 :9221, 127.0.0.1 via SSH) — 200|404
 if probe_http "127.0.0.1" "$PVE_EXPORTER_PORT" "/" "$PVE_EXPORTER_EXPECTED" "" "$CT116_SSH_HOST"; then
     echo "  ✅ PVE Exporter: alive"
 else

scripts/test_infra_monitoring.sh

@@ -121,12 +121,32 @@ assert "Port 9405 NOT in any curl call" \
     '! grep -q ":9405" "$CURL_LOG"'
 
 # ── 5. Docker Stats / PVE Exporter: SSH-probed at correct ports ────────────
-assert "Docker Stats probed at port 9323 via SSH" \
-    'grep -q "9323" "$SSH_LOG"'
-
-assert "PVE Exporter probed at port 9324 via SSH" \
+assert "Docker Stats probed at port 9324 via SSH" \
     'grep -q "9324" "$SSH_LOG"'
 
+assert "PVE Exporter probed at port 9221 via SSH" \
+    'grep -q "9221" "$SSH_LOG"'
+
+# ── 5a. Per-leg assertions (proves which leg owns which port) ──────────────
+# The script source must show Docker Stats using $DOCKER_STATS_PORT and
+# PVE Exporter using $PVE_EXPORTER_PORT in the correct leg sections
+assert "Docker Stats leg uses DOCKER_STATS_PORT constant" \
+    'grep -A 3 "# 6. Docker Stats" "$SCRIPT" | grep -q "\$DOCKER_STATS_PORT"'
+
+assert "PVE Exporter leg uses PVE_EXPORTER_PORT constant" \
+    'grep -A 3 "# 7. PVE Exporter" "$SCRIPT" | grep -q "\$PVE_EXPORTER_PORT"'
+
+# Verify the constants themselves are set to the correct values
+assert "DOCKER_STATS_PORT constant set to 9324" \
+    'grep -q "^DOCKER_STATS_PORT=\"9324\"" "$SCRIPT"'
+
+assert "PVE_EXPORTER_PORT constant set to 9221" \
+    'grep -q "^PVE_EXPORTER_PORT=\"9221\"" "$SCRIPT"'
+
+# ── 5b. Stale port 9323 (dockerd) NOT probed ──────────────────────────────
+assert "Port 9323 (dockerd) NOT in SSH log" \
+    '! grep -q "9323" "$SSH_LOG"'
+
 # ── 6. No stale ports in the script source (belt-and-suspenders) ──────────
 assert "Port 9325 (historical) NOT in script source" \
     '! grep -q "9325" "$SCRIPT"'