From a13457bcd61cf553dc06a2370ed36d2837200dfc Mon Sep 17 00:00:00 2001 From: root Date: Sat, 19 Sep 2026 02:32:00 +0000 Subject: [PATCH] fix(infra): Fix Docker Stats (9324) and PVE Exporter (9221) probe ports Previously probed wrong ports: - Docker Stats was at 9323 (dockerd metrics) but should be 9324 (harness-docker-stats, docker_container_* metrics) - PVE Exporter was at 9324 (harness-docker-stats) but should be 9221 (harness-pve-exporter, 5 pve_* metrics) Both exporters bind to 127.0.0.1 on CT 116 and must be probed via SSH. Updated infrastructure-monitoring.prose.md to document the correct ports. Added test assertions verifying the exact ports are probed. Branch: fix/infra-monitoring-probe-ports-20260919 --- infrastructure-monitoring.prose.md | 21 +++++++++++++++++++++ scripts/infra-monitoring.sh | 4 ++-- scripts/test_infra_monitoring.sh | 12 ++++++++---- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/infrastructure-monitoring.prose.md b/infrastructure-monitoring.prose.md index 04e62ce..52b1ac6 100644 --- a/infrastructure-monitoring.prose.md +++ b/infrastructure-monitoring.prose.md @@ -277,6 +277,27 @@ code (or failure kind with retry details). Apply the standing probe rules: any HTTP status = ALIVE; only 000/timeout/refused = probe-failed. A redirect is not a failure. +### Docker Stats and PVE Exporter Ports + +These two exporters bind to 127.0.0.1 on CT 116 (localhost-only) and must be probed via SSH: + +| Exporter | Port | Container | Metrics | +|----------|------|-----------|---------| +| **Docker Stats** | **9324** | harness-docker-stats | `docker_container_*` (per-container CPU/mem/network) | +| **PVE Exporter** | **9221** | harness-pve-exporter | `pve_*` (5 cluster-level metrics) | + +**IMPORTANT**: Do not confuse with port 9323, which is owned by dockerd and serves the Docker Engine's own metrics (`builder_builds_*`, `containerd_build_info_*`). + +```bash +# Docker Stats (harness-docker-stats) +ssh root@192.168.68.116 "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9324/metrics" +# Expected: 200 or 404 (any HTTP status = ALIVE) + +# PVE Exporter (harness-pve-exporter) +ssh root@192.168.68.116 "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 http://127.0.0.1:9221/metrics" +# Expected: 200 or 404 (any HTTP status = ALIVE) +``` + ### Phase 1: GPU Exporters **NVIDIA (.8 and .110)**: diff --git a/scripts/infra-monitoring.sh b/scripts/infra-monitoring.sh index bd20718..18a565a 100755 --- a/scripts/infra-monitoring.sh +++ b/scripts/infra-monitoring.sh @@ -60,8 +60,8 @@ GPU_PATH="/metrics" GPU_EXPECTED="200" # Docker Stats and PVE Exporter bind to 127.0.0.1 on CT 116 -DOCKER_STATS_PORT="9323" -PVE_EXPORTER_PORT="9324" +DOCKER_STATS_PORT="9324" # harness-docker-stats (docker_container_* metrics) +PVE_EXPORTER_PORT="9221" # harness-pve-exporter (5 pve_* metrics) CT116_SSH_HOST="192.168.68.116" # Both are bare-200: 404 = container not yet started DOCKER_STATS_EXPECTED="200|404" diff --git a/scripts/test_infra_monitoring.sh b/scripts/test_infra_monitoring.sh index c3d5567..3b3226f 100755 --- a/scripts/test_infra_monitoring.sh +++ b/scripts/test_infra_monitoring.sh @@ -121,12 +121,16 @@ assert "Port 9405 NOT in any curl call" \ '! grep -q ":9405" "$CURL_LOG"' # ── 5. Docker Stats / PVE Exporter: SSH-probed at correct ports ──────────── -assert "Docker Stats probed at port 9323 via SSH" \ - 'grep -q "9323" "$SSH_LOG"' - -assert "PVE Exporter probed at port 9324 via SSH" \ +assert "Docker Stats probed at port 9324 via SSH" \ 'grep -q "9324" "$SSH_LOG"' +assert "PVE Exporter probed at port 9221 via SSH" \ + 'grep -q "9221" "$SSH_LOG"' + +# ── 5a. Stale port 9323 (dockerd) NOT probed ────────────────────────────── +assert "Port 9323 (dockerd) NOT in SSH log" \ + '! grep -q "9323" "$SSH_LOG"' + # ── 6. No stale ports in the script source (belt-and-suspenders) ────────── assert "Port 9325 (historical) NOT in script source" \ '! grep -q "9325" "$SCRIPT"'