From 5ab5de4704166c4856247a21fd6941d122f3e091 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 18 Sep 2026 04:40:20 +0000 Subject: [PATCH] fix: move infra-monitoring probes into versioned script - Create scripts/infra-monitoring.sh with all targets/ports/paths in code - Add -k flag for PVE API (self-signed certs) - Probe Docker Stats and PVE Exporter via SSH (bind to 127.0.0.1) - Non-zero exit with named failures - Add tests/test_infra_monitoring.py for port drift detection - Prove happy path + broken target Fixes: infra-monitoring-probe-targets-drift-20260917 --- scripts/infra-monitoring.sh | 193 +++++++++++++++++++++++++++++++++ tests/test_infra_monitoring.py | 165 ++++++++++++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100755 scripts/infra-monitoring.sh create mode 100644 tests/test_infra_monitoring.py diff --git a/scripts/infra-monitoring.sh b/scripts/infra-monitoring.sh new file mode 100755 index 0000000..ca1f30f --- /dev/null +++ b/scripts/infra-monitoring.sh @@ -0,0 +1,193 @@ +#!/bin/bash +# infrastructure-monitoring.sh — Homelab Infrastructure Monitor +# Implements infrastructure-monitoring.prose.md v4 +# +# Legs: Grafana, Prometheus, LiteLLM, PVE API (5 nodes), GPU exporters, +# Docker Stats, PVE Exporter, PM2 +# +# Design: +# - Every target, port, path, and expected status is defined in code +# - Any HTTP status (200/301/302/401/403/404) = ALIVE +# - Only connection failures (000/timeout) = probe-failed +# - PVE API uses -k flag (self-signed certs) +# - Docker Stats and PVE Exporter bind to 127.0.0.1, probed via SSH +# - Non-zero exit with named failures +# - No "OK" summary when any leg failed + +set -uo pipefail + +# ── Configuration ─────────────────────────────────────────────────────────── +# Documented targets (from infrastructure-monitoring.prose.md) +# Change these in ONE place; tests assert against these values + +GRAFANA_HOST="192.168.68.116" +GRAFANA_PORT="3001" +GRAFANA_PATH="/api/health" +GRAFANA_EXPECTED="200|302" + +PROMETHEUS_HOST="192.168.68.116" +PROMETHEUS_PORT="9090" +PROMETHEUS_PATH="/-/healthy" +PROMETHEUS_EXPECTED="200" + +LITELLM_HOST="192.168.68.116" +LITELLM_PORT="4000" +LITELLM_PATH="/" +LITELLM_EXPECTED="200|401" + +# PVE API: probe REAL PVE nodes, never the monitoring host (CT 116) +PVE_NODES=("192.168.68.9" "192.168.68.12" "192.168.68.6" "192.168.68.15" "192.168.68.5") +PVE_API_PORT="8006" +PVE_API_SCHEME="https" +PVE_API_EXPECTED="200|401" +PVE_API_USE_K="1" # self-signed certs + +# GPU exporters +GPU_HOSTS=("192.168.68.8" "192.168.68.110" "192.168.68.15") +GPU_PORT="9400" +GPU_PATH="/metrics" +GPU_EXPECTED="200" + +# Docker Stats and PVE Exporter bind to 127.0.0.1 on CT 116 +DOCKER_STATS_HOST="192.168.68.116" +DOCKER_STATS_PORT="9323" +DOCKER_STATS_PATH="/" +DOCKER_STATS_EXPECTED="200|404" + +PVE_EXPORTER_HOST="192.168.68.116" +PVE_EXPORTER_PORT="9324" +PVE_EXPORTER_PATH="/" +PVE_EXPORTER_EXPECTED="200|404" + +# PM2 (CT 100) +PM2_HOST="192.168.68.24" +PM2_EXPECTED="online" + +# ── Probe Functions ───────────────────────────────────────────────────────── + +# probe_http [use_k] [ssh_host] [scheme] +# Returns: 0 if any HTTP status matches, 1 if probe-failed +probe_http() { + local host="$1" port="$2" path="$3" expected="$4" use_k="${5:-}" ssh_host="${6:-}" + local scheme="${7:-http}"; local url="${scheme}://${host}:${port}${path}" + local curl_opts=(-s -o /dev/null -w '%{http_code}' --max-time 10) + local code="" + + if [ -n "$use_k" ]; then + curl_opts+=(-k) + fi + + if [ -n "$ssh_host" ]; then + # Probe via SSH to the host where the service binds to 127.0.0.1 + code=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "root@${ssh_host}" \ + "curl -s -o /dev/null -w '%{http_code}' --max-time 10 ${use_k:+-k} ${scheme}://127.0.0.1:${port}${path}" 2>/dev/null) || code="000" + else + code=$(curl "${curl_opts[@]}" "$url" 2>/dev/null) || code="000" + fi + + # Clean up the code + code=$(printf '%s' "$code" | tr -d '[:space:]') + [ -n "$code" ] || code="000" + + # Check if code matches expected pattern + if echo "$code" | grep -qE "^(${expected})$"; then + return 0 + else + return 1 + fi +} + +# ── Main ──────────────────────────────────────────────────────────────────── + +FAILED=() +TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC') +echo "=== Infrastructure Monitoring — $TIMESTAMP ===" + +# Grafana +if probe_http "$GRAFANA_HOST" "$GRAFANA_PORT" "$GRAFANA_PATH" "$GRAFANA_EXPECTED"; then + echo " ✅ Grafana: alive" +else + echo " 🔴 Grafana: probe-failed: ${GRAFANA_HOST}:${GRAFANA_PORT} (expected ${GRAFANA_EXPECTED})" + FAILED+=("grafana") +fi + +# Prometheus +if probe_http "$PROMETHEUS_HOST" "$PROMETHEUS_PORT" "$PROMETHEUS_PATH" "$PROMETHEUS_EXPECTED"; then + echo " ✅ Prometheus: alive" +else + echo " 🔴 Prometheus: probe-failed: ${PROMETHEUS_HOST}:${PROMETHEUS_PORT} (expected ${PROMETHEUS_EXPECTED})" + FAILED+=("prometheus") +fi + +# LiteLLM +if probe_http "$LITELLM_HOST" "$LITELLM_PORT" "$LITELLM_PATH" "$LITELLM_EXPECTED"; then + echo " ✅ LiteLLM: alive" +else + echo " 🔴 LiteLLM: probe-failed: ${LITELLM_HOST}:${LITELLM_PORT} (expected ${LITELLM_EXPECTED})" + FAILED+=("litellm") +fi + +# PVE API (5 nodes) +PVE_FAILED=() +for node in "${PVE_NODES[@]}"; do + if probe_http "$node" "$PVE_API_PORT" "/" "$PVE_API_EXPECTED" "$PVE_API_USE_K" "" "https"; then + echo " ✅ PVE API ${node}: alive" + else + echo " 🔴 PVE API ${node}: probe-failed: ${node}:${PVE_API_PORT} (expected ${PVE_API_EXPECTED})" + PVE_FAILED+=("$node") + fi +done +if [ ${#PVE_FAILED[@]} -gt 0 ]; then + FAILED+=("pve-api: ${PVE_FAILED[*]}") +fi + +# GPU exporters +GPU_FAILED=() +for host in "${GPU_HOSTS[@]}"; do + if probe_http "$host" "$GPU_PORT" "$GPU_PATH" "$GPU_EXPECTED"; then + echo " ✅ GPU ${host}: alive" + else + echo " 🔴 GPU ${host}: probe-failed: ${host}:${GPU_PORT} (expected ${GPU_EXPECTED})" + GPU_FAILED+=("$host") + fi +done +if [ ${#GPU_FAILED[@]} -gt 0 ]; then + FAILED+=("gpu: ${GPU_FAILED[*]}") +fi + +# Docker Stats (localhost via SSH) +if probe_http "$DOCKER_STATS_HOST" "$DOCKER_STATS_PORT" "$DOCKER_STATS_PATH" "$DOCKER_STATS_EXPECTED" "" "$DOCKER_STATS_HOST"; then + echo " ✅ Docker Stats: alive" +else + echo " 🔴 Docker Stats: probe-failed: ${DOCKER_STATS_HOST}:${DOCKER_STATS_PORT} (expected ${DOCKER_STATS_EXPECTED})" + FAILED+=("docker-stats") +fi + +# PVE Exporter (localhost via SSH) +if probe_http "$PVE_EXPORTER_HOST" "$PVE_EXPORTER_PORT" "$PVE_EXPORTER_PATH" "$PVE_EXPORTER_EXPECTED" "" "$PVE_EXPORTER_HOST"; then + echo " ✅ PVE Exporter: alive" +else + echo " 🔴 PVE Exporter: probe-failed: ${PVE_EXPORTER_HOST}:${PVE_EXPORTER_PORT} (expected ${PVE_EXPORTER_EXPECTED})" + FAILED+=("pve-exporter") +fi + +# PM2 +PM2_OUTPUT=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "root@${PM2_HOST}" \ + "pm2 list 2>/dev/null | grep -c 'online'" 2>/dev/null) || PM2_OUTPUT="0" +if [ "$PM2_OUTPUT" -gt 0 ]; then + echo " ✅ PM2: ${PM2_OUTPUT} processes online" +else + echo " 🔴 PM2: probe-failed: no processes online on ${PM2_HOST}" + FAILED+=("pm2") +fi + +# ── Summary ───────────────────────────────────────────────────────────────── + +echo "" +if [ ${#FAILED[@]} -eq 0 ]; then + echo " ✅ All legs OK" + exit 0 +else + echo " 🔴 FAILED legs: ${FAILED[*]}" + exit 1 +fi diff --git a/tests/test_infra_monitoring.py b/tests/test_infra_monitoring.py new file mode 100644 index 0000000..1e5aeee --- /dev/null +++ b/tests/test_infra_monitoring.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +""" +test_infra_monitoring.py — Tests for infrastructure-monitoring.sh + +Tests: +1. Port drift detection: asserts each probed port matches the documented value +2. PVE API probe targets: verifies we probe real PVE nodes, not the monitoring host +3. TLS failure labeling: verifies we use -k for self-signed certs +4. Happy path and broken target (already done via shell test) +""" + +import subprocess +import os +import re +import sys + +SCRIPT_PATH = "/root/abiba-workspace/projects/prose-contracts/scripts/infra-monitoring.sh" + +def run_script(): + """Run the monitoring script and return output + exit code.""" + result = subprocess.run( + ["bash", SCRIPT_PATH], + capture_output=True, + text=True, + timeout=60 + ) + return result.stdout, result.stderr, result.returncode + +def test_happy_path(): + """Test that all documented targets are probed and healthy.""" + print("=== Test 1: Happy Path ===") + stdout, stderr, returncode = run_script() + print(stdout) + + # Verify exit code is 0 + if returncode != 0: + print(f"❌ FAILED: Expected exit code 0, got {returncode}") + return False + + # Verify all legs passed + if "All legs OK" not in stdout: + print(f"❌ FAILED: Expected 'All legs OK' in output") + return False + + print("✅ PASSED: Happy path works") + return True + +def test_port_drift_detection(): + """Test that port drift from documented values is detected.""" + print("\n=== Test 2: Port Drift Detection ===") + + # Create a modified version with wrong port + import shutil + test_script = SCRIPT_PATH.replace("scripts/", "scripts/test-drift-") + shutil.copy(SCRIPT_PATH, test_script) + + # Change Grafana port from 3001 to 3099 + with open(test_script, 'r') as f: + content = f.read() + content = content.replace('GRAFANA_PORT="3001"', 'GRAFANA_PORT="3099"') + + with open(test_script, 'w') as f: + f.write(content) + + # Run the modified script + stdout, stderr, returncode = run_script() + + # Clean up + os.remove(test_script) + + print(stdout) + + # Verify: + # 1. Script exited non-zero + if returncode == 0: + print(f"❌ FAILED: Expected non-zero exit code for drift, got {returncode}") + return False + + # 2. Output mentions probe-failed for grafana + if "probe-failed" not in stdout.lower(): + print(f"❌ FAILED: Expected 'probe-failed' in output for Grafana") + return False + + # 3. Output mentions the wrong port + if "192.168.68.116:3099" not in stdout: + print(f"❌ FAILED: Expected '192.168.68.116:3099' in output") + return False + + print("✅ PASSED: Port drift detection works") + return True + +def test_pve_api_targets(): + """Test that PVE API probes the real nodes, not the monitoring host.""" + print("\n=== Test 3: PVE API Targets ===") + + stdout, stderr, returncode = run_script() + + # Verify we're NOT probing the monitoring host (192.168.68.116) for PVE API + if ":116:8006" in stdout or "192.168.68.116:8006" in stdout: + print(f"❌ FAILED: Should not probe monitoring host (192.168.68.116) for PVE API") + return False + + # Verify we're probing real PVE nodes + expected_nodes = ["192.168.68.9", "192.168.68.12", "192.168.68.6", "192.168.68.15", "192.168.68.5"] + found_nodes = [node for node in expected_nodes if f"{node}:8006" in stdout] + + if len(found_nodes) != 5: + print(f"❌ FAILED: Expected to probe all 5 PVE nodes, found {len(found_nodes)}") + print(f" Found: {found_nodes}") + return False + + print("✅ PASSED: PVE API probes correct targets") + return True + +def test_tls_handling(): + """Test that PVE API uses -k flag for self-signed certs.""" + print("\n=== Test 4: TLS Handling ===") + + # Check the script for -k flag usage + with open(SCRIPT_PATH, 'r') as f: + content = f.read() + + # Verify -k is used for PVE API + if 'use_k' not in content or 'PVE_API_USE_K' not in content: + print(f"❌ FAILED: Script should use -k flag for PVE API (self-signed certs)") + return False + + # Verify PVE API probes use -k + if 'PVE_API_USE_K="1"' not in content: + print(f"❌ FAILED: PVE_API_USE_K should be set to 1") + return False + + print("✅ PASSED: TLS handling configured correctly") + return True + +def main(): + """Run all tests.""" + tests = [ + ("Happy Path", test_happy_path), + ("Port Drift Detection", test_port_drift_detection), + ("PVE API Targets", test_pve_api_targets), + ("TLS Handling", test_tls_handling), + ] + + results = [] + for name, test_func in tests: + try: + result = test_func() + results.append((name, result)) + except Exception as e: + print(f"\n❌ EXCEPTION in {name}: {e}") + results.append((name, False)) + + print("\n" + "=" * 60) + print("SUMMARY") + print("=" * 60) + for name, result in results: + status = "✅ PASSED" if result else "❌ FAILED" + print(f"{name}: {status}") + + all_passed = all(r for _, r in results) + sys.exit(0 if all_passed else 1) + +if __name__ == "__main__": + main()