scripts/infra-monitoring.sh

@@ -18,7 +18,7 @@
 #
 # Output shape per leg:
 #   ✅ <name>: alive
-#   🔴 <name>: probe-failed: <host>:<port> (expected <pattern>)
+#   🔴 <name>: probe-failed: <host>:<port> (expected <pattern>) (<kind>)
 #
 # Failure kinds: timeout | refused | tls | unexpected:<code> (printed in the failure line)
 
@@ -84,26 +84,16 @@ probe_http() {
     local code="" kind=""
     LAST_KIND=""
 
-    # First attempt
-    if [ -n "$ssh_host" ]; then
-        code=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "root@${ssh_host}" \
-            "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 --max-time 15 ${use_k:+-k} ${url}" 2>/dev/null)
-    else
-        code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 --max-time 15 ${use_k:+-k} "$url" 2>/dev/null)
-    fi
-
-    code=$(printf '%s' "$code" | tr -d '[:space:]')
-
-    # Capture curl exit code for TLS error detection
+    # Single invocation that captures both output and status
     if [ -n "$ssh_host" ]; then
         out=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "root@${ssh_host}" \
             "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 --max-time 15 ${use_k:+-k} ${url}" 2>/dev/null)
-        code=$(printf '%s' "$out" | tr -d '[:space:]')
+        rc=$?
     else
         out=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 --max-time 15 ${use_k:+-k} "$url" 2>/dev/null)
-        code=$(printf '%s' "$out" | tr -d '[:space:]')
+        rc=$?
     fi
-    rc=$?
+    code=$(printf '%s' "$out" | tr -d '[:space:]')
 
     # Classify failure kind and retry if needed
     if [ -z "$code" ] || [ "$code" = "000" ]; then
@@ -112,18 +102,19 @@ probe_http() {
             35|51|58|59|60|77|83) kind="tls" ;;
             *) kind="timeout" ;;
         esac
+        # Retry once at longer timeout (25s connect, 30s max)
         if [ -n "$ssh_host" ]; then
-            # FIX C2: SSH retry at longer timeout (25s connect, 30s max)
             code=$(ssh -o ConnectTimeout=5 -o BatchMode=yes "root@${ssh_host}" \
                 "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 25 --max-time 30 ${use_k:+-k} ${url}" 2>/dev/null)
+            rc=$?
             code=$(printf '%s' "$code" | tr -d '[:space:]')
         else
-            # Retry once with longer timeout to distinguish
             code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 25 --max-time 30 ${use_k:+-k} "$url" 2>/dev/null)
+            rc=$?
             code=$(printf '%s' "$code" | tr -d '[:space:]')
-            # Classify: timeout if still 000, refused if we get an unexpected response
+            # On retry, classify: still 000 = keep existing kind (or timeout if empty), unexpected status = refused
             if [ -z "$code" ] || [ "$code" = "000" ]; then
-                kind="timeout"
+                [ -z "$kind" ] && kind="timeout"
             elif ! echo "$code" | grep -qE "^(${expected})$"; then
                 kind="refused"
             fi

scripts/test_infra_monitoring.sh

@@ -135,8 +135,10 @@ assert "Port 9405 (historical) NOT in script source" \
     '! grep -q "9405" "$SCRIPT"'
 
 # ── 7. Failure-line content includes non-empty kind ────────────────────────
-# Stub curl to return 500 (unexpected status) and verify failure line has (<kind>)
 TMP_DIR=$(mktemp -d)
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+# Test 7a: Unexpected status (500) → kind should be unexpected:500
 cat > "$TMP_DIR/curl" << 'EOF'
 #!/bin/bash
 # Stub: return 500 for Grafana port, 200 otherwise
@@ -150,20 +152,49 @@ echo "200"
 exit 0
 EOF
 chmod +x "$TMP_DIR/curl"
-# Run script with stubbed curl and capture output
 OUT=$(PATH="$TMP_DIR:$PATH" bash "$SCRIPT" 2>&1)
-# Find the Grafana failure line and verify it has a non-empty kind
 GRAFANA_FAIL=$(echo "$OUT" | grep "Grafana: probe-failed")
-assert "Grafana failure line exists" \
+assert "Grafana failure line exists (unexpected status)" \
     '[[ -n "$GRAFANA_FAIL" ]]' 
-
-# Extract the kind from the failure line (should be <unexpected:500>)
 KIND=$(echo "$GRAFANA_FAIL" | grep -oP '\(<[^>]+>\)' | tr -d '()<>')
-assert "Grafana failure kind is non-empty" \
+assert "Grafana failure kind is non-empty (unexpected status)" \
     '[[ -n "$KIND" ]]' 
 
-# Cleanup
-rm -rf "$TMP_DIR"
+# Test 7b: TLS error (000 + exit 60) → kind should be tls
+cat > "$TMP_DIR/curl" << 'EOF'
+#!/bin/bash
+# Stub: return 000 and exit 60 for Grafana port (TLS error) on ALL invocations
+for arg in "$@"; do
+    if [[ "$arg" == *":3001"* ]]; then
+        echo "000"
+        exit 60
+    fi
+done
+echo "200"
+exit 0
+EOF
+chmod +x "$TMP_DIR/curl"
+# Also stub ssh to return 000 + exit 60 for the retry
+cat > "$TMP_DIR/ssh" << 'EOF'
+#!/bin/bash
+for arg in "$@"; do
+    if [[ "$arg" == curl* ]]; then
+        echo "000"
+        exit 60
+    fi
+done
+echo "200"
+exit 0
+EOF
+chmod +x "$TMP_DIR/ssh"
+export PATH="$TMP_DIR:$PATH"
+OUT=$(bash "$SCRIPT" 2>&1)
+GRAFANA_FAIL=$(echo "$OUT" | grep "Grafana: probe-failed")
+assert "Grafana failure line exists (TLS error)" \
+    '[[ -n "$GRAFANA_FAIL" ]]' 
+KIND=$(echo "$GRAFANA_FAIL" | grep -oP '\(<[^>]+>\)' | tr -d '()<>')
+assert "Grafana failure kind is tls" \
+    '[[ "$KIND" == "tls" ]]'
 
 # ── Summary ─────────────────────────────────────────────────────────────────
 echo ""