infrastructure-monitoring.prose.md

@@ -146,7 +146,7 @@ leg failed. Port drift is caught by `scripts/test_infra_monitoring.sh` which
 asserts every probed port matches the documented value.
 
 **PROBE SHAPE (per standing rules above):**
-- Every probe prints: `✅ <name>: alive` on success, or `🔴 <name>: probe-failed: <host>:<port> (expected <pattern>)` on failure
+- Every probe prints: `✅ <name>: alive` on success, or `🔴 <name>: probe-failed: <host>:<port> (expected <pattern>) (<kind>)` on failure
 - PVE API failures include `(any-HTTP liveness, -k for self-signed)` to distinguish TLS vs connection
 - Retry once on connection failure at longer timeout (25s connect, 30s max)
 - Any HTTP status = ALIVE; only 000/timeout/refused = probe-failed

scripts/infra-monitoring.sh

@@ -76,11 +76,13 @@ PVE_EXPORTER_EXPECTED="200|404"
 # FIX C1: The kind value is computed and printed in the failure line.
 # FIX C2: SSH retry logic is in the first attempt branch (not unreachable).
 
+LAST_KIND=""
 probe_http() {
     local host="$1" port="$2" path="$3" expected="$4"
     local use_k="${5:-}" ssh_host="${6:-}" scheme="${7:-http}" liveness="${8:-0}"
     local url="${scheme}://${host}:${port}${path}"
     local code="" kind=""
+    LAST_KIND=""
 
     # First attempt
     if [ -n "$ssh_host" ]; then
@@ -105,10 +107,10 @@ probe_http() {
             # Retry once with longer timeout to distinguish
             code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 25 --max-time 30 ${use_k:+-k} "$url" 2>/dev/null)
             code=$(printf '%s' "$code" | tr -d '[:space:]')
-            # Classify: timeout if still 000, refused if we get a response
+            # Classify: timeout if still 000, refused if we get an unexpected response
             if [ -z "$code" ] || [ "$code" = "000" ]; then
                 kind="timeout"
-            else
+            elif ! echo "$code" | grep -qE "^(${expected})$"; then
                 kind="refused"
             fi
         fi
@@ -130,6 +132,7 @@ probe_http() {
         fi
     else
         [ -z "$kind" ] && kind="refused"
+        LAST_KIND="$kind"
         return 1
     fi
 }
@@ -137,6 +140,7 @@ probe_http() {
 # ── Main ────────────────────────────────────────────────────────────────────
 
 FAILED=()
+FAILED_KIND=()
 TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
 echo "=== Infrastructure Monitoring — $TIMESTAMP ==="
 echo "Executed from: $(pwd -P)"
@@ -146,7 +150,7 @@ echo ""
 if probe_http "$GRAFANA_HOST" "$GRAFANA_PORT" "$GRAFANA_PATH" "$GRAFANA_EXPECTED"; then
     echo "  ✅ Grafana: alive"
 else
-    echo "  🔴 Grafana: probe-failed: ${GRAFANA_HOST}:${GRAFANA_PORT} (expected ${GRAFANA_EXPECTED})"
+    echo "  🔴 Grafana: probe-failed: ${GRAFANA_HOST}:${GRAFANA_PORT} (expected ${GRAFANA_EXPECTED}) (<${LAST_KIND}>)"
     FAILED+=("grafana")
 fi
 
@@ -154,7 +158,7 @@ fi
 if probe_http "$PROMETHEUS_HOST" "$PROMETHEUS_PORT" "$PROMETHEUS_PATH" "$PROMETHEUS_EXPECTED"; then
     echo "  ✅ Prometheus: alive"
 else
-    echo "  🔴 Prometheus: probe-failed: ${PROMETHEUS_HOST}:${PROMETHEUS_PORT} (expected ${PROMETHEUS_EXPECTED})"
+    echo "  🔴 Prometheus: probe-failed: ${PROMETHEUS_HOST}:${PROMETHEUS_PORT} (expected ${PROMETHEUS_EXPECTED}) (<${LAST_KIND}>)"
     FAILED+=("prometheus")
 fi
 
@@ -162,7 +166,7 @@ fi
 if probe_http "$LITELLM_HOST" "$LITELLM_PORT" "$LITELLM_PATH" "" "" "" "http" "$LITELLM_LIVENESS"; then
     echo "  ✅ LiteLLM: alive"
 else
-    echo "  🔴 LiteLLM: probe-failed: ${LITELLM_HOST}:${LITELLM_PORT}${LITELLM_PATH} (any-HTTP liveness)"
+    echo "  🔴 LiteLLM: probe-failed: ${LITELLM_HOST}:${LITELLM_PORT}${LITELLM_PATH} (any-HTTP liveness) (<${LAST_KIND}>)"
     FAILED+=("litellm")
 fi
 
@@ -172,7 +176,7 @@ for node in "${PVE_NODES[@]}"; do
     if probe_http "$node" "$PVE_API_PORT" "$PVE_API_PATH" "" "$PVE_API_USE_K" "" "https" "$PVE_API_LIVENESS"; then
         echo "  ✅ PVE API ${node}: alive"
     else
-        echo "  🔴 PVE API ${node}: probe-failed: ${node}:${PVE_API_PORT} (any-HTTP liveness, -k for self-signed)"
+        echo "  🔴 PVE API ${node}: probe-failed: ${node}:${PVE_API_PORT} (any-HTTP liveness, -k for self-signed) (<${LAST_KIND}>)"
         PVE_FAILED+=("$node")
     fi
 done
@@ -186,7 +190,7 @@ for host in "${GPU_HOSTS[@]}"; do
     if probe_http "$host" "$GPU_PORT" "$GPU_PATH" "$GPU_EXPECTED"; then
         echo "  ✅ GPU exporter ${host}: alive"
     else
-        echo "  🔴 GPU exporter ${host}: probe-failed: ${host}:${GPU_PORT} (expected 200)"
+        echo "  🔴 GPU exporter ${host}: probe-failed: ${host}:${GPU_PORT} (expected 200) (<${LAST_KIND}>)"
         GPU_FAILED+=("$host")
     fi
 done
@@ -198,7 +202,7 @@ fi
 if probe_http "127.0.0.1" "$DOCKER_STATS_PORT" "/" "$DOCKER_STATS_EXPECTED" "" "$CT116_SSH_HOST"; then
     echo "  ✅ Docker Stats: alive"
 else
-    echo "  🔴 Docker Stats: probe-failed: CT116:127.0.0.1:${DOCKER_STATS_PORT} (expected 200|404)"
+    echo "  🔴 Docker Stats: probe-failed: CT116:127.0.0.1:${DOCKER_STATS_PORT} (expected 200|404) (<${LAST_KIND}>)"
     FAILED+=("docker-stats")
 fi
 
@@ -206,7 +210,7 @@ fi
 if probe_http "127.0.0.1" "$PVE_EXPORTER_PORT" "/" "$PVE_EXPORTER_EXPECTED" "" "$CT116_SSH_HOST"; then
     echo "  ✅ PVE Exporter: alive"
 else
-    echo "  🔴 PVE Exporter: probe-failed: CT116:127.0.0.1:${PVE_EXPORTER_PORT} (expected 200|404)"
+    echo "  🔴 PVE Exporter: probe-failed: CT116:127.0.0.1:${PVE_EXPORTER_PORT} (expected 200|404) (<${LAST_KIND}>)"
     FAILED+=("pve-exporter")
 fi
 
@@ -221,4 +225,4 @@ else
         echo "  🔴 FAILED: $f"
     done
     exit 1
-fi
+fi

scripts/zulip-monitor.sh

@@ -9,7 +9,8 @@ set -euo pipefail
 
 # Credentials sourced from environment variable ZULIP_API_KEY (set by vault-backed start script)
 # Never fall back to a literal key.
-# When unset/placeholder, the server leg is skipped (not the whole script) — the pi/Tanko/kagentz
+# When unset/placeholder, the server leg is still probed (200 without auth is expected) —
+# only notify() is gated on credential. The pi/Tanko/kagentz
 # legs do not need the Zulip API key. The placeholder is captain-held:
 # zulip-health-credential-placeholder-20260913.
 ZULIP_API_KEY="${ZULIP_API_KEY:-}"
@@ -47,25 +48,24 @@ notify() {
       -d "type=stream&to=%5B7%5D&topic=zulip-health&content=$(printf '%s' "${stream_content}" | python3 -c "import sys,urllib.parse; print(urllib.parse.quote_from_bytes(sys.stdin.buffer.read()))")" \
       > /dev/null 2>&1 \
       || echo "  WARN: stream alert to #agent-hub (zulip-health) delivery failed (curl exit $?)">> "$LOG"
+  else
+    echo "  ALERT SUPPRESSED (no credential): ${severity} ${msg}" >> "$LOG"
   fi
 }
 
 # ── Global: Zulip Server ──
-# Skip when credential is placeholder/absent (captain-held item)
-if [ "$ZULIP_CRED_OK" -eq 0 ]; then
-  echo "  Server: ⏭ skipped: credential placeholder, held for captain (zulip-health-credential-placeholder-20260913)" >> "$LOG"
+# F3: Always probe server regardless of credential — 200 without auth is expected
+# (verified live: server_settings returns 200 with no credential or wrong key).
+SERVER_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 \
+  https://chat.sysloggh.net/api/v1/server_settings \
+  -u "${ZULIP_EMAIL}:${ZULIP_API_KEY}" 2>/dev/null) || SERVER_CODE="000"
+SERVER_CODE=$(printf '%s' "$SERVER_CODE" | tr -d '[:space:]')
+[ -n "$SERVER_CODE" ] || SERVER_CODE="000"
+if [ "$SERVER_CODE" != "200" ]; then
+  notify "🔴" "Zulip server returned HTTP $SERVER_CODE"
+  ISSUES=$((ISSUES + 1))
 else
-  SERVER_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 \
-    https://chat.sysloggh.net/api/v1/server_settings \
-    -u "${ZULIP_EMAIL}:${ZULIP_API_KEY}" 2>/dev/null) || SERVER_CODE="000"
-  SERVER_CODE=$(printf '%s' "$SERVER_CODE" | tr -d '[:space:]')
-  [ -n "$SERVER_CODE" ] || SERVER_CODE="000"
-  if [ "$SERVER_CODE" != "200" ]; then
-    notify "🔴" "Zulip server returned HTTP $SERVER_CODE"
-    ISSUES=$((ISSUES + 1))
-  else
-    echo "  Server: ✅ HTTP 200" >> "$LOG"
-  fi
+  echo "  Server: ✅ HTTP 200" >> "$LOG"
 fi
 
 # ── Platform A: pi (Abiba) ──
@@ -199,7 +199,7 @@ fi
 # ── Summary ──
 if [ "$ISSUES" -eq 0 ]; then
   if [ "$ZULIP_CRED_OK" -eq 0 ]; then
-    echo "  Result: ✅ All healthy (server leg skipped: credential placeholder)" >> "$LOG"
+    echo "  Result: ✅ All healthy" >> "$LOG"
   else
     echo "  Result: ✅ All healthy" >> "$LOG"
   fi