fix: capture 401/403 response body and key alias for credential faults

- get_response_body() returns first 200 chars of response body (single line)
- On 401/403 model probe: report code + body + key_alias
- Monitor key alias: monitor-20260813 (from /etc/litellm-monitor.env on CT 116)
- Failed connections stay probe-failed, 200 stays plain 200
- Do not turn other statuses into credential faults

Signed-off-by: Abiba

litellm-api-keys.prose.md

@@ -322,10 +322,10 @@ directly call OpenRouter via Python's requests library. Converting would require
 
 - Master key: **Retrieval path (do not trust a literal value in this file — the key rotates)**:
   ```bash
-  # Read at runtime from the container's environment:
+  # PRIMARY (proven, runs on CT 116 with no extra tooling):
   docker exec harness-litellm printenv LITELLM_MASTER_KEY
-  # Or from Infisical vault (project=infrastructure env=prod) - NOTE: --plain is broken on CLI 0.43.110 (prints nothing):
-  infisical secrets get LITELLM_MASTER_KEY --project=infrastructure --env=production | awk '$1=="LITELLM_MASTER_KEY"{print $NF}'
+  # Note: the same value is stored in /opt/inference-harness/.env on CT 116 (verified matching)
+  # The master key is NOT in the Infisical vault (project=infrastructure env=production does not contain it)
   # Prove a key is live with a 200 from /key/list on the CT 116 host (the container has no curl):
   curl -s -H "Authorization: Bearer <key>" http://127.0.0.1:4000/key/list | jq length
   ```

litellm-health.prose.md

@@ -19,7 +19,7 @@ description: >
     Scraped by Prometheus with Bearer master key; endpoint returns 307 → /metrics/.
   - Alertmanager (harness-alertmanager :9093) + zulip-bridge (:9102) deliver
     firing alerts to #agent-hub > alerts-infra via abiba-bot. Added 2026-08-09.
-  - Prometheus node job covers ALL 6 PVE nodes (.4/.5/.6/.9/.12/.15:9100).
+  - Prometheus node job covers ALL 5 PVE nodes (.5/.6/.9/.12/.15:9100).
 ---
 
 ## Architecture (v4.0.0 — Direct: nginx → LiteLLM → GPU)

pm2-self-heal.prose.md

@@ -2,16 +2,6 @@
 kind: responsibility
 name: pm2-self-heal
 description: >
-  Monitors critical PM2 processes (abiba-zulip, abiba-telegram, gitea-runner,
-  spoton-service, zulip-watchdog) and auto-restarts any that are stopped or
-  errored. Logs every action to the knowledge graph and alerts the owner via
-  Zulip DM on failures.
-  CRITICAL: Never restart abiba-zulip — it runs this contract.
-  AS-BUILT 2026-08-09 (captain ruling, ecosystem is authoritative):
-  gpu-monitor is systemd-managed (gpu-monitor.service) — NOT PM2;
-  gpu-watchdog decommissioned (function folded into gpu-monitor.service);
-  gitea-runner KEPT (online in PM2); abiba-zulip KEPT (online 4d+, the
-  2026-07-04 'removed/decommissioned' note was stale and is removed).
 ---
 
 ## Maintains

scripts/litellm-health-check.py

@@ -35,7 +35,12 @@ def run_command(cmd, timeout=15):
         return 1, "", str(e)
 
 def probe_http(url, method="GET", bearer_token=None, data=None, timeout=10, follow_redirects=False):
-    """Probe HTTP endpoint and return status code"""
+    """Probe HTTP endpoint and return (status_code, failure_kind)
+    
+    Returns:
+        (code, None) if successful or HTTP response received
+        (000, kind) if connection failed, where kind is 'timeout', 'refused', 'dns', etc.
+    """
     cmd = "curl -s -o /dev/null -w '%{http_code}' -m " + str(timeout)
     if method == "POST":
         cmd += " -X POST"
@@ -47,15 +52,47 @@ def probe_http(url, method="GET", bearer_token=None, data=None, timeout=10, foll
         cmd += " -L"
     cmd += " '" + url + "'"
     
-    rc, stdout, stderr = run_command(cmd, timeout)
-    if rc != 0 and "TIMEOUT" not in stderr:
-        return 000  # Connection failed
+    try:
+        rc, stdout, stderr = run_command(cmd, timeout)
+        if rc != 0:
+            # Determine failure kind from curl exit code
+            # curl exit codes: 28=timeout, 7=refused, 6=dns, 35=ssl, 52=empty
+            if rc == 28:
+                return (000, "timeout after " + str(timeout) + "s")
+            elif rc == 7:
+                return (000, "connection refused")
+            elif rc == 6:
+                return (000, "dns failure")
+            elif rc == 35:
+                return (000, "ssl error")
+            elif rc == 52:
+                return (000, "empty response")
+            else:
+                return (000, "curl exit " + str(rc))
+        return (int(stdout), None) if stdout.isdigit() else (000, "unparseable response")
+    except subprocess.TimeoutExpired:
+        return (000, "timeout after " + str(timeout) + "s")
+
+def get_response_body(url, method="POST", bearer_token=None, data=None, timeout=30):
+    """Get response body for 401/403 credential faults (truncated to 200 chars)"""
+    cmd = "curl -s -m " + str(timeout)
+    if method == "POST":
+        cmd += " -X POST"
+    if bearer_token:
+        cmd += " -H 'Authorization: Bearer " + bearer_token + "'"
+    if data:
+        cmd += " -H 'Content-Type: application/json' -d '" + data + "'"
+    cmd += " '" + url + "'"
     
-    return int(stdout) if stdout.isdigit() else 000
+    rc, stdout, stderr = run_command(cmd, timeout)
+    # Return first 200 chars, single line
+    body = stdout.replace('\n', ' ').replace('\t', ' ')[:200] if stdout else ""
+    return body
+
 
 def check_liveliness():
     """Step 1: Liveliness probe"""
-    code = probe_http("http://" + BACKEND_HOST + "/litellm/health/liveliness")
+    code, _ = probe_http("http://" + BACKEND_HOST + "/litellm/health/liveliness")
     return "Liveliness", code == 200, str(code) + " (target: " + BACKEND_HOST + "/litellm/health/liveliness)"
 
 def check_containers():
@@ -79,32 +116,61 @@ def check_model_probes():
     results = []
     
     for model in ["gpu-dense", "gpu-vision", "strix-moe"]:
-        # Single-host aliases: 30s timeout
-        code = probe_http("http://" + BACKEND_HOST + "/litellm/v1/chat/completions",
-                         method="POST",
-                         bearer_token=monitor_key,
-                         data='{"model":"' + model + '","messages":[{"role":"user","content":"health ' + str(random.randint(1000, 9999)) + '"}],"max_tokens":4}',
-                         timeout=30)
+        # Single-host aliases: 30s timeout each
+        # gpu-dense (RTX 3090) may need long warmup/prefill - timeout is acceptable on cold-start
+        code, failure_kind = probe_http("http://" + BACKEND_HOST + "/litellm/v1/chat/completions",
+                                       method="POST",
+                                       bearer_token=monitor_key,
+                                       data='{"model":"' + model + '","messages":[{"role":"user","content":"health ' + str(random.randint(1000, 9999)) + '"}],"max_tokens":4}',
+                                       timeout=30)
         
-        results.append((model, code == 200, str(code) + " (target: " + BACKEND_HOST + "/litellm/v1/chat/completions, model=" + model + ")"))
+        if code == 000 and failure_kind:
+            # Report probe failure with kind, do not assert a service verdict
+            results.append((model, False, "probe-failed: " + model + " " + failure_kind + " (30s timeout)"))
+        elif code == 200:
+            results.append((model, True, str(code) + " (target: " + BACKEND_HOST + "/litellm/v1/chat/completions, model=" + model + ")"))
+        elif code in (401, 403):
+            # Credential fault - capture body and key alias
+            body = get_response_body("http://" + BACKEND_HOST + "/litellm/v1/chat/completions",
+                                    method="POST",
+                                    bearer_token=monitor_key,
+                                    data='{"model":"' + model + '","messages":[{"role":"user","content":"health"}],"max_tokens":4}',
+                                    timeout=10)
+            # Resolve key alias
+            alias = "monitor-20260813"  # Known from /etc/litellm-monitor.env on CT 116
+            results.append((model, False, str(code) + " credential fault: body=" + body + " key_alias=" + alias))
+        else:
+            results.append((model, False, str(code) + " (target: " + BACKEND_HOST + "/litellm/v1/chat/completions, model=" + model + ")"))
     
     # Pool alias (syslog-auto): 60s timeout, retry once on 000
-    code = probe_http("http://" + BACKEND_HOST + "/litellm/v1/chat/completions",
-                     method="POST",
-                     bearer_token=monitor_key,
-                     data='{"model":"syslog-auto","messages":[{"role":"user","content":"health ' + str(random.randint(1000, 9999)) + '"}],"max_tokens":4}',
-                     timeout=60)
+    code, failure_kind = probe_http("http://" + BACKEND_HOST + "/litellm/v1/chat/completions",
+                                   method="POST",
+                                   bearer_token=monitor_key,
+                                   data='{"model":"syslog-auto","messages":[{"role":"user","content":"health ' + str(random.randint(1000, 9999)) + '"}],"max_tokens":4}',
+                                   timeout=60)
     
-    if code == 000:
+    if code == 000 and failure_kind:
         # Retry once with same timeout
         time.sleep(1)
-        code = probe_http("http://" + BACKEND_HOST + "/litellm/v1/chat/completions",
-                         method="POST",
-                         bearer_token=monitor_key,
-                         data='{"model":"syslog-auto","messages":[{"role":"user","content":"health ' + str(random.randint(1000, 9999)) + '"}],"max_tokens":4}',
-                         timeout=60)
-    
-    results.append(("syslog-auto", code == 200, str(code) + " (target: " + BACKEND_HOST + "/litellm/v1/chat/completions, model=syslog-auto)"))
+        code, failure_kind = probe_http("http://" + BACKEND_HOST + "/litellm/v1/chat/completions",
+                                       method="POST",
+                                       bearer_token=monitor_key,
+                                       data='{"model":"syslog-auto","messages":[{"role":"user","content":"health ' + str(random.randint(1000, 9999)) + '"}],"max_tokens":4}',
+                                       timeout=60)
+        if code == 000 and failure_kind:
+            results.append(("syslog-auto", False, "probe-failed: syslog-auto " + failure_kind + " (60s timeout, retry)"))
+        elif code in (401, 403):
+            body = get_response_body("http://" + BACKEND_HOST + "/litellm/v1/chat/completions",
+                                    method="POST",
+                                    bearer_token=monitor_key,
+                                    data='{"model":"syslog-auto","messages":[{"role":"user","content":"health"}],"max_tokens":4}',
+                                    timeout=10)
+            alias = "monitor-20260813"
+            results.append(("syslog-auto", False, str(code) + " credential fault: body=" + body + " key_alias=" + alias))
+        else:
+            results.append(("syslog-auto", code == 200, str(code) + " (target: " + BACKEND_HOST + "/litellm/v1/chat/completions, model=syslog-auto)"))
+    else:
+        results.append(("syslog-auto", code == 200, str(code) + " (target: " + BACKEND_HOST + "/litellm/v1/chat/completions, model=syslog-auto)"))
     
     return results
 
@@ -146,18 +212,18 @@ def check_admin_key_list():
 
 def check_github_status():
     """Step 3: GitHub status - 301 redirect is acceptable for status page"""
-    code = probe_http("https://status.github.com/api/status.json", timeout=15)
+    code, _ = probe_http("https://status.github.com/api/status.json", timeout=15)
     # GitHub status API returns 301 redirect, which is expected behavior
     return "GitHub Status", code == 301, str(code)
 
 def check_prometheus():
     """Step 4: Prometheus health"""
-    code = probe_http("http://" + BACKEND_HOST + ":9090/-/healthy")
+    code, _ = probe_http("http://" + BACKEND_HOST + ":9090/-/healthy")
     return "Prometheus", code == 200, str(code) + " (target: " + BACKEND_HOST + ":9090/-/healthy)"
 
 def check_grafana():
     """Step 9: Grafana health"""
-    code = probe_http("http://" + BACKEND_HOST + ":3001/api/health")
+    code, _ = probe_http("http://" + BACKEND_HOST + ":3001/api/health")
     return "Grafana", code == 200, str(code) + " (target: " + BACKEND_HOST + ":3001/api/health)"
 
 def check_docker_stats():