no-mistakes(document): Document llmuser SSH user for .8 GPU health probe

agent-health-check.prose.md

@@ -24,7 +24,7 @@ Runs every 4 hours (2, 6, 10, 14, 18, 22 UTC at :35) via cron (`35 2,6,10,14,18,
 ## Requires
 
 - **LiteLLM admin key** for key validation (retrieved from `/root/.pi/agent/env.sh`)
-- **SSH access** to GPU hosts (.8, .110, .15) and agent CTs (.122, .129, .114, .24)
+- **SSH access** to GPU hosts — `llmuser` on .8 (owns `llama-server`), `root` on .110 and .15 — and agent CTs (.122, .129, .114, .24)
 - **Python 3** for script execution
 - **Network access** to LiteLLM (:4000), GPU exporters (:9400), and gateway endpoints
 

scripts/agent-health-check.py

@@ -50,6 +50,11 @@ Changelog:
     (kagentz CT 105 on minipve, .14, dedicated `hermes` user) and is monitored
     from her side. This script must not probe mumuni or .24 — the v2 changelog
     roster line was the last reference still placing her at .24 / CT100.
+  v6 (2026-09-28): .8 GPU health probe now runs as `llmuser` instead of `root`.
+    Root SSH to .8 was lost when the guest was rebuilt, so every .8 leg read as
+    UNREACHABLE for a healthy host. llmuser owns llama-server and can read
+    `systemctl is-active`, `systemctl show -p MainPID`, and the :8080 pid.
+    .110 and .15 keep the default `root` user.
 """
 
 import subprocess, json, sys, os, time, re, io, contextlib

tests/test_probe_drift.py

@@ -124,6 +124,39 @@ def test_tanko_ct112_is_probed_on_minipve(ahc, monkeypatch, capsys):
         ahc.REPORT_ONLY.clear()
 
 
+def test_gpu_rtx3090_probe_uses_llmuser_not_root(ahc, monkeypatch, capsys):
+    # 2026-09-28: root SSH to .8 was lost when the guest was rebuilt; llmuser
+    # owns llama-server and can read systemctl status and the :8080 pid. A root
+    # probe reads as UNREACHABLE for a healthy host (the reported bug). Execute
+    # check_gpu_ports() against an SSH boundary that only accepts llmuser@.8 and
+    # assert the .8 leg does not produce the false UNREACHABLE failure.
+    seen = []
+
+    def fake_ssh(host, cmd, user="root"):
+        seen.append((host, user))
+        if host == "192.168.68.8" and user != "llmuser":
+            return None  # root SSH denied -> baseline false UNREACHABLE
+        if cmd.startswith("systemctl is-active"):
+            return "active"
+        if cmd.startswith("ss -tlnp"):
+            return "48351"
+        if cmd.startswith("curl"):
+            return '{"status":"ok"}'
+        return None
+
+    monkeypatch.setattr(ahc, "ssh", fake_ssh)
+    ahc.FAIL.clear()
+    try:
+        ahc.check_gpu_ports()
+        out = capsys.readouterr().out
+        assert "gpu-unreachable:192.168.68.8" not in ahc.FAIL
+        assert "\u2705 gpu-rtx3090 (.8): healthy" in out
+        assert ("192.168.68.8", "llmuser") in seen
+        assert not any(host == "192.168.68.8" and user == "root" for host, user in seen)
+    finally:
+        ahc.FAIL.clear()
+
+
 def test_report_only_legs_never_count_as_failures(ahc):
     for agent, report_only in (("koby", True), ("koonimo", False), ("tanko", False)):
         ahc.FAIL.clear()