diff --git a/agent-health-check.prose.md b/agent-health-check.prose.md index 3b3c83a..2cceb04 100644 --- a/agent-health-check.prose.md +++ b/agent-health-check.prose.md @@ -24,7 +24,7 @@ Runs every 4 hours (2, 6, 10, 14, 18, 22 UTC at :35) via cron (`35 2,6,10,14,18, ## Requires - **LiteLLM admin key** for key validation (retrieved from `/root/.pi/agent/env.sh`) -- **SSH access** to GPU hosts (.8, .110, .15) and agent CTs (.122, .129, .114, .24) +- **SSH access** to GPU hosts — `llmuser` on .8 (owns `llama-server`), `root` on .110 and .15 — and agent CTs (.122, .129, .114, .24) - **Python 3** for script execution - **Network access** to LiteLLM (:4000), GPU exporters (:9400), and gateway endpoints diff --git a/scripts/agent-health-check.py b/scripts/agent-health-check.py index 02c98b6..a194843 100755 --- a/scripts/agent-health-check.py +++ b/scripts/agent-health-check.py @@ -50,6 +50,11 @@ Changelog: (kagentz CT 105 on minipve, .14, dedicated `hermes` user) and is monitored from her side. This script must not probe mumuni or .24 — the v2 changelog roster line was the last reference still placing her at .24 / CT100. + v6 (2026-09-28): .8 GPU health probe now runs as `llmuser` instead of `root`. + Root SSH to .8 was lost when the guest was rebuilt, so every .8 leg read as + UNREACHABLE for a healthy host. llmuser owns llama-server and can read + `systemctl is-active`, `systemctl show -p MainPID`, and the :8080 pid. + .110 and .15 keep the default `root` user. """ import subprocess, json, sys, os, time, re, io, contextlib diff --git a/tests/test_probe_drift.py b/tests/test_probe_drift.py index 7f3e41d..8270ee8 100644 --- a/tests/test_probe_drift.py +++ b/tests/test_probe_drift.py @@ -124,6 +124,39 @@ def test_tanko_ct112_is_probed_on_minipve(ahc, monkeypatch, capsys): ahc.REPORT_ONLY.clear() +def test_gpu_rtx3090_probe_uses_llmuser_not_root(ahc, monkeypatch, capsys): + # 2026-09-28: root SSH to .8 was lost when the guest was rebuilt; llmuser + # owns llama-server and can read systemctl status and the :8080 pid. A root + # probe reads as UNREACHABLE for a healthy host (the reported bug). Execute + # check_gpu_ports() against an SSH boundary that only accepts llmuser@.8 and + # assert the .8 leg does not produce the false UNREACHABLE failure. + seen = [] + + def fake_ssh(host, cmd, user="root"): + seen.append((host, user)) + if host == "192.168.68.8" and user != "llmuser": + return None # root SSH denied -> baseline false UNREACHABLE + if cmd.startswith("systemctl is-active"): + return "active" + if cmd.startswith("ss -tlnp"): + return "48351" + if cmd.startswith("curl"): + return '{"status":"ok"}' + return None + + monkeypatch.setattr(ahc, "ssh", fake_ssh) + ahc.FAIL.clear() + try: + ahc.check_gpu_ports() + out = capsys.readouterr().out + assert "gpu-unreachable:192.168.68.8" not in ahc.FAIL + assert "\u2705 gpu-rtx3090 (.8): healthy" in out + assert ("192.168.68.8", "llmuser") in seen + assert not any(host == "192.168.68.8" and user == "root" for host, user in seen) + finally: + ahc.FAIL.clear() + + def test_report_only_legs_never_count_as_failures(ahc): for agent, report_only in (("koby", True), ("koonimo", False), ("tanko", False)): ahc.FAIL.clear()