Merge remote-tracking branch 'origin/master' into update/docker-ecosystems-20260908

tests/fixtures/zulip-health-connected.json

@@ -0,0 +1,25 @@
+{
+    "status": "ok",
+    "platform": "pi",
+    "agent": "abiba",
+    "zulip": {
+        "connected": true,
+        "site": "https://chat.sysloggh.net",
+        "email": "abiba-bot@chat.sysloggh.net",
+        "queue_id": "ee7f8b6d-9d53-48a7-ad58-f6e999771001",
+        "bot_user_id": 21,
+        "messages_processed": 0,
+        "skipped": 0,
+        "last_error": null
+    },
+    "circuit_breaker": {
+        "state": "CLOSED",
+        "failures": 0,
+        "successes": 5,
+        "totalRequests": 5,
+        "failureRate": "0.000",
+        "openedAt": null
+    },
+    "workers": [],
+    "worker_count": 0
+}

tests/fixtures/zulip-health-disconnected.json

@@ -0,0 +1,25 @@
+{
+    "status": "down",
+    "platform": "pi",
+    "agent": "abiba",
+    "zulip": {
+        "connected": false,
+        "site": "https://chat.sysloggh.net",
+        "email": "abiba-bot@chat.sysloggh.net",
+        "queue_id": null,
+        "bot_user_id": null,
+        "messages_processed": 0,
+        "skipped": 0,
+        "last_error": "Zulip API error 401: queue registration failed"
+    },
+    "circuit_breaker": {
+        "state": "CLOSED",
+        "failures": 0,
+        "successes": 0,
+        "totalRequests": 0,
+        "failureRate": "0.000",
+        "openedAt": null
+    },
+    "workers": [],
+    "worker_count": 0
+}

tests/test_mumuni_monitor_removal.py

@@ -0,0 +1,352 @@
+"""Regression tests for the 2026-09-10 retirement of the Mumuni monitoring leg.
+
+WHY THIS FILE EXISTS: captain ruling 2026-09-10 — Mumuni moved off this host
+onto her own container (kagentz CT 105 on minipve, 192.168.68.14, dedicated
+`hermes` user) and is monitored from her side. The monitor nevertheless kept
+ssh'ing to root@192.168.68.24 for `~/.hermes/gateway_state.json` on the
+decommissioned deployment, read "unknown" on every run, and posted a false 🔴
+"Mumuni (Hermes) Zulip state: unknown" DM + #agent-hub stream alert to the
+captain. The daily infra digest published a matching `mumuni:unknown` row.
+
+CONTRACT UNDER TEST:
+  * `scripts/zulip-monitor.sh` carries NO Mumuni probe and NO 192.168.68.24
+    reference; it never ssh'es .24, and even on a failing run it emits no Mumuni
+    notify (stdout alert, Zulip payload, or log line).
+  * The Abiba (pi — the Zulip bridge), Tanko (DSH) and Agent Zero (kagentz) legs
+    still work: deleting the Mumuni leg must not have gutted the rest.
+  * `scripts/daily-infra-report.py` no longer probes .24 for a Hermes gateway
+    state and no longer emits a `mumuni` agent entry.
+  * `scripts/agent-health-check.py`'s AGENTS roster has no mumuni entry. This is
+    a pin, not a behavior change — verify the probe was already gone.
+  * `zulip-health.prose.md` retires the Mumuni-only steps and says explicitly
+    that Mumuni is not monitored from this host.
+
+HOW: behavioral execution plus one named deliverable-text contract. The sandbox
+copies the shipped monitor verbatim and rewrites only its LOG constant, then
+runs it with stub ssh/curl on PATH; the ssh stub records every host it is asked
+to reach, so "never probes .24" and "no Mumuni notify" are asserted from
+observed behavior. The daily digest is pinned by importing it and exercising
+collect() and build_html() directly. The single source-text assertion is the
+deliverable-text contract the captain acceptance names for the shipped monitor.
+
+Usage: python3 -m pytest tests/test_mumuni_monitor_removal.py
+"""
+from __future__ import annotations
+
+import importlib.util
+import os
+import pathlib
+import stat
+import subprocess
+
+import pytest
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+ZULIP_MONITOR = ROOT / "scripts" / "zulip-monitor.sh"
+DAILY_REPORT = ROOT / "scripts" / "daily-infra-report.py"
+AHC = ROOT / "scripts" / "agent-health-check.py"
+HEALTH_CONTRACT = ROOT / "zulip-health.prose.md"
+CONNECTED_FIXTURE = ROOT / "tests" / "fixtures" / "zulip-health-connected.json"
+
+MUMUNI_IP = "192.168.68.24"       # Mumuni's old (decommissioned) deployment
+TANKO_VANTAGE = "192.168.68.15"   # amdpve — Tanko CT 112 via pct exec
+AGENT_ZERO_HOST = "192.168.68.14"  # kagentz host, Agent Zero docker
+
+
+# ── scripts/zulip-monitor.sh: deliverable-text contract ─────────────
+
+def test_zulip_monitor_deliverable_text_contract():
+    """Owned deliverable-text contract for scripts/zulip-monitor.sh.
+
+    Captain acceptance requires the shipped monitor to contain no Mumuni probe
+    identifier and no 192.168.68.24 literal. Behavioral proof that the monitor
+    never contacts that host and never emits a Mumuni notify lives in the
+    sandbox tests below; this only pins the named text contract.
+    """
+    text = ZULIP_MONITOR.read_text()
+    assert "mumuni" not in text.lower()
+    assert MUMUNI_IP not in text
+
+
+# ── scripts/zulip-monitor.sh: behavioral sandbox ─────────────────────
+
+SSH_STUB = r"""#!/usr/bin/env bash
+# Stub ssh: record the target host, then answer by host + remote command.
+printf '%s\n' "$*" >> "$RECORD_DIR/ssh.calls"
+host=""
+for a in "$@"; do
+  case "$a" in
+    *@192.168.*) host="${a##*@}" ;;
+  esac
+done
+printf '%s\n' "$host" >> "$RECORD_DIR/ssh.hosts"
+cmd="${*: -1}"
+case "$host" in
+  192.168.68.15)
+    case "$cmd" in
+      *"systemctl is-active"*) printf '%s' "$TANKO_SVC" ;;
+      *curl*)                  printf '%s' "$TANKO_HTTP" ;;
+    esac ;;
+  192.168.68.14)
+    case "$cmd" in
+      *agent.json*) printf '%s' "$AZ_A2A" ;;
+      *"ps aux"*)   printf '%s\n' "$AZ_PS" ;;
+    esac ;;
+  *)
+    printf 'UNEXPECTED-SSH-HOST %s\n' "$host" >> "$RECORD_DIR/unexpected-ssh" ;;
+esac
+exit 0
+"""
+
+CURL_STUB = r"""#!/usr/bin/env bash
+# Stub curl: serve the Abiba health fixture and the Zulip server 200, and
+# record every call (including notify) payloads.
+printf '%s\n' "$*" >> "$RECORD_DIR/curl.calls"
+case "$*" in
+  *:9200/health*)
+    case " $* " in
+      *" -w "*) printf '%s' "$PI_HTTP" ;;   # -w '%{http_code}' probe
+      *)        printf '%s' "$PI_BODY" ;;   # body probe
+    esac ;;
+  *server_settings*)
+    printf '%s' "$SERVER_HTTP" ;;
+esac
+exit 0
+"""
+
+
+def _write_exec(path: pathlib.Path, body: str) -> None:
+    path.write_text(body)
+    path.chmod(path.stat().st_mode
+               | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+
+def _run_monitor(tmp_path, *, tanko_svc="active", tanko_http="200",
+                 az_a2a='{"name":"kagentz"}',
+                 az_ps="root 111 0.1 0.2 /opt/venv-a0/bin/python3 -u adapter.py"):
+    """Run the shipped monitor in a sandbox; return (proc, record_dir, log_path).
+
+    Only the LOG constant is rewritten (to keep the run inside the worktree).
+    Everything else — legs, labels, notify logic — is the shipped script.
+    """
+    sandbox = tmp_path / "sandbox"
+    bindir = sandbox / "bin"
+    record = sandbox / "record"
+    bindir.mkdir(parents=True)
+    record.mkdir()
+
+    _write_exec(bindir / "ssh", SSH_STUB)
+    _write_exec(bindir / "curl", CURL_STUB)
+
+    source = ZULIP_MONITOR.read_text()
+    log_line = 'LOG="/root/zulip-health-monitor.log"'
+    assert log_line in source, "LOG constant moved — update the sandbox harness"
+    log_path = sandbox / "zulip-health-monitor.log"
+    script = sandbox / "zulip-monitor.sh"
+    script.write_text(source.replace(log_line, f'LOG="{log_path}"'))
+
+    env = dict(os.environ)
+    env.update({
+        "PATH": f"{bindir}:{env['PATH']}",
+        "RECORD_DIR": str(record),
+        "TANKO_SVC": tanko_svc,
+        "TANKO_HTTP": tanko_http,
+        "AZ_A2A": az_a2a,
+        "AZ_PS": az_ps,
+        "PI_HTTP": "200",
+        "PI_BODY": CONNECTED_FIXTURE.read_text(),
+        "SERVER_HTTP": "200",
+    })
+    proc = subprocess.run(["bash", str(script)], cwd=sandbox, env=env,
+                          capture_output=True, text=True)
+    return proc, record, log_path
+
+
+def test_healthy_run_is_quiet_and_never_reaches_mumuni(tmp_path):
+    proc, record, log_path = _run_monitor(tmp_path)
+    assert proc.returncode == 0, proc.stderr
+    log = log_path.read_text()
+
+    # Every retained leg actually ran and passed.
+    assert "Server: ✅ HTTP 200" in log
+    assert "Abiba: ✅ Connected" in log
+    assert "Tanko: ✅ service=active http=200" in log
+    assert "kagentz: ✅ A2A alive" in log
+    assert "kagentz: ✅ Adapter running" in log
+    assert "Result: ✅ All healthy" in log
+
+    # A healthy run emits no notify at all — and certainly no Mumuni one.
+    assert proc.stdout == ""
+    assert "Mumuni" not in log
+    assert "🔴" not in log
+
+    # Observed behavior: .24 is never resolved, only Tanko's vantage and the
+    # Agent Zero host are contacted.
+    hosts = record.joinpath("ssh.hosts").read_text().split()
+    assert MUMUNI_IP not in hosts
+    assert set(hosts) == {TANKO_VANTAGE, AGENT_ZERO_HOST}
+    assert not record.joinpath("unexpected-ssh").exists()
+
+
+def test_failing_run_alerts_on_tanko_but_never_on_mumuni(tmp_path):
+    # Failure path: exercises notify() end to end so "no Mumuni notify" is
+    # proven on the alert path, not only on the quiet healthy path.
+    proc, record, log_path = _run_monitor(tmp_path, tanko_svc="inactive",
+                                          tanko_http="000")
+    assert proc.returncode == 0, proc.stderr
+
+    alerts = proc.stdout
+    assert "Tanko (DSH dsh-web) service state: inactive" in alerts
+    assert "1 issue(s) found" in alerts
+
+    # No Mumuni text in stdout, the log, or any Zulip DM/stream payload.
+    assert "Mumuni" not in alerts
+    assert "Mumuni" not in log_path.read_text()
+    assert MUMUNI_IP not in alerts + log_path.read_text()
+    payloads = record.joinpath("curl.calls").read_text()
+    assert "Mumuni" not in payloads
+    assert MUMUNI_IP not in payloads
+
+    # The rest of the monitor still ran alongside the failing Tanko leg.
+    log = log_path.read_text()
+    assert "Abiba: ✅ Connected" in log
+    assert "kagentz: ✅ A2A alive" in log
+    assert "Result: 🔴 1 issue(s) found" in log
+
+
+# ── scripts/daily-infra-report.py: behavioral digest checks ──────────
+
+@pytest.fixture(scope="module")
+def daily():
+    spec = importlib.util.spec_from_file_location("daily_infra_report", DAILY_REPORT)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+DAILY_AGENTS = {
+    "abiba": {
+        "platform": "pi", "ct": 100, "ip": MUMUNI_IP,
+        "zulip_connected": True, "zulip_processed": 5,
+        "pm2_status": "online", "pm2_restarts": "0", "pm2_uptime": "1h",
+    },
+    "tanko": {
+        "platform": "dsh", "ct": 112, "ip": "192.168.68.122",
+        "gateway_state": "n/a (DSH)", "zulip_state": "connected",
+        "telegram_state": "unknown", "gateway_pid": None, "updated_at": "",
+    },
+}
+
+
+def _fabricated_report(agents):
+    return {
+        "nodes": {},
+        "node_count": 1,
+        "nodes_online": 1,
+        "total_vms": 0,
+        "running_vms": 0,
+        "stopped_vms": [],
+        "vms_by_node": {n: [] for n in
+                        ["amdpve", "minipve", "storepve", "acerpve", "ocupve"]},
+        "storage": [],
+        "docker_vm": {"total": 0, "running": 0, "unhealthy": [],
+                      "containers": [], "reclaimable": "", "disk_used": "1%"},
+        "docker_syslog": {"total": 0, "running": 0, "containers": []},
+        "docker_netbird": {"total": 0, "running": 0, "containers": []},
+        "endpoints": [],
+        "litellm": {"checks": []},
+        "nfs": [],
+        "zulip_ext": {
+            "connected": True, "queue_id": "queue", "last_error": None,
+            "messages_processed": 0, "retry_count": 0, "pm2": {},
+            "pm2_healthy": True, "bot_skipped_15min": 0, "finalized_1h": 0,
+            "failed_finalize_1h": 0, "finalize_fail_pct": 0,
+            "server_status": "200",
+        },
+        "agents": agents,
+    }
+
+
+def _agent_status_card(html):
+    start = html.index("🤖 Agent Status")
+    end = html.index("💬 Zulip Extension")
+    return html[start:end]
+
+
+def test_daily_report_renders_only_abiba_and_tanko_agents(daily):
+    """build_html() over a Mumuni-free agent set must render no Mumuni row and
+    no Mumuni gateway-unknown issue, while abiba and tanko rows still render."""
+    html = daily.build_html(_fabricated_report(dict(DAILY_AGENTS)))
+    card = _agent_status_card(html)
+    assert "mumuni" not in card.lower()
+    assert "abiba" in card
+    assert "tanko" in card
+    assert "mumuni" not in html.lower()
+
+
+def test_daily_report_collect_never_probes_mumuni(monkeypatch, daily):
+    """collect() with ssh stubbed must add no mumuni agent and must never ssh
+    its decommissioned .24 host."""
+    probed = []
+
+    class _NoSubprocess:
+        @staticmethod
+        def check_output(*args, **kwargs):
+            return b""
+
+    def fake_ssh(host, cmd):
+        probed.append(host)
+        return ""
+
+    monkeypatch.setattr(daily, "pve_get", lambda path: [])
+    monkeypatch.setattr(daily, "ssh_jerome", lambda host, cmd: "")
+    monkeypatch.setattr(daily, "ssh", fake_ssh)
+    monkeypatch.setattr(daily, "http_get",
+                        lambda url, auth=None, timeout=10: "200")
+    monkeypatch.setattr(daily, "http_get_body",
+                        lambda url, auth=None, timeout=10: "")
+    monkeypatch.setattr(daily, "count_in_log", lambda *a, **k: 0)
+    monkeypatch.setattr(daily, "subprocess", _NoSubprocess)
+
+    report = daily.collect()
+    assert "mumuni" not in report["agents"]
+    assert MUMUNI_IP not in probed
+
+
+# ── scripts/agent-health-check.py: roster pin ───────────────────────
+
+@pytest.fixture(scope="module")
+def ahc():
+    spec = importlib.util.spec_from_file_location("agent_health_check_roster", AHC)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_agent_health_roster_has_no_mumuni_entry(ahc):
+    assert "mumuni" not in ahc.AGENTS
+
+
+# ── zulip-health.prose.md: contract reconciliation ──────────────────
+
+def test_health_contract_retires_mumuni_only_steps():
+    text = HEALTH_CONTRACT.read_text()
+    assert MUMUNI_IP not in text
+    for step in ("**B4:", "**B5:", "**B6:"):
+        assert step not in text
+
+
+def test_health_contract_states_mumuni_is_not_monitored_from_this_host():
+    text = HEALTH_CONTRACT.read_text()
+    assert "Mumuni is NOT monitored from this host" in text
+    assert "monitored on her side" in text
+    assert "her own container" in text
+
+
+def test_health_contract_keeps_tanko_agent_zero_and_bridge_steps():
+    text = HEALTH_CONTRACT.read_text()
+    for marker in ("**B1:", "**B2:", "**B3:", "Step 4: Platform C",
+                   "Step 2: Platform A", "Step 1: Zulip Server Liveness"):
+        assert marker in text, marker

tests/test_probe_drift.py

@@ -0,0 +1,466 @@
+"""Regression tests for the 2026-09-09/10 probe-drift corrections.
+
+WHY THIS FILE EXISTS: the monitoring contracts kept emitting false alarms from
+stale expectations rather than live faults.
+
+  * agent-health-check v3 reported 6 failures that were all stale expectations:
+    abiba (pi-only since the harness purge) was tested as a Hermes host, koby
+    (report-only per the captain's 2026-08-17 ruling) was counted as repairable,
+    koby's CT 111 was probed on amdpve where it does not exist (it runs on
+    storepve .6), and the wrapper infisical check had two bugs — it read only
+    the first 20 lines, so koonimo's wrapper (which references /usr/bin/infisical
+    past line 20) false-failed, and it treated koby's genuine no-infisical
+    (~/.hermes/.env) wrapper as broken.
+  * gpu-monitor emitted "DEGRADED — GPU-rtx3090 000, GPU-rtx5070 000" three
+    times from probing bare port 80 on GPU hosts while :8080 answered 200.
+  * infrastructure-monitoring probed CT 116 for the PVE API (no pveproxy ->
+    000) instead of the five real cluster nodes, which answer 401 = alive.
+
+These tests execute the health script (with SSH/vault stubbed) and the real
+provenance consumer (scripts/prose-lint.sh), and parse the contracts' executable
+check-health probe blocks into normalized probe sets. No live network, vault, or
+SSH access is required.
+"""
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import pathlib
+import re
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+AHC = ROOT / "scripts" / "agent-health-check.py"
+LINT = ROOT / "scripts" / "prose-lint.sh"
+GPU = ROOT / "gpu-monitor.prose.md"
+INFRA = ROOT / "infrastructure-monitoring.prose.md"
+
+PVE_NODE_IPS = {
+    "192.168.68.9",
+    "192.168.68.5",
+    "192.168.68.15",
+    "192.168.68.6",
+    "192.168.68.12",
+}
+
+
+@pytest.fixture(scope="module")
+def ahc():
+    """Import agent-health-check.py without live network/SSH side effects."""
+    spec = importlib.util.spec_from_file_location("agent_health_check", AHC)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# ── helpers: execute the health script with SSH/vault stubbed ─────────
+
+def _run_main(ahc, monkeypatch, capsys, argv, ssh_result=None):
+    ahc.FAIL.clear()
+    ahc.REPORT_ONLY.clear()
+    monkeypatch.setattr(ahc, "load_agent_keys", lambda: None)
+    monkeypatch.setattr(ahc, "ssh", lambda *a, **k: ssh_result)
+    monkeypatch.setattr(sys, "argv", ["agent-health-check.py", "--no-deploy", *argv])
+    with pytest.raises(SystemExit) as exc:
+        ahc.main()
+    return exc.value.code, capsys.readouterr().out
+
+
+def _json_payload(out):
+    for line in reversed(out.splitlines()):
+        if line.startswith('{"timestamp"'):
+            return json.loads(line)
+    raise AssertionError(f"no JSON payload in output:\n{out}")
+
+
+# ── agent-health-check: stale-expectation legs ───────────────────────
+
+def test_import_does_not_contact_vault(ahc):
+    # Keys are loaded in main() via load_agent_keys(); importing must stay inert.
+    assert callable(ahc.load_agent_keys)
+    assert all(agent.get("key") is None for agent in ahc.AGENTS.values())
+
+
+def test_abiba_is_pi_only_runtime(ahc):
+    # .24 has run pi-only since the harness purge: no Hermes gateway, config, or
+    # wrapper. Probing those legs produced false failures.
+    assert ahc.AGENTS["abiba"]["runtime"] == "pi"
+
+
+def test_koby_is_report_only(ahc):
+    # Captain's 2026-08-17 ruling (Rule 17): detect and report, never repair.
+    assert ahc.AGENTS["koby"]["report_only"] is True
+
+
+def test_koby_ct111_is_on_storepve(ahc):
+    # Live-verified 2026-09-10: `pct status 111` = running on storepve (.6);
+    # amdpve has no lxc/111.conf, which is what false-failed before.
+    assert ahc.AGENTS["koby"]["pve"] == "storepve"
+
+
+def test_report_only_legs_never_count_as_failures(ahc):
+    for agent, report_only in (("koby", True), ("koonimo", False), ("tanko", False)):
+        ahc.FAIL.clear()
+        ahc.REPORT_ONLY.clear()
+        ahc._fail(f"probe:{agent}", agent)
+        if report_only:
+            assert ahc.FAIL == []
+            assert ahc.REPORT_ONLY == [f"probe:{agent}"]
+        else:
+            assert ahc.FAIL == [f"probe:{agent}"]
+            assert ahc.REPORT_ONLY == []
+    ahc.FAIL.clear()
+    ahc.REPORT_ONLY.clear()
+
+
+def test_failure_recording_accepts_agentless_keys(ahc):
+    ahc.FAIL.clear()
+    try:
+        ahc._fail("gpu-no-port:gpu-rtx3090 (.8)")
+        assert ahc.FAIL == ["gpu-no-port:gpu-rtx3090 (.8)"]
+    finally:
+        ahc.FAIL.clear()
+
+
+def test_json_reports_absolute_execution_provenance(ahc, monkeypatch, capsys):
+    code, out = _run_main(ahc, monkeypatch, capsys, ["--json"])
+    payload = _json_payload(out)
+    assert payload["execution_path"] == os.path.abspath(str(AHC))
+    assert payload["cwd"] == os.getcwd()
+    assert code == 1  # stubbed SSH fails every leg, but provenance is still emitted
+
+
+def test_quiet_run_still_carries_provenance_on_the_alert_path(ahc, monkeypatch, capsys):
+    # The cron runs --quiet; a failure report must still carry provenance. The
+    # header line is suppressed in quiet mode, so the ALERT line is the carrier.
+    code, out = _run_main(ahc, monkeypatch, capsys, ["--quiet"])
+    assert code == 1
+    assert "📍 executed from:" not in out
+    alerts = [ln for ln in out.splitlines() if ln.startswith("ALERT agent-health:")]
+    assert alerts, out
+    assert f"script={os.path.abspath(str(AHC))}" in alerts[0]
+    assert f"cwd={os.getcwd()}" in alerts[0]
+
+
+def test_quiet_healthy_run_emits_no_stdout(ahc, monkeypatch, capsys):
+    # --quiet is documented as "only output on failure": a run with no fleet
+    # failures must produce no stdout at all (the production cron runs --quiet).
+    for name in ("check_keys", "check_gpu_ports", "check_agents", "check_ct_liveness",
+                 "check_config_integrity", "check_wrapper_integrity", "check_vault_secrets"):
+        monkeypatch.setattr(ahc, name, lambda: None)
+    code, out = _run_main(ahc, monkeypatch, capsys, ["--quiet"])
+    assert code == 0
+    assert out == ""
+
+
+def test_json_surfaces_report_only_findings_separately(ahc, monkeypatch, capsys):
+    # Koby's down legs are reported but must not count as fleet failures; the
+    # --json payload exposes them in their own array (item 1 + f8).
+    _, out = _run_main(ahc, monkeypatch, capsys, ["--json"])
+    payload = _json_payload(out)
+    assert isinstance(payload["report_only"], list)
+    assert any(key.startswith(("gateway-down:koby", "ct-unreachable:koby"))
+               for key in payload["report_only"])
+    assert not any("koby" in key for key in payload["failures"])
+
+
+# ── agent-health-check: wrapper infisical behavior (f3) ───────────────
+
+def _stub_wrapper_ssh(ahc, monkeypatch, wrapper_body, test_x_result="OK", command_v="/usr/local/bin/infisical"):
+    def fake_ssh(host, cmd, user="root"):
+        if cmd.startswith("cat /root/.local/bin/hermes"):
+            return wrapper_body
+        if cmd.startswith("ls -la /root/.local/bin/hermes "):
+            return "-rwxr-xr-x 1 root root 0 Jan 1 00:00 /root/.local/bin/hermes"
+        if cmd.startswith("ls -la /root/.local/bin/hermes-real") or "venv/bin/hermes" in cmd:
+            return "-rwxr-xr-x 1 root root 0 Jan 1 00:00 /root/.local/bin/hermes-real"
+        if cmd.startswith("grep -c 'LITELLM_API_KEY'"):
+            return "1"
+        if cmd.startswith("test -x "):
+            path = cmd[len("test -x "):].split()[0]
+            if isinstance(test_x_result, dict):
+                return test_x_result.get(path, "MISS")
+            return test_x_result
+        if cmd.startswith("command -v infisical"):
+            return command_v
+        return None
+
+    monkeypatch.setattr(ahc, "ssh", fake_ssh)
+    monkeypatch.setattr(ahc, "AGENTS", {"koonimo": dict(ahc.AGENTS["koonimo"])})
+    ahc.FAIL.clear()
+    ahc.REPORT_ONLY.clear()
+
+
+def test_env_based_wrapper_without_infisical_is_not_failed(ahc, monkeypatch, capsys):
+    _stub_wrapper_ssh(ahc, monkeypatch,
+                      "#!/bin/bash\nsource ~/.hermes/.env\nexec hermes-real \"$@\"\n")
+    ahc.check_wrapper_integrity()
+    out = capsys.readouterr().out
+    assert ahc.FAIL == []
+    assert "wrapper resolves creds without infisical" in out
+
+
+def test_dangling_absolute_infisical_path_is_failed(ahc, monkeypatch, capsys):
+    # Wrapper hardcodes /usr/bin/infisical, which is absent, while PATH resolves
+    # infisical to /usr/local/bin/infisical. The literal path must be verified,
+    # not inferred from PATH resolution.
+    _stub_wrapper_ssh(ahc, monkeypatch,
+                      "#!/bin/bash\n/usr/bin/infisical run -- hermes-real \"$@\"\n",
+                      test_x_result="MISS", command_v="/usr/local/bin/infisical")
+    ahc.check_wrapper_integrity()
+    assert "wrapper-infisical-path:koonimo" in ahc.FAIL
+
+
+def test_existing_absolute_infisical_path_passes(ahc, monkeypatch, capsys):
+    _stub_wrapper_ssh(ahc, monkeypatch,
+                      "#!/bin/bash\n/usr/bin/infisical run -- hermes-real \"$@\"\n",
+                      test_x_result="OK")
+    ahc.check_wrapper_integrity()
+    out = capsys.readouterr().out
+    assert ahc.FAIL == []
+    assert "wrapper infisical path OK" in out
+
+
+def test_comment_mentioning_removed_infisical_path_is_not_failed(ahc, monkeypatch, capsys):
+    # litellm-api-keys.prose.md documents `rm -f /usr/local/bin/infisical`; a
+    # wrapper comment about that migration must not manufacture a dangling path
+    # when the real invocation (/usr/bin/infisical) is present and executable.
+    _stub_wrapper_ssh(ahc, monkeypatch,
+                      "#!/bin/bash\n# migrated from /usr/local/bin/infisical\n"
+                      "exec /usr/bin/infisical run -- hermes-real \"$@\"\n",
+                      test_x_result={"/usr/bin/infisical": "OK",
+                                     "/usr/local/bin/infisical": "MISS"})
+    ahc.check_wrapper_integrity()
+    out = capsys.readouterr().out
+    assert ahc.FAIL == []
+    assert "wrapper infisical path OK" in out
+
+
+def test_comment_only_infisical_mention_does_not_reach_path_check(ahc, monkeypatch, capsys):
+    # A comment-only mention of a removed infisical path on a healthy .env-based
+    # wrapper is not an invocation: it must not fall through to the `command -v`
+    # PATH check and false-FAIL `wrapper-no-infisical`.
+    _stub_wrapper_ssh(ahc, monkeypatch,
+                      "#!/bin/bash\n# migrated from /usr/local/bin/infisical\n"
+                      "source ~/.hermes/.env\nexec hermes-real \"$@\"\n",
+                      test_x_result="MISS", command_v=None)
+    ahc.check_wrapper_integrity()
+    out = capsys.readouterr().out
+    assert ahc.FAIL == []
+    assert "wrapper resolves creds without infisical" in out
+
+
+# ── item 4: prose-lint enforces report provenance (real consumer) ─────
+
+GOOD_CONTRACT = textwrap.dedent("""\
+    ---
+    kind: function
+    name: good
+    description: fixture with provenance
+    ---
+
+    ## Parameters
+
+    - x: y
+
+    ## Returns
+
+    ok
+
+    ### check-health
+
+    ```bash
+    pwd -P
+    ```
+
+    **Report format**: Begin with the absolute path the probe executed from.
+    """)
+
+DECOY_CONTRACT = textwrap.dedent("""\
+    ---
+    kind: function
+    name: decoy
+    description: fixture with provenance only outside the report format
+    ---
+
+    ## Parameters
+
+    - x: y
+
+    ## Returns
+
+    ok
+
+    The absolute path of the config is /etc/foo.
+
+    ### check-health
+
+    ```bash
+    true
+    ```
+
+    **Report format**: Summarize actual results from each probe.
+    """)
+
+MISSING_CONTRACT = textwrap.dedent("""\
+    ---
+    kind: function
+    name: missing
+    description: check-health contract with no report format
+    ---
+
+    ## Parameters
+
+    - x: y
+
+    ## Returns
+
+    ok
+
+    ### check-health
+
+    ```bash
+    pwd -P
+    ```
+    """)
+
+
+def _run_lint(tmp_path, text, name):
+    (tmp_path / name).write_text(text)
+    return subprocess.run(["bash", str(LINT)], cwd=tmp_path,
+                          capture_output=True, text=True)
+
+
+def test_prose_lint_accepts_report_format_with_provenance(tmp_path):
+    result = _run_lint(tmp_path, GOOD_CONTRACT, "good.prose.md")
+    assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_prose_lint_rejects_report_format_without_provenance(tmp_path):
+    result = _run_lint(tmp_path, DECOY_CONTRACT, "decoy.prose.md")
+    assert result.returncode == 1, result.stdout
+    assert "lacks execution provenance" in result.stdout
+
+
+def test_prose_lint_requires_report_format_on_check_health_contract(tmp_path):
+    result = _run_lint(tmp_path, MISSING_CONTRACT, "missing.prose.md")
+    assert result.returncode == 1, result.stdout
+    assert "no **Report format** paragraph" in result.stdout
+
+
+# ── contracts: parse the executable check-health probe block ─────────
+
+def _check_health_block(contract):
+    """Extract the bash probe block under ### check-health (the probe interface)."""
+    text = contract.read_text()
+    marker = "### check-health"
+    assert marker in text, f"{contract.name} has no {marker}"
+    after = text.split(marker, 1)[1]
+    match = re.search(r"```bash\n(.*?)```", after, re.S)
+    assert match, f"{contract.name} check-health has no bash probe block"
+    return match.group(1)
+
+
+def _loop_nodes(block):
+    nodes = []
+    for line in block.splitlines():
+        match = re.match(r"\s*for\s+\w+\s+in\s+(.+?);?\s*do\b", line)
+        if match:
+            nodes = match.group(1).split()
+    return nodes
+
+
+def _record(url):
+    """Normalize a URL into a probe record: host, port, path, expected status."""
+    match = re.match(r"https?://([^/\s\"')]+)(/[^\s\"')]*)?", url)
+    assert match, f"unparseable probe URL: {url}"
+    hostport = match.group(1)
+    if "@" in hostport:
+        hostport = hostport.split("@", 1)[1]
+    if hostport.startswith("["):
+        host, port = hostport[1:hostport.index("]")], None
+    elif ":" in hostport:
+        host, raw_port = hostport.rsplit(":", 1)
+        port = int(raw_port) if raw_port.isdigit() else None
+    else:
+        host, port = hostport, None
+    return {"host": host, "port": port, "path": match.group(2) or "/",
+            "expected": None}
+
+
+def _probes(block):
+    """Parse the executable check-health bash block into a normalized probe model.
+
+    Comments are not probes; an `# Expected: <status>` comment annotates the
+    preceding probe. URLs using the block's shell-loop variable `$node` are
+    expanded over the loop's node list.
+    """
+    loop_nodes = _loop_nodes(block)
+    probes = []
+    last = None
+    for raw in block.splitlines():
+        stripped = raw.strip()
+        if stripped.startswith("#"):
+            expected = re.search(r"Expected:\s*(\d{3})", stripped, re.I)
+            if expected and last is not None:
+                last["expected"] = int(expected.group(1))
+            continue
+        for url in re.findall(r"https?://[^\s\"')]+", raw):
+            hosts = loop_nodes if "$node" in url else [None]
+            for node in hosts:
+                record = _record(url.replace("$node", node) if node else url)
+                probes.append(record)
+                last = record
+    return probes
+
+
+def test_gpu_monitor_probes_every_gpu_health_on_8080():
+    probes = _probes(_check_health_block(GPU))
+    targets = {(p["host"], p["port"], p["path"]) for p in probes}
+    assert ("192.168.68.8", 8080, "/health") in targets
+    assert ("192.168.68.110", 8080, "/health") in targets
+
+
+def test_gpu_monitor_never_probes_bare_port_80_on_gpu_hosts():
+    probes = _probes(_check_health_block(GPU))
+    gpu_hosts = {"192.168.68.8", "192.168.68.110", "192.168.68.15"}
+    offenders = [p for p in probes
+                 if p["host"] in gpu_hosts and p["port"] in (None, 80)]
+    assert offenders == []
+
+
+def test_probe_model_flags_explicit_port_80_on_gpu_host():
+    # Regression: a bare-port probe may be spelled with an explicit :80.
+    block = ("curl -s -o /dev/null -w '%{http_code}' "
+             "http://192.168.68.8:80/health\n")
+    gpu_hosts = {"192.168.68.8", "192.168.68.110", "192.168.68.15"}
+    offenders = [p for p in _probes(block)
+                 if p["host"] in gpu_hosts and p["port"] in (None, 80)]
+    assert offenders and offenders[0]["port"] == 80
+
+
+def test_gpu_monitor_treats_router_301_as_alive():
+    probes = _probes(_check_health_block(GPU))
+    unified = [p for p in probes
+               if p["host"] == "192.168.68.116" and p["path"] == "/health/unified"]
+    assert unified, "router /health/unified probe missing"
+    assert unified[0]["expected"] == 301
+
+
+def test_infra_monitoring_probes_every_real_pve_node():
+    probes = _probes(_check_health_block(INFRA))
+    pve = {(p["host"], p["port"], p["path"]) for p in probes if p["port"] == 8006}
+    assert {host for host, _, _ in pve} == PVE_NODE_IPS
+    assert {path for _, _, path in pve} == {"/api2/json/version"}
+
+
+def test_infra_monitoring_does_not_probe_ct116_for_pve_api():
+    probes = _probes(_check_health_block(INFRA))
+    assert not any(p["host"] == "192.168.68.116" and p["port"] == 8006
+                   for p in probes)

tests/zulip-monitor-abiba.sh

@@ -0,0 +1,211 @@
+#!/bin/bash
+# tests/zulip-monitor-abiba.sh — regression test pinning the producer→consumer
+# contract between the pi Zulip extension's :9200/health payload and the Abiba
+# leg of scripts/zulip-monitor.sh.
+#
+# WHY THIS TEST EXISTS: 2026-09-09 live incident. The monitor parsed the health
+# payload at the WRONG nesting level (d.get('connected') at top level, while the
+# extension serves zulip.connected) so PI_CONNECTED was always False and every
+# monitor run restarted a healthy bot: pm2 showed restarts=8 with the process
+# created 2026-09-09T09:35:09Z, the monitor log recorded four ❌ Abiba verdicts
+# (04:23, 05:35, 06:55, 09:35 UTC) and zero ✅, while the Zulip server answered
+# HTTP 200 and the bot logged a clean connect plus continuing heartbeats. The
+# watchdog was the fault, not the connection. This test makes that class of
+# regression fail loudly instead of silently restarting healthy services.
+#
+# CONTRACT UNDER TEST (must hold for scripts/zulip-monitor.sh):
+#   * Connection state is NESTED: zulip.connected (boolean) and zulip.last_error
+#     live inside the `zulip` object. There is NO top-level `connected` and NO
+#     retry counter anywhere in the payload (verified against the extension's
+#     startHealthServer handler) — the old retry_count branch was dropped.
+#   * zulip.connected=true                  -> log "✅ Connected", NO pm2 restart.
+#   * zulip.connected=false                 -> alert, pm2 restart abiba-zulip.
+#   * fetch error / non-2xx / empty body / unparseable body / missing or
+#     non-boolean zulip.connected           -> "⚠️ Probe failed" alert with a
+#     "NOT restarting" label, NO pm2 restart. A parse miss must never kill a
+#     healthy service.
+#   * zulip.connected=true with last_error  -> degraded 🟡 warning, no restart.
+#
+# HOW: the Abiba leg of the shipped script sits between the
+# `# -- abiba-leg-start` / `# -- abiba-leg-end` marker comments. This runner
+# extracts that block verbatim and executes it with a stubbed curl (fixture body
+# + HTTP code), recorded notify()/pm2 shims, and a temp $LOG. If the markers
+# disappear (fix reverted or renamed) extraction yields nothing and the suite
+# fails — the bug cannot return silently.
+#
+# Usage: bash tests/zulip-monitor-abiba.sh [path/to/zulip-monitor.sh]
+# Exit 0 iff every check passes.
+#
+# shellcheck disable=SC2034,SC2329,SC1090
+# LOG/ISSUES and the notify/pm2/curl stubs below are consumed at runtime by
+# the leg extracted between the marker comments and `source`d in each case;
+# the static analyzer cannot see across that dynamic source, so it flags them.
+set -uo pipefail
+
+ROOT=$(cd "$(dirname "$0")/.." && pwd)
+SCRIPT=${1:-"$ROOT/scripts/zulip-monitor.sh"}
+FIXTURES="$ROOT/tests/fixtures"
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+PASS=0
+FAIL=0
+ok()  { PASS=$((PASS + 1)); printf '  \033[32m✔\033[0m %s\n' "$1"; }
+bad() { FAIL=$((FAIL + 1)); printf '  \033[31m✘\033[0m %s\n' "$1"; }
+
+echo "== tests/zulip-monitor-abiba.sh — Abiba leg vs :9200/health producer contract =="
+echo "target script: $SCRIPT"
+
+# --- structural guards -------------------------------------------------------
+if ! grep -q '^# -- abiba-leg-start' "$SCRIPT"; then
+  echo "✘ FATAL: $SCRIPT has no '# -- abiba-leg-start' marker — the fix has been reverted or renamed."
+  exit 1
+fi
+if ! grep -q '^# -- abiba-leg-end' "$SCRIPT"; then
+  echo "✘ FATAL: $SCRIPT has no '# -- abiba-leg-end' marker."
+  exit 1
+fi
+
+LEG="$TMP/leg.sh"
+awk '/^# -- abiba-leg-start/{f=1; next}
+     /^# -- abiba-leg-end/{f=0; next}
+     f' "$SCRIPT" > "$LEG"
+if [ ! -s "$LEG" ]; then
+  echo "✘ FATAL: extracted Abiba leg is empty."
+  exit 1
+fi
+echo "== structural =="
+if bash -n "$SCRIPT"; then ok "syntax: bash -n $SCRIPT"; else bad "syntax: bash -n $SCRIPT failed"; fi
+if bash -n "$LEG"; then ok "syntax: extracted leg parses (bash -n)"; else bad "syntax: extracted leg fails bash -n"; fi
+
+# --- per-case harness ---------------------------------------------------------
+CURRENT_NAME=""
+CURRENT_DIR=""
+
+# $1 case name, $2 http-code, $3 body (file path or literal)
+run_case() {
+  local name="$1" http="$2" body_src="$3" body
+  CURRENT_NAME="$name"
+  CURRENT_DIR=$(mktemp -d "$TMP/case.XXXXXX")
+  if [ -f "$body_src" ]; then
+    body=$(cat "$body_src")
+  else
+    body="$body_src"
+  fi
+  (
+    LOG="$CURRENT_DIR/log"; ISSUES=0
+    notify() { printf 'ALERT [%s] %s\n' "$1" "$2" >> "$CURRENT_DIR/alerts"; }
+    pm2()    { printf 'PM2 %s\n' "$*"    >> "$CURRENT_DIR/pm2"; }
+    curl() {
+      local url=""
+      for a in "$@"; do case "$a" in http*) url="$a";; esac; done
+      case "$url" in
+        *:9200/health*)
+          case " $* " in
+            *"-w"*) printf '%s' "$http" ;;   # -w '%{http_code}' code probe
+            *)      printf '%s' "$body"  ;;   # body probe
+          esac ;;
+        *)
+          printf 'UNEXPECTED-CURL %s\n' "$*" >> "$CURRENT_DIR/unexpected-curl"
+          return 7 ;;
+      esac
+      return 0
+    }
+    source "$LEG"
+  )
+}
+
+assert_log_has() {
+  if grep -qF -- "$1" "$CURRENT_DIR/log"; then ok "$CURRENT_NAME — log has: $1"; else bad "$CURRENT_NAME — log MISSING: $1"; fi
+}
+assert_log_lacks() {
+  if grep -qF -- "$1" "$CURRENT_DIR/log"; then bad "$CURRENT_NAME — log must NOT contain: $1"; else ok "$CURRENT_NAME — log correctly lacks: $1"; fi
+}
+assert_alert_has() {
+  if grep -qF -- "$1" "$CURRENT_DIR/alerts"; then ok "$CURRENT_NAME — alert sent: $1"; else bad "$CURRENT_NAME — alert MISSING: $1"; fi
+}
+assert_alert_empty() {
+  if [ ! -s "$CURRENT_DIR/alerts" ]; then ok "$CURRENT_NAME — no alert sent (quiet healthy path)"; else bad "$CURRENT_NAME — unexpected alert: $(cat "$CURRENT_DIR/alerts")"; fi
+}
+assert_pm2_restarted() {
+  if grep -qF "PM2 restart abiba-zulip" "$CURRENT_DIR/pm2"; then ok "$CURRENT_NAME — pm2 restart abiba-zulip was called"; else bad "$CURRENT_NAME — expected pm2 restart abiba-zulip, pm2 log: $(cat "$CURRENT_DIR/pm2" 2>/dev/null)"; fi
+}
+assert_no_restart() {
+  if [ ! -s "$CURRENT_DIR/pm2" ]; then ok "$CURRENT_NAME — NO pm2 restart (fail-safe holds)"; else bad "$CURRENT_NAME — pm2 was called but must NOT be: $(cat "$CURRENT_DIR/pm2")"; fi
+}
+assert_no_unexpected_curl() {
+  if [ ! -s "$CURRENT_DIR/unexpected-curl" ]; then ok "$CURRENT_NAME — only :9200/health was probed"; else bad "$CURRENT_NAME — unexpected curl: $(cat "$CURRENT_DIR/unexpected-curl")"; fi
+}
+
+# --- case 1: real payload shape, zulip.connected=true -> healthy, no restart --
+echo "== case 1: connected (real producer payload: nested zulip.connected=true) =="
+run_case "connected" 200 "$FIXTURES/zulip-health-connected.json"
+assert_log_has "Abiba: ✅ Connected (processed=0)"
+assert_log_lacks "Disconnected"
+assert_alert_empty
+assert_no_restart
+assert_no_unexpected_curl
+
+# --- case 2: zulip.connected=false -> disconnected, restart -------------------
+echo "== case 2: disconnected (nested zulip.connected=false triggers restart) =="
+run_case "disconnected" 200 "$FIXTURES/zulip-health-disconnected.json"
+assert_log_has "Abiba: ❌ Disconnected — restarted"
+assert_alert_has "DISCONNECTED — restarting"
+assert_pm2_restarted
+assert_no_unexpected_curl
+
+# --- cases 3-9: probe failures must alert and MUST NOT restart ----------------
+echo "== probe-failure cases: alert 'NOT restarting', zero pm2 restarts =="
+
+run_case "empty body" 200 ""
+assert_log_has "Abiba: ⚠️ Probe failed"
+assert_log_lacks "❌ Disconnected"
+assert_alert_has "NOT restarting"
+assert_no_restart
+
+run_case "garbage body" 200 '{not valid json!!'
+assert_log_has "Abiba: ⚠️ Probe failed"
+assert_alert_has "NOT restarting"
+assert_no_restart
+
+run_case "missing zulip key" 200 '{"status":"ok","platform":"pi","agent":"abiba"}'
+assert_log_has "Probe failed"
+assert_alert_has "NOT restarting"
+assert_no_restart
+
+run_case "zulip without connected" 200 '{"status":"ok","zulip":{"last_error":null}}'
+assert_log_has "Probe failed"
+assert_alert_has "NOT restarting"
+assert_no_restart
+
+run_case "non-boolean connected" 200 '{"status":"ok","zulip":{"connected":"true"}}'
+assert_log_has "Probe failed"
+assert_alert_has "NOT restarting"
+assert_no_restart
+
+run_case "fetch failure http 000" 000 ""
+assert_log_has "Probe failed"
+assert_alert_has "NOT restarting"
+assert_no_restart
+
+run_case "non-2xx http 500" 500 '{"error":"boom"}'
+assert_log_has "Probe failed"
+assert_alert_has "NOT restarting"
+assert_no_restart
+
+# --- case 10: connected but last_error set -> degraded 🟡, no restart ---------
+echo "== case 10: degraded (connected=true but last_error set) warns, no restart =="
+run_case "degraded" 200 '{"status":"ok","zulip":{"connected":true,"last_error":"transient queue hiccup","messages_processed":3}}'
+assert_log_has "Abiba: 🟡 Error: transient queue hiccup"
+assert_log_lacks "❌ Disconnected"
+assert_no_restart
+
+# --- summary -------------------------------------------------------------------
+echo ""
+if [ "$FAIL" -eq 0 ]; then
+  echo "✅ ALL CHECKS PASSED ($PASS/$PASS) — tests/zulip-monitor-abiba.sh"
+  exit 0
+else
+  echo "❌ $FAIL CHECK(S) FAILED ($PASS passed) — tests/zulip-monitor-abiba.sh"
+  exit 1
+fi