"""Regression tests for the 2026-09-10 retirement of the Mumuni monitoring leg. WHY THIS FILE EXISTS: captain ruling 2026-09-10 — Mumuni moved off this host onto her own container (kagentz CT 105 on minipve, 192.168.68.14, dedicated `hermes` user) and is monitored from her side. The monitor nevertheless kept ssh'ing to root@192.168.68.24 for `~/.hermes/gateway_state.json` on the decommissioned deployment, read "unknown" on every run, and posted a false 🔴 "Mumuni (Hermes) Zulip state: unknown" DM + #agent-hub stream alert to the captain. The daily infra digest published a matching `mumuni:unknown` row. CONTRACT UNDER TEST: * `scripts/zulip-monitor.sh` carries NO Mumuni probe and NO 192.168.68.24 reference; it never ssh'es .24, and even on a failing run it emits no Mumuni notify (stdout alert, Zulip payload, or log line). * The Abiba (pi — the Zulip bridge), Tanko (DSH) and Agent Zero (kagentz) legs still work: deleting the Mumuni leg must not have gutted the rest. * `scripts/daily-infra-report.py` no longer probes .24 for a Hermes gateway state and no longer emits a `mumuni` agent entry. * `scripts/agent-health-check.py`'s AGENTS roster has no mumuni entry. This is a pin, not a behavior change — verify the probe was already gone. * `zulip-health.prose.md` retires the Mumuni-only steps and says explicitly that Mumuni is not monitored from this host. HOW: behavioral execution plus one named deliverable-text contract. The sandbox copies the shipped monitor verbatim and rewrites only its LOG constant, then runs it with stub ssh/curl on PATH; the ssh stub records every host it is asked to reach, so "never probes .24" and "no Mumuni notify" are asserted from observed behavior. The daily digest is pinned by importing it and exercising collect() and build_html() directly. The single source-text assertion is the deliverable-text contract the captain acceptance names for the shipped monitor. Usage: python3 -m pytest tests/test_mumuni_monitor_removal.py """ from __future__ import annotations import importlib.util import os import pathlib import stat import subprocess import pytest ROOT = pathlib.Path(__file__).resolve().parents[1] ZULIP_MONITOR = ROOT / "scripts" / "zulip-monitor.sh" DAILY_REPORT = ROOT / "scripts" / "daily-infra-report.py" AHC = ROOT / "scripts" / "agent-health-check.py" HEALTH_CONTRACT = ROOT / "zulip-health.prose.md" CONNECTED_FIXTURE = ROOT / "tests" / "fixtures" / "zulip-health-connected.json" MUMUNI_IP = "192.168.68.24" # Mumuni's old (decommissioned) deployment TANKO_VANTAGE = "192.168.68.15" # amdpve — Tanko CT 112 via pct exec AGENT_ZERO_HOST = "192.168.68.14" # kagentz host, Agent Zero docker # ── scripts/zulip-monitor.sh: deliverable-text contract ───────────── def test_zulip_monitor_deliverable_text_contract(): """Owned deliverable-text contract for scripts/zulip-monitor.sh. Captain acceptance requires the shipped monitor to contain no Mumuni probe identifier and no 192.168.68.24 literal. Behavioral proof that the monitor never contacts that host and never emits a Mumuni notify lives in the sandbox tests below; this only pins the named text contract. """ text = ZULIP_MONITOR.read_text() assert "mumuni" not in text.lower() assert MUMUNI_IP not in text # ── scripts/zulip-monitor.sh: behavioral sandbox ───────────────────── SSH_STUB = r"""#!/usr/bin/env bash # Stub ssh: record the target host, then answer by host + remote command. printf '%s\n' "$*" >> "$RECORD_DIR/ssh.calls" host="" for a in "$@"; do case "$a" in *@192.168.*) host="${a##*@}" ;; esac done printf '%s\n' "$host" >> "$RECORD_DIR/ssh.hosts" cmd="${*: -1}" case "$host" in 192.168.68.15) case "$cmd" in *"systemctl is-active"*) printf '%s' "$TANKO_SVC" ;; *curl*) printf '%s' "$TANKO_HTTP" ;; esac ;; 192.168.68.14) case "$cmd" in *"/a2a/"*) printf '%s' "$AZ_A2A_CODE"; exit "$AZ_A2A_EXIT" ;; esac ;; *) printf 'UNEXPECTED-SSH-HOST %s\n' "$host" >> "$RECORD_DIR/unexpected-ssh" ;; esac exit 0 """ CURL_STUB = r"""#!/usr/bin/env bash # Stub curl: serve the Abiba health fixture and the Zulip server 200, and # record every call (including notify) payloads. printf '%s\n' "$*" >> "$RECORD_DIR/curl.calls" case "$*" in *:9200/health*) case " $* " in *" -w "*) printf '%s' "$PI_HTTP" ;; # -w '%{http_code}' probe *) printf '%s' "$PI_BODY" ;; # body probe esac ;; *server_settings*) printf '%s' "$SERVER_HTTP" ;; esac exit 0 """ def _write_exec(path: pathlib.Path, body: str) -> None: path.write_text(body) path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) def _run_monitor(tmp_path, *, tanko_svc="active", tanko_http="200", az_a2a_code="401", az_a2a_exit=0): """Run the shipped monitor in a sandbox; return (proc, record_dir, log_path). Only the LOG constant is rewritten (to keep the run inside the worktree). Everything else — legs, labels, notify logic — is the shipped script. """ sandbox = tmp_path / "sandbox" bindir = sandbox / "bin" record = sandbox / "record" bindir.mkdir(parents=True) record.mkdir() _write_exec(bindir / "ssh", SSH_STUB) _write_exec(bindir / "curl", CURL_STUB) source = ZULIP_MONITOR.read_text() log_line = 'LOG="/root/zulip-health-monitor.log"' assert log_line in source, "LOG constant moved — update the sandbox harness" log_path = sandbox / "zulip-health-monitor.log" script = sandbox / "zulip-monitor.sh" script.write_text(source.replace(log_line, f'LOG="{log_path}"')) env = dict(os.environ) env.update({ "PATH": f"{bindir}:{env['PATH']}", "RECORD_DIR": str(record), "TANKO_SVC": tanko_svc, "TANKO_HTTP": tanko_http, "AZ_A2A_CODE": az_a2a_code, "AZ_A2A_EXIT": str(az_a2a_exit), "PI_HTTP": "200", "PI_BODY": CONNECTED_FIXTURE.read_text(), "SERVER_HTTP": "200", }) proc = subprocess.run(["bash", str(script)], cwd=sandbox, env=env, capture_output=True, text=True) return proc, record, log_path def test_healthy_run_is_quiet_and_never_reaches_mumuni(tmp_path): proc, record, log_path = _run_monitor(tmp_path) assert proc.returncode == 0, proc.stderr log = log_path.read_text() # Every retained leg actually ran and passed. assert "Server: ✅ HTTP 200" in log assert "Abiba: ✅ Connected" in log assert "Tanko: ✅ service=active http=200" in log assert "kagentz: ✅ A2A alive (HTTP 401)" in log assert "Result: ✅ All healthy" in log # A healthy run emits no notify at all — and certainly no Mumuni one. assert proc.stdout == "" assert "Mumuni" not in log assert "🔴" not in log # Observed behavior: .24 is never resolved, only Tanko's vantage and the # Agent Zero host are contacted. hosts = record.joinpath("ssh.hosts").read_text().split() assert MUMUNI_IP not in hosts assert set(hosts) == {TANKO_VANTAGE, AGENT_ZERO_HOST} assert not record.joinpath("unexpected-ssh").exists() def test_failing_run_alerts_on_tanko_but_never_on_mumuni(tmp_path): # Failure path: exercises notify() end to end so "no Mumuni notify" is # proven on the alert path, not only on the quiet healthy path. proc, record, log_path = _run_monitor(tmp_path, tanko_svc="inactive", tanko_http="000") assert proc.returncode == 0, proc.stderr alerts = proc.stdout assert "Tanko (DSH dsh-web) service state: inactive" in alerts assert "1 issue(s) found" in alerts # No Mumuni text in stdout, the log, or any Zulip DM/stream payload. assert "Mumuni" not in alerts assert "Mumuni" not in log_path.read_text() assert MUMUNI_IP not in alerts + log_path.read_text() payloads = record.joinpath("curl.calls").read_text() assert "Mumuni" not in payloads assert MUMUNI_IP not in payloads # The rest of the monitor still ran alongside the failing Tanko leg. log = log_path.read_text() assert "Abiba: ✅ Connected" in log assert "kagentz: ✅ A2A alive" in log assert "Result: 🔴 1 issue(s) found" in log def test_unexpected_a2a_status_is_an_issue_not_healthy(tmp_path): proc, record, log_path = _run_monitor(tmp_path, az_a2a_code="500") assert proc.returncode == 0, proc.stderr log = log_path.read_text() assert "kagentz: 🟡 A2A unexpected http=500 (running, warning)" in log assert "kagentz: ✅ A2A alive" not in log assert "Result: 🔴 1 issue(s) found" in log assert "kagentz A2A server answered HTTP 500" in proc.stdout def test_a2a_connection_failure_is_down_not_unexpected(tmp_path): # curl prints the http_code before failing, so the ssh stub exits non-zero # with "000" on stdout — exercising the real outage path. proc, record, log_path = _run_monitor(tmp_path, az_a2a_code="000", az_a2a_exit=7) assert proc.returncode == 0, proc.stderr log = log_path.read_text() assert "kagentz: ❌ A2A down (HTTP 000)" in log assert "kagentz: ✅ A2A alive" not in log assert "unexpected" not in log assert "Result: 🔴 1 issue(s) found" in log assert "kagentz A2A server DOWN (connection failed)" in proc.stdout # ── scripts/daily-infra-report.py: behavioral digest checks ────────── @pytest.fixture(scope="module") def daily(): spec = importlib.util.spec_from_file_location("daily_infra_report", DAILY_REPORT) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module DAILY_AGENTS = { "abiba": { "platform": "pi", "ct": 100, "ip": MUMUNI_IP, "zulip_connected": True, "zulip_processed": 5, "pm2_status": "online", "pm2_restarts": "0", "pm2_uptime": "1h", }, "tanko": { "platform": "dsh", "ct": 112, "ip": "192.168.68.122", "gateway_state": "n/a (DSH)", "zulip_state": "connected", "telegram_state": "unknown", "gateway_pid": None, "updated_at": "", }, } def _fabricated_report(agents): return { "nodes": {}, "node_count": 1, "nodes_online": 1, "total_vms": 0, "running_vms": 0, "stopped_vms": [], "vms_by_node": {n: [] for n in ["amdpve", "minipve", "storepve", "acerpve", "ocupve"]}, "storage": [], "docker_vm": {"total": 0, "running": 0, "unhealthy": [], "containers": [], "reclaimable": "", "disk_used": "1%"}, "docker_syslog": {"total": 0, "running": 0, "containers": []}, "docker_netbird": {"total": 0, "running": 0, "containers": []}, "endpoints": [], "litellm": {"checks": []}, "nfs": [], "zulip_ext": { "connected": True, "queue_id": "queue", "last_error": None, "messages_processed": 0, "retry_count": 0, "pm2": {}, "pm2_healthy": True, "bot_skipped_15min": 0, "finalized_1h": 0, "failed_finalize_1h": 0, "finalize_fail_pct": 0, "server_status": "200", }, "agents": agents, } def _agent_status_card(html): start = html.index("🤖 Agent Status") end = html.index("💬 Zulip Extension") return html[start:end] def test_daily_report_renders_only_abiba_and_tanko_agents(daily): """build_html() over a Mumuni-free agent set must render no Mumuni row and no Mumuni gateway-unknown issue, while abiba and tanko rows still render.""" html = daily.build_html(_fabricated_report(dict(DAILY_AGENTS))) card = _agent_status_card(html) assert "mumuni" not in card.lower() assert "abiba" in card assert "tanko" in card assert "mumuni" not in html.lower() def test_daily_report_collect_never_probes_mumuni(monkeypatch, daily): """collect() with ssh stubbed must add no mumuni agent and must never ssh its decommissioned .24 host.""" probed = [] class _NoSubprocess: @staticmethod def check_output(*args, **kwargs): return b"" def fake_ssh(host, cmd): probed.append(host) return "" monkeypatch.setattr(daily, "pve_get", lambda path: []) monkeypatch.setattr(daily, "ssh_jerome", lambda host, cmd: "") monkeypatch.setattr(daily, "ssh", fake_ssh) monkeypatch.setattr(daily, "http_get", lambda url, auth=None, timeout=10: "200") monkeypatch.setattr(daily, "http_get_body", lambda url, auth=None, timeout=10: "") monkeypatch.setattr(daily, "count_in_log", lambda *a, **k: 0) monkeypatch.setattr(daily, "subprocess", _NoSubprocess) report = daily.collect() assert "mumuni" not in report["agents"] assert MUMUNI_IP not in probed # ── scripts/agent-health-check.py: roster pin ─────────────────────── @pytest.fixture(scope="module") def ahc(): spec = importlib.util.spec_from_file_location("agent_health_check_roster", AHC) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def test_agent_health_roster_has_no_mumuni_entry(ahc): assert "mumuni" not in ahc.AGENTS # ── zulip-health.prose.md: contract reconciliation ────────────────── def test_health_contract_retires_mumuni_only_steps(): text = HEALTH_CONTRACT.read_text() assert MUMUNI_IP not in text for step in ("**B4: Gateway Process**", "**B5: Heartbeat Verification**", "**B6: Response Delivery**"): assert step not in text def test_health_contract_states_mumuni_is_not_monitored_from_this_host(): text = HEALTH_CONTRACT.read_text() assert "Mumuni is NOT monitored from this host" in text assert "monitored on her side" in text assert "her own container" in text def test_health_contract_keeps_tanko_agent_zero_and_bridge_steps(): text = HEALTH_CONTRACT.read_text() for marker in ("**B1:", "**B2:", "**B3:", "Step 4: Platform C", "Step 2: Platform A", "Step 1: Zulip Server Liveness"): assert marker in text, marker