prose-contracts/tests/test_mumuni_monitor_removal.py

"""Regression tests for the 2026-09-10 retirement of the Mumuni monitoring leg.

WHY THIS FILE EXISTS: captain ruling 2026-09-10 — Mumuni moved off this host
onto her own container (kagentz CT 105 on minipve, 192.168.68.14, dedicated
`hermes` user) and is monitored from her side. The monitor nevertheless kept
ssh'ing to root@192.168.68.24 for `~/.hermes/gateway_state.json` on the
decommissioned deployment, read "unknown" on every run, and posted a false 🔴
"Mumuni (Hermes) Zulip state: unknown" DM + #agent-hub stream alert to the
captain. The daily infra digest published a matching `mumuni:unknown` row.

CONTRACT UNDER TEST:
  * `scripts/zulip-monitor.sh` carries NO Mumuni probe and NO 192.168.68.24
    reference; it never ssh'es .24, and even on a failing run it emits no Mumuni
    notify (stdout alert, Zulip payload, or log line).
  * The Abiba (pi — the Zulip bridge), Tanko (DSH) and Agent Zero (kagentz) legs
    still work: deleting the Mumuni leg must not have gutted the rest.
  * `scripts/daily-infra-report.py` no longer probes .24 for a Hermes gateway
    state and no longer emits a `mumuni` agent entry.
  * `scripts/agent-health-check.py`'s AGENTS roster has no mumuni entry. This is
    a pin, not a behavior change — verify the probe was already gone.
  * `zulip-health.prose.md` retires the Mumuni-only steps and says explicitly
    that Mumuni is not monitored from this host.

HOW: behavioral execution plus one named deliverable-text contract. The sandbox
copies the shipped monitor verbatim and rewrites only its LOG constant, then
runs it with stub ssh/curl on PATH; the ssh stub records every host it is asked
to reach, so "never probes .24" and "no Mumuni notify" are asserted from
observed behavior. The daily digest is pinned by importing it and exercising
collect() and build_html() directly. The single source-text assertion is the
deliverable-text contract the captain acceptance names for the shipped monitor.

Usage: python3 -m pytest tests/test_mumuni_monitor_removal.py
"""
from __future__ import annotations

import importlib.util
import os
import pathlib
import stat
import subprocess

import pytest

ROOT = pathlib.Path(__file__).resolve().parents[1]
ZULIP_MONITOR = ROOT / "scripts" / "zulip-monitor.sh"
DAILY_REPORT = ROOT / "scripts" / "daily-infra-report.py"
AHC = ROOT / "scripts" / "agent-health-check.py"
HEALTH_CONTRACT = ROOT / "zulip-health.prose.md"
CONNECTED_FIXTURE = ROOT / "tests" / "fixtures" / "zulip-health-connected.json"

MUMUNI_IP = "192.168.68.24"       # Mumuni's old (decommissioned) deployment
TANKO_VANTAGE = "192.168.68.15"   # amdpve — Tanko CT 112 via pct exec
AGENT_ZERO_HOST = "192.168.68.14"  # kagentz host, Agent Zero docker


# ── scripts/zulip-monitor.sh: deliverable-text contract ─────────────

def test_zulip_monitor_deliverable_text_contract():
    """Owned deliverable-text contract for scripts/zulip-monitor.sh.

    Captain acceptance requires the shipped monitor to contain no Mumuni probe
    identifier and no 192.168.68.24 literal. Behavioral proof that the monitor
    never contacts that host and never emits a Mumuni notify lives in the
    sandbox tests below; this only pins the named text contract.
    """
    text = ZULIP_MONITOR.read_text()
    assert "mumuni" not in text.lower()
    assert MUMUNI_IP not in text


# ── scripts/zulip-monitor.sh: behavioral sandbox ─────────────────────

SSH_STUB = r"""#!/usr/bin/env bash
# Stub ssh: record the target host, then answer by host + remote command.
printf '%s\n' "$*" >> "$RECORD_DIR/ssh.calls"
host=""
for a in "$@"; do
  case "$a" in
    *@192.168.*) host="${a##*@}" ;;
  esac
done
printf '%s\n' "$host" >> "$RECORD_DIR/ssh.hosts"
cmd="${*: -1}"
case "$host" in
  192.168.68.15)
    case "$cmd" in
      *"systemctl is-active"*) printf '%s' "$TANKO_SVC" ;;
      *curl*)                  printf '%s' "$TANKO_HTTP" ;;
    esac ;;
  192.168.68.14)
    case "$cmd" in
      *"/a2a/"*) printf '%s' "$AZ_A2A_CODE"; exit "$AZ_A2A_EXIT" ;;
    esac ;;
  *)
    printf 'UNEXPECTED-SSH-HOST %s\n' "$host" >> "$RECORD_DIR/unexpected-ssh" ;;
esac
exit 0
"""

CURL_STUB = r"""#!/usr/bin/env bash
# Stub curl: serve the Abiba health fixture, the Zulip server 200, and the
# kagentz C3 public URL, and record every call (including notify) payloads.
printf '%s\n' "$*" >> "$RECORD_DIR/curl.calls"
case "$*" in
  *:9200/health*)
    case " $* " in
      *" -w "*) printf '%s' "$PI_HTTP" ;;   # -w '%{http_code}' probe
      *)        printf '%s' "$PI_BODY" ;;   # body probe
    esac ;;
  *server_settings*)
    printf '%s' "$SERVER_HTTP" ;;
  *kagentz.sysloggh.net*)
    printf '%s' "$KAGENTZ_PUBLIC_CODE" ;;
esac
exit 0
"""


def _write_exec(path: pathlib.Path, body: str) -> None:
    path.write_text(body)
    path.chmod(path.stat().st_mode
               | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)


def _run_monitor(tmp_path, *, tanko_svc="active", tanko_http="200",
                 az_a2a_code="401", az_a2a_exit=0,
                 kagentz_public_code="302"):
    """Run the shipped monitor in a sandbox; return (proc, record_dir, log_path).

    Only the LOG constant is rewritten (to keep the run inside the worktree).
    Everything else — legs, labels, notify logic — is the shipped script.
    """
    sandbox = tmp_path / "sandbox"
    bindir = sandbox / "bin"
    record = sandbox / "record"
    bindir.mkdir(parents=True)
    record.mkdir()

    _write_exec(bindir / "ssh", SSH_STUB)
    _write_exec(bindir / "curl", CURL_STUB)

    source = ZULIP_MONITOR.read_text()
    log_line = 'LOG="/root/zulip-health-monitor.log"'
    assert log_line in source, "LOG constant moved — update the sandbox harness"
    log_path = sandbox / "zulip-health-monitor.log"
    script = sandbox / "zulip-monitor.sh"
    script.write_text(source.replace(log_line, f'LOG="{log_path}"'))

    env = dict(os.environ)
    env.update({
        "PATH": f"{bindir}:{env['PATH']}",
        "RECORD_DIR": str(record),
        "TANKO_SVC": tanko_svc,
        "TANKO_HTTP": tanko_http,
        "AZ_A2A_CODE": az_a2a_code,
        "AZ_A2A_EXIT": str(az_a2a_exit),
        "PI_HTTP": "200",
        "PI_BODY": CONNECTED_FIXTURE.read_text(),
        "SERVER_HTTP": "200",
        "KAGENTZ_PUBLIC_CODE": kagentz_public_code,
    })
    proc = subprocess.run(["bash", str(script)], cwd=sandbox, env=env,
                          capture_output=True, text=True)
    return proc, record, log_path


def test_healthy_run_is_quiet_and_never_reaches_mumuni(tmp_path):
    proc, record, log_path = _run_monitor(tmp_path)
    assert proc.returncode == 0, proc.stderr
    log = log_path.read_text()

    # Every retained leg actually ran and passed.
    assert "Server: ✅ HTTP 200" in log
    assert "Abiba: ✅ Connected" in log
    assert "Tanko: ✅ service=active http=200" in log
    assert "kagentz C1: ✅ A2A alive (HTTP 401)" in log
    assert "kagentz C3: ✅ public URL alive (HTTP 302)" in log
    assert "Result: ✅ 0 issues (all healthy)" in log

    # A healthy run emits no notify at all — and certainly no Mumuni one.
    assert proc.stdout == ""
    assert "Mumuni" not in log
    assert "🔴" not in log

    # Observed behavior: .24 is never resolved, only Tanko's vantage and the
    # Agent Zero host are contacted.
    hosts = record.joinpath("ssh.hosts").read_text().split()
    assert MUMUNI_IP not in hosts
    assert set(hosts) == {TANKO_VANTAGE, AGENT_ZERO_HOST}
    assert not record.joinpath("unexpected-ssh").exists()


def test_failing_run_alerts_on_tanko_but_never_on_mumuni(tmp_path):
    # Failure path: exercises notify() end to end so "no Mumuni notify" is
    # proven on the alert path, not only on the quiet healthy path.
    proc, record, log_path = _run_monitor(tmp_path, tanko_svc="inactive",
                                          tanko_http="000")
    assert proc.returncode == 0, proc.stderr

    alerts = proc.stdout
    assert "Tanko (DSH dsh-web) service state: inactive" in alerts
    assert "1 issue(s) found" in alerts

    # No Mumuni text in stdout, the log, or any Zulip DM/stream payload.
    assert "Mumuni" not in alerts
    assert "Mumuni" not in log_path.read_text()
    assert MUMUNI_IP not in alerts + log_path.read_text()
    payloads = record.joinpath("curl.calls").read_text()
    assert "Mumuni" not in payloads
    assert MUMUNI_IP not in payloads

    # The rest of the monitor still ran alongside the failing Tanko leg.
    log = log_path.read_text()
    assert "Abiba: ✅ Connected" in log
    assert "kagentz C1: ✅ A2A alive" in log
    assert "Result: 🔴 INCIDENT — 1 issue(s) found" in log


def test_unexpected_a2a_status_is_an_issue_not_healthy(tmp_path):
    proc, record, log_path = _run_monitor(tmp_path, az_a2a_code="500")
    assert proc.returncode == 0, proc.stderr
    log = log_path.read_text()

    assert "kagentz C1: 🟡 A2A unexpected http=500 (running, warning)" in log
    assert "kagentz C1: ✅ A2A alive" not in log
    assert "Result: 🔴 INCIDENT — 1 issue(s) found" in log
    assert "kagentz A2A server answered HTTP 500" in proc.stdout


def test_a2a_connection_failure_is_down_not_unexpected(tmp_path):
    # curl prints the http_code before failing, so the ssh stub exits non-zero
    # with "000" on stdout — exercising the real outage path.
    proc, record, log_path = _run_monitor(tmp_path, az_a2a_code="000",
                                          az_a2a_exit=7)
    assert proc.returncode == 0, proc.stderr
    log = log_path.read_text()

    assert "kagentz C1: ❌ A2A down (HTTP 000)" in log
    assert "kagentz C1: ✅ A2A alive" not in log
    assert "unexpected" not in log
    assert "Result: 🔴 INCIDENT — 1 issue(s) found" in log
    assert "kagentz A2A server DOWN (connection failed)" in proc.stdout


# ── scripts/daily-infra-report.py: behavioral digest checks ──────────

@pytest.fixture(scope="module")
def daily():
    spec = importlib.util.spec_from_file_location("daily_infra_report", DAILY_REPORT)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


DAILY_AGENTS = {
    "abiba": {
        "platform": "pi", "ct": 100, "ip": MUMUNI_IP,
        "zulip_connected": True, "zulip_processed": 5,
        "pm2_status": "online", "pm2_restarts": "0", "pm2_uptime": "1h",
    },
    "tanko": {
        "platform": "dsh", "ct": 112, "ip": "192.168.68.122",
        "gateway_state": "n/a (DSH)", "zulip_state": "connected",
        "telegram_state": "unknown", "gateway_pid": None, "updated_at": "",
    },
}


def _fabricated_report(agents):
    return {
        "nodes": {},
        "node_count": 1,
        "nodes_online": 1,
        "total_vms": 0,
        "running_vms": 0,
        "stopped_vms": [],
        "vms_by_node": {n: [] for n in
                        ["amdpve", "minipve", "storepve", "acerpve", "ocupve"]},
        "storage": [],
        "docker_vm": {"total": 0, "running": 0, "unhealthy": [],
                      "containers": [], "reclaimable": "", "disk_used": "1%"},
        "docker_syslog": {"total": 0, "running": 0, "containers": []},
        "docker_netbird": {"total": 0, "running": 0, "containers": []},
        "endpoints": [],
        "litellm": {"checks": []},
        "nfs": [],
        "zulip_ext": {
            "connected": True, "queue_id": "queue", "last_error": None,
            "messages_processed": 0, "retry_count": 0, "pm2": {},
            "pm2_healthy": True, "bot_skipped_15min": 0, "finalized_1h": 0,
            "failed_finalize_1h": 0, "finalize_fail_pct": 0,
            "server_status": "200",
        },
        "agents": agents,
    }


def _agent_status_card(html):
    start = html.index("🤖 Agent Status")
    end = html.index("💬 Zulip Extension")
    return html[start:end]


def test_daily_report_renders_only_abiba_and_tanko_agents(daily):
    """build_html() over a Mumuni-free agent set must render no Mumuni row and
    no Mumuni gateway-unknown issue, while abiba and tanko rows still render."""
    html = daily.build_html(_fabricated_report(dict(DAILY_AGENTS)))
    card = _agent_status_card(html)
    assert "mumuni" not in card.lower()
    assert "abiba" in card
    assert "tanko" in card
    assert "mumuni" not in html.lower()


def test_daily_report_collect_never_probes_mumuni(monkeypatch, daily):
    """collect() with ssh stubbed must add no mumuni agent and must never ssh
    its decommissioned .24 host."""
    probed = []

    class _NoSubprocess:
        @staticmethod
        def check_output(*args, **kwargs):
            return b""

    def fake_ssh(host, cmd):
        probed.append(host)
        return ""

    monkeypatch.setattr(daily, "pve_get", lambda path: [])
    monkeypatch.setattr(daily, "ssh_jerome", lambda host, cmd: "")
    monkeypatch.setattr(daily, "ssh", fake_ssh)
    monkeypatch.setattr(daily, "http_get",
                        lambda url, auth=None, timeout=10: "200")
    monkeypatch.setattr(daily, "http_get_body",
                        lambda url, auth=None, timeout=10: "")
    monkeypatch.setattr(daily, "count_in_log", lambda *a, **k: 0)
    monkeypatch.setattr(daily, "subprocess", _NoSubprocess)

    report = daily.collect()
    assert "mumuni" not in report["agents"]
    assert MUMUNI_IP not in probed


# ── scripts/agent-health-check.py: roster pin ───────────────────────

@pytest.fixture(scope="module")
def ahc():
    spec = importlib.util.spec_from_file_location("agent_health_check_roster", AHC)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def test_agent_health_roster_has_no_mumuni_entry(ahc):
    assert "mumuni" not in ahc.AGENTS


# ── zulip-health.prose.md: contract reconciliation ──────────────────

def test_health_contract_retires_mumuni_only_steps():
    text = HEALTH_CONTRACT.read_text()
    assert MUMUNI_IP not in text
    for step in ("**B4: Gateway Process**", "**B5: Heartbeat Verification**",
                 "**B6: Response Delivery**"):
        assert step not in text


def test_health_contract_states_mumuni_is_not_monitored_from_this_host():
    text = HEALTH_CONTRACT.read_text()
    assert "Mumuni is NOT monitored from this host" in text
    assert "monitored on her side" in text
    assert "her own container" in text


def test_health_contract_keeps_tanko_agent_zero_and_bridge_steps():
    text = HEALTH_CONTRACT.read_text()
    for marker in ("**B1:", "**B2:", "**B3:", "Step 4: Platform C",
                   "Step 2: Platform A", "Step 1: Zulip Server Liveness"):
        assert marker in text, marker