From 568fec2efaf904e5e5c984c6e46efbcb11ecfae2 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 19 Sep 2026 22:25:03 +0000 Subject: [PATCH] test(zulip-kagentz): Replace string-presence tests with behavioural sandbox tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - C3 502 → INCIDENT - C3 000 → INCIDENT - C1 401 + C3 302 → 0 issues, all healthy - C1 000 → INCIDENT Each test asserts from the run's own log/verdict, not from file text. Prose assertions kept as secondary. Proven to bite: run against pre-fix script (origin/master) shows all 4 behavioural cases fail because C3 leg doesn't exist and Result line doesn't say 'INCIDENT'. --- tests/test_zulip_kagentz_legs.py | 330 ++++++++++++++++++++----------- 1 file changed, 213 insertions(+), 117 deletions(-) diff --git a/tests/test_zulip_kagentz_legs.py b/tests/test_zulip_kagentz_legs.py index 91a497f..0254a86 100644 --- a/tests/test_zulip_kagentz_legs.py +++ b/tests/test_zulip_kagentz_legs.py @@ -1,137 +1,233 @@ #!/usr/bin/env python3 -""" -Tests for zulip-monitor.sh kagentz A2A and public access path legs. +"""Behavioural tests for zulip-monitor.sh kagentz C1/C3 legs and the Result verdict. -Covers: -- (b) Run verdict is non-optimistic: when ISSUES > 0, the Result line says "INCIDENT" -- (d) Public access path leg: 200/302/401 = alive, 502 = incident, 000 = incident -- (c) C1/C2/C3 distinction documented in prose +WHY THIS FILE EXISTS: the 2026-09-19 kagentz A2A outage was correctly detected +by the monitor (C1 returned 000, the run logged an issue) but the lane's own +summarization in ops.status wrote "OK" with the note "A2A server DOWN — expected +(no credentials configured)". The optimistic verdict came from the lane, not the +script. The fix adds a C3 public-access-path leg and makes the Result line say +"INCIDENT" when issues are found, so the lane can quote it verbatim. -These tests parse the script and prose to verify the expected structure. +CONTRACT UNDER TEST: + * C1 (A2A liveness, no credential): 000 → INCIDENT. + * C3 (public access path, no credential): 502 → INCIDENT, 000 → INCIDENT, + 200/302/401 → alive. + * Result verdict: when ISSUES > 0, the log's Result line says "INCIDENT", + not just "issues found". + * Healthy control: C1 401 + C3 302 → 0 issues, "all healthy". + +HOW: behavioural execution using the sandbox pattern already in this repo +(tests/test_mumuni_monitor_removal.py). The sandbox copies the shipped monitor +verbatim, rewrites only its LOG constant, and runs it with stub ssh/curl on +PATH. Each test asserts from the run's own log/verdict, not from file text. + +Usage: python3 -m pytest tests/test_zulip_kagentz_legs.py """ +from __future__ import annotations import os -import re +import pathlib +import stat import subprocess -import sys -from pathlib import Path -# Paths -SCRIPT = Path(__file__).parent.parent / "scripts" / "zulip-monitor.sh" -PROSE = Path(__file__).parent.parent / "zulip-health.prose.md" +import pytest -def read_file(path): - return path.read_text() +ROOT = pathlib.Path(__file__).resolve().parents[1] +ZULIP_MONITOR = ROOT / "scripts" / "zulip-monitor.sh" +CONNECTED_FIXTURE = ROOT / "tests" / "fixtures" / "zulip-health-connected.json" -def test_script_has_c1_c2_c3_legs(): - """Script should have C1, C2, C3 leg markers.""" - content = read_file(SCRIPT) - assert "C1: A2A liveness" in content, "Missing C1 leg comment" - assert "C3: Public access path" in content, "Missing C3 leg comment" - print("✓ Script has C1, C2, C3 leg markers") +TANKO_VANTAGE = "192.168.68.15" # amdpve — Tanko CT 112 via pct exec +AGENT_ZERO_HOST = "192.168.68.14" # kagentz host, Agent Zero docker -def test_c1_no_credential_needed(): - """C1 should be documented as needing no credential.""" - prose = read_file(PROSE) - assert "C1: A2A Server Health (no credential needed)" in prose, \ - "C1 header should say 'no credential needed'" - assert "INCIDENT" in prose, "C1 000 should be marked as INCIDENT" - print("✓ C1 documented as no-credential, 000 = INCIDENT") -def test_c2_requires_litellm_key(): - """C2 should be documented as requiring LITELLM_KEY.""" - prose = read_file(PROSE) - assert "C2: A2A Response Verification (requires LITELLM_KEY)" in prose, \ - "C2 header should say 'requires LITELLM_KEY'" - assert "credential issue, NOT a server-down incident" in prose, \ - "C2 401 should be documented as credential issue, not server-down" - print("✓ C2 documented as requiring LITELLM_KEY") +# ── Stub ssh: answers Tanko and Agent Zero probes by env vars ────────── -def test_c3_public_access_path(): - """C3 should probe https://kagentz.sysloggh.net/.""" - prose = read_file(PROSE) - script = read_file(SCRIPT) - assert "C3: Public Access Path" in prose, "Missing C3 section in prose" - assert "https://kagentz.sysloggh.net/" in prose, "C3 should probe the public URL" - assert "502" in prose, "C3 should document 502 as incident" - assert "KAGENTZ_PUBLIC_CODE" in script, "Script should have KAGENTZ_PUBLIC_CODE variable" - print("✓ C3 public access path leg present") +SSH_STUB = r"""#!/usr/bin/env bash +# Stub ssh: record the target host, then answer by host + remote command. +printf '%s\n' "$*" >> "$RECORD_DIR/ssh.calls" +host="" +for a in "$@"; do + case "$a" in + *@192.168.*) host="${a##*@}" ;; + esac +done +printf '%s\n' "$host" >> "$RECORD_DIR/ssh.hosts" +cmd="${*: -1}" +case "$host" in + 192.168.68.15) + case "$cmd" in + *"systemctl is-active"*) printf '%s' "$TANKO_SVC" ;; + *curl*) printf '%s' "$TANKO_HTTP" ;; + esac ;; + 192.168.68.14) + case "$cmd" in + *"/a2a/"*) printf '%s' "$AZ_A2A_CODE"; exit "${AZ_A2A_EXIT:-0}" ;; + esac ;; + *) + printf 'UNEXPECTED-SSH-HOST %s\n' "$host" >> "$RECORD_DIR/unexpected-ssh" ;; +esac +exit 0 +""" -def test_result_line_non_optimistic(): - """When ISSUES > 0, the Result line should say INCIDENT, not just 'issues found'.""" - script = read_file(SCRIPT) - # The summary section should have "INCIDENT" in the non-zero branch - assert "Result: 🔴 INCIDENT" in script, \ - "Result line should say 'INCIDENT' when ISSUES > 0" - assert "Result: ✅ 0 issues (all healthy)" in script, \ - "Result line should say '0 issues (all healthy)' when ISSUES = 0" - print("✓ Result line is non-optimistic (INCIDENT when issues > 0)") -def test_c3_502_is_incident(): - """C3 should treat 502 as an incident.""" - script = read_file(SCRIPT) - # The script should have a branch for 502 - assert 'elif [ "$KAGENTZ_PUBLIC_CODE" = "502" ]' in script, \ - "Script should have explicit 502 branch" - assert "upstream refused" in script, \ - "502 should be documented as 'upstream refused'" - print("✓ C3 502 treated as incident") +# ── Stub curl: serves Zulip server, Abiba health, and C3 public URL ─── -def test_c3_000_is_incident(): - """C3 should treat 000 as an incident.""" - script = read_file(SCRIPT) - assert 'if [ "$KAGENTZ_PUBLIC_CODE" = "000" ]' in script, \ - "Script should have explicit 000 branch" - assert "public URL DOWN" in script, \ - "000 should be reported as 'public URL DOWN'" - print("✓ C3 000 treated as incident") +CURL_STUB = r"""#!/usr/bin/env bash +# Stub curl: serve the Abiba health fixture, the Zulip server 200, and the +# C3 public URL probe (https://kagentz.sysloggh.net/). Record every call. +printf '%s\n' "$*" >> "$RECORD_DIR/curl.calls" +case "$*" in + *:9200/health*) + case " $* " in + *" -w "*) printf '%s' "$PI_HTTP" ;; # -w '%{http_code}' probe + *) printf '%s' "$PI_BODY" ;; # body probe + esac ;; + *server_settings*) + printf '%s' "$SERVER_HTTP" ;; + *kagentz.sysloggh.net*) + printf '%s' "$KAGENTZ_PUBLIC_CODE" ;; +esac +exit 0 +""" -def test_c3_alive_statuses(): - """C3 should treat 200/302/401 as alive.""" - script = read_file(SCRIPT) - assert "200|302|401" in script, \ - "Script should classify 200/302/401 as alive" - assert "public URL alive" in script, \ - "Alive statuses should be reported as 'public URL alive'" - print("✓ C3 200/302/401 treated as alive") -def test_prose_documents_c3_actions(): - """Prose should document C3 actions in the Platform C Actions table.""" - prose = read_file(PROSE) - assert "C3 public URL returns `502`" in prose, \ - "Platform C Actions table should have C3 502 row" - assert "C3 public URL returns `000`" in prose, \ - "Platform C Actions table should have C3 000 row" - print("✓ Prose documents C3 actions") +def _write_exec(path: pathlib.Path, body: str) -> None: + path.write_text(body) + path.chmod(path.stat().st_mode + | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) -def run_all_tests(): - tests = [ - test_script_has_c1_c2_c3_legs, - test_c1_no_credential_needed, - test_c2_requires_litellm_key, - test_c3_public_access_path, - test_result_line_non_optimistic, - test_c3_502_is_incident, - test_c3_000_is_incident, - test_c3_alive_statuses, - test_prose_documents_c3_actions, - ] - passed = 0 - failed = 0 - for test in tests: - try: - test() - passed += 1 - except AssertionError as e: - print(f"✗ {test.__name__}: {e}") - failed += 1 - print(f"\n{'='*50}") - print(f"Tests passed: {passed}/{len(tests)}") - if failed > 0: - print(f"Tests failed: {failed}/{len(tests)}") - sys.exit(1) - else: - print("All tests passed!") -if __name__ == "__main__": - run_all_tests() +def _run_monitor(tmp_path, *, tanko_svc="active", tanko_http="200", + az_a2a_code="401", az_a2a_exit=0, + kagentz_public_code="302"): + """Run the shipped monitor in a sandbox; return (proc, record_dir, log_path). + + Only the LOG constant is rewritten (to keep the run inside the worktree). + Everything else — legs, labels, notify logic — is the shipped script. + """ + sandbox = tmp_path / "sandbox" + bindir = sandbox / "bin" + record = sandbox / "record" + bindir.mkdir(parents=True) + record.mkdir() + + _write_exec(bindir / "ssh", SSH_STUB) + _write_exec(bindir / "curl", CURL_STUB) + + source = ZULIP_MONITOR.read_text() + log_line = 'LOG="/root/zulip-health-monitor.log"' + assert log_line in source, "LOG constant moved — update the sandbox harness" + log_path = sandbox / "zulip-health-monitor.log" + script = sandbox / "zulip-monitor.sh" + script.write_text(source.replace(log_line, f'LOG="{log_path}"')) + + env = dict(os.environ) + env.update({ + "PATH": f"{bindir}:{env['PATH']}", + "RECORD_DIR": str(record), + "TANKO_SVC": tanko_svc, + "TANKO_HTTP": tanko_http, + "AZ_A2A_CODE": az_a2a_code, + "AZ_A2A_EXIT": str(az_a2a_exit), + "PI_HTTP": "200", + "PI_BODY": CONNECTED_FIXTURE.read_text(), + "SERVER_HTTP": "200", + "KAGENTZ_PUBLIC_CODE": kagentz_public_code, + }) + proc = subprocess.run(["bash", str(script)], cwd=sandbox, env=env, + capture_output=True, text=True) + return proc, record, log_path + + +# ── Required behavioural cases ───────────────────────────────────────── + +def test_c3_502_is_incident(tmp_path): + """C3 public leg returns 502 → the run's verdict is an INCIDENT, + the C3 line names the 502, and the run is not summarised as healthy.""" + proc, record, log_path = _run_monitor(tmp_path, kagentz_public_code="502") + assert proc.returncode == 0, proc.stderr + log = log_path.read_text() + + # The C3 line names the 502. + assert "kagentz C3: ❌ public URL 502 (upstream refused)" in log + # The verdict is an INCIDENT, not healthy. + assert "Result: 🔴 INCIDENT" in log + assert "all healthy" not in log + # The run is not summarised as healthy. + assert "✅ 0 issues" not in log + + +def test_c3_000_is_incident(tmp_path): + """C3 public leg returns 000 → INCIDENT.""" + proc, record, log_path = _run_monitor(tmp_path, kagentz_public_code="000") + assert proc.returncode == 0, proc.stderr + log = log_path.read_text() + + # The C3 line reports the connection failure. + assert "kagentz C3: ❌ public URL down (HTTP 000)" in log + # The verdict is an INCIDENT. + assert "Result: 🔴 INCIDENT" in log + assert "all healthy" not in log + assert "✅ 0 issues" not in log + + +def test_healthy_control_c1_401_c3_302(tmp_path): + """Healthy control: C1 401 plus C3 302 → 0 issues and a healthy verdict, + proving the new leg cannot cry wolf.""" + proc, record, log_path = _run_monitor(tmp_path, + az_a2a_code="401", + kagentz_public_code="302") + assert proc.returncode == 0, proc.stderr + log = log_path.read_text() + + # Both legs report alive. + assert "kagentz C1: ✅ A2A alive (HTTP 401)" in log + assert "kagentz C3: ✅ public URL alive (HTTP 302)" in log + # Zero issues, healthy verdict. + assert "Result: ✅ 0 issues (all healthy)" in log + # No INCIDENT. + assert "INCIDENT" not in log + # No notify fired for kagentz. + assert "kagentz public URL" not in proc.stdout + assert "kagentz A2A server" not in proc.stdout + + +def test_c1_000_is_incident(tmp_path): + """C1 returns 000 → INCIDENT, keeping the leg that actually caught this + outage covered behaviourally.""" + proc, record, log_path = _run_monitor(tmp_path, + az_a2a_code="000", + az_a2a_exit=7, + kagentz_public_code="302") + assert proc.returncode == 0, proc.stderr + log = log_path.read_text() + + # The C1 line reports the A2A down. + assert "kagentz C1: ❌ A2A down (HTTP 000)" in log + # The verdict is an INCIDENT (even though C3 is healthy). + assert "Result: 🔴 INCIDENT" in log + assert "all healthy" not in log + # The notify fired for the A2A down. + assert "kagentz A2A server DOWN" in proc.stdout + + +# ── Prose assertions (kept as secondary, do not replace behavioural) ── + +def test_prose_c1_no_credential_needed(): + """C1 header should say 'no credential needed'.""" + prose = (ROOT / "zulip-health.prose.md").read_text() + assert "C1: A2A Server Health (no credential needed)" in prose + + +def test_prose_c2_requires_litellm_key(): + """C2 header should say 'requires LITELLM_KEY'.""" + prose = (ROOT / "zulip-health.prose.md").read_text() + assert "C2: A2A Response Verification (requires LITELLM_KEY)" in prose + + +def test_prose_c3_public_access_path(): + """C3 section should exist and document 502/000 as incidents.""" + prose = (ROOT / "zulip-health.prose.md").read_text() + assert "C3: Public Access Path" in prose + assert "https://kagentz.sysloggh.net/" in prose + assert "502" in prose