diff --git a/scripts/zulip-monitor.sh b/scripts/zulip-monitor.sh index c1423e3..05b6a42 100755 --- a/scripts/zulip-monitor.sh +++ b/scripts/zulip-monitor.sh @@ -176,6 +176,11 @@ fi # never contact her former host. # ── Platform C: Agent Zero (kagentz) ── +# C1: A2A liveness (no credential needed) — probes the container's internal :80/a2a/ +# C2: A2A response verification (needs LITELLM_KEY) — probes POST /a2a with auth +# C3: Public access path (no credential needed) — probes https://kagentz.sysloggh.net/ + +# C1: A2A liveness (container-internal probe) AZ_A2A_CODE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ "docker exec agent-zero curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:80/a2a/ 2>/dev/null" 2>/dev/null) || AZ_A2A_CODE="000" AZ_A2A_CODE=$(printf '%s' "$AZ_A2A_CODE" | tr -d '[:space:]') @@ -184,27 +189,53 @@ AZ_A2A_CODE=$(printf '%s' "$AZ_A2A_CODE" | tr -d '[:space:]') if [ "$AZ_A2A_CODE" = "000" ]; then notify "🔴" "kagentz A2A server DOWN (connection failed)" ISSUES=$((ISSUES + 1)) - echo " kagentz: ❌ A2A down (HTTP 000)" >> "$LOG" + echo " kagentz C1: ❌ A2A down (HTTP 000)" >> "$LOG" else case "$AZ_A2A_CODE" in 200|401) - echo " kagentz: ✅ A2A alive (HTTP $AZ_A2A_CODE)" >> "$LOG" ;; + echo " kagentz C1: ✅ A2A alive (HTTP $AZ_A2A_CODE)" >> "$LOG" ;; *) notify "🟡" "kagentz A2A server answered HTTP $AZ_A2A_CODE — running, unexpected status" ISSUES=$((ISSUES + 1)) - echo " kagentz: 🟡 A2A unexpected http=$AZ_A2A_CODE (running, warning)" >> "$LOG" ;; + echo " kagentz C1: 🟡 A2A unexpected http=$AZ_A2A_CODE (running, warning)" >> "$LOG" ;; + esac +fi + +# C3: Public access path (the captain's point of view) +# Probes the public URL that NetBird proxies to the container. 200/302/401 = alive, +# 502 = proxy's "upstream refused" page (incident), connection failed = incident. +# Never restarts anything — the contract forbids restarting the platform. +KAGENTZ_PUBLIC_CODE=$(curl -s -o /dev/null --connect-timeout 10 --max-time 15 \ + -w '%{http_code}' https://kagentz.sysloggh.net/ 2>/dev/null) || KAGENTZ_PUBLIC_CODE="000" +KAGENTZ_PUBLIC_CODE=$(printf '%s' "$KAGENTZ_PUBLIC_CODE" | tr -d '[:space:]') +[ -n "$KAGENTZ_PUBLIC_CODE" ] || KAGENTZ_PUBLIC_CODE="000" + +if [ "$KAGENTZ_PUBLIC_CODE" = "000" ]; then + notify "🔴" "kagentz public URL DOWN (connection failed)" + ISSUES=$((ISSUES + 1)) + echo " kagentz C3: ❌ public URL down (HTTP 000)" >> "$LOG" +elif [ "$KAGENTZ_PUBLIC_CODE" = "502" ]; then + notify "🔴" "kagentz public URL 502 (upstream refused)" + ISSUES=$((ISSUES + 1)) + echo " kagentz C3: ❌ public URL 502 (upstream refused)" >> "$LOG" +else + case "$KAGENTZ_PUBLIC_CODE" in + 200|302|401) + echo " kagentz C3: ✅ public URL alive (HTTP $KAGENTZ_PUBLIC_CODE)" >> "$LOG" ;; + *) + notify "🟡" "kagentz public URL answered HTTP $KAGENTZ_PUBLIC_CODE — running, unexpected status" + ISSUES=$((ISSUES + 1)) + echo " kagentz C3: 🟡 public URL unexpected http=$KAGENTZ_PUBLIC_CODE (running, warning)" >> "$LOG" ;; esac fi # ── Summary ── +# The run verdict is non-optimistic: when ISSUES > 0, the run is an INCIDENT. +# The lane must quote this Result line verbatim in its status report. if [ "$ISSUES" -eq 0 ]; then - if [ "$ZULIP_CRED_OK" -eq 0 ]; then - echo " Result: ✅ All healthy" >> "$LOG" - else - echo " Result: ✅ All healthy" >> "$LOG" - fi + echo " Result: ✅ 0 issues (all healthy)" >> "$LOG" else - echo " Result: 🔴 $ISSUES issue(s) found" >> "$LOG" + echo " Result: 🔴 INCIDENT — $ISSUES issue(s) found" >> "$LOG" notify "🔴" "$ISSUES issue(s) found — check /root/zulip-health-monitor.log" fi diff --git a/tests/test_zulip_kagentz_legs.py b/tests/test_zulip_kagentz_legs.py new file mode 100644 index 0000000..91a497f --- /dev/null +++ b/tests/test_zulip_kagentz_legs.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +""" +Tests for zulip-monitor.sh kagentz A2A and public access path legs. + +Covers: +- (b) Run verdict is non-optimistic: when ISSUES > 0, the Result line says "INCIDENT" +- (d) Public access path leg: 200/302/401 = alive, 502 = incident, 000 = incident +- (c) C1/C2/C3 distinction documented in prose + +These tests parse the script and prose to verify the expected structure. +""" + +import os +import re +import subprocess +import sys +from pathlib import Path + +# Paths +SCRIPT = Path(__file__).parent.parent / "scripts" / "zulip-monitor.sh" +PROSE = Path(__file__).parent.parent / "zulip-health.prose.md" + +def read_file(path): + return path.read_text() + +def test_script_has_c1_c2_c3_legs(): + """Script should have C1, C2, C3 leg markers.""" + content = read_file(SCRIPT) + assert "C1: A2A liveness" in content, "Missing C1 leg comment" + assert "C3: Public access path" in content, "Missing C3 leg comment" + print("✓ Script has C1, C2, C3 leg markers") + +def test_c1_no_credential_needed(): + """C1 should be documented as needing no credential.""" + prose = read_file(PROSE) + assert "C1: A2A Server Health (no credential needed)" in prose, \ + "C1 header should say 'no credential needed'" + assert "INCIDENT" in prose, "C1 000 should be marked as INCIDENT" + print("✓ C1 documented as no-credential, 000 = INCIDENT") + +def test_c2_requires_litellm_key(): + """C2 should be documented as requiring LITELLM_KEY.""" + prose = read_file(PROSE) + assert "C2: A2A Response Verification (requires LITELLM_KEY)" in prose, \ + "C2 header should say 'requires LITELLM_KEY'" + assert "credential issue, NOT a server-down incident" in prose, \ + "C2 401 should be documented as credential issue, not server-down" + print("✓ C2 documented as requiring LITELLM_KEY") + +def test_c3_public_access_path(): + """C3 should probe https://kagentz.sysloggh.net/.""" + prose = read_file(PROSE) + script = read_file(SCRIPT) + assert "C3: Public Access Path" in prose, "Missing C3 section in prose" + assert "https://kagentz.sysloggh.net/" in prose, "C3 should probe the public URL" + assert "502" in prose, "C3 should document 502 as incident" + assert "KAGENTZ_PUBLIC_CODE" in script, "Script should have KAGENTZ_PUBLIC_CODE variable" + print("✓ C3 public access path leg present") + +def test_result_line_non_optimistic(): + """When ISSUES > 0, the Result line should say INCIDENT, not just 'issues found'.""" + script = read_file(SCRIPT) + # The summary section should have "INCIDENT" in the non-zero branch + assert "Result: 🔴 INCIDENT" in script, \ + "Result line should say 'INCIDENT' when ISSUES > 0" + assert "Result: ✅ 0 issues (all healthy)" in script, \ + "Result line should say '0 issues (all healthy)' when ISSUES = 0" + print("✓ Result line is non-optimistic (INCIDENT when issues > 0)") + +def test_c3_502_is_incident(): + """C3 should treat 502 as an incident.""" + script = read_file(SCRIPT) + # The script should have a branch for 502 + assert 'elif [ "$KAGENTZ_PUBLIC_CODE" = "502" ]' in script, \ + "Script should have explicit 502 branch" + assert "upstream refused" in script, \ + "502 should be documented as 'upstream refused'" + print("✓ C3 502 treated as incident") + +def test_c3_000_is_incident(): + """C3 should treat 000 as an incident.""" + script = read_file(SCRIPT) + assert 'if [ "$KAGENTZ_PUBLIC_CODE" = "000" ]' in script, \ + "Script should have explicit 000 branch" + assert "public URL DOWN" in script, \ + "000 should be reported as 'public URL DOWN'" + print("✓ C3 000 treated as incident") + +def test_c3_alive_statuses(): + """C3 should treat 200/302/401 as alive.""" + script = read_file(SCRIPT) + assert "200|302|401" in script, \ + "Script should classify 200/302/401 as alive" + assert "public URL alive" in script, \ + "Alive statuses should be reported as 'public URL alive'" + print("✓ C3 200/302/401 treated as alive") + +def test_prose_documents_c3_actions(): + """Prose should document C3 actions in the Platform C Actions table.""" + prose = read_file(PROSE) + assert "C3 public URL returns `502`" in prose, \ + "Platform C Actions table should have C3 502 row" + assert "C3 public URL returns `000`" in prose, \ + "Platform C Actions table should have C3 000 row" + print("✓ Prose documents C3 actions") + +def run_all_tests(): + tests = [ + test_script_has_c1_c2_c3_legs, + test_c1_no_credential_needed, + test_c2_requires_litellm_key, + test_c3_public_access_path, + test_result_line_non_optimistic, + test_c3_502_is_incident, + test_c3_000_is_incident, + test_c3_alive_statuses, + test_prose_documents_c3_actions, + ] + passed = 0 + failed = 0 + for test in tests: + try: + test() + passed += 1 + except AssertionError as e: + print(f"✗ {test.__name__}: {e}") + failed += 1 + print(f"\n{'='*50}") + print(f"Tests passed: {passed}/{len(tests)}") + if failed > 0: + print(f"Tests failed: {failed}/{len(tests)}") + sys.exit(1) + else: + print("All tests passed!") + +if __name__ == "__main__": + run_all_tests() diff --git a/zulip-health.prose.md b/zulip-health.prose.md index 54229f8..097e2ec 100644 --- a/zulip-health.prose.md +++ b/zulip-health.prose.md @@ -471,7 +471,7 @@ ssh root@192.168.68.15 "pct exec 112 -- curl -s -b /tmp/dsh-new.jar -o /dev/null > heartbeat/queue, or adapter-restart step. Agent Zero is probed for A2A > liveness only, and a probe must never restart a platform. -**C1: A2A Server Health** +**C1: A2A Server Health (no credential needed)** ```bash # A2A listens on :80 inside the agent-zero container (host-mapped to :50080) and @@ -479,9 +479,9 @@ ssh root@192.168.68.15 "pct exec 112 -- curl -s -b /tmp/dsh-new.jar -o /dev/null ssh root@192.168.68.14 "docker exec agent-zero curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:80/a2a/" ``` -Expected: `401` (auth-gated, A2A server is up and responding) or `200` (if no auth required). Connection refused (`000`) → A2A server down. Any other status → running but unexpected: log/report it, never restart. +Expected: `401` (auth-gated, A2A server is up and responding) or `200` (if no auth required). Connection refused (`000`) → A2A server down (INCIDENT). Any other status → running but unexpected: log/report it, never restart. -**C2: A2A Response Verification** +**C2: A2A Response Verification (requires LITELLM_KEY)** ```bash # A2A listens on :80 inside the container and is auth-gated (401 expected unauthenticated). @@ -491,15 +491,30 @@ ssh root@192.168.68.14 "docker exec agent-zero curl -s -X POST http://127.0.0.1: -d '{\"jsonrpc\":\"2.0\",\"method\":\"tasks/send\",\"params\":{\"message\":{\"role\":\"user\",\"parts\":[{\"text\":\"ping\"}]}},\"id\":1}'" ``` -Expected: task ID with "working" status. Poll for completion with `tasks/get`. If 401, check LITELLM_KEY is set. +Expected: task ID with "working" status. Poll for completion with `tasks/get`. If 401, check LITELLM_KEY is set (this is a credential issue, NOT a server-down incident). + +**C3: Public Access Path (no credential needed)** + +```bash +# Probes the public URL that NetBird proxies to the agent-zero container. +# This is the captain's point of view: if the captain can't reach it, it's down. +# 200/302/401 = alive, 502 = proxy's "upstream refused" page (INCIDENT), +# connection failed (000) = INCIDENT. Never restarts anything. +curl -s -o /dev/null --connect-timeout 10 --max-time 15 -w '%{http_code}' https://kagentz.sysloggh.net/ +``` + +Expected: `200` (Agent Zero login page), `302` (redirect), or `401` (auth-gated) = alive. `502` = NetBird proxy's "upstream refused" page (the container's port 80 is not listening) = INCIDENT. `000` (connection failed) = INCIDENT. Any other status → running but unexpected: log/report it, never restart. **Platform C Actions** | Condition | Action | |-----------|--------| -| A2A returns `000` (connection refused/timeout) | Alert only — never restart the platform; investigate the agent-zero container | -| A2A returns a status other than `200`/`401` | Log/report as a warning — reported, never healed on | -| LiteLLM 401 | Check API key in a2a_agent.py `LITELLM_KEY` | +| C1 A2A returns `000` (connection refused/timeout) | Alert only — never restart the platform; investigate the agent-zero container | +| C1 A2A returns a status other than `200`/`401` | Log/report as a warning — reported, never healed on | +| C2 A2A returns 401 | Check `LITELLM_KEY` is set (credential issue, NOT server-down) | +| C3 public URL returns `502` (upstream refused) | Alert only — the container's port 80 is not listening; investigate the agent-zero container | +| C3 public URL returns `000` (connection failed) | Alert only — the public path is down; investigate the NetBird proxy or the container | +| C3 public URL returns a status other than `200`/`302`/`401` | Log/report as a warning — reported, never healed on | ### Step 5: Global Checks