diff --git a/contract-registry.yaml b/contract-registry.yaml index 7135893..93f398a 100644 --- a/contract-registry.yaml +++ b/contract-registry.yaml @@ -628,7 +628,7 @@ contracts: sensitivity: high status: active owner: abiba - version: 3.2.0 + version: 3.3.0 trigger: type: scheduled cadence: '*/15 * * * *' diff --git a/scripts/zulip-monitor.sh b/scripts/zulip-monitor.sh index a6fd6f2..45838db 100755 --- a/scripts/zulip-monitor.sh +++ b/scripts/zulip-monitor.sh @@ -41,7 +41,9 @@ notify() { # ── Global: Zulip Server ── SERVER_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 \ https://chat.sysloggh.net/api/v1/server_settings \ - -u 'abiba-bot@chat.sysloggh.net:cKTDMZAPW08dk3zl05sStzO7HRztzyn8' 2>/dev/null || echo "000") + -u 'abiba-bot@chat.sysloggh.net:cKTDMZAPW08dk3zl05sStzO7HRztzyn8' 2>/dev/null) || SERVER_CODE="000" +SERVER_CODE=$(printf '%s' "$SERVER_CODE" | tr -d '[:space:]') +[ -n "$SERVER_CODE" ] || SERVER_CODE="000" if [ "$SERVER_CODE" != "200" ]; then notify "🔴" "Zulip server returned HTTP $SERVER_CODE" ISSUES=$((ISSUES + 1)) @@ -59,7 +61,9 @@ fi # zulip.connected is a PROBE FAILURE: it alerts and NEVER calls pm2 restart. # pm2 restart runs ONLY on affirmative zulip.connected=false. # -- abiba-leg-start (verbatim-extracted by tests/zulip-monitor-abiba.sh) -PI_HTTP=$(curl -s -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' http://localhost:9200/health 2>/dev/null || echo "000") +PI_HTTP=$(curl -s -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' http://localhost:9200/health 2>/dev/null) || PI_HTTP="000" +PI_HTTP=$(printf '%s' "$PI_HTTP" | tr -d '[:space:]') +[ -n "$PI_HTTP" ] || PI_HTTP="000" PI_BODY=$(curl -s --connect-timeout 5 --max-time 10 http://localhost:9200/health 2>/dev/null || true) PI_STATE=$(printf '%s' "$PI_BODY" | python3 -c ' import sys, json @@ -156,29 +160,23 @@ fi # ── Platform C: Agent Zero (kagentz) ── AZ_A2A_CODE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ - "docker exec agent-zero curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:80/a2a/ 2>/dev/null" 2>/dev/null || echo "000") + "docker exec agent-zero curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:80/a2a/ 2>/dev/null" 2>/dev/null) || AZ_A2A_CODE="000" +AZ_A2A_CODE=$(printf '%s' "$AZ_A2A_CODE" | tr -d '[:space:]') +[ -n "$AZ_A2A_CODE" ] || AZ_A2A_CODE="000" if [ "$AZ_A2A_CODE" = "000" ]; then - notify "🔴" "kagentz A2A server DOWN (connection failed) — restarting" - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ - "docker exec agent-zero bash -c 'pkill -9 -f a2a_agent; sleep 1; cd /a0 && /opt/venv-a0/bin/python3 -u /a0/usr/a2a_agent.py > /tmp/a2a.log 2>&1 &'" 2>/dev/null || true + notify "🔴" "kagentz A2A server DOWN (connection failed)" ISSUES=$((ISSUES + 1)) - echo " kagentz: ❌ A2A down — restarted" >> "$LOG" + echo " kagentz: ❌ A2A down (HTTP 000)" >> "$LOG" else - echo " kagentz: ✅ A2A alive" >> "$LOG" - - # Check adapter process - AZ_ADAPTER=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ - "docker exec agent-zero ps aux 2>/dev/null | grep adapter | grep -v grep | wc -l" 2>/dev/null || echo "0") - if [ "$AZ_ADAPTER" -lt 1 ]; then - notify "🔴" "kagentz Zulip adapter DOWN — restarting" - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ - "docker exec agent-zero bash -c 'cd /a0/usr/kagentz-zulip && ZULIP_SITE=https://chat.sysloggh.net ZULIP_EMAIL=kagentz-bot@chat.sysloggh.net ZULIP_API_KEY=E9q9PXJTxftPYBkb5pBDWupDO7KK21ty ZULIP_AGENT_NAME=kagentz A2A_URL=http://localhost:80/a2a A2A_TOKEN=8zNgdOEXzYxjQvTl /opt/venv-a0/bin/python3 -u adapter.py > /tmp/zulip-adapter.log 2>&1 &'" 2>/dev/null || true - ISSUES=$((ISSUES + 1)) - echo " kagentz: ❌ Adapter down — restarted" >> "$LOG" - else - echo " kagentz: ✅ Adapter running" >> "$LOG" - fi + case "$AZ_A2A_CODE" in + 200|401) + echo " kagentz: ✅ A2A alive (HTTP $AZ_A2A_CODE)" >> "$LOG" ;; + *) + notify "🟡" "kagentz A2A server answered HTTP $AZ_A2A_CODE — running, unexpected status" + ISSUES=$((ISSUES + 1)) + echo " kagentz: 🟡 A2A unexpected http=$AZ_A2A_CODE (running, warning)" >> "$LOG" ;; + esac fi # ── Summary ── diff --git a/tests/test_mumuni_monitor_removal.py b/tests/test_mumuni_monitor_removal.py index 5f9b9eb..df55352 100644 --- a/tests/test_mumuni_monitor_removal.py +++ b/tests/test_mumuni_monitor_removal.py @@ -89,8 +89,7 @@ case "$host" in esac ;; 192.168.68.14) case "$cmd" in - *agent.json*) printf '%s' "$AZ_A2A" ;; - *"ps aux"*) printf '%s\n' "$AZ_PS" ;; + *"/a2a/"*) printf '%s' "$AZ_A2A_CODE"; exit "$AZ_A2A_EXIT" ;; esac ;; *) printf 'UNEXPECTED-SSH-HOST %s\n' "$host" >> "$RECORD_DIR/unexpected-ssh" ;; @@ -122,8 +121,7 @@ def _write_exec(path: pathlib.Path, body: str) -> None: def _run_monitor(tmp_path, *, tanko_svc="active", tanko_http="200", - az_a2a='{"name":"kagentz"}', - az_ps="root 111 0.1 0.2 /opt/venv-a0/bin/python3 -u adapter.py"): + az_a2a_code="401", az_a2a_exit=0): """Run the shipped monitor in a sandbox; return (proc, record_dir, log_path). Only the LOG constant is rewritten (to keep the run inside the worktree). @@ -151,8 +149,8 @@ def _run_monitor(tmp_path, *, tanko_svc="active", tanko_http="200", "RECORD_DIR": str(record), "TANKO_SVC": tanko_svc, "TANKO_HTTP": tanko_http, - "AZ_A2A": az_a2a, - "AZ_PS": az_ps, + "AZ_A2A_CODE": az_a2a_code, + "AZ_A2A_EXIT": str(az_a2a_exit), "PI_HTTP": "200", "PI_BODY": CONNECTED_FIXTURE.read_text(), "SERVER_HTTP": "200", @@ -171,8 +169,7 @@ def test_healthy_run_is_quiet_and_never_reaches_mumuni(tmp_path): assert "Server: ✅ HTTP 200" in log assert "Abiba: ✅ Connected" in log assert "Tanko: ✅ service=active http=200" in log - assert "kagentz: ✅ A2A alive" in log - assert "kagentz: ✅ Adapter running" in log + assert "kagentz: ✅ A2A alive (HTTP 401)" in log assert "Result: ✅ All healthy" in log # A healthy run emits no notify at all — and certainly no Mumuni one. @@ -214,6 +211,32 @@ def test_failing_run_alerts_on_tanko_but_never_on_mumuni(tmp_path): assert "Result: 🔴 1 issue(s) found" in log +def test_unexpected_a2a_status_is_an_issue_not_healthy(tmp_path): + proc, record, log_path = _run_monitor(tmp_path, az_a2a_code="500") + assert proc.returncode == 0, proc.stderr + log = log_path.read_text() + + assert "kagentz: 🟡 A2A unexpected http=500 (running, warning)" in log + assert "kagentz: ✅ A2A alive" not in log + assert "Result: 🔴 1 issue(s) found" in log + assert "kagentz A2A server answered HTTP 500" in proc.stdout + + +def test_a2a_connection_failure_is_down_not_unexpected(tmp_path): + # curl prints the http_code before failing, so the ssh stub exits non-zero + # with "000" on stdout — exercising the real outage path. + proc, record, log_path = _run_monitor(tmp_path, az_a2a_code="000", + az_a2a_exit=7) + assert proc.returncode == 0, proc.stderr + log = log_path.read_text() + + assert "kagentz: ❌ A2A down (HTTP 000)" in log + assert "kagentz: ✅ A2A alive" not in log + assert "unexpected" not in log + assert "Result: 🔴 1 issue(s) found" in log + assert "kagentz A2A server DOWN (connection failed)" in proc.stdout + + # ── scripts/daily-infra-report.py: behavioral digest checks ────────── @pytest.fixture(scope="module") @@ -334,7 +357,8 @@ def test_agent_health_roster_has_no_mumuni_entry(ahc): def test_health_contract_retires_mumuni_only_steps(): text = HEALTH_CONTRACT.read_text() assert MUMUNI_IP not in text - for step in ("**B4:", "**B5:", "**B6:"): + for step in ("**B4: Gateway Process**", "**B5: Heartbeat Verification**", + "**B6: Response Delivery**"): assert step not in text diff --git a/zulip-health.prose.md b/zulip-health.prose.md index 4e9cfb6..46ca1e4 100644 --- a/zulip-health.prose.md +++ b/zulip-health.prose.md @@ -1,9 +1,9 @@ --- kind: responsibility name: zulip-health -description: Multi-platform health monitor for the Zulip messaging mesh spanning Platform A (pi/Abiba Zulip bridge), Platform B (Tanko on DSH), and Platform C (Agent Zero Docker). Verifies bot registration, DM delivery, and cross-platform connectivity. Mumuni is no longer monitored from this host — she runs on her own container (kagentz CT 105 on minipve, .14) and is monitored on her side. +description: Multi-platform health monitor for the Zulip messaging mesh spanning Platform A (pi/Abiba Zulip bridge), Platform B (Tanko on DSH), and Platform C (Agent Zero Docker). Verifies bot registration, DM delivery, and cross-platform connectivity. The kagentz Zulip adapter leg is retired (its code no longer exists) — Agent Zero is probed for A2A liveness only. Mumuni is no longer monitored from this host — she runs on her own container (kagentz CT 105 on minipve, .14) and is monitored on her side. title: Zulip Mesh Health Monitor — Multi-Platform -version: 3.2.0 +version: 3.3.0 runtime_contract: 2 agent: abiba report_only_agents: @@ -435,36 +435,29 @@ ssh root@192.168.68.15 "pct exec 112 -- curl -s -b /tmp/dsh-new.jar -o /dev/null ### Step 4: Platform C — Agent Zero (kagentz, CT 105 via Docker host .14) +> **The kagentz Zulip adapter leg is retired (2026-09-12).** Its code +> (`/a0/usr/kagentz-zulip/`) no longer exists in the agent-zero container, so +> the former adapter-process and heartbeat/queue checks always failed and the +> monitor issued a restart for something that could not start, posting a false +> kagentz-adapter-down alert on every run. Do NOT re-add an adapter-process, +> heartbeat/queue, or adapter-restart step. Agent Zero is probed for A2A +> liveness only, and a probe must never restart a platform. + **C1: A2A Server Health** ```bash -# A2A server is on :50080 (not :8001) and is auth-gated (401 expected for unauthenticated) -ssh root@192.168.68.14 "curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:50080/a2a/" +# A2A listens on :80 inside the agent-zero container (host-mapped to :50080) and +# is auth-gated: an unauthenticated probe gets 401, which means the server is up. +ssh root@192.168.68.14 "docker exec agent-zero curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:80/a2a/" ``` -Expected: `401` (auth-gated, A2A server is up and responding) or `200` (if no auth required). Connection refused (000) → A2A server down. +Expected: `401` (auth-gated, A2A server is up and responding) or `200` (if no auth required). Connection refused (`000`) → A2A server down. Any other status → running but unexpected: log/report it, never restart. -**C2: Adapter Process** +**C2: A2A Response Verification** ```bash -ssh root@192.168.68.14 "docker exec agent-zero ps aux | grep adapter | grep -v grep" -``` - -Adapter should be running. Missing → restart inside container. - -**C3: Heartbeat & Queue** - -```bash -ssh root@192.168.68.14 "docker exec agent-zero grep Heartbeat /tmp/zulip-adapter.log | tail -3" -``` - -Check: `processed=N` incrementing, `silence < 600s`, `reconnects` ≈ 0. - -**C4: A2A Response Verification** - -```bash -# A2A server is on :50080 (not :8001) and is auth-gated (401 expected for unauthenticated) -ssh root@192.168.68.14 "curl -s -X POST http://127.0.0.1:50080/a2a \ +# A2A listens on :80 inside the container and is auth-gated (401 expected unauthenticated). +ssh root@192.168.68.14 "docker exec agent-zero curl -s -X POST http://127.0.0.1:80/a2a \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer $LITELLM_KEY' \ -d '{\"jsonrpc\":\"2.0\",\"method\":\"tasks/send\",\"params\":{\"message\":{\"role\":\"user\",\"parts\":[{\"text\":\"ping\"}]}},\"id\":1}'" @@ -476,9 +469,8 @@ Expected: task ID with "working" status. Poll for completion with `tasks/get`. I | Condition | Action | |-----------|--------| -| A2A `.well-known/agent.json` fails | `docker exec agent-zero bash -c "pkill -9 -f a2a_agent; cd /a0 && /opt/venv-a0/bin/python3 -u /a0/usr/a2a_agent.py > /tmp/a2a.log 2>&1 &"` | -| Adapter process missing | Restart adapter inside container with env vars | -| Silence > 600s | Restart adapter (auto-reconnect handles BAD_EVENT_QUEUE_ID) | +| A2A returns `000` (connection refused/timeout) | Alert only — never restart the platform; investigate the agent-zero container | +| A2A returns a status other than `200`/`401` | Log/report as a warning — reported, never healed on | | LiteLLM 401 | Check API key in a2a_agent.py `LITELLM_KEY` | ### Step 5: Global Checks @@ -488,7 +480,6 @@ Expected: task ID with "working" status. Poll for completion with `tasks/get`. I Check each agent's log for excessive bot-to-bot chatter: - Abiba: `Skipped.*bot msgs` count - Tanko: Repeated DM exchanges between bots -- kagentz: Adapter log for bot DMs being processed If any bot processes >50 bot-originated messages in 15min → warning. diff --git a/zulip-mention-reliability.prose.md b/zulip-mention-reliability.prose.md index 2e0a490..21e0f2a 100644 --- a/zulip-mention-reliability.prose.md +++ b/zulip-mention-reliability.prose.md @@ -10,8 +10,9 @@ description: > > **⚠️ RETIRED** — The pi Zulip extension (`~/.pi/agent/extensions/zulip/`) and > PM2 process (`abiba-zulip`) have been decommissioned. All mention/reliability -> monitoring now happens through Telegram. Agents (Mumuni on Hermes, Tanko on DSH) and -> Agent Zero (kagentz) continue to use Zulip. +> monitoring now happens through Telegram. Mumuni (Hermes) and Tanko (DSH) +> continue to use Zulip; Agent Zero's Zulip adapter is retired — see +> `zulip-health.prose.md` for current Platform C (Agent Zero) state. ## Maintains