fix(zulip-health): retire the dead kagentz Zulip adapter leg #78

Merged
abiba-bot merged 4 commits from fix/zulip-kagentz-adapter-20260912 into master 2026-09-12 14:34:09 +00:00
5 changed files with 75 additions and 61 deletions
+1 -1
View File
@@ -628,7 +628,7 @@ contracts:
sensitivity: high
status: active
owner: abiba
version: 3.2.0
version: 3.3.0
trigger:
type: scheduled
cadence: '*/15 * * * *'
+19 -21
View File
@@ -41,7 +41,9 @@ notify() {
# ── Global: Zulip Server ──
SERVER_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 \
https://chat.sysloggh.net/api/v1/server_settings \
-u 'abiba-bot@chat.sysloggh.net:cKTDMZAPW08dk3zl05sStzO7HRztzyn8' 2>/dev/null || echo "000")
-u 'abiba-bot@chat.sysloggh.net:cKTDMZAPW08dk3zl05sStzO7HRztzyn8' 2>/dev/null) || SERVER_CODE="000"
SERVER_CODE=$(printf '%s' "$SERVER_CODE" | tr -d '[:space:]')
[ -n "$SERVER_CODE" ] || SERVER_CODE="000"
if [ "$SERVER_CODE" != "200" ]; then
notify "🔴" "Zulip server returned HTTP $SERVER_CODE"
ISSUES=$((ISSUES + 1))
@@ -59,7 +61,9 @@ fi
# zulip.connected is a PROBE FAILURE: it alerts and NEVER calls pm2 restart.
# pm2 restart runs ONLY on affirmative zulip.connected=false.
# -- abiba-leg-start (verbatim-extracted by tests/zulip-monitor-abiba.sh)
PI_HTTP=$(curl -s -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' http://localhost:9200/health 2>/dev/null || echo "000")
PI_HTTP=$(curl -s -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' http://localhost:9200/health 2>/dev/null) || PI_HTTP="000"
PI_HTTP=$(printf '%s' "$PI_HTTP" | tr -d '[:space:]')
[ -n "$PI_HTTP" ] || PI_HTTP="000"
PI_BODY=$(curl -s --connect-timeout 5 --max-time 10 http://localhost:9200/health 2>/dev/null || true)
PI_STATE=$(printf '%s' "$PI_BODY" | python3 -c '
import sys, json
@@ -156,29 +160,23 @@ fi
# ── Platform C: Agent Zero (kagentz) ──
AZ_A2A_CODE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \
"docker exec agent-zero curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:80/a2a/ 2>/dev/null" 2>/dev/null || echo "000")
"docker exec agent-zero curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:80/a2a/ 2>/dev/null" 2>/dev/null) || AZ_A2A_CODE="000"
AZ_A2A_CODE=$(printf '%s' "$AZ_A2A_CODE" | tr -d '[:space:]')
[ -n "$AZ_A2A_CODE" ] || AZ_A2A_CODE="000"
if [ "$AZ_A2A_CODE" = "000" ]; then
notify "🔴" "kagentz A2A server DOWN (connection failed) — restarting"
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \
"docker exec agent-zero bash -c 'pkill -9 -f a2a_agent; sleep 1; cd /a0 && /opt/venv-a0/bin/python3 -u /a0/usr/a2a_agent.py > /tmp/a2a.log 2>&1 &'" 2>/dev/null || true
notify "🔴" "kagentz A2A server DOWN (connection failed)"
ISSUES=$((ISSUES + 1))
echo " kagentz: ❌ A2A down — restarted" >> "$LOG"
echo " kagentz: ❌ A2A down (HTTP 000)" >> "$LOG"
else
echo " kagentz: ✅ A2A alive" >> "$LOG"
# Check adapter process
AZ_ADAPTER=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \
"docker exec agent-zero ps aux 2>/dev/null | grep adapter | grep -v grep | wc -l" 2>/dev/null || echo "0")
if [ "$AZ_ADAPTER" -lt 1 ]; then
notify "🔴" "kagentz Zulip adapter DOWN — restarting"
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \
"docker exec agent-zero bash -c 'cd /a0/usr/kagentz-zulip && ZULIP_SITE=https://chat.sysloggh.net ZULIP_EMAIL=kagentz-bot@chat.sysloggh.net ZULIP_API_KEY=E9q9PXJTxftPYBkb5pBDWupDO7KK21ty ZULIP_AGENT_NAME=kagentz A2A_URL=http://localhost:80/a2a A2A_TOKEN=8zNgdOEXzYxjQvTl /opt/venv-a0/bin/python3 -u adapter.py > /tmp/zulip-adapter.log 2>&1 &'" 2>/dev/null || true
ISSUES=$((ISSUES + 1))
echo " kagentz: ❌ Adapter down — restarted" >> "$LOG"
else
echo " kagentz: ✅ Adapter running" >> "$LOG"
fi
case "$AZ_A2A_CODE" in
200|401)
echo " kagentz: ✅ A2A alive (HTTP $AZ_A2A_CODE)" >> "$LOG" ;;
*)
notify "🟡" "kagentz A2A server answered HTTP $AZ_A2A_CODE — running, unexpected status"
ISSUES=$((ISSUES + 1))
echo " kagentz: 🟡 A2A unexpected http=$AZ_A2A_CODE (running, warning)" >> "$LOG" ;;
esac
fi
# ── Summary ──
+33 -9
View File
@@ -89,8 +89,7 @@ case "$host" in
esac ;;
192.168.68.14)
case "$cmd" in
*agent.json*) printf '%s' "$AZ_A2A" ;;
*"ps aux"*) printf '%s\n' "$AZ_PS" ;;
*"/a2a/"*) printf '%s' "$AZ_A2A_CODE"; exit "$AZ_A2A_EXIT" ;;
esac ;;
*)
printf 'UNEXPECTED-SSH-HOST %s\n' "$host" >> "$RECORD_DIR/unexpected-ssh" ;;
@@ -122,8 +121,7 @@ def _write_exec(path: pathlib.Path, body: str) -> None:
def _run_monitor(tmp_path, *, tanko_svc="active", tanko_http="200",
az_a2a='{"name":"kagentz"}',
az_ps="root 111 0.1 0.2 /opt/venv-a0/bin/python3 -u adapter.py"):
az_a2a_code="401", az_a2a_exit=0):
"""Run the shipped monitor in a sandbox; return (proc, record_dir, log_path).
Only the LOG constant is rewritten (to keep the run inside the worktree).
@@ -151,8 +149,8 @@ def _run_monitor(tmp_path, *, tanko_svc="active", tanko_http="200",
"RECORD_DIR": str(record),
"TANKO_SVC": tanko_svc,
"TANKO_HTTP": tanko_http,
"AZ_A2A": az_a2a,
"AZ_PS": az_ps,
"AZ_A2A_CODE": az_a2a_code,
"AZ_A2A_EXIT": str(az_a2a_exit),
"PI_HTTP": "200",
"PI_BODY": CONNECTED_FIXTURE.read_text(),
"SERVER_HTTP": "200",
@@ -171,8 +169,7 @@ def test_healthy_run_is_quiet_and_never_reaches_mumuni(tmp_path):
assert "Server: ✅ HTTP 200" in log
assert "Abiba: ✅ Connected" in log
assert "Tanko: ✅ service=active http=200" in log
assert "kagentz: ✅ A2A alive" in log
assert "kagentz: ✅ Adapter running" in log
assert "kagentz: ✅ A2A alive (HTTP 401)" in log
assert "Result: ✅ All healthy" in log
# A healthy run emits no notify at all — and certainly no Mumuni one.
@@ -214,6 +211,32 @@ def test_failing_run_alerts_on_tanko_but_never_on_mumuni(tmp_path):
assert "Result: 🔴 1 issue(s) found" in log
def test_unexpected_a2a_status_is_an_issue_not_healthy(tmp_path):
proc, record, log_path = _run_monitor(tmp_path, az_a2a_code="500")
assert proc.returncode == 0, proc.stderr
log = log_path.read_text()
assert "kagentz: 🟡 A2A unexpected http=500 (running, warning)" in log
assert "kagentz: ✅ A2A alive" not in log
assert "Result: 🔴 1 issue(s) found" in log
assert "kagentz A2A server answered HTTP 500" in proc.stdout
def test_a2a_connection_failure_is_down_not_unexpected(tmp_path):
# curl prints the http_code before failing, so the ssh stub exits non-zero
# with "000" on stdout — exercising the real outage path.
proc, record, log_path = _run_monitor(tmp_path, az_a2a_code="000",
az_a2a_exit=7)
assert proc.returncode == 0, proc.stderr
log = log_path.read_text()
assert "kagentz: ❌ A2A down (HTTP 000)" in log
assert "kagentz: ✅ A2A alive" not in log
assert "unexpected" not in log
assert "Result: 🔴 1 issue(s) found" in log
assert "kagentz A2A server DOWN (connection failed)" in proc.stdout
# ── scripts/daily-infra-report.py: behavioral digest checks ──────────
@pytest.fixture(scope="module")
@@ -334,7 +357,8 @@ def test_agent_health_roster_has_no_mumuni_entry(ahc):
def test_health_contract_retires_mumuni_only_steps():
text = HEALTH_CONTRACT.read_text()
assert MUMUNI_IP not in text
for step in ("**B4:", "**B5:", "**B6:"):
for step in ("**B4: Gateway Process**", "**B5: Heartbeat Verification**",
"**B6: Response Delivery**"):
assert step not in text
+19 -28
View File
@@ -1,9 +1,9 @@
---
kind: responsibility
name: zulip-health
description: Multi-platform health monitor for the Zulip messaging mesh spanning Platform A (pi/Abiba Zulip bridge), Platform B (Tanko on DSH), and Platform C (Agent Zero Docker). Verifies bot registration, DM delivery, and cross-platform connectivity. Mumuni is no longer monitored from this host — she runs on her own container (kagentz CT 105 on minipve, .14) and is monitored on her side.
description: Multi-platform health monitor for the Zulip messaging mesh spanning Platform A (pi/Abiba Zulip bridge), Platform B (Tanko on DSH), and Platform C (Agent Zero Docker). Verifies bot registration, DM delivery, and cross-platform connectivity. The kagentz Zulip adapter leg is retired (its code no longer exists) — Agent Zero is probed for A2A liveness only. Mumuni is no longer monitored from this host — she runs on her own container (kagentz CT 105 on minipve, .14) and is monitored on her side.
title: Zulip Mesh Health Monitor — Multi-Platform
version: 3.2.0
version: 3.3.0
runtime_contract: 2
agent: abiba
report_only_agents:
@@ -435,36 +435,29 @@ ssh root@192.168.68.15 "pct exec 112 -- curl -s -b /tmp/dsh-new.jar -o /dev/null
### Step 4: Platform C — Agent Zero (kagentz, CT 105 via Docker host .14)
> **The kagentz Zulip adapter leg is retired (2026-09-12).** Its code
> (`/a0/usr/kagentz-zulip/`) no longer exists in the agent-zero container, so
> the former adapter-process and heartbeat/queue checks always failed and the
> monitor issued a restart for something that could not start, posting a false
> kagentz-adapter-down alert on every run. Do NOT re-add an adapter-process,
> heartbeat/queue, or adapter-restart step. Agent Zero is probed for A2A
> liveness only, and a probe must never restart a platform.
**C1: A2A Server Health**
```bash
# A2A server is on :50080 (not :8001) and is auth-gated (401 expected for unauthenticated)
ssh root@192.168.68.14 "curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:50080/a2a/"
# A2A listens on :80 inside the agent-zero container (host-mapped to :50080) and
# is auth-gated: an unauthenticated probe gets 401, which means the server is up.
ssh root@192.168.68.14 "docker exec agent-zero curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:80/a2a/"
```
Expected: `401` (auth-gated, A2A server is up and responding) or `200` (if no auth required). Connection refused (000) → A2A server down.
Expected: `401` (auth-gated, A2A server is up and responding) or `200` (if no auth required). Connection refused (`000`) → A2A server down. Any other status → running but unexpected: log/report it, never restart.
**C2: Adapter Process**
**C2: A2A Response Verification**
```bash
ssh root@192.168.68.14 "docker exec agent-zero ps aux | grep adapter | grep -v grep"
```
Adapter should be running. Missing → restart inside container.
**C3: Heartbeat & Queue**
```bash
ssh root@192.168.68.14 "docker exec agent-zero grep Heartbeat /tmp/zulip-adapter.log | tail -3"
```
Check: `processed=N` incrementing, `silence < 600s`, `reconnects` ≈ 0.
**C4: A2A Response Verification**
```bash
# A2A server is on :50080 (not :8001) and is auth-gated (401 expected for unauthenticated)
ssh root@192.168.68.14 "curl -s -X POST http://127.0.0.1:50080/a2a \
# A2A listens on :80 inside the container and is auth-gated (401 expected unauthenticated).
ssh root@192.168.68.14 "docker exec agent-zero curl -s -X POST http://127.0.0.1:80/a2a \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer $LITELLM_KEY' \
-d '{\"jsonrpc\":\"2.0\",\"method\":\"tasks/send\",\"params\":{\"message\":{\"role\":\"user\",\"parts\":[{\"text\":\"ping\"}]}},\"id\":1}'"
@@ -476,9 +469,8 @@ Expected: task ID with "working" status. Poll for completion with `tasks/get`. I
| Condition | Action |
|-----------|--------|
| A2A `.well-known/agent.json` fails | `docker exec agent-zero bash -c "pkill -9 -f a2a_agent; cd /a0 && /opt/venv-a0/bin/python3 -u /a0/usr/a2a_agent.py > /tmp/a2a.log 2>&1 &"` |
| Adapter process missing | Restart adapter inside container with env vars |
| Silence > 600s | Restart adapter (auto-reconnect handles BAD_EVENT_QUEUE_ID) |
| A2A returns `000` (connection refused/timeout) | Alert only — never restart the platform; investigate the agent-zero container |
| A2A returns a status other than `200`/`401` | Log/report as a warning — reported, never healed on |
| LiteLLM 401 | Check API key in a2a_agent.py `LITELLM_KEY` |
### Step 5: Global Checks
@@ -488,7 +480,6 @@ Expected: task ID with "working" status. Poll for completion with `tasks/get`. I
Check each agent's log for excessive bot-to-bot chatter:
- Abiba: `Skipped.*bot msgs` count
- Tanko: Repeated DM exchanges between bots
- kagentz: Adapter log for bot DMs being processed
If any bot processes >50 bot-originated messages in 15min → warning.
+3 -2
View File
@@ -10,8 +10,9 @@ description: >
> **⚠️ RETIRED** — The pi Zulip extension (`~/.pi/agent/extensions/zulip/`) and
> PM2 process (`abiba-zulip`) have been decommissioned. All mention/reliability
> monitoring now happens through Telegram. Agents (Mumuni on Hermes, Tanko on DSH) and
> Agent Zero (kagentz) continue to use Zulip.
> monitoring now happens through Telegram. Mumuni (Hermes) and Tanko (DSH)
> continue to use Zulip; Agent Zero's Zulip adapter is retired — see
> `zulip-health.prose.md` for current Platform C (Agent Zero) state.
## Maintains