diff --git a/scripts/zulip-monitor.sh b/scripts/zulip-monitor.sh index ab4d4cd..5596c5c 100755 --- a/scripts/zulip-monitor.sh +++ b/scripts/zulip-monitor.sh @@ -66,22 +66,33 @@ else echo " Abiba: ✅ Connected (processed=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('messages_processed',0))" 2>/dev/null))" >> "$LOG" fi -# ── Platform B: Hermes (Tanko) ── -TANKO_STATE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 jerome@192.168.68.122 \ - "cat ~/.hermes/gateway_state.json 2>/dev/null" 2>/dev/null || echo "{}") -TANKO_ZULIP=$(echo "$TANKO_STATE" | python3 -c " -import sys,json -d=json.load(sys.stdin) -p=d.get('platforms',{}).get('zulip',{}) -print(p.get('state','unknown')) -" 2>/dev/null) +# ── Platform B: Tanko (DSH dsh-web on amdpve CT 112) ── +# Direct SSH to 192.168.68.122 is retired after the agent migrations. Tanko's +# Zulip gateway runs as the dsh-web systemd unit inside CT 112 on amdpve +# (192.168.68.15), reachable only via `pct exec`. The gateway binds +# 127.0.0.1:3080 loopback-only by design — a remote :3080 probe is refused and +# is NOT a fault. +TANKO_SVC=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.15 \ + "pct exec 112 -- systemctl is-active dsh-web" 2>/dev/null || echo "unknown") +TANKO_HTTP=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.15 \ + "pct exec 112 -- curl -s --connect-timeout 5 --max-time 10 -o /dev/null -w '%{http_code}' http://127.0.0.1:3080/" 2>/dev/null || echo "000") -if [ "$TANKO_ZULIP" != "connected" ]; then - notify "🔴" "Tanko (Hermes) Zulip state: $TANKO_ZULIP — needs restart" +if [ "$TANKO_SVC" != "active" ]; then + notify "🔴" "Tanko (DSH dsh-web) service state: $TANKO_SVC — needs restart" ISSUES=$((ISSUES + 1)) - echo " Tanko: ❌ state=$TANKO_ZULIP" >> "$LOG" + echo " Tanko: ❌ service=$TANKO_SVC" >> "$LOG" +elif [ "$TANKO_HTTP" = "000" ]; then + notify "🔴" "Tanko (DSH dsh-web) HTTP :3080 connection refused/timeout — needs restart" + ISSUES=$((ISSUES + 1)) + echo " Tanko: ❌ http=000 (refused/timeout)" >> "$LOG" else - echo " Tanko: ✅ Zulip connected" >> "$LOG" + case "$TANKO_HTTP" in + 200|301|302|307|308|401|403) + echo " Tanko: ✅ service=active http=$TANKO_HTTP" >> "$LOG" ;; + *) + notify "🟡" "Tanko (DSH dsh-web) HTTP :3080 answered $TANKO_HTTP — running, unexpected status" + echo " Tanko: 🟡 service=active http=$TANKO_HTTP (running, warning)" >> "$LOG" ;; + esac fi # ── Platform B: Hermes (Mumuni) ── diff --git a/zulip-health.prose.md b/zulip-health.prose.md index 883c2f5..ead8402 100644 --- a/zulip-health.prose.md +++ b/zulip-health.prose.md @@ -216,23 +216,31 @@ Expected: `active`. Anything else → gateway service down → apply the Tanko h **B2: Gateway HTTP Liveness (Tanko — loopback-only :3080)** ```bash -ssh root@192.168.68.15 "pct exec 112 -- curl -s --connect-timeout 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:3080/" +ssh root@192.168.68.15 "pct exec 112 -- curl -s --connect-timeout 5 --max-time 10 -o /dev/null -w '%{http_code}' http://127.0.0.1:3080/" ``` -Alive = **ANY** HTTP status of `200`/`301`/`302`/`307`/`308`/`401`/`403` — the -gateway UI is token-gated and legitimately answers with redirects/auth-challenges, -so never require a bare `200`. Down = connection refused (`000`) or timeout only. +Alive = **ANY** HTTP status response from the endpoint — the expected set is +`200`/`301`/`302`/`307`/`308`/`401`/`403` (the gateway UI is token-gated and +legitimately answers with redirects/auth-challenges, so never require a bare +`200`), and any other status, including `404`/`5xx`, also counts alive: a +process answering `503` is running and self-heal must NOT restart-loop it. +Down = connection refused (`000`) or timeout only. Statuses outside the +expected set are logged/reported as a warning — reported, never healed on. **B3: Public-URL Fallback Probe (Tanko — for nodes without pct/ssh access to amdpve)** ```bash -curl -s --connect-timeout 10 -o /dev/null -w '%{http_code}' https://tankodhs.sysloggh.net/ +curl -s --connect-timeout 10 --max-time 15 -o /dev/null -w '%{http_code}' https://tankodhs.sysloggh.net/ ``` Fallback only — used when the monitoring node has no pct/SSH path to amdpve. -Healthy = `302` (authentik proxy-auth redirect) or `401` (auth-gated). Down = -connection refused, timeout, or 5xx. Never expect a bare `200` — the public URL -terminates in the token-gated authentik chain. +Alive = **ANY** HTTP status response from the endpoint — healthy signals are +`302` (authentik proxy-auth redirect) and `401` (auth-gated), and any other +status, including `404`/`5xx`, also counts alive: the endpoint is up and +answering and must NOT be restart-looped. Down = connection refused (`000`) or +timeout only. Never expect a bare `200` — the public URL terminates in the +token-gated authentik chain. Statuses outside the healthy set are +logged/reported as a warning — reported, never healed on. **B4: Gateway Process** (Hermes agent Mumuni only — Tanko runs no Hermes gateway)