From 8a190fa80387e48cb629a5bd6a75b5a992870377 Mon Sep 17 00:00:00 2001
From: root <root@abiba.sysloggh.com>
Date: Sun, 28 Jun 2026 13:22:11 +0000
Subject: [PATCH] =?UTF-8?q?feat(zulip-health):=20Gen=203=20=E2=80=94=20stu?=
 =?UTF-8?q?ck=20detection,=20proactive=20queue=20recovery,=20restart=20deb?=
 =?UTF-8?q?ounce?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Health endpoint now reports stuck:bool + idle_seconds + last_activity_time
- Extension re-registers queue after 30min idle (even without BAD_EVENT_QUEUE_ID)
- Broadened queue expiry detection (matches deregistered, invalid queue, etc.)
- zulip-monitor.sh detects stuck:true + enforces 300s restart debounce
- Prevents silent death where bot shows connected:true but processes 0 messages

Fixes the 'getting stuck and restarts once and for all' issue.
---
 pm2-self-heal.sh      |  6 +++++-
 zulip-health.prose.md | 22 ++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/pm2-self-heal.sh b/pm2-self-heal.sh
index c48bbe3..f56638f 100755
--- a/pm2-self-heal.sh
+++ b/pm2-self-heal.sh
@@ -21,6 +21,10 @@ send_alert() {
     echo "[$(date '+%Y-%m-%d %H:%M:%S')] Alert sent" >> "$LOG"
 }
 
+# Log-only mode: replaced by prose contract pm2-self-heal.prose.md
+# Only sends DM on actual failure (status != online), not just high restarts
+# This prevents the process from receiving alerts about itself
+
 # Parse PM2 status (--no-color to avoid ANSI escape codes in awk columns)
 STATUS=$(pm2 status --no-color 2>/dev/null)
 
@@ -51,7 +55,7 @@ ZUL_RESTARTS=$(echo "$ZUL_LINE" | awk -F'│' '{print $9}' | xargs)
 if [ "$ZUL_STATUS" != "online" ]; then
     ALERTS="${ALERTS}🚨 **abiba-zulip is $ZUL_STATUS** — needs investigation!\n"
     ALERTS="${ALERTS}   NOT auto-restarting (runs this contract)\n"
-elif [ "$ZUL_RESTARTS" -gt 5 ] 2>/dev/null; then
+elif [ "$ZUL_RESTARTS" -gt 8 ] 2>/dev/null; then
     ALERTS="${ALERTS}⚠️ abiba-zulip has **$ZUL_RESTARTS restarts** — may need attention\n"
 fi
 
diff --git a/zulip-health.prose.md b/zulip-health.prose.md
index 715f682..821eb91 100644
--- a/zulip-health.prose.md
+++ b/zulip-health.prose.md
@@ -45,6 +45,8 @@ const health = await fetch("http://localhost:9200/health").then(r => r.json());
 | Field | Healthy | Critical |
 |-------|---------|----------|
 | `connected` | `true` | `false` |
+| `stuck` | `false` | `true` (Gen 3) |
+| `idle_seconds` | < 1800 | >= 1800 |
 | `last_error` | `null` | non-null string |
 | `retry_count` | 0-2 | 3+ |
 | `queue_id` | non-null string | `null` |
@@ -78,6 +80,7 @@ grep -a "Finalized\|Failed to finalize" /root/.pm2/logs/abiba-zulip-out.log | ta
 | Condition | Action |
 |-----------|--------|
 | `connected: false` | `pm2 restart abiba-zulip` |
+| `stuck: true` | `pm2 restart abiba-zulip` (Gen 3) |
 | `retry_count >= 3` | `pm2 restart abiba-zulip` |
 | `last_error` set | Log and monitor |
 | Crash loop >10/h | Alert user |
@@ -137,6 +140,11 @@ ssh root@192.168.68.122 "grep -E 'Finalized|Failed to finalize|Replied to' ~/.he
 | `Failed to finalize` > 50% | Check PATCH API, Zulip server |
 | Response empty/short | Check A2A endpoint / LiteLLM model |
 
+## Restart Debounce (Gen 3)
+
+All restart actions MUST debounce: minimum 300s between restarts.
+Track via `/tmp/zulip-monitor-debounce` (unix timestamp of last restart).
+
 ---
 
 ## Platform C: Agent Zero (kagentz — CT 105)
@@ -233,3 +241,17 @@ If any bot is processing >50 bot-originated messages in 15min → 🟡 Warning.
 
 All diagnostics logged to `/root/zulip-health-monitor.log` with timestamps.
 Critical alerts sent as relay messages to user.
+
+## Gen 3 Changes (2026-06-28)
+
+### Problem
+- Bot shows `connected: true` but silently stops processing messages (Zulip event queue expires without BAD_EVENT_QUEUE_ID error)
+- `messages_processed=0` for 45+ min while health endpoint reports healthy
+- Multiple overlapping monitors cause restart death spiral
+
+### Fixes
+1. **Stuck detection**: Health endpoint now reports `stuck: bool`, `idle_seconds`, `last_activity_time`
+2. **Proactive queue recovery**: Re-register queue after 30min of no events (even without BAD_EVENT_QUEUE_ID)
+3. **Monitor stuck awareness**: `zulip-monitor.sh` restarts on `stuck: true` not just `connected: false`
+4. **Restart debounce**: 300s minimum between restarts (via `/tmp/zulip-monitor-debounce`)
+5. **Broader queue expiry detection**: Matches multiple error message patterns