Merge pull request #128 from decisions-2026-08-03-rebased

pm2-self-heal.prose.md

@@ -2,10 +2,10 @@
 kind: responsibility
 name: pm2-self-heal
 description: >
-  PM2 process health check for abiba-telegram, abiba-zulip, gitea-runner, and zulip-watchdog.
-  gpu-monitor is systemd-managed (gpu-monitor.service), NOT PM2.
-  gpu-watchdog is decommissioned and folded into gpu-monitor.service.
-  gitea-runner is KEPT. abiba-zulip is KEPT (online for days).
+  Monitors critical PM2 processes (abiba-telegram, abiba-zulip, gitea-runner, zulip-watchdog)
+  and auto-restarts any that are stopped or errored. Logs every action to
+  Gitea health-logs and alerts the owner via Telegram (primary) or Zulip DM (secondary).
+  Abiba-zulip is the live Zulip bridge and may be restarted; alert owner on failure.
 ---
 
 ## Maintains
@@ -16,6 +16,8 @@ description: >
 - zulip-watchdog: { status: "online", uptime: string, restarts: number }
 - last_check: timestamp
 
+> **Status (2026-08-03):** `abiba-zulip` fully restored — live Zulip bridge, heartbeating, monitored. `gpu-monitor` runs via systemd only (gpu-monitor.service); NOT PM2-tracked. `gpu-watchdog` retired from PM2 (folded into gpu-monitor.service). `zulip-watchdog` remains live and PM2-managed.
+
 
 ## Continuity
 
@@ -40,7 +42,7 @@ description: >
   crash-loop that never toggles status to "stopped"/"errored" (e.g. the 10k-restarts
   spoton incident). Alerts include the restart count.
 - **AS-BUILT (2026-09-15)**: spoton-service was deleted with its app; the live PM2 set is
-  four processes (abiba-telegram, abiba-zulip, gitea-runner, zulip-watchdog). The spoton
+  four processes (abiba-telegram, abiba-zulip, gitea-runner, gpu-monitor). The spoton
   reference above is historical context for the crash-loop guard, not a live process.
 - **Escalate**: Only when restarts > 30 — alerts to Zulip DM
 - **Historical fix**: Previous cycles were caused by abiba-zulip extension's
@@ -51,17 +53,25 @@ description: >
 ## Execution
 
 1. **Check PM2 status** — Run `pm2 status --no-color` and parse the table (5th data column = PID, 8th = restarts, 9th = status)
-2. **Check abiba-telegram**:
+2. **Check abiba-telegram** (safe to auto-restart):
    - If status is "online" → pass
    - If status is "stopped" or "errored" → apply Rule 1
-   - If restarts > 5 → alert owner
+   - If restarts > 1000 → apply Rule 2 (crash-loop guard)
 3. **Check abiba-zulip** (live Zulip bridge, heartbeating):
    - If status is "online" → pass, log restarts count
    - If status is "stopped" or "errored" → restart (`pm2 restart abiba-zulip` — fully restored)
    - If restarts > 5 in last hour → alert owner with full diagnostics
-4. **Log results** — Append to `SyslogSolution/health-logs/pm2/{timestamp}.md` in Gitea (not knowledge graph — hard rule)
-5. **Alert** — Send Zulip DM to owner if escalation needed (do NOT run pm2 commands during alerting)
-6. **Wait 5 min** → repeat from step 1
+4. **Check gitea-runner**:
+   - If status is "online" → pass
+   - If status is "stopped" or "errored" → apply Rule 1
+   - If restarts > 5 → alert owner
+5. **Check zulip-watchdog**:
+   - If status is "online" → pass
+   - If status is "stopped" or "errored" → apply Rule 1
+   - If restarts > 5 → alert owner
+6. **Log results** — Append to `SyslogSolution/health-logs/pm2/{timestamp}.md` in Gitea (not knowledge graph — hard rule)
+7. **Alert** — Send Telegram (primary) or Zulip DM (secondary) to owner if escalation needed (do NOT run pm2 commands during alerting)
+8. **Wait 5 min** → repeat from step 1
 
 ## Example Output (when healthy)
 

scripts/pm2-self-heal.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # pm2-self-heal — hourly PM2 process check
 # Part of the pm2-self-heal prose contract
-# Alerts via Telegram (abiba-zulip decommissioned 2026-07-04)
+# Alerts via Telegram (primary) and Zulip DM (secondary, abiba-zulip restored 2026-08-03)
 # Field positions (awk -F'│'): $7=pid $8=uptime $9=restarts $10=status
 
 TELEGRAM_BOT_TOKEN="$(grep TELEGRAM_BOT_TOKEN /root/.pi/agent/extensions/telegram/.env 2>/dev/null | cut -d= -f2 || echo '')"
@@ -46,9 +46,75 @@ if [ "$TEL_STATUS" != "online" ] || [ "$TEL_RESTARTS" -gt 1000 ]; then
     fi
 fi
 
+# Check abiba-zulip (live Zulip bridge, heartbeating)
+ZULIP_LINE=$(echo "$STATUS" | grep "abiba-zulip")
+ZULIP_STATUS=$(echo "$ZULIP_LINE" | awk -F'│' '{print $10}' | xargs)
+ZULIP_RESTARTS=$(echo "$ZULIP_LINE" | awk -F'│' '{print $9}' | xargs)
+
+if [ "$ZULIP_STATUS" != "online" ]; then
+    pm2 restart abiba-zulip > /dev/null 2>&1
+    sleep 3
+    ZULIP_LINE2=$(pm2 status --no-color 2>/dev/null | grep "abiba-zulip")
+    ZULIP_STATUS2=$(echo "$ZULIP_LINE2" | awk -F'│' '{print $10}' | xargs)
+    if [ "$ZULIP_STATUS2" = "online" ]; then
+        msg="⚠️ abiba-zulip was **$ZULIP_STATUS** → restarted to online"
+        ALERTS="${ALERTS}${msg}\n"
+    else
+        msg="🚨 abiba-zulip **failed restart** (was $ZULIP_STATUS, still $ZULIP_STATUS2)"
+        ALERTS="${ALERTS}${msg}\n"
+    fi
+elif [ "$ZULIP_RESTARTS" -gt 5 ]; then
+    msg="⚠️ abiba-zulip has **$ZULIP_RESTARTS** restarts (high count)"
+    ALERTS="${ALERTS}${msg}\n"
+fi
+
+# Check gitea-runner
+GITEA_LINE=$(echo "$STATUS" | grep "gitea-runner")
+GITEA_STATUS=$(echo "$GITEA_LINE" | awk -F'│' '{print $10}' | xargs)
+GITEA_RESTARTS=$(echo "$GITEA_LINE" | awk -F'│' '{print $9}' | xargs)
+
+if [ "$GITEA_STATUS" != "online" ]; then
+    pm2 restart gitea-runner > /dev/null 2>&1
+    sleep 3
+    GITEA_LINE2=$(pm2 status --no-color 2>/dev/null | grep "gitea-runner")
+    GITEA_STATUS2=$(echo "$GITEA_LINE2" | awk -F'│' '{print $10}' | xargs)
+    if [ "$GITEA_STATUS2" = "online" ]; then
+        msg="⚠️ gitea-runner was **$GITEA_STATUS** → restarted to online"
+        ALERTS="${ALERTS}${msg}\n"
+    else
+        msg="🚨 gitea-runner **failed restart** (was $GITEA_STATUS, still $GITEA_STATUS2)"
+        ALERTS="${ALERTS}${msg}\n"
+    fi
+elif [ "$GITEA_RESTARTS" -gt 5 ]; then
+    msg="⚠️ gitea-runner has **$GITEA_RESTARTS** restarts (high count)"
+    ALERTS="${ALERTS}${msg}\n"
+fi
+
+# Check zulip-watchdog
+WATCHDOG_LINE=$(echo "$STATUS" | grep "zulip-watchdog")
+WATCHDOG_STATUS=$(echo "$WATCHDOG_LINE" | awk -F'│' '{print $10}' | xargs)
+WATCHDOG_RESTARTS=$(echo "$WATCHDOG_LINE" | awk -F'│' '{print $9}' | xargs)
+
+if [ "$WATCHDOG_STATUS" != "online" ]; then
+    pm2 restart zulip-watchdog > /dev/null 2>&1
+    sleep 3
+    WATCHDOG_LINE2=$(pm2 status --no-color 2>/dev/null | grep "zulip-watchdog")
+    WATCHDOG_STATUS2=$(echo "$WATCHDOG_LINE2" | awk -F'│' '{print $10}' | xargs)
+    if [ "$WATCHDOG_STATUS2" = "online" ]; then
+        msg="⚠️ zulip-watchdog was **$WATCHDOG_STATUS** → restarted to online"
+        ALERTS="${ALERTS}${msg}\n"
+    else
+        msg="🚨 zulip-watchdog **failed restart** (was $WATCHDOG_STATUS, still $WATCHDOG_STATUS2)"
+        ALERTS="${ALERTS}${msg}\n"
+    fi
+elif [ "$WATCHDOG_RESTARTS" -gt 5 ]; then
+    msg="⚠️ zulip-watchdog has **$WATCHDOG_RESTARTS** restarts (high count)"
+    ALERTS="${ALERTS}${msg}\n"
+fi
+
 # Log check
 {
-    echo "[$(date '+%Y-%m-%d %H:%M:%S')] tel=$TEL_STATUS alerts=${ALERTS:+yes}"
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] tel=$TEL_STATUS zulip=$ZULIP_STATUS gitea=$GITEA_STATUS watchdog=$WATCHDOG_STATUS alerts=${ALERTS:+yes}"
     [ -n "$ALERTS" ] && echo "$ALERTS"
 } >> "$LOG"