root
8a190fa803
- Health endpoint now reports stuck:bool + idle_seconds + last_activity_time - Extension re-registers queue after 30min idle (even without BAD_EVENT_QUEUE_ID) - Broadened queue expiry detection (matches deregistered, invalid queue, etc.) - zulip-monitor.sh detects stuck:true + enforces 300s restart debounce - Prevents silent death where bot shows connected:true but processes 0 messages Fixes the 'getting stuck and restarts once and for all' issue.
69 lines
2.6 KiB
Bash
Executable File
69 lines
2.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# pm2-self-heal — hourly PM2 process check
|
|
# Part of the pm2-self-heal prose contract
|
|
# Only sends Zulip DM when action is taken or escalation needed
|
|
# Field positions (awk -F'│'): $7=pid $8=uptime $9=restarts $10=status
|
|
|
|
ZULIP_BOT="abiba-bot@chat.sysloggh.net"
|
|
ZULIP_KEY="cKTDMZAPW08dk3zl05sStzO7HRztzyn8"
|
|
ZULIP_SITE="https://chat.sysloggh.net"
|
|
OWNER_ID=9
|
|
LOG="/tmp/pm2-self-heal.log"
|
|
|
|
send_alert() {
|
|
local subject="$1"
|
|
local body="$2"
|
|
curl -s -X POST "$ZULIP_SITE/api/v1/messages" \
|
|
-u "$ZULIP_BOT:$ZULIP_KEY" \
|
|
-d "type=private" \
|
|
-d "to=[$OWNER_ID]" \
|
|
-d "content=$subject\n$body" > /dev/null 2>&1
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Alert sent" >> "$LOG"
|
|
}
|
|
|
|
# Log-only mode: replaced by prose contract pm2-self-heal.prose.md
|
|
# Only sends DM on actual failure (status != online), not just high restarts
|
|
# This prevents the process from receiving alerts about itself
|
|
|
|
# Parse PM2 status (--no-color to avoid ANSI escape codes in awk columns)
|
|
STATUS=$(pm2 status --no-color 2>/dev/null)
|
|
|
|
ALERTS=""
|
|
|
|
# Check abiba-telegram (safe to auto-restart)
|
|
TEL_LINE=$(echo "$STATUS" | grep "abiba-telegram")
|
|
TEL_STATUS=$(echo "$TEL_LINE" | awk -F'│' '{print $10}' | xargs)
|
|
TEL_RESTARTS=$(echo "$TEL_LINE" | awk -F'│' '{print $9}' | xargs)
|
|
|
|
if [ "$TEL_STATUS" != "online" ]; then
|
|
pm2 restart abiba-telegram > /dev/null 2>&1
|
|
sleep 3
|
|
TEL_LINE2=$(pm2 status --no-color 2>/dev/null | grep "abiba-telegram")
|
|
TEL_STATUS2=$(echo "$TEL_LINE2" | awk -F'│' '{print $10}' | xargs)
|
|
if [ "$TEL_STATUS2" = "online" ]; then
|
|
ALERTS="${ALERTS}⚠️ abiba-telegram was **$TEL_STATUS** → restarted to online\n"
|
|
else
|
|
ALERTS="${ALERTS}🚨 abiba-telegram **failed restart** (was $TEL_STATUS, still $TEL_STATUS2)\n"
|
|
fi
|
|
fi
|
|
|
|
# Check abiba-zulip (read-only — never restart)
|
|
ZUL_LINE=$(echo "$STATUS" | grep "abiba-zulip")
|
|
ZUL_STATUS=$(echo "$ZUL_LINE" | awk -F'│' '{print $10}' | xargs)
|
|
ZUL_RESTARTS=$(echo "$ZUL_LINE" | awk -F'│' '{print $9}' | xargs)
|
|
|
|
if [ "$ZUL_STATUS" != "online" ]; then
|
|
ALERTS="${ALERTS}🚨 **abiba-zulip is $ZUL_STATUS** — needs investigation!\n"
|
|
ALERTS="${ALERTS} NOT auto-restarting (runs this contract)\n"
|
|
elif [ "$ZUL_RESTARTS" -gt 8 ] 2>/dev/null; then
|
|
ALERTS="${ALERTS}⚠️ abiba-zulip has **$ZUL_RESTARTS restarts** — may need attention\n"
|
|
fi
|
|
|
|
# Log check
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] tel=$TEL_STATUS zul=$ZUL_STATUS alerts=${ALERTS:+yes}" >> "$LOG"
|
|
|
|
# Only DM when there's something to report
|
|
if [ -n "$ALERTS" ]; then
|
|
send_alert "**PM2 Self-Heal — $(date '+%H:%M UTC')**" "$ALERTS"
|
|
fi
|