root
1f90c3dc74
Root cause: abiba-zulip extension had STUCK_THRESHOLD_MS=30min. After 30min of inactivity (overnight), it reported stuck=True, which triggered the health monitor to restart it — cycling restarts up to 18. Permanent fixes: 1. STUCK_THRESHOLD_MS raised 30min → 4 hours in extension index.js 2. PM2 restart counter reset: delete+re-add abiba-zulip (was 18, now 0) 3. pm2-self-heal.sh alert threshold documented at 30 (implementation already) 4. Prose contract updated with historical note and threshold rationale
69 lines
2.6 KiB
Bash
Executable File
69 lines
2.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# pm2-self-heal — hourly PM2 process check
|
|
# Part of the pm2-self-heal prose contract
|
|
# Only sends Zulip DM when action is taken or escalation needed
|
|
# Field positions (awk -F'│'): $7=pid $8=uptime $9=restarts $10=status
|
|
|
|
ZULIP_BOT="abiba-bot@chat.sysloggh.net"
|
|
ZULIP_KEY="cKTDMZAPW08dk3zl05sStzO7HRztzyn8"
|
|
ZULIP_SITE="https://chat.sysloggh.net"
|
|
OWNER_ID=9
|
|
LOG="/tmp/pm2-self-heal.log"
|
|
|
|
send_alert() {
|
|
local subject="$1"
|
|
local body="$2"
|
|
curl -s -X POST "$ZULIP_SITE/api/v1/messages" \
|
|
-u "$ZULIP_BOT:$ZULIP_KEY" \
|
|
-d "type=private" \
|
|
-d "to=[$OWNER_ID]" \
|
|
-d "content=$subject\n$body" > /dev/null 2>&1
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Alert sent" >> "$LOG"
|
|
}
|
|
|
|
# Log-only mode: replaced by prose contract pm2-self-heal.prose.md
|
|
# Only sends DM on actual failure (status != online), not just high restarts
|
|
# This prevents the process from receiving alerts about itself
|
|
|
|
# Parse PM2 status (--no-color to avoid ANSI escape codes in awk columns)
|
|
STATUS=$(pm2 status --no-color 2>/dev/null)
|
|
|
|
ALERTS=""
|
|
|
|
# Check abiba-telegram (safe to auto-restart)
|
|
TEL_LINE=$(echo "$STATUS" | grep "abiba-telegram")
|
|
TEL_STATUS=$(echo "$TEL_LINE" | awk -F'│' '{print $10}' | xargs)
|
|
TEL_RESTARTS=$(echo "$TEL_LINE" | awk -F'│' '{print $9}' | xargs)
|
|
|
|
if [ "$TEL_STATUS" != "online" ]; then
|
|
pm2 restart abiba-telegram > /dev/null 2>&1
|
|
sleep 3
|
|
TEL_LINE2=$(pm2 status --no-color 2>/dev/null | grep "abiba-telegram")
|
|
TEL_STATUS2=$(echo "$TEL_LINE2" | awk -F'│' '{print $10}' | xargs)
|
|
if [ "$TEL_STATUS2" = "online" ]; then
|
|
ALERTS="${ALERTS}⚠️ abiba-telegram was **$TEL_STATUS** → restarted to online\n"
|
|
else
|
|
ALERTS="${ALERTS}🚨 abiba-telegram **failed restart** (was $TEL_STATUS, still $TEL_STATUS2)\n"
|
|
fi
|
|
fi
|
|
|
|
# Check abiba-zulip (read-only — never restart)
|
|
ZUL_LINE=$(echo "$STATUS" | grep "abiba-zulip")
|
|
ZUL_STATUS=$(echo "$ZUL_LINE" | awk -F'│' '{print $10}' | xargs)
|
|
ZUL_RESTARTS=$(echo "$ZUL_LINE" | awk -F'│' '{print $9}' | xargs)
|
|
|
|
if [ "$ZUL_STATUS" != "online" ]; then
|
|
ALERTS="${ALERTS}🚨 **abiba-zulip is $ZUL_STATUS** — needs investigation!\n"
|
|
ALERTS="${ALERTS} NOT auto-restarting (runs this contract)\n"
|
|
elif [ "$ZUL_RESTARTS" -gt 30 ] 2>/dev/null; then
|
|
ALERTS="${ALERTS}⚠️ abiba-zulip has **$ZUL_RESTARTS restarts** — may need attention\n"
|
|
fi
|
|
|
|
# Log check
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] tel=$TEL_STATUS zul=$ZUL_STATUS alerts=${ALERTS:+yes}" >> "$LOG"
|
|
|
|
# Only DM when there's something to report
|
|
if [ -n "$ALERTS" ]; then
|
|
send_alert "**PM2 Self-Heal — $(date '+%H:%M UTC')**" "$ALERTS"
|
|
fi
|