diff --git a/pm2-self-heal.prose.md b/pm2-self-heal.prose.md index ae986c2..1878a2a 100644 --- a/pm2-self-heal.prose.md +++ b/pm2-self-heal.prose.md @@ -5,6 +5,7 @@ description: > Monitors critical PM2 processes (abiba-zulip, abiba-telegram) and auto-restarts any that are stopped or errored. Logs every action to the knowledge graph and alerts the owner via Zulip DM on failures. + CRITICAL: Never restart abiba-zulip — it runs this contract. --- ## Maintains @@ -33,14 +34,18 @@ description: > ## Execution -1. **Check PM2 status** — Run `pm2 status` and parse the table -2. **For each critical process** (abiba-zulip, abiba-telegram): +1. **Check PM2 status** — Run `pm2 status --no-color` and parse the table (5th data column = PID, 8th = restarts, 9th = status) +2. **Check abiba-telegram**: - If status is "online" → pass - If status is "stopped" or "errored" → apply Rule 1 - - If restarts > 5 → apply Rule 2 -3. **Log results** — Create `[LEARN]` node in knowledge graph for any actions taken -4. **Alert owner** — Send Zulip DM if escalation needed -5. **Wait 5 min** → repeat from step 1 + - If restarts > 5 → alert owner +3. **Check abiba-zulip** (self-process, read-only): + - If status is "online" → pass, log restarts count + - If status is "stopped" or "errored" → **DO NOT RESTART** — alert owner immediately + - If restarts > 5 in last hour → alert owner with full diagnostics +4. **Log results** — Create `[LEARN]` node in knowledge graph for any actions taken +5. **Alert** — Send Zulip DM to owner if escalation needed (do NOT run pm2 commands during alerting) +6. **Wait 5 min** → repeat from step 1 ## Example Output (when healthy) @@ -62,10 +67,11 @@ description: > ```json { "processes": { - "abiba-zulip": { "status": "errored → restarted → online" } + "abiba-telegram": { "status": "errored → restarted → online" }, + "abiba-zulip": { "status": "online", "uptime": "2h", "restarts": 0 } }, "actions_taken": [ - { "target": "abiba-zulip", "action": "pm2 restart", "result": "online" } + { "target": "abiba-telegram", "action": "pm2 restart", "result": "online" } ], "overall": "fixed" } diff --git a/pm2-self-heal.sh b/pm2-self-heal.sh new file mode 100755 index 0000000..c48bbe3 --- /dev/null +++ b/pm2-self-heal.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# pm2-self-heal — hourly PM2 process check +# Part of the pm2-self-heal prose contract +# Only sends Zulip DM when action is taken or escalation needed +# Field positions (awk -F'│'): $7=pid $8=uptime $9=restarts $10=status + +ZULIP_BOT="abiba-bot@chat.sysloggh.net" +ZULIP_KEY="cKTDMZAPW08dk3zl05sStzO7HRztzyn8" +ZULIP_SITE="https://chat.sysloggh.net" +OWNER_ID=9 +LOG="/tmp/pm2-self-heal.log" + +send_alert() { + local subject="$1" + local body="$2" + curl -s -X POST "$ZULIP_SITE/api/v1/messages" \ + -u "$ZULIP_BOT:$ZULIP_KEY" \ + -d "type=private" \ + -d "to=[$OWNER_ID]" \ + -d "content=$subject\n$body" > /dev/null 2>&1 + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Alert sent" >> "$LOG" +} + +# Parse PM2 status (--no-color to avoid ANSI escape codes in awk columns) +STATUS=$(pm2 status --no-color 2>/dev/null) + +ALERTS="" + +# Check abiba-telegram (safe to auto-restart) +TEL_LINE=$(echo "$STATUS" | grep "abiba-telegram") +TEL_STATUS=$(echo "$TEL_LINE" | awk -F'│' '{print $10}' | xargs) +TEL_RESTARTS=$(echo "$TEL_LINE" | awk -F'│' '{print $9}' | xargs) + +if [ "$TEL_STATUS" != "online" ]; then + pm2 restart abiba-telegram > /dev/null 2>&1 + sleep 3 + TEL_LINE2=$(pm2 status --no-color 2>/dev/null | grep "abiba-telegram") + TEL_STATUS2=$(echo "$TEL_LINE2" | awk -F'│' '{print $10}' | xargs) + if [ "$TEL_STATUS2" = "online" ]; then + ALERTS="${ALERTS}⚠️ abiba-telegram was **$TEL_STATUS** → restarted to online\n" + else + ALERTS="${ALERTS}🚨 abiba-telegram **failed restart** (was $TEL_STATUS, still $TEL_STATUS2)\n" + fi +fi + +# Check abiba-zulip (read-only — never restart) +ZUL_LINE=$(echo "$STATUS" | grep "abiba-zulip") +ZUL_STATUS=$(echo "$ZUL_LINE" | awk -F'│' '{print $10}' | xargs) +ZUL_RESTARTS=$(echo "$ZUL_LINE" | awk -F'│' '{print $9}' | xargs) + +if [ "$ZUL_STATUS" != "online" ]; then + ALERTS="${ALERTS}🚨 **abiba-zulip is $ZUL_STATUS** — needs investigation!\n" + ALERTS="${ALERTS} NOT auto-restarting (runs this contract)\n" +elif [ "$ZUL_RESTARTS" -gt 5 ] 2>/dev/null; then + ALERTS="${ALERTS}⚠️ abiba-zulip has **$ZUL_RESTARTS restarts** — may need attention\n" +fi + +# Log check +echo "[$(date '+%Y-%m-%d %H:%M:%S')] tel=$TEL_STATUS zul=$ZUL_STATUS alerts=${ALERTS:+yes}" >> "$LOG" + +# Only DM when there's something to report +if [ -n "$ALERTS" ]; then + send_alert "**PM2 Self-Heal — $(date '+%H:%M UTC')**" "$ALERTS" +fi