#!/bin/bash
# pm2-self-heal — hourly PM2 process check
# Part of the pm2-self-heal prose contract
# Alerts via Telegram (primary) and Zulip DM (secondary, abiba-zulip restored 2026-08-03)
# Field positions (awk -F'│'): $7=pid $8=uptime $9=restarts $10=status

TELEGRAM_BOT_TOKEN="$(grep TELEGRAM_BOT_TOKEN /root/.pi/agent/extensions/telegram/.env 2>/dev/null | cut -d= -f2 || echo '')"
LOG="/root/pm2-self-heal.log"
TELEGRAM_CHAT_ID="5822977936"

notify_tg() {
    local subject="$1"
    local body="$2"
    local msg="${subject}\n${body}"
    curl -sf -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
        -d "chat_id=${TELEGRAM_CHAT_ID}" \
        -d "text=${msg}" \
        -d "parse_mode=HTML" > /dev/null 2>&1 || true
}

# Log-only mode: replaced by prose contract pm2-self-heal.prose.md
# Only alerts Telegram on actual failure (status != online)
# This prevents the process from receiving alerts about itself

# Parse PM2 status (--no-color to avoid ANSI escape codes in awk columns)
STATUS=$(pm2 status --no-color 2>/dev/null)

ALERTS=""

# Check abiba-telegram (safe to auto-restart)
TEL_LINE=$(echo "$STATUS" | grep "abiba-telegram")
TEL_STATUS=$(echo "$TEL_LINE" | awk -F'│' '{print $10}' | xargs)
TEL_RESTARTS=$(echo "$TEL_LINE" | awk -F'│' '{print $9}' | xargs)

if [ "$TEL_STATUS" != "online" ] || [ "$TEL_RESTARTS" -gt 1000 ]; then
    pm2 restart abiba-telegram > /dev/null 2>&1
    sleep 3
    TEL_LINE2=$(pm2 status --no-color 2>/dev/null | grep "abiba-telegram")
    TEL_STATUS2=$(echo "$TEL_LINE2" | awk -F'│' '{print $10}' | xargs)
    if [ "$TEL_STATUS2" = "online" ]; then
        msg="⚠️ abiba-telegram was **$TEL_STATUS** → restarted to online"
        ALERTS="${ALERTS}${msg}\n"
    else
        msg="🚨 abiba-telegram **failed restart** (was $TEL_STATUS, still $TEL_STATUS2)"
        ALERTS="${ALERTS}${msg}\n"
    fi
fi

# Check abiba-zulip (live Zulip bridge, heartbeating)
ZULIP_LINE=$(echo "$STATUS" | grep "abiba-zulip")
ZULIP_STATUS=$(echo "$ZULIP_LINE" | awk -F'│' '{print $10}' | xargs)
ZULIP_RESTARTS=$(echo "$ZULIP_LINE" | awk -F'│' '{print $9}' | xargs)

if [ "$ZULIP_STATUS" != "online" ]; then
    pm2 restart abiba-zulip > /dev/null 2>&1
    sleep 3
    ZULIP_LINE2=$(pm2 status --no-color 2>/dev/null | grep "abiba-zulip")
    ZULIP_STATUS2=$(echo "$ZULIP_LINE2" | awk -F'│' '{print $10}' | xargs)
    if [ "$ZULIP_STATUS2" = "online" ]; then
        msg="⚠️ abiba-zulip was **$ZULIP_STATUS** → restarted to online"
        ALERTS="${ALERTS}${msg}\n"
    else
        msg="🚨 abiba-zulip **failed restart** (was $ZULIP_STATUS, still $ZULIP_STATUS2)"
        ALERTS="${ALERTS}${msg}\n"
    fi
elif [ "$ZULIP_RESTARTS" -gt 5 ]; then
    msg="⚠️ abiba-zulip has **$ZULIP_RESTARTS** restarts (high count)"
    ALERTS="${ALERTS}${msg}\n"
fi

# Check gitea-runner
GITEA_LINE=$(echo "$STATUS" | grep "gitea-runner")
GITEA_STATUS=$(echo "$GITEA_LINE" | awk -F'│' '{print $10}' | xargs)
GITEA_RESTARTS=$(echo "$GITEA_LINE" | awk -F'│' '{print $9}' | xargs)

if [ "$GITEA_STATUS" != "online" ]; then
    pm2 restart gitea-runner > /dev/null 2>&1
    sleep 3
    GITEA_LINE2=$(pm2 status --no-color 2>/dev/null | grep "gitea-runner")
    GITEA_STATUS2=$(echo "$GITEA_LINE2" | awk -F'│' '{print $10}' | xargs)
    if [ "$GITEA_STATUS2" = "online" ]; then
        msg="⚠️ gitea-runner was **$GITEA_STATUS** → restarted to online"
        ALERTS="${ALERTS}${msg}\n"
    else
        msg="🚨 gitea-runner **failed restart** (was $GITEA_STATUS, still $GITEA_STATUS2)"
        ALERTS="${ALERTS}${msg}\n"
    fi
elif [ "$GITEA_RESTARTS" -gt 5 ]; then
    msg="⚠️ gitea-runner has **$GITEA_RESTARTS** restarts (high count)"
    ALERTS="${ALERTS}${msg}\n"
fi

# Check zulip-watchdog
WATCHDOG_LINE=$(echo "$STATUS" | grep "zulip-watchdog")
WATCHDOG_STATUS=$(echo "$WATCHDOG_LINE" | awk -F'│' '{print $10}' | xargs)
WATCHDOG_RESTARTS=$(echo "$WATCHDOG_LINE" | awk -F'│' '{print $9}' | xargs)

if [ "$WATCHDOG_STATUS" != "online" ]; then
    pm2 restart zulip-watchdog > /dev/null 2>&1
    sleep 3
    WATCHDOG_LINE2=$(pm2 status --no-color 2>/dev/null | grep "zulip-watchdog")
    WATCHDOG_STATUS2=$(echo "$WATCHDOG_LINE2" | awk -F'│' '{print $10}' | xargs)
    if [ "$WATCHDOG_STATUS2" = "online" ]; then
        msg="⚠️ zulip-watchdog was **$WATCHDOG_STATUS** → restarted to online"
        ALERTS="${ALERTS}${msg}\n"
    else
        msg="🚨 zulip-watchdog **failed restart** (was $WATCHDOG_STATUS, still $WATCHDOG_STATUS2)"
        ALERTS="${ALERTS}${msg}\n"
    fi
elif [ "$WATCHDOG_RESTARTS" -gt 5 ]; then
    msg="⚠️ zulip-watchdog has **$WATCHDOG_RESTARTS** restarts (high count)"
    ALERTS="${ALERTS}${msg}\n"
fi

# Log check
{
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] tel=$TEL_STATUS zulip=$ZULIP_STATUS gitea=$GITEA_STATUS watchdog=$WATCHDOG_STATUS alerts=${ALERTS:+yes}"
    [ -n "$ALERTS" ] && echo "$ALERTS"
} >> "$LOG"

# Only Telegram DM when there's something to report
if [ -n "$ALERTS" ]; then
    notify_tg "PM2 Self-Heal — $(date '+%H:%M UTC')" "$ALERTS"
fi