Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
27bc6bafba | ||
|
|
97dd261123 | ||
|
|
a179164f1f | ||
|
|
c4626c3512 | ||
|
|
b799d46596 |
@@ -4,7 +4,8 @@ name: pm2-self-heal
|
|||||||
description: >
|
description: >
|
||||||
Monitors critical PM2 processes (abiba-zulip, abiba-telegram, gitea-runner,
|
Monitors critical PM2 processes (abiba-zulip, abiba-telegram, gitea-runner,
|
||||||
spoton-service, zulip-watchdog) and auto-restarts any that are stopped or
|
spoton-service, zulip-watchdog) and auto-restarts any that are stopped or
|
||||||
errored. Logs every action to the knowledge graph and alerts the owner via
|
errored. Logs every action to Gitea (SyslogSolution/health-logs — not
|
||||||
|
knowledge graph, hard rule) and alerts the owner via
|
||||||
Zulip DM on failures.
|
Zulip DM on failures.
|
||||||
CRITICAL: Never restart abiba-zulip — it runs this contract.
|
CRITICAL: Never restart abiba-zulip — it runs this contract.
|
||||||
AS-BUILT 2026-08-09 (captain ruling, ecosystem is authoritative):
|
AS-BUILT 2026-08-09 (captain ruling, ecosystem is authoritative):
|
||||||
|
|||||||
+11
-26
@@ -9,10 +9,11 @@ ZULIP_EMAIL="abiba-bot@chat.sysloggh.net"
|
|||||||
ZULIP_KEY="cKTDMZAPW08dk3zl05sStzO7HRztzyn8"
|
ZULIP_KEY="cKTDMZAPW08dk3zl05sStzO7HRztzyn8"
|
||||||
OWNER_ZULIP_ID="9"
|
OWNER_ZULIP_ID="9"
|
||||||
|
|
||||||
# Email config
|
|
||||||
GMAIL_USER="jtabiri@gmail.com"
|
LOG="/root/zulip-health-monitor.log"
|
||||||
GMAIL_PASS="rgbuomwcydxwbszd"
|
TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
|
||||||
EMAIL_TO="jerome@sysloggh.com"
|
ISSUES=0
|
||||||
|
echo "=== Zulip Health Check — $TIMESTAMP ===" >> "$LOG"
|
||||||
|
|
||||||
notify() {
|
notify() {
|
||||||
local severity="$1" msg="$2"
|
local severity="$1" msg="$2"
|
||||||
@@ -24,30 +25,14 @@ notify() {
|
|||||||
curl -sf -X POST "${ZULIP_SITE}/api/v1/messages" \
|
curl -sf -X POST "${ZULIP_SITE}/api/v1/messages" \
|
||||||
-u "${ZULIP_EMAIL}:${ZULIP_KEY}" \
|
-u "${ZULIP_EMAIL}:${ZULIP_KEY}" \
|
||||||
-d "${form}" > /dev/null 2>&1 || true
|
-d "${form}" > /dev/null 2>&1 || true
|
||||||
|
# Zulip stream post to #agent-hub on topic 'zulip-health'
|
||||||
# Email alert
|
local stream_content="${severity} Zulip Monitor: ${msg}"
|
||||||
local subject="${severity} Zulip Monitor Alert"
|
curl -sf -X POST "${ZULIP_SITE}/api/v1/messages" \
|
||||||
python3 -c "
|
-u "${ZULIP_EMAIL}:${ZULIP_KEY}" \
|
||||||
import smtplib
|
-d "type=stream\&to=%5B7%5D\&topic=zulip-health\&content=$(python3 -c "import urllib.parse; print(urllib.parse.quote(str()))")" \
|
||||||
from email.mime.text import MIMEText
|
> /dev/null 2>&1 || true
|
||||||
m = MIMEText('''${msg}''')
|
|
||||||
m['From'] = 'abiba@sysloggh.com'
|
|
||||||
m['To'] = '${EMAIL_TO}'
|
|
||||||
m['Subject'] = '${subject}'
|
|
||||||
s = smtplib.SMTP('smtp.gmail.com', 587)
|
|
||||||
s.starttls()
|
|
||||||
s.login('${GMAIL_USER}', '${GMAIL_PASS}')
|
|
||||||
s.sendmail('abiba@sysloggh.com', ['${EMAIL_TO}'], m.as_string())
|
|
||||||
s.quit()
|
|
||||||
" 2>/dev/null || true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
|
|
||||||
ISSUES=0
|
|
||||||
LOG="/root/zulip-health-monitor.log"
|
|
||||||
|
|
||||||
echo "=== Zulip Health Check — $TIMESTAMP ===" >> "$LOG"
|
|
||||||
|
|
||||||
# ── Global: Zulip Server ──
|
# ── Global: Zulip Server ──
|
||||||
SERVER_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 \
|
SERVER_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 \
|
||||||
https://chat.sysloggh.net/api/v1/server_settings \
|
https://chat.sysloggh.net/api/v1/server_settings \
|
||||||
|
|||||||
@@ -324,6 +324,7 @@ Track via `/tmp/zulip-monitor-debounce` (unix timestamp of last restart).
|
|||||||
- **Persistent failure threshold**: Only restart after 3 consecutive failed checks (45 min) AND no self-healing in progress
|
- **Persistent failure threshold**: Only restart after 3 consecutive failed checks (45 min) AND no self-healing in progress
|
||||||
- **Debounce gate**: Skip all checks entirely if recently restarted
|
- **Debounce gate**: Skip all checks entirely if recently restarted
|
||||||
|
|
||||||
|
4. **Zulip stream delivery** — Alerts posted to #agent-hub on topic 'zulip-health' (stream_id 7) instead of email, keeping DM to captain for critical (🔴) alerts
|
||||||
### Gen 4 (2026-06-29) — Response Pipeline Fix
|
### Gen 4 (2026-06-29) — Response Pipeline Fix
|
||||||
|
|
||||||
**Root Cause**: LLM hangs due to GPU saturation (503 QUEUE_TIMEOUT) → pi's `agent_end` never fires → pending Zulip replies accumulate with no timeout. Health endpoint showed `connected: true, stuck: false` while user experienced complete silence.
|
**Root Cause**: LLM hangs due to GPU saturation (503 QUEUE_TIMEOUT) → pi's `agent_end` never fires → pending Zulip replies accumulate with no timeout. Health endpoint showed `connected: true, stuck: false` while user experienced complete silence.
|
||||||
|
|||||||
@@ -75,9 +75,8 @@ Track via `/tmp/zulip-heal-debounce-<agent>` (unix timestamp of last restart).
|
|||||||
|
|
||||||
## Reporting
|
## Reporting
|
||||||
|
|
||||||
Every cycle produces a knowledge graph node:
|
This contract is RETIRED — health-check logs are NOT knowledge graph content.
|
||||||
- Title: `[LEARN] zulip-self-heal: <timestamp>`
|
No graph nodes are created. Logs go to Gitea (SyslogSolution/health-logs).
|
||||||
- metadata: { type: "remediation", status: "fixed" | "escalated" | "healthy" }
|
|
||||||
- Issues fixed → DM: "🛠 Zulip Self-Heal: fixed <issue>"
|
- Issues fixed → DM: "🛠 Zulip Self-Heal: fixed <issue>"
|
||||||
- Issues escalated → DM: "⚠️ Zulip Self-Heal: <issue> needs attention"
|
- Issues escalated → DM: "⚠️ Zulip Self-Heal: <issue> needs attention"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user