Abiba (pi)
33453b8204
Root cause: Workers hung because default model was DeepSeek v4-pro (reasoning model that produces empty content with normal token limits). pi RPC workers waited forever for content that never arrived. Changes: - Circuit breaker (CLOSED→OPEN→HALF_OPEN) around Zulip API calls - Retry with exponential backoff + jitter for transient errors - Queue lifecycle management (idle_queue_timeout, auto re-register on BAD_EVENT_QUEUE_ID) - External supervisor watchdog (restarts router after 3 health check failures) - Busy worker timeout (SIGKILL after 5 min + error DM) - PM2 hardening (max_restarts=100, max_memory_restart=500M) - Crash handlers (uncaughtException + unhandledRejection → reconnect, not die) - Fixed default model: deepseek-v4-pro → syslog-harness/syslog-auto - Enhanced health endpoint with circuit breaker stats Architecture: Research-backed from Zulip event system docs + Node.js resilience patterns. Inline circuit breaker (no dependency). Separate watchdog process (Hermes pattern). Verified: Circuit breaker trips on outage, recovers gracefully. End-to-end DM processed in 4 seconds. Watchdog monitoring every 30s.
77 lines
2.2 KiB
JavaScript
77 lines
2.2 KiB
JavaScript
/**
|
|
* PM2 Ecosystem Config — Zulip Gateway v3 (Resilience)
|
|
*
|
|
* Deploy: pm2 start /root/.pm2/ecosystem.config.cjs
|
|
* Status: pm2 status
|
|
* Logs: pm2 logs abiba-zulip
|
|
*/
|
|
|
|
module.exports = {
|
|
apps: [
|
|
{
|
|
// ── Router (main Zulip gateway) ──
|
|
name: "abiba-zulip",
|
|
script: "/bin/pi",
|
|
args: "--mode rpc --session-id zulip-service",
|
|
cwd: "/root",
|
|
|
|
// Resilience hardening — up from pi defaults
|
|
max_restarts: 100, // Crash loops won't exhaust PM2 (was default 10)
|
|
min_uptime: "10s", // Must survive 10s to count as "alive"
|
|
max_memory_restart: "500M", // OOM protection — restart before swap thrash
|
|
restart_delay: 5000, // 5s cooldown between restarts
|
|
kill_timeout: 15000, // 15s SIGTERM grace before SIGKILL
|
|
listen_timeout: 30000, // 30s to bind health port
|
|
|
|
// Logging
|
|
log_date_format: "YYYY-MM-DD HH:mm:ss Z",
|
|
error_file: "/root/.pm2/logs/abiba-zulip-error.log",
|
|
out_file: "/root/.pm2/logs/abiba-zulip-out.log",
|
|
merge_logs: true,
|
|
log_type: "json",
|
|
|
|
// Process management
|
|
autorestart: true,
|
|
watch: false,
|
|
instances: 1,
|
|
exec_mode: "fork",
|
|
|
|
// Environment
|
|
env: {
|
|
ZULIP_ROLE: "router",
|
|
ZULIP_SITE: "https://chat.sysloggh.net",
|
|
ZULIP_EMAIL: "abiba-bot@chat.sysloggh.net",
|
|
ZULIP_API_KEY: "cKTDMZAPW08dk3zl05sStzO7HRztzyn8",
|
|
AGENT_NAME: "abiba",
|
|
AGENT_OWNER_EMAIL: "jerome@sysloggh.com",
|
|
NODE_ENV: "production",
|
|
},
|
|
},
|
|
{
|
|
// ── Supervisor (external watchdog) ──
|
|
name: "zulip-watchdog",
|
|
script: "/root/.pi/agent/extensions/zulip/watchdog.js",
|
|
cwd: "/root",
|
|
|
|
max_restarts: 10,
|
|
min_uptime: "3s",
|
|
restart_delay: 3000,
|
|
kill_timeout: 5000,
|
|
|
|
log_date_format: "YYYY-MM-DD HH:mm:ss Z",
|
|
error_file: "/root/.pm2/logs/zulip-watchdog-error.log",
|
|
out_file: "/root/.pm2/logs/zulip-watchdog-out.log",
|
|
merge_logs: true,
|
|
|
|
autorestart: true,
|
|
watch: false,
|
|
instances: 1,
|
|
exec_mode: "fork",
|
|
|
|
env: {
|
|
NODE_ENV: "production",
|
|
},
|
|
},
|
|
],
|
|
};
|