Abiba (pi)
33453b8204
Root cause: Workers hung because default model was DeepSeek v4-pro (reasoning model that produces empty content with normal token limits). pi RPC workers waited forever for content that never arrived. Changes: - Circuit breaker (CLOSED→OPEN→HALF_OPEN) around Zulip API calls - Retry with exponential backoff + jitter for transient errors - Queue lifecycle management (idle_queue_timeout, auto re-register on BAD_EVENT_QUEUE_ID) - External supervisor watchdog (restarts router after 3 health check failures) - Busy worker timeout (SIGKILL after 5 min + error DM) - PM2 hardening (max_restarts=100, max_memory_restart=500M) - Crash handlers (uncaughtException + unhandledRejection → reconnect, not die) - Fixed default model: deepseek-v4-pro → syslog-harness/syslog-auto - Enhanced health endpoint with circuit breaker stats Architecture: Research-backed from Zulip event system docs + Node.js resilience patterns. Inline circuit breaker (no dependency). Separate watchdog process (Hermes pattern). Verified: Circuit breaker trips on outage, recovers gracefully. End-to-end DM processed in 4 seconds. Watchdog monitoring every 30s.
85 lines
2.8 KiB
JavaScript
85 lines
2.8 KiB
JavaScript
/**
|
|
* Zulip Gateway Supervisor — external watchdog process.
|
|
*
|
|
* Monitors the router health endpoint every 30s. If 3 consecutive checks fail,
|
|
* restarts the abiba-zulip PM2 process gracefully.
|
|
*
|
|
* This is the pattern Hermes uses: an external supervisor that can recover
|
|
* the gateway even when the gateway process itself is hung (not just crashed).
|
|
*
|
|
* Deployed via PM2 as a separate process in ecosystem.config.cjs.
|
|
*/
|
|
|
|
const HEALTH_URL = "http://127.0.0.1:9200/health";
|
|
const CHECK_INTERVAL_MS = 30_000;
|
|
const MAX_FAILURES = 3;
|
|
const RESTART_GRACE_MS = 15_000;
|
|
|
|
let failures = 0;
|
|
|
|
async function check() {
|
|
try {
|
|
const res = await fetch(HEALTH_URL, {
|
|
signal: AbortSignal.timeout(5000),
|
|
});
|
|
|
|
if (res.ok) {
|
|
const data = await res.json();
|
|
|
|
if (data.status === "ok" && data.zulip?.connected) {
|
|
if (failures > 0) {
|
|
console.log(`[watchdog] Router recovered after ${failures} failure(s)`);
|
|
}
|
|
failures = 0;
|
|
// Silent health log every 10 checks (~5 min) for monitoring
|
|
if (Math.random() < 0.1) {
|
|
console.log(`[watchdog] Router healthy (uptime: ${Math.round(process.uptime())}s)`);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Connected but degraded
|
|
console.warn(`[watchdog] Router degraded: status=${data.status}, connected=${data.zulip?.connected}`);
|
|
failures++;
|
|
} else {
|
|
console.warn(`[watchdog] Health check returned ${res.status}`);
|
|
failures++;
|
|
}
|
|
} catch (err) {
|
|
failures++;
|
|
console.warn(`[watchdog] Health check ${failures}/${MAX_FAILURES}: ${err.message}`);
|
|
}
|
|
|
|
if (failures >= MAX_FAILURES) {
|
|
console.error(`[watchdog] ${MAX_FAILURES} consecutive failures — restarting abiba-zulip`);
|
|
|
|
const { execSync } = await import("node:child_process");
|
|
try {
|
|
execSync("pm2 restart abiba-zulip", { timeout: 30000, encoding: "utf-8" });
|
|
console.log("[watchdog] Restart command sent successfully");
|
|
} catch (e) {
|
|
console.error(`[watchdog] Restart failed: ${e.message}`);
|
|
|
|
// Fallback: try resurrect if restart fails (process may be deleted)
|
|
try {
|
|
execSync("pm2 resurrect", { timeout: 30000 });
|
|
console.log("[watchdog] PM2 resurrected (fallback)");
|
|
} catch (e2) {
|
|
console.error(`[watchdog] Resurrect also failed: ${e2.message}`);
|
|
}
|
|
}
|
|
|
|
failures = 0;
|
|
// Wait for restart to fully initialize before checking again
|
|
await new Promise((r) => setTimeout(r, RESTART_GRACE_MS));
|
|
}
|
|
}
|
|
|
|
console.log("[watchdog] Zulip gateway supervisor started");
|
|
console.log(`[watchdog] Monitoring ${HEALTH_URL} every ${CHECK_INTERVAL_MS / 1000}s`);
|
|
console.log(`[watchdog] Max failures before restart: ${MAX_FAILURES}`);
|
|
|
|
// Immediate first check, then periodic
|
|
check();
|
|
setInterval(check, CHECK_INTERVAL_MS);
|