/** * Zulip Gateway Supervisor — external watchdog process. * * Monitors the router health endpoint every 30s. If 3 consecutive checks fail, * restarts the abiba-zulip PM2 process gracefully. * * This is the pattern Hermes uses: an external supervisor that can recover * the gateway even when the gateway process itself is hung (not just crashed). * * Deployed via PM2 as a separate process in ecosystem.config.cjs. */ const HEALTH_URL = "http://127.0.0.1:9200/health"; const CHECK_INTERVAL_MS = 30_000; const MAX_FAILURES = 3; const RESTART_GRACE_MS = 15_000; let failures = 0; async function check() { try { const res = await fetch(HEALTH_URL, { signal: AbortSignal.timeout(5000), }); if (res.ok) { const data = await res.json(); if (data.status === "ok" && data.zulip?.connected) { if (failures > 0) { console.log(`[watchdog] Router recovered after ${failures} failure(s)`); } failures = 0; // Silent health log every 10 checks (~5 min) for monitoring if (Math.random() < 0.1) { console.log(`[watchdog] Router healthy (uptime: ${Math.round(process.uptime())}s)`); } return; } // Connected but degraded console.warn(`[watchdog] Router degraded: status=${data.status}, connected=${data.zulip?.connected}`); failures++; } else { console.warn(`[watchdog] Health check returned ${res.status}`); failures++; } } catch (err) { failures++; console.warn(`[watchdog] Health check ${failures}/${MAX_FAILURES}: ${err.message}`); } if (failures >= MAX_FAILURES) { console.error(`[watchdog] ${MAX_FAILURES} consecutive failures — restarting abiba-zulip`); const { execSync } = await import("node:child_process"); try { execSync("pm2 restart abiba-zulip", { timeout: 30000, encoding: "utf-8" }); console.log("[watchdog] Restart command sent successfully"); } catch (e) { console.error(`[watchdog] Restart failed: ${e.message}`); // Fallback: try resurrect if restart fails (process may be deleted) try { execSync("pm2 resurrect", { timeout: 30000 }); console.log("[watchdog] PM2 resurrected (fallback)"); } catch (e2) { console.error(`[watchdog] Resurrect also failed: ${e2.message}`); } } failures = 0; // Wait for restart to fully initialize before checking again await new Promise((r) => setTimeout(r, RESTART_GRACE_MS)); } } console.log("[watchdog] Zulip gateway supervisor started"); console.log(`[watchdog] Monitoring ${HEALTH_URL} every ${CHECK_INTERVAL_MS / 1000}s`); console.log(`[watchdog] Max failures before restart: ${MAX_FAILURES}`); // Immediate first check, then periodic check(); setInterval(check, CHECK_INTERVAL_MS);