Merge pull request 'feat: Zulip Gateway v3 — Production Resilience Rewrite' (#34) from feat/zulip-v3-resilience into main

pi-zulip-extension/config/ecosystem.abiba.config.cjs

@@ -1,26 +1,76 @@
+/**
+ * PM2 Ecosystem Config — Zulip Gateway v3 (Resilience)
+ *
+ * Deploy: pm2 start /root/.pm2/ecosystem.config.cjs
+ * Status:  pm2 status
+ * Logs:    pm2 logs abiba-zulip
+ */
+
 module.exports = {
   apps: [
     {
+      // ── Router (main Zulip gateway) ──
       name: "abiba-zulip",
-      script: "/usr/bin/pi",
+      script: "/bin/pi",
       args: "--mode rpc --session-id zulip-service",
       cwd: "/root",
+
+      // Resilience hardening — up from pi defaults
+      max_restarts: 100,            // Crash loops won't exhaust PM2 (was default 10)
+      min_uptime: "10s",            // Must survive 10s to count as "alive"
+      max_memory_restart: "500M",   // OOM protection — restart before swap thrash
+      restart_delay: 5000,          // 5s cooldown between restarts
+      kill_timeout: 15000,          // 15s SIGTERM grace before SIGKILL
+      listen_timeout: 30000,        // 30s to bind health port
+
+      // Logging
+      log_date_format: "YYYY-MM-DD HH:mm:ss Z",
+      error_file: "/root/.pm2/logs/abiba-zulip-error.log",
+      out_file: "/root/.pm2/logs/abiba-zulip-out.log",
+      merge_logs: true,
+      log_type: "json",
+
+      // Process management
+      autorestart: true,
+      watch: false,
+      instances: 1,
+      exec_mode: "fork",
+
+      // Environment
       env: {
         ZULIP_ROLE: "router",
-        ZULIP_EXTENSION_ACTIVE: "true",
+        ZULIP_SITE: "https://chat.sysloggh.net",
-        NODE_OPTIONS: "--max-old-space-size=512",
+        ZULIP_EMAIL: "abiba-bot@chat.sysloggh.net",
+        ZULIP_API_KEY: "cKTDMZAPW08dk3zl05sStzO7HRztzyn8",
+        AGENT_NAME: "abiba",
+        AGENT_OWNER_EMAIL: "jerome@sysloggh.com",
+        NODE_ENV: "production",
       },
-      // Prevent rapid crash-looping: restart with backoff, limit retries
-      min_uptime: "30s",
-      max_restarts: 15,
-      restart_delay: 10000,
-      kill_timeout: 5000,
-      autorestart: true,
     },
     {
-      name: "abiba-telegram",
+      // ── Supervisor (external watchdog) ──
-      script: "/usr/bin/pitg",
+      name: "zulip-watchdog",
+      script: "/root/.pi/agent/extensions/zulip/watchdog.js",
       cwd: "/root",
+
+      max_restarts: 10,
+      min_uptime: "3s",
+      restart_delay: 3000,
+      kill_timeout: 5000,
+
+      log_date_format: "YYYY-MM-DD HH:mm:ss Z",
+      error_file: "/root/.pm2/logs/zulip-watchdog-error.log",
+      out_file: "/root/.pm2/logs/zulip-watchdog-out.log",
+      merge_logs: true,
+
+      autorestart: true,
+      watch: false,
+      instances: 1,
+      exec_mode: "fork",
+
+      env: {
+        NODE_ENV: "production",
+      },
     },
   ],
 };

pi-zulip-extension/config/settings.json

@@ -0,0 +1,11 @@
+{
+  "lastChangelogVersion": "0.80.6",
+  "defaultProvider": "syslog-harness",
+  "defaultModel": "syslog-auto",
+  "defaultThinkingLevel": "high",
+  "extensions": [
+    "+extensions/mcp/index.ts",
+    "+extensions/zulip/index.js"
+  ],
+  "theme": "dark"
+}

pi-zulip-extension/extension-src/watchdog.js

@@ -0,0 +1,84 @@
+/**
+ * Zulip Gateway Supervisor — external watchdog process.
+ *
+ * Monitors the router health endpoint every 30s. If 3 consecutive checks fail,
+ * restarts the abiba-zulip PM2 process gracefully.
+ *
+ * This is the pattern Hermes uses: an external supervisor that can recover
+ * the gateway even when the gateway process itself is hung (not just crashed).
+ *
+ * Deployed via PM2 as a separate process in ecosystem.config.cjs.
+ */
+
+const HEALTH_URL = "http://127.0.0.1:9200/health";
+const CHECK_INTERVAL_MS = 30_000;
+const MAX_FAILURES = 3;
+const RESTART_GRACE_MS = 15_000;
+
+let failures = 0;
+
+async function check() {
+  try {
+    const res = await fetch(HEALTH_URL, {
+      signal: AbortSignal.timeout(5000),
+    });
+
+    if (res.ok) {
+      const data = await res.json();
+
+      if (data.status === "ok" && data.zulip?.connected) {
+        if (failures > 0) {
+          console.log(`[watchdog] Router recovered after ${failures} failure(s)`);
+        }
+        failures = 0;
+        // Silent health log every 10 checks (~5 min) for monitoring
+        if (Math.random() < 0.1) {
+          console.log(`[watchdog] Router healthy (uptime: ${Math.round(process.uptime())}s)`);
+        }
+        return;
+      }
+
+      // Connected but degraded
+      console.warn(`[watchdog] Router degraded: status=${data.status}, connected=${data.zulip?.connected}`);
+      failures++;
+    } else {
+      console.warn(`[watchdog] Health check returned ${res.status}`);
+      failures++;
+    }
+  } catch (err) {
+    failures++;
+    console.warn(`[watchdog] Health check ${failures}/${MAX_FAILURES}: ${err.message}`);
+  }
+
+  if (failures >= MAX_FAILURES) {
+    console.error(`[watchdog] ${MAX_FAILURES} consecutive failures — restarting abiba-zulip`);
+
+    const { execSync } = await import("node:child_process");
+    try {
+      execSync("pm2 restart abiba-zulip", { timeout: 30000, encoding: "utf-8" });
+      console.log("[watchdog] Restart command sent successfully");
+    } catch (e) {
+      console.error(`[watchdog] Restart failed: ${e.message}`);
+
+      // Fallback: try resurrect if restart fails (process may be deleted)
+      try {
+        execSync("pm2 resurrect", { timeout: 30000 });
+        console.log("[watchdog] PM2 resurrected (fallback)");
+      } catch (e2) {
+        console.error(`[watchdog] Resurrect also failed: ${e2.message}`);
+      }
+    }
+
+    failures = 0;
+    // Wait for restart to fully initialize before checking again
+    await new Promise((r) => setTimeout(r, RESTART_GRACE_MS));
+  }
+}
+
+console.log("[watchdog] Zulip gateway supervisor started");
+console.log(`[watchdog] Monitoring ${HEALTH_URL} every ${CHECK_INTERVAL_MS / 1000}s`);
+console.log(`[watchdog] Max failures before restart: ${MAX_FAILURES}`);
+
+// Immediate first check, then periodic
+check();
+setInterval(check, CHECK_INTERVAL_MS);