Compare commits
20
Commits
main
..
6eaf175d2c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6eaf175d2c | ||
|
|
6e1be6b0dc | ||
|
|
312197a12b | ||
|
|
517d840e0d | ||
|
|
9597980d42 | ||
|
|
6938b6172a | ||
|
|
55ca15d846 | ||
|
|
aa014229fd | ||
|
|
20271f0eb3 | ||
|
|
f205230e84 | ||
|
|
08d0b7371b | ||
|
|
28dabe8834 | ||
|
|
92f281c43a | ||
|
|
7474ab1dc4 | ||
|
|
7047749ef0 | ||
|
|
553d469174 | ||
|
|
080d2ad756 | ||
|
|
9611f935bc | ||
|
|
84b80179ec | ||
|
|
5300c0f998 |
@@ -14,10 +14,10 @@ jobs:
|
|||||||
validate:
|
validate:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
- run: python3 --version
|
- run: python3 --version
|
||||||
- run: node --version
|
- run: node --version
|
||||||
- run: echo "Runner works!"
|
- run: echo "Runner works!"
|
||||||
- run: git clone --depth 1 http://abiba-bot:***REMOVED***@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
|
||||||
|
|
||||||
- name: Python syntax check
|
- name: Python syntax check
|
||||||
run: |
|
run: |
|
||||||
@@ -45,8 +45,8 @@ print('✅ config.yaml.example valid')
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [validate]
|
needs: [validate]
|
||||||
steps:
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
- run: echo "🚀 Deploy tag $(echo $GITHUB_REF_NAME)"
|
- run: echo "🚀 Deploy tag $(echo $GITHUB_REF_NAME)"
|
||||||
- run: git clone --depth 1 http://abiba-bot:***REMOVED***@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
|
||||||
|
|
||||||
- name: Deploy to Tanko (canary)
|
- name: Deploy to Tanko (canary)
|
||||||
run: ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 jerome@192.168.68.122 "cd /root && bash -s" < scripts/deploy.sh --ct=tanko --mode=native "$GITHUB_REF_NAME" 2>&1 || echo "⚠️ Tanko deploy skipped"
|
run: ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 jerome@192.168.68.122 "cd /root && bash -s" < scripts/deploy.sh --ct=tanko --mode=native "$GITHUB_REF_NAME" 2>&1 || echo "⚠️ Tanko deploy skipped"
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
run: git clone --depth 1 http://abiba-bot:***REMOVED***@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
run: git clone --depth 1 http://abiba-bot:mipjoq-tybbox-2ciHru@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
||||||
- name: Python syntax
|
- name: Python syntax
|
||||||
run: |
|
run: |
|
||||||
python3 -m py_compile plugins/platforms/zulip/adapter.py 2>/dev/null
|
python3 -m py_compile plugins/platforms/zulip/adapter.py 2>/dev/null
|
||||||
@@ -37,7 +37,7 @@ jobs:
|
|||||||
environment: canary
|
environment: canary
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
run: git clone --depth 1 http://abiba-bot:***REMOVED***@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
run: git clone --depth 1 http://abiba-bot:mipjoq-tybbox-2ciHru@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
||||||
- name: Deploy to Tanko
|
- name: Deploy to Tanko
|
||||||
env:
|
env:
|
||||||
TAG: ${{ github.ref_name }}
|
TAG: ${{ github.ref_name }}
|
||||||
@@ -66,7 +66,7 @@ jobs:
|
|||||||
environment: production
|
environment: production
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
run: git clone --depth 1 http://abiba-bot:***REMOVED***@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
run: git clone --depth 1 http://abiba-bot:mipjoq-tybbox-2ciHru@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
||||||
- name: Deploy to all Hermes agents
|
- name: Deploy to all Hermes agents
|
||||||
env:
|
env:
|
||||||
TAG: ${{ github.ref_name }}
|
TAG: ${{ github.ref_name }}
|
||||||
@@ -97,7 +97,7 @@ jobs:
|
|||||||
environment: agent-zero
|
environment: agent-zero
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
run: git clone --depth 1 http://abiba-bot:***REMOVED***@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
run: git clone --depth 1 http://abiba-bot:mipjoq-tybbox-2ciHru@192.168.68.17:3000/SyslogSolution/zulip-platform-plugins.git .
|
||||||
- name: Deploy to kagentz
|
- name: Deploy to kagentz
|
||||||
env:
|
env:
|
||||||
TAG: ${{ github.ref_name }}
|
TAG: ${{ github.ref_name }}
|
||||||
|
|||||||
@@ -1,76 +1,26 @@
|
|||||||
/**
|
|
||||||
* PM2 Ecosystem Config — Zulip Gateway v3 (Resilience)
|
|
||||||
*
|
|
||||||
* Deploy: pm2 start /root/.pm2/ecosystem.config.cjs
|
|
||||||
* Status: pm2 status
|
|
||||||
* Logs: pm2 logs abiba-zulip
|
|
||||||
*/
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
apps: [
|
apps: [
|
||||||
{
|
{
|
||||||
// ── Router (main Zulip gateway) ──
|
|
||||||
name: "abiba-zulip",
|
name: "abiba-zulip",
|
||||||
script: "/bin/pi",
|
script: "/usr/bin/pi",
|
||||||
args: "--mode rpc --session-id zulip-service",
|
args: "--mode rpc --session-id zulip-service",
|
||||||
cwd: "/root",
|
cwd: "/root",
|
||||||
|
|
||||||
// Resilience hardening — up from pi defaults
|
|
||||||
max_restarts: 100, // Crash loops won't exhaust PM2 (was default 10)
|
|
||||||
min_uptime: "10s", // Must survive 10s to count as "alive"
|
|
||||||
max_memory_restart: "500M", // OOM protection — restart before swap thrash
|
|
||||||
restart_delay: 5000, // 5s cooldown between restarts
|
|
||||||
kill_timeout: 15000, // 15s SIGTERM grace before SIGKILL
|
|
||||||
listen_timeout: 30000, // 30s to bind health port
|
|
||||||
|
|
||||||
// Logging
|
|
||||||
log_date_format: "YYYY-MM-DD HH:mm:ss Z",
|
|
||||||
error_file: "/root/.pm2/logs/abiba-zulip-error.log",
|
|
||||||
out_file: "/root/.pm2/logs/abiba-zulip-out.log",
|
|
||||||
merge_logs: true,
|
|
||||||
log_type: "json",
|
|
||||||
|
|
||||||
// Process management
|
|
||||||
autorestart: true,
|
|
||||||
watch: false,
|
|
||||||
instances: 1,
|
|
||||||
exec_mode: "fork",
|
|
||||||
|
|
||||||
// Environment
|
|
||||||
env: {
|
env: {
|
||||||
ZULIP_ROLE: "router",
|
ZULIP_ROLE: "router",
|
||||||
ZULIP_SITE: "https://chat.sysloggh.net",
|
ZULIP_EXTENSION_ACTIVE: "true",
|
||||||
ZULIP_EMAIL: "abiba-bot@chat.sysloggh.net",
|
NODE_OPTIONS: "--max-old-space-size=512",
|
||||||
ZULIP_API_KEY: "cKTDMZAPW08dk3zl05sStzO7HRztzyn8",
|
|
||||||
AGENT_NAME: "abiba",
|
|
||||||
AGENT_OWNER_EMAIL: "jerome@sysloggh.com",
|
|
||||||
NODE_ENV: "production",
|
|
||||||
},
|
},
|
||||||
|
// Prevent rapid crash-looping: restart with backoff, limit retries
|
||||||
|
min_uptime: "30s",
|
||||||
|
max_restarts: 15,
|
||||||
|
restart_delay: 10000,
|
||||||
|
kill_timeout: 5000,
|
||||||
|
autorestart: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// ── Supervisor (external watchdog) ──
|
name: "abiba-telegram",
|
||||||
name: "zulip-watchdog",
|
script: "/usr/bin/pitg",
|
||||||
script: "/root/.pi/agent/extensions/zulip/watchdog.js",
|
|
||||||
cwd: "/root",
|
cwd: "/root",
|
||||||
|
|
||||||
max_restarts: 10,
|
|
||||||
min_uptime: "3s",
|
|
||||||
restart_delay: 3000,
|
|
||||||
kill_timeout: 5000,
|
|
||||||
|
|
||||||
log_date_format: "YYYY-MM-DD HH:mm:ss Z",
|
|
||||||
error_file: "/root/.pm2/logs/zulip-watchdog-error.log",
|
|
||||||
out_file: "/root/.pm2/logs/zulip-watchdog-out.log",
|
|
||||||
merge_logs: true,
|
|
||||||
|
|
||||||
autorestart: true,
|
|
||||||
watch: false,
|
|
||||||
instances: 1,
|
|
||||||
exec_mode: "fork",
|
|
||||||
|
|
||||||
env: {
|
|
||||||
NODE_ENV: "production",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
{
|
|
||||||
"lastChangelogVersion": "0.80.6",
|
|
||||||
"defaultProvider": "syslog-harness",
|
|
||||||
"defaultModel": "syslog-auto",
|
|
||||||
"defaultThinkingLevel": "high",
|
|
||||||
"extensions": [
|
|
||||||
"+extensions/mcp/index.ts",
|
|
||||||
"+extensions/zulip/index.js"
|
|
||||||
],
|
|
||||||
"theme": "dark"
|
|
||||||
}
|
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
/**
|
|
||||||
* Zulip Gateway Supervisor — external watchdog process.
|
|
||||||
*
|
|
||||||
* Monitors the router health endpoint every 30s. If 3 consecutive checks fail,
|
|
||||||
* restarts the abiba-zulip PM2 process gracefully.
|
|
||||||
*
|
|
||||||
* This is the pattern Hermes uses: an external supervisor that can recover
|
|
||||||
* the gateway even when the gateway process itself is hung (not just crashed).
|
|
||||||
*
|
|
||||||
* Deployed via PM2 as a separate process in ecosystem.config.cjs.
|
|
||||||
*/
|
|
||||||
|
|
||||||
const HEALTH_URL = "http://127.0.0.1:9200/health";
|
|
||||||
const CHECK_INTERVAL_MS = 30_000;
|
|
||||||
const MAX_FAILURES = 3;
|
|
||||||
const RESTART_GRACE_MS = 15_000;
|
|
||||||
|
|
||||||
let failures = 0;
|
|
||||||
|
|
||||||
async function check() {
|
|
||||||
try {
|
|
||||||
const res = await fetch(HEALTH_URL, {
|
|
||||||
signal: AbortSignal.timeout(5000),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (res.ok) {
|
|
||||||
const data = await res.json();
|
|
||||||
|
|
||||||
if (data.status === "ok" && data.zulip?.connected) {
|
|
||||||
if (failures > 0) {
|
|
||||||
console.log(`[watchdog] Router recovered after ${failures} failure(s)`);
|
|
||||||
}
|
|
||||||
failures = 0;
|
|
||||||
// Silent health log every 10 checks (~5 min) for monitoring
|
|
||||||
if (Math.random() < 0.1) {
|
|
||||||
console.log(`[watchdog] Router healthy (uptime: ${Math.round(process.uptime())}s)`);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Connected but degraded
|
|
||||||
console.warn(`[watchdog] Router degraded: status=${data.status}, connected=${data.zulip?.connected}`);
|
|
||||||
failures++;
|
|
||||||
} else {
|
|
||||||
console.warn(`[watchdog] Health check returned ${res.status}`);
|
|
||||||
failures++;
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
failures++;
|
|
||||||
console.warn(`[watchdog] Health check ${failures}/${MAX_FAILURES}: ${err.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (failures >= MAX_FAILURES) {
|
|
||||||
console.error(`[watchdog] ${MAX_FAILURES} consecutive failures — restarting abiba-zulip`);
|
|
||||||
|
|
||||||
const { execSync } = await import("node:child_process");
|
|
||||||
try {
|
|
||||||
execSync("pm2 restart abiba-zulip", { timeout: 30000, encoding: "utf-8" });
|
|
||||||
console.log("[watchdog] Restart command sent successfully");
|
|
||||||
} catch (e) {
|
|
||||||
console.error(`[watchdog] Restart failed: ${e.message}`);
|
|
||||||
|
|
||||||
// Fallback: try resurrect if restart fails (process may be deleted)
|
|
||||||
try {
|
|
||||||
execSync("pm2 resurrect", { timeout: 30000 });
|
|
||||||
console.log("[watchdog] PM2 resurrected (fallback)");
|
|
||||||
} catch (e2) {
|
|
||||||
console.error(`[watchdog] Resurrect also failed: ${e2.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
failures = 0;
|
|
||||||
// Wait for restart to fully initialize before checking again
|
|
||||||
await new Promise((r) => setTimeout(r, RESTART_GRACE_MS));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log("[watchdog] Zulip gateway supervisor started");
|
|
||||||
console.log(`[watchdog] Monitoring ${HEALTH_URL} every ${CHECK_INTERVAL_MS / 1000}s`);
|
|
||||||
console.log(`[watchdog] Max failures before restart: ${MAX_FAILURES}`);
|
|
||||||
|
|
||||||
// Immediate first check, then periodic
|
|
||||||
check();
|
|
||||||
setInterval(check, CHECK_INTERVAL_MS);
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -67,6 +67,7 @@ DEFAULT_STREAM = "agent-hub"
|
|||||||
DEFAULT_ALL_BOTS_USER_ID = 1
|
DEFAULT_ALL_BOTS_USER_ID = 1
|
||||||
DEFAULT_POLL_INTERVAL = 3.0
|
DEFAULT_POLL_INTERVAL = 3.0
|
||||||
MAX_ZULIP_MESSAGE = 10000
|
MAX_ZULIP_MESSAGE = 10000
|
||||||
|
TRUNCATION_NOTICE = "\n\n[...truncated at Zulip limit]"
|
||||||
ECHO_TAG_PREFIX = "hermes-zulip-"
|
ECHO_TAG_PREFIX = "hermes-zulip-"
|
||||||
RECONNECT_BACKOFF = [2, 5, 10, 30, 60]
|
RECONNECT_BACKOFF = [2, 5, 10, 30, 60]
|
||||||
DEDUP_WINDOW = 300 # 5 minutes
|
DEDUP_WINDOW = 300 # 5 minutes
|
||||||
@@ -120,7 +121,7 @@ def _truncate(text: str, limit: int = MAX_ZULIP_MESSAGE) -> str:
|
|||||||
"""Truncate to Zulip's message limit with notice."""
|
"""Truncate to Zulip's message limit with notice."""
|
||||||
if len(text) <= limit:
|
if len(text) <= limit:
|
||||||
return text
|
return text
|
||||||
return text[:limit] + "\n\n[...truncated at Zulip limit]"
|
return text[:limit - len(TRUNCATION_NOTICE)] + TRUNCATION_NOTICE
|
||||||
|
|
||||||
|
|
||||||
def _parse_zulip_timestamp(ts: Any) -> datetime:
|
def _parse_zulip_timestamp(ts: Any) -> datetime:
|
||||||
|
|||||||
Reference in New Issue
Block a user