diff --git a/scripts/daily-infra-report.py b/scripts/daily-infra-report.py new file mode 100755 index 0000000..a4802c1 --- /dev/null +++ b/scripts/daily-infra-report.py @@ -0,0 +1,708 @@ +#!/usr/bin/env python3 +""" +/root/scripts/daily-infra-report.py — v2.0.0 +Daily infrastructure dashboard emailed to jerome@sysloggh.com +Runs at 6:00 AM daily via cron. +Integrated with prose contracts: infrastructure-control, litellm-health, zulip-health. + +Usage: + python3 daily-infra-report.py # Generate and email report + python3 daily-infra-report.py --test-email # Send a test email for review + python3 daily-infra-report.py --json # Output raw JSON to stdout +""" + +import smtplib, json, subprocess, os, sys, datetime, re +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart + +PVE = "https://minipve.sysloggh.net" +AUTH = "Authorization: PVEAPIToken=monitoring@pve!mumuni=eafd56c5-93d4-4d40-a41d-e688be0987f3" + +# ── Shared credentials —─ + +ZULIP_SITE = "https://chat.sysloggh.net" +ZULIP_EMAIL = "abiba-bot@chat.sysloggh.net" +ZULIP_KEY = "cKTDMZAPW08dk3zl05sStzO7HRztzyn8" +ZULIP_AUTH = f"{ZULIP_EMAIL}:{ZULIP_KEY}" + +LITELLM_PUBLIC = "https://litellm.sysloggh.net" +LITELLM_BACKEND = "192.168.68.116" +AUTH_HOST = "192.168.68.11" + +SYNTHETIC_API_KEY = "sk-U_ydi3B-wfGU-_xESkoU1Q" + +NOW = datetime.datetime.now() +DATE_STR = NOW.strftime("%Y-%m-%d") +TIME_STR = NOW.strftime("%Y-%m-%d %H:%M UTC") + +# ── Helpers ── + +def pve_get(path): + cmd = f'curl -sfk --connect-timeout 10 "{PVE}{path}" -H "{AUTH}"' + try: + return json.loads(subprocess.check_output(cmd, shell=True))["data"] + except: return [] + +def ssh(host, cmd): + try: + return subprocess.check_output( + f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@{host} "{cmd}"', + shell=True, stderr=subprocess.DEVNULL).decode().strip() + except: return "" + +def ssh_jerome(host, cmd): + try: + return subprocess.check_output( + f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 jerome@{host} "{cmd}"', + shell=True, stderr=subprocess.DEVNULL).decode().strip() + except: return "" + +def http_get(url, auth=None, timeout=10): + try: + cmd = f'curl -sfk --connect-timeout {timeout} -o /dev/null -w "%{{http_code}}" "{url}"' + if auth: + cmd = cmd.replace('"', '\\"') + cmd = f'curl -sfk --connect-timeout {timeout} -u "{auth}" -o /dev/null -w "%{{http_code}}" "{url}"' + r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout+2) + return r.stdout.strip() or "000" + except: + return "timeout" + +def http_get_body(url, auth=None, timeout=10): + try: + cmd = f'curl -sfk --connect-timeout {timeout} "{url}"' + if auth: + cmd = f'curl -sfk --connect-timeout {timeout} -u "{auth}" "{url}"' + r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout+2) + if r.returncode == 0: + return r.stdout.strip() + return "" + except: + return "" + +def count_in_log(filepath, pattern, minutes=15): + """Count occurrences of a pattern in a log file within the last N minutes.""" + try: + since = (NOW - datetime.timedelta(minutes=minutes)).strftime("%Y-%m-%d %H:%M") + cmd = f"""grep -a "{pattern}" {filepath} 2>/dev/null | awk '$0 >= "{since}"' | wc -l""" + count = subprocess.check_output(cmd, shell=True).decode().strip() + return int(count) + except: + return 0 + +# ── Data Collection ── + +def collect(): + report = {} + + # ── Proxmox Nodes ── + nodes = pve_get("/api2/json/nodes") + report["nodes"] = {n["node"]: { + "cpu_pct": round(n.get('cpu',0)*100, 1), + "ram": f"{n.get('mem',0)//1024//1024}/{n.get('maxmem',0)//1024//1024}MB", + "ram_pct": round(n.get('mem',0)/n.get('maxmem',1)*100, 0), + "disk": f"{n.get('disk',0)//1024//1024//1024}/{n.get('maxdisk',0)//1024//1024//1024}GB", + "disk_pct": round(n.get('disk',0)/n.get('maxdisk',1)*100, 0), + "uptime_h": n.get('uptime',0)//3600, + "status": n["status"] + } for n in nodes} + report["node_count"] = len(nodes) + report["nodes_online"] = sum(1 for n in nodes if n["status"] == "online") + + # ── VMs/CTs ── + resources = pve_get("/api2/json/cluster/resources") + vms = [r for r in resources if r.get("type") in ("qemu","lxc")] + report["total_vms"] = len(vms) + report["running_vms"] = sum(1 for v in vms if v.get("status") == "running") + stopped = [v for v in vms if v.get("status") != "running"] + report["stopped_vms"] = [f"{v['type']} {v['vmid']} {v.get('name','?')}" for v in stopped] + + report["vms_by_node"] = {} + for n in ["amdpve", "minipve", "storepve", "acerpve", "ocupve"]: + node_vms = [v for v in vms if v.get("node") == n] + node_vms.sort(key=lambda x: int(x.get("vmid",0))) + report["vms_by_node"][n] = [{ + "type": v.get("type"), "vmid": v.get("vmid"), "name": v.get("name","?"), + "status": v.get("status"), + "ram": f"{v.get('mem',0)//1024//1024}/{v.get('maxmem',0)//1024//1024}MB", + "disk": f"{v.get('disk',0)//1024//1024//1024}/{v.get('maxdisk',0)//1024//1024//1024}GB", + "cpu": v.get("cpus", v.get("maxcpu","?")), + } for v in node_vms] + + # ── Storage ── + storages = pve_get("/api2/json/nodes/storepve/storage") + report["storage"] = [] + for s in storages: + total = s.get("total",0) or 1 + used = s.get("used",0) + pct = used/total*100 + report["storage"].append({ + "name": s.get("storage","?"), "used": f"{used//1024//1024//1024}GB", + "total": f"{total//1024//1024//1024}GB", "pct": round(pct, 0), "type": s.get("type","") + }) + + # ── Docker: docker-vm (.7) ── + docker_raw = ssh("192.168.68.7", "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'") + containers = [] + for line in docker_raw.strip().split("\n"): + if "|" in line: + parts = line.split("|", 2) + containers.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""}) + report["docker_vm"] = {"total": len(containers), "running": sum(1 for c in containers if "Up" in c["status"]), + "unhealthy": [c for c in containers if "unhealthy" in c["status"]], "containers": containers} + report["docker_vm"]["reclaimable"] = ssh("192.168.68.7", "docker system df 2>/dev/null | tail -1 | awk '{print $5}'") + report["docker_vm"]["disk_used"] = ssh("192.168.68.7", "df -h / | tail -1 | awk '{print $5}'") + + # ── Docker: CT 116 (syslog-api — LiteLLM stack) ── + docker2_raw = ssh(LITELLM_BACKEND, "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'") + containers2 = [] + for line in docker2_raw.strip().split("\n"): + if "|" in line: + parts = line.split("|", 2) + containers2.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""}) + report["docker_syslog"] = {"total": len(containers2), "running": sum(1 for c in containers2 if "Up" in c["status"]), + "containers": containers2} + + # ── Docker: Netbird (72.61.0.17) ── + docker3_raw = ssh("72.61.0.17", "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'") + containers3 = [] + for line in docker3_raw.strip().split("\n"): + if "|" in line: + parts = line.split("|", 2) + containers3.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""}) + report["docker_netbird"] = {"total": len(containers3), "running": sum(1 for c in containers3 if "Up" in c["status"]), + "containers": containers3} + + # ── Public Endpoints ── + endpoints = [ + ("LiteLLM Admin UI", f"{LITELLM_PUBLIC}/ui/"), + ("LiteLLM Swagger", f"{LITELLM_PUBLIC}/docs"), + ("LiteLLM ReDoc", f"{LITELLM_PUBLIC}/redoc"), + ("LiteLLM OpenAPI", f"{LITELLM_PUBLIC}/openapi.json"), + ("Gitea", "https://git.sysloggh.net"), + ("Authentik", "https://auth.sysloggh.net"), + ("Zulip", "https://chat.sysloggh.net"), + ("Pulse", "https://pulse.sysloggh.net"), + ("Proxmox", "https://minipve.sysloggh.net"), + ("SearXNG", "http://192.168.68.7:8888"), + ("Firecrawl", "http://192.168.68.7:3002/health"), + ] + report["endpoints"] = [] + for name, url in endpoints: + code = http_get(url) + report["endpoints"].append({"name": name, "code": code}) + + # ── LiteLLM Specific Checks (from litellm-health prose contract) ── + report["litellm"] = {"checks": []} + + # Check 1: LiteLLM aggregate health endpoint (router binds to 127.0.0.1, check via SSH) + health_unified = ssh(LITELLM_BACKEND, "curl -sf http://127.0.0.1:9000/health/unified -o /dev/null -w '%{http_code}' 2>/dev/null") + report["litellm"]["health_unified"] = health_unified or "000" + report["litellm"]["checks"].append({"name": "unified-health", "status": "pass" if health_unified == "200" else "fail", "code": health_unified or "000"}) + + # Check 2: Nginx-proxied internal endpoints + for path, name in [("/litellm/ui/", "nginx-ui"), ("/litellm/docs", "nginx-docs")]: + code = http_get(f"http://{LITELLM_BACKEND}{path}") + report["litellm"]["checks"].append({"name": name, "status": "pass" if code == "200" else "fail", "code": code}) + + # Check 3: Docker container health for LiteLLM stack + expected_containers = ["harness-litellm", "harness-nginx", "harness-router", + "harness-postgres", "harness-redis", "harness-dashboard"] + actual_names = [c["name"] for c in containers2] + report["litellm"]["expected_containers"] = expected_containers + report["litellm"]["missing_containers"] = [e for e in expected_containers if e not in actual_names] + report["litellm"]["checks"].append({ + "name": "container-health", + "status": "pass" if not report["litellm"]["missing_containers"] else "fail", + "missing": report["litellm"]["missing_containers"], + "healthy": sum(1 for c in containers2 if "healthy" in c.get("status","")), + "total": len(expected_containers) + }) + + # Check 4: OIDC Auth endpoint + auth_code = http_get(f"https://auth.sysloggh.net") + report["litellm"]["auth_status"] = auth_code + report["litellm"]["checks"].append({"name": "oidc-auth", "status": "pass" if auth_code in ("200","302") else "fail", "code": auth_code}) + + # Check 5: Synthetic API call through LiteLLM + api_check = http_get(f"{LITELLM_PUBLIC}/v1/models", auth=SYNTHETIC_API_KEY) + report["litellm"]["api_models"] = api_check + report["litellm"]["checks"].append({"name": "api-endpoint", "status": "pass" if api_check == "200" else "fail", "code": api_check}) + + # ── NFS Mounts ── + nfs = ssh("192.168.68.7", "df -h /media/storage /media/mediastore 2>/dev/null | tail -n +2") + report["nfs"] = [] + for line in nfs.strip().split("\n"): + parts = line.split() + if len(parts) >= 6: + report["nfs"].append({"mount": parts[5], "size": parts[1], "used": parts[2], "pct": parts[4], "avail": parts[3]}) + + # ── Zulip Extension Health (from zulip-health prose contract) ── + report["zulip_ext"] = {} + + # Phase 1: Health endpoint + health_body = http_get_body("http://localhost:9200/health") + zulip_health = {} + try: + zulip_health = json.loads(health_body) if health_body else {} + except: + zulip_health = {} + report["zulip_ext"]["connected"] = zulip_health.get("connected", False) + report["zulip_ext"]["queue_id"] = zulip_health.get("queue_id") + report["zulip_ext"]["last_error"] = zulip_health.get("last_error") + report["zulip_ext"]["messages_processed"] = zulip_health.get("messages_processed", 0) + report["zulip_ext"]["retry_count"] = zulip_health.get("retry_count", 0) + + # Phase 2: PM2 process check + pm2_raw = subprocess.check_output( + "pm2 show abiba-zulip --no-color 2>/dev/null | grep -E 'status|restarts|uptime'", + shell=True).decode().strip() + pm2 = {} + for line in pm2_raw.split("\n"): + if "│" in line: + parts = line.split("│") + if len(parts) >= 3: + key = parts[1].strip() + val = parts[2].strip() + pm2[key] = val + report["zulip_ext"]["pm2"] = pm2 + report["zulip_ext"]["pm2_healthy"] = pm2.get("status") == "online" + + # Phase 3: Echo loop detection + skipped_count = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Skipped.*bot msgs", 15) + report["zulip_ext"]["bot_skipped_15min"] = skipped_count + + # Phase 4: Response delivery stats + finalized = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Finalized", 60) + failed = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Failed to finalize", 60) + report["zulip_ext"]["finalized_1h"] = finalized + report["zulip_ext"]["failed_finalize_1h"] = failed + report["zulip_ext"]["finalize_fail_pct"] = round(failed / (finalized + failed) * 100, 0) if (finalized + failed) > 0 else 0 + + # Phase 5: Zulip server liveness + zulip_server_code = http_get(f"{ZULIP_SITE}/api/v1/server_settings", auth=ZULIP_AUTH) + report["zulip_ext"]["server_status"] = zulip_server_code + + # ── Agent Status ── + report["agents"] = {} + + # Abiba (pi) + report["agents"]["abiba"] = { + "platform": "pi", "ct": 100, "ip": "192.168.68.24", + "zulip_connected": zulip_health.get("connected", False), + "zulip_processed": zulip_health.get("messages_processed", 0), + "pm2_status": pm2.get("status", "unknown"), + "pm2_restarts": pm2.get("restarts", "?"), + "pm2_uptime": pm2.get("uptime", "?"), + } + + # Tanko (CT 122) + tanko_state = ssh_jerome("192.168.68.122", "cat ~/.hermes/gateway_state.json 2>/dev/null") + tanko_data = {} + try: + tanko_data = json.loads(tanko_state) if tanko_state else {} + except: + tanko_data = {} + platforms = tanko_data.get("platforms", {}) + report["agents"]["tanko"] = { + "platform": "hermes", "ct": 112, "ip": "192.168.68.122", + "gateway_state": tanko_data.get("gateway_state", "unknown"), + "zulip_state": platforms.get("zulip", {}).get("state", "unknown"), + "telegram_state": platforms.get("telegram", {}).get("state", "unknown"), + "gateway_pid": tanko_data.get("pid"), + "updated_at": tanko_data.get("updated_at"), + } + + # Mumuni (CT 114, IP 192.168.68.123) + mumuni_state = ssh("192.168.68.123", "cat ~/.hermes/gateway_state.json 2>/dev/null") + mumuni_data = {} + try: + mumuni_data = json.loads(mumuni_state) if mumuni_state else {} + except: + mumuni_data = {} + mumuni_platforms = mumuni_data.get("platforms", {}) + report["agents"]["mumuni"] = { + "platform": "hermes", "ct": 114, "ip": "192.168.68.123", + "gateway_state": mumuni_data.get("gateway_state", "unknown"), + "telegram_state": mumuni_platforms.get("telegram", {}).get("state", "unknown"), + "zulip_state": mumuni_platforms.get("zulip", {}).get("state", "not_installed"), + "email_state": mumuni_platforms.get("email", {}).get("state", "unknown"), + "hermes_version": "", + } + # Get Hermes version + ver = ssh("192.168.68.123", "hermes --version 2>/dev/null | head -1") + if ver: + report["agents"]["mumuni"]["hermes_version"] = ver.split("·")[0].replace("Hermes Agent ","").strip() + + return report + + +# ── HTML Dashboard ── + +def build_html(r): + issues = [] + + # Proxmox issues + if r["nodes_online"] < r["node_count"]: + issues.append(f"🔴 {r['node_count'] - r['nodes_online']} Proxmox node(s) offline") + if r["stopped_vms"]: + issues.append(f"🔴 {len(r['stopped_vms'])} VM(s)/CT(s) stopped — {', '.join(r['stopped_vms'][:3])}") + + # Endpoint issues + for ep in r["endpoints"]: + if ep["code"] in ("000", "timeout"): + issues.append(f"🔴 {ep['name']} — unreachable") + elif ep["code"] >= "500": + issues.append(f"🟡 {ep['name']} — HTTP {ep['code']}") + + # Docker issues + for c in r["docker_vm"].get("unhealthy", []): + issues.append(f"🟡 docker-vm: {c['name']} — unhealthy") + + # Small cluster mode warning + if r["docker_vm"].get("total", 0) == 0: + issues.append("🟡 docker-vm: no containers reported — host may be down") + + # LiteLLM issues + for check in r.get("litellm", {}).get("checks", []): + if check["status"] == "fail": + issues.append(f"🔴 LiteLLM: {check['name']} — {'missing: ' + ', '.join(check.get('missing',[])) if check.get('missing') else 'HTTP ' + check.get('code','?')}") + + # Zulip extension issues + z = r.get("zulip_ext", {}) + if not z.get("connected"): + issues.append("🔴 Zulip extension: disconnected") + if z.get("last_error"): + issues.append(f"🟡 Zulip extension: {z['last_error'][:80]}") + if z.get("retry_count", 0) >= 3: + issues.append(f"🟡 Zulip extension: {z['retry_count']} retries") + if z.get("finalize_fail_pct", 0) > 50: + issues.append(f"🔴 Zulip extension: {z['finalize_fail_pct']:.0f}% edit failures") + if z.get("bot_skipped_15min", 0) > 100: + issues.append(f"🟡 Zulip echo loop: {z['bot_skipped_15min']} bot msgs skipped in 15min") + if z.get("server_status") and z["server_status"] not in ("200", "302"): + issues.append(f"🔴 Zulip server: HTTP {z['server_status']}") + + # Agent issues + def agent_gateway(agent): + return agent.get("gateway_state") or agent.get("pm2_status") or "unknown" + for name, agent in r.get("agents", {}).items(): + gs = agent_gateway(agent) + if gs == "unknown": + issues.append(f"🟡 {name}: gateway state unknown (unreachable)") + elif gs != "running" and gs != "online": + issues.append(f"🔴 {name}: gateway {gs}") + + status = "🟢 All Healthy" if not issues else f"{'🔴' if sum(1 for i in issues if '🔴' in i) > 0 else '🟡'} {len(issues)} Issue(s)" + + # ── Build HTML ── + html = f""" +
+{TIME_STR}
+ +{status}
++{r['node_count']} PVE nodes · {r['total_vms']} VMs/CTs · {r['running_vms']} running · +{r['docker_vm']['total'] + r['docker_syslog']['total'] + r['docker_netbird']['total']} containers · +{len(r['endpoints'])} endpoints · {len(r.get('agents',{}))} agents +
+{i}
' + html += '| Check | Status |
|---|---|
| {check["name"]} | {detail} |
| Container | Status | Image |
|---|---|---|
| {c["name"]} | {c["status"]} | {c["image"]} |
Stopped: {", ".join(r["stopped_vms"])}
' + html += '{badge}
' + html += '| ID | Type | Name | Status | CPU | RAM | Disk |
|---|---|---|---|---|---|---|
| {v["vmid"]} | {v["type"]} | {v["name"]} | {v["status"]} | {v["cpu"]} | {v["ram"]} | {v["disk"]} |
| Pool | Used | Total | % | Type |
|---|---|---|---|---|
| {s["name"]} | {s["used"]} | {s["total"]} | {s["pct"]:.0f}% | {s["type"]} |
Reclaimable Docker space: {d["reclaimable"]}
' + html += '| Container | Status |
|---|---|
| {c["name"]} | {c["status"][:50]} |
| Agent | Platform | CT | Gateway | Zulip | Processed |
|---|---|---|---|---|---|
| {name} | {agent["platform"]} | {agent["ct"]} | {gateway} | {zulip_state} | {processed} |
| Check | Status |
|---|---|
| {name} | {val} |
| Service | Status |
|---|---|
| {ep["name"]} | HTTP {ep["code"]} |
| Mount | Size | Used | Avail | % |
|---|---|---|---|---|
| {m["mount"]} | {m["size"]} | {m["used"]} | {m.get("avail","?")} | {pct_str} |
Syslog Infrastructure Monitor v2.0 · Generated ' + TIME_STR + '
' + html += '' + return html + + +# ── Send Email ── + +def send_email(html_content, subject_prefix=""): + FROM = "abiba@sysloggh.com" + TO = "jerome@sysloggh.com" + SUBJECT = f"{subject_prefix}{'🏗️ Infrastructure Report — ' + DATE_STR}" + + msg = MIMEMultipart("alternative") + msg["From"] = FROM + msg["To"] = TO + msg["Subject"] = SUBJECT + msg.attach(MIMEText("Infrastructure report in HTML format — enable images to view.", "plain")) + msg.attach(MIMEText(html_content, "html")) + + try: + EMAIL_PASSWORD = "rgbuomwcydxwbszd" + GMAIL_EMAIL = "jtabiri@gmail.com" + + server = smtplib.SMTP("smtp.gmail.com", 587) + server.starttls() + server.login(GMAIL_EMAIL, EMAIL_PASSWORD) + server.sendmail(FROM, [TO], msg.as_string()) + server.quit() + return True, "✅ Email sent to jerome@sysloggh.com" + except Exception as e: + return False, f"❌ Email failed: {e}" + + +# ── Main ── + +if __name__ == "__main__": + is_test = "--test-email" in sys.argv + + print(f"{'🧪 TEST MODE' if is_test else '📊'} Collecting infrastructure data...") + report = collect() + + if "--json" in sys.argv: + print(json.dumps(report, indent=2, default=str)) + sys.exit(0) + + print(" Building dashboard...") + html = build_html(report) + + if is_test: + prefix = "🧪 TEST — " + print(" Sending test email...") + else: + prefix = "" + print(" Sending email...") + + ok, msg = send_email(html, subject_prefix=prefix) + print(f" {msg}") + + # Show summary + issues = sum(1 for i in ["red"] if report.get("zulip_ext", {}).get("connected") == False) + print(f"\n📋 Summary:") + print(f" Proxmox: {report['nodes_online']}/{report['node_count']} nodes online") + print(f" VMs/CTs: {report['running_vms']}/{report['total_vms']} running") + print(f" Zulip Ext: {'✅' if report.get('zulip_ext',{}).get('connected') else '❌'}") + print(f" LiteLLM: {sum(1 for c in report.get('litellm',{}).get('checks',[]) if c['status']=='pass')}/{len(report.get('litellm',{}).get('checks',[]))} checks pass") + agent_parts = [] +for k,v in report.get('agents',{}).items(): + agent_parts.append(f"{k}:{v.get('gateway_state',v.get('pm2_status','?'))}") +print(f" Agents: {', '.join(agent_parts)}") diff --git a/scripts/zulip-monitor.sh b/scripts/zulip-monitor.sh new file mode 100755 index 0000000..0d77d80 --- /dev/null +++ b/scripts/zulip-monitor.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# /root/scripts/zulip-monitor.sh — Zulip Mesh Health Monitor +# Implements zulip-health.prose.md v2 +# Runs every 15 min via cron. Alerts via Telegram. +set -euo pipefail + +ZULIP_SITE="https://chat.sysloggh.net" +ZULIP_EMAIL="abiba-bot@chat.sysloggh.net" +ZULIP_KEY="cKTDMZAPW08dk3zl05sStzO7HRztzyn8" +OWNER_ZULIP_ID="9" + +# Email config +GMAIL_USER="jtabiri@gmail.com" +GMAIL_PASS="rgbuomwcydxwbszd" +EMAIL_TO="jerome@sysloggh.com" + +notify() { + local severity="$1" msg="$2" + echo "[$severity] $msg" + + # Zulip DM to owner + local content="${severity} Zulip Monitor: ${msg}" + local form="type=private&to=%5B${OWNER_ZULIP_ID}%5D&content=$(python3 -c "import urllib.parse; print(urllib.parse.quote('''${content}'''))")" + curl -sf -X POST "${ZULIP_SITE}/api/v1/messages" \ + -u "${ZULIP_EMAIL}:${ZULIP_KEY}" \ + -d "${form}" > /dev/null 2>&1 || true + + # Email alert + local subject="${severity} Zulip Monitor Alert" + python3 -c " +import smtplib +from email.mime.text import MIMEText +m = MIMEText('''${msg}''') +m['From'] = 'abiba@sysloggh.com' +m['To'] = '${EMAIL_TO}' +m['Subject'] = '${subject}' +s = smtplib.SMTP('smtp.gmail.com', 587) +s.starttls() +s.login('${GMAIL_USER}', '${GMAIL_PASS}') +s.sendmail('abiba@sysloggh.com', ['${EMAIL_TO}'], m.as_string()) +s.quit() +" 2>/dev/null || true +} + +TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC') +ISSUES=0 +LOG="/root/zulip-health-monitor.log" + +echo "=== Zulip Health Check — $TIMESTAMP ===" >> "$LOG" + +# ── Global: Zulip Server ── +SERVER_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 \ + https://chat.sysloggh.net/api/v1/server_settings \ + -u 'abiba-bot@chat.sysloggh.net:cKTDMZAPW08dk3zl05sStzO7HRztzyn8' 2>/dev/null || echo "000") +if [ "$SERVER_CODE" != "200" ]; then + notify "🔴" "Zulip server returned HTTP $SERVER_CODE" + ISSUES=$((ISSUES + 1)) +else + echo " Server: ✅ HTTP 200" >> "$LOG" +fi + +# ── Platform A: pi (Abiba) ── +PI_HEALTH=$(curl -sf --connect-timeout 5 http://localhost:9200/health 2>/dev/null || echo "{}") +PI_CONNECTED=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('connected',False))" 2>/dev/null) +PI_ERROR=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('last_error') or '')" 2>/dev/null) +PI_RETRIES=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('retry_count',0))" 2>/dev/null) + +if [ "$PI_CONNECTED" != "True" ]; then + notify "🔴" "Abiba pi extension DISCONNECTED — restarting" + pm2 restart abiba-zulip 2>/dev/null || true + ISSUES=$((ISSUES + 1)) + echo " Abiba: ❌ Disconnected — restarted" >> "$LOG" +elif [ -n "$PI_ERROR" ]; then + notify "🟡" "Abiba pi extension error: ${PI_ERROR:0:100}" + echo " Abiba: 🟡 Error: ${PI_ERROR:0:100}" >> "$LOG" +elif [ "$PI_RETRIES" -ge 3 ]; then + notify "🟡" "Abiba pi extension: $PI_RETRIES retries — restarting" + pm2 restart abiba-zulip 2>/dev/null || true + echo " Abiba: 🟡 $PI_RETRIES retries — restarted" >> "$LOG" +else + echo " Abiba: ✅ Connected (processed=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('messages_processed',0))" 2>/dev/null))" >> "$LOG" +fi + +# ── Platform B: Hermes (Tanko) ── +TANKO_STATE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 jerome@192.168.68.122 \ + "cat ~/.hermes/gateway_state.json 2>/dev/null" 2>/dev/null || echo "{}") +TANKO_ZULIP=$(echo "$TANKO_STATE" | python3 -c " +import sys,json +d=json.load(sys.stdin) +p=d.get('platforms',{}).get('zulip',{}) +print(p.get('state','unknown')) +" 2>/dev/null) + +if [ "$TANKO_ZULIP" != "connected" ]; then + notify "🔴" "Tanko (Hermes) Zulip state: $TANKO_ZULIP — needs restart" + ISSUES=$((ISSUES + 1)) + echo " Tanko: ❌ state=$TANKO_ZULIP" >> "$LOG" +else + echo " Tanko: ✅ Zulip connected" >> "$LOG" +fi + +# ── Platform B: Hermes (Mumuni) ── +MUMUNI_STATE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.123 \ + "cat ~/.hermes/gateway_state.json 2>/dev/null" 2>/dev/null || echo "{}") +MUMUNI_ZULIP=$(echo "$MUMUNI_STATE" | python3 -c " +import sys,json +d=json.load(sys.stdin) +p=d.get('platforms',{}).get('zulip',{}) +print(p.get('state','unknown')) +" 2>/dev/null) + +if [ "$MUMUNI_ZULIP" != "connected" ]; then + notify "🔴" "Mumuni (Hermes) Zulip state: $MUMUNI_ZULIP" + ISSUES=$((ISSUES + 1)) + echo " Mumuni: ❌ state=$MUMUNI_ZULIP" >> "$LOG" +else + echo " Mumuni: ✅ Zulip connected" >> "$LOG" +fi + +# ── Platform C: Agent Zero (kagentz) ── +AZ_A2A=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ + "docker exec agent-zero curl -s --connect-timeout 5 http://127.0.0.1:8001/.well-known/agent.json 2>/dev/null" 2>/dev/null || echo "") +AZ_ALIVE=$(echo "$AZ_A2A" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('name',''))" 2>/dev/null) + +if [ "$AZ_ALIVE" != "kagentz" ]; then + notify "🔴" "kagentz A2A server DOWN — restarting" + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ + "docker exec agent-zero bash -c 'pkill -9 -f a2a_agent; sleep 1; cd /a0 && /opt/venv-a0/bin/python3 -u /a0/usr/a2a_agent.py > /tmp/a2a.log 2>&1 &'" 2>/dev/null || true + ISSUES=$((ISSUES + 1)) + echo " kagentz: ❌ A2A down — restarted" >> "$LOG" +else + echo " kagentz: ✅ A2A alive" >> "$LOG" + + # Check adapter process + AZ_ADAPTER=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ + "docker exec agent-zero ps aux 2>/dev/null | grep adapter | grep -v grep | wc -l" 2>/dev/null || echo "0") + if [ "$AZ_ADAPTER" -lt 1 ]; then + notify "🔴" "kagentz Zulip adapter DOWN — restarting" + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \ + "docker exec agent-zero bash -c 'cd /a0/usr/kagentz-zulip && ZULIP_SITE=https://chat.sysloggh.net ZULIP_EMAIL=kagentz-bot@chat.sysloggh.net ZULIP_API_KEY=E9q9PXJTxftPYBkb5pBDWupDO7KK21ty ZULIP_AGENT_NAME=kagentz A2A_URL=http://localhost:8001/a2a A2A_TOKEN=8zNgdOEXzYxjQvTl /opt/venv-a0/bin/python3 -u adapter.py > /tmp/zulip-adapter.log 2>&1 &'" 2>/dev/null || true + ISSUES=$((ISSUES + 1)) + echo " kagentz: ❌ Adapter down — restarted" >> "$LOG" + else + echo " kagentz: ✅ Adapter running" >> "$LOG" + fi +fi + +# ── Summary ── +if [ "$ISSUES" -eq 0 ]; then + echo " Result: ✅ All healthy" >> "$LOG" +else + echo " Result: 🔴 $ISSUES issue(s) found" >> "$LOG" + notify "🔴" "$ISSUES issue(s) found — check /root/zulip-health-monitor.log" +fi + +echo "" >> "$LOG"