feat(zulip-health): monitor script + infra report — cron-ready implementations of prose contracts

scripts/daily-infra-report.py

@@ -0,0 +1,708 @@
+#!/usr/bin/env python3
+"""
+/root/scripts/daily-infra-report.py — v2.0.0
+Daily infrastructure dashboard emailed to jerome@sysloggh.com
+Runs at 6:00 AM daily via cron.
+Integrated with prose contracts: infrastructure-control, litellm-health, zulip-health.
+
+Usage:
+  python3 daily-infra-report.py              # Generate and email report
+  python3 daily-infra-report.py --test-email  # Send a test email for review
+  python3 daily-infra-report.py --json        # Output raw JSON to stdout
+"""
+
+import smtplib, json, subprocess, os, sys, datetime, re
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+
+PVE = "https://minipve.sysloggh.net"
+AUTH = "Authorization: PVEAPIToken=monitoring@pve!mumuni=eafd56c5-93d4-4d40-a41d-e688be0987f3"
+
+# ── Shared credentials —─
+
+ZULIP_SITE = "https://chat.sysloggh.net"
+ZULIP_EMAIL = "abiba-bot@chat.sysloggh.net"
+ZULIP_KEY = "cKTDMZAPW08dk3zl05sStzO7HRztzyn8"
+ZULIP_AUTH = f"{ZULIP_EMAIL}:{ZULIP_KEY}"
+
+LITELLM_PUBLIC = "https://litellm.sysloggh.net"
+LITELLM_BACKEND = "192.168.68.116"
+AUTH_HOST = "192.168.68.11"
+
+SYNTHETIC_API_KEY = "sk-U_ydi3B-wfGU-_xESkoU1Q"
+
+NOW = datetime.datetime.now()
+DATE_STR = NOW.strftime("%Y-%m-%d")
+TIME_STR = NOW.strftime("%Y-%m-%d %H:%M UTC")
+
+# ── Helpers ──
+
+def pve_get(path):
+    cmd = f'curl -sfk --connect-timeout 10 "{PVE}{path}" -H "{AUTH}"'
+    try:
+        return json.loads(subprocess.check_output(cmd, shell=True))["data"]
+    except: return []
+
+def ssh(host, cmd):
+    try:
+        return subprocess.check_output(
+            f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@{host} "{cmd}"',
+            shell=True, stderr=subprocess.DEVNULL).decode().strip()
+    except: return ""
+
+def ssh_jerome(host, cmd):
+    try:
+        return subprocess.check_output(
+            f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 jerome@{host} "{cmd}"',
+            shell=True, stderr=subprocess.DEVNULL).decode().strip()
+    except: return ""
+
+def http_get(url, auth=None, timeout=10):
+    try:
+        cmd = f'curl -sfk --connect-timeout {timeout} -o /dev/null -w "%{{http_code}}" "{url}"'
+        if auth:
+            cmd = cmd.replace('"', '\\"')
+            cmd = f'curl -sfk --connect-timeout {timeout} -u "{auth}" -o /dev/null -w "%{{http_code}}" "{url}"'
+        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout+2)
+        return r.stdout.strip() or "000"
+    except:
+        return "timeout"
+
+def http_get_body(url, auth=None, timeout=10):
+    try:
+        cmd = f'curl -sfk --connect-timeout {timeout} "{url}"'
+        if auth:
+            cmd = f'curl -sfk --connect-timeout {timeout} -u "{auth}" "{url}"'
+        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout+2)
+        if r.returncode == 0:
+            return r.stdout.strip()
+        return ""
+    except:
+        return ""
+
+def count_in_log(filepath, pattern, minutes=15):
+    """Count occurrences of a pattern in a log file within the last N minutes."""
+    try:
+        since = (NOW - datetime.timedelta(minutes=minutes)).strftime("%Y-%m-%d %H:%M")
+        cmd = f"""grep -a "{pattern}" {filepath} 2>/dev/null | awk '$0 >= "{since}"' | wc -l"""
+        count = subprocess.check_output(cmd, shell=True).decode().strip()
+        return int(count)
+    except:
+        return 0
+
+# ── Data Collection ──
+
+def collect():
+    report = {}
+    
+    # ── Proxmox Nodes ──
+    nodes = pve_get("/api2/json/nodes")
+    report["nodes"] = {n["node"]: {
+        "cpu_pct": round(n.get('cpu',0)*100, 1),
+        "ram": f"{n.get('mem',0)//1024//1024}/{n.get('maxmem',0)//1024//1024}MB",
+        "ram_pct": round(n.get('mem',0)/n.get('maxmem',1)*100, 0),
+        "disk": f"{n.get('disk',0)//1024//1024//1024}/{n.get('maxdisk',0)//1024//1024//1024}GB",
+        "disk_pct": round(n.get('disk',0)/n.get('maxdisk',1)*100, 0),
+        "uptime_h": n.get('uptime',0)//3600,
+        "status": n["status"]
+    } for n in nodes}
+    report["node_count"] = len(nodes)
+    report["nodes_online"] = sum(1 for n in nodes if n["status"] == "online")
+    
+    # ── VMs/CTs ──
+    resources = pve_get("/api2/json/cluster/resources")
+    vms = [r for r in resources if r.get("type") in ("qemu","lxc")]
+    report["total_vms"] = len(vms)
+    report["running_vms"] = sum(1 for v in vms if v.get("status") == "running")
+    stopped = [v for v in vms if v.get("status") != "running"]
+    report["stopped_vms"] = [f"{v['type']} {v['vmid']} {v.get('name','?')}" for v in stopped]
+    
+    report["vms_by_node"] = {}
+    for n in ["amdpve", "minipve", "storepve", "acerpve", "ocupve"]:
+        node_vms = [v for v in vms if v.get("node") == n]
+        node_vms.sort(key=lambda x: int(x.get("vmid",0)))
+        report["vms_by_node"][n] = [{
+            "type": v.get("type"), "vmid": v.get("vmid"), "name": v.get("name","?"),
+            "status": v.get("status"),
+            "ram": f"{v.get('mem',0)//1024//1024}/{v.get('maxmem',0)//1024//1024}MB",
+            "disk": f"{v.get('disk',0)//1024//1024//1024}/{v.get('maxdisk',0)//1024//1024//1024}GB",
+            "cpu": v.get("cpus", v.get("maxcpu","?")),
+        } for v in node_vms]
+    
+    # ── Storage ──
+    storages = pve_get("/api2/json/nodes/storepve/storage")
+    report["storage"] = []
+    for s in storages:
+        total = s.get("total",0) or 1
+        used = s.get("used",0)
+        pct = used/total*100
+        report["storage"].append({
+            "name": s.get("storage","?"), "used": f"{used//1024//1024//1024}GB",
+            "total": f"{total//1024//1024//1024}GB", "pct": round(pct, 0), "type": s.get("type","")
+        })
+    
+    # ── Docker: docker-vm (.7) ──
+    docker_raw = ssh("192.168.68.7", "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'")
+    containers = []
+    for line in docker_raw.strip().split("\n"):
+        if "|" in line:
+            parts = line.split("|", 2)
+            containers.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""})
+    report["docker_vm"] = {"total": len(containers), "running": sum(1 for c in containers if "Up" in c["status"]),
+        "unhealthy": [c for c in containers if "unhealthy" in c["status"]], "containers": containers}
+    report["docker_vm"]["reclaimable"] = ssh("192.168.68.7", "docker system df 2>/dev/null | tail -1 | awk '{print $5}'")
+    report["docker_vm"]["disk_used"] = ssh("192.168.68.7", "df -h / | tail -1 | awk '{print $5}'")
+    
+    # ── Docker: CT 116 (syslog-api — LiteLLM stack) ──
+    docker2_raw = ssh(LITELLM_BACKEND, "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'")
+    containers2 = []
+    for line in docker2_raw.strip().split("\n"):
+        if "|" in line:
+            parts = line.split("|", 2)
+            containers2.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""})
+    report["docker_syslog"] = {"total": len(containers2), "running": sum(1 for c in containers2 if "Up" in c["status"]),
+        "containers": containers2}
+    
+    # ── Docker: Netbird (72.61.0.17) ──
+    docker3_raw = ssh("72.61.0.17", "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'")
+    containers3 = []
+    for line in docker3_raw.strip().split("\n"):
+        if "|" in line:
+            parts = line.split("|", 2)
+            containers3.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""})
+    report["docker_netbird"] = {"total": len(containers3), "running": sum(1 for c in containers3 if "Up" in c["status"]),
+        "containers": containers3}
+    
+    # ── Public Endpoints ──
+    endpoints = [
+        ("LiteLLM Admin UI", f"{LITELLM_PUBLIC}/ui/"),
+        ("LiteLLM Swagger", f"{LITELLM_PUBLIC}/docs"),
+        ("LiteLLM ReDoc", f"{LITELLM_PUBLIC}/redoc"),
+        ("LiteLLM OpenAPI", f"{LITELLM_PUBLIC}/openapi.json"),
+        ("Gitea", "https://git.sysloggh.net"),
+        ("Authentik", "https://auth.sysloggh.net"),
+        ("Zulip", "https://chat.sysloggh.net"),
+        ("Pulse", "https://pulse.sysloggh.net"),
+        ("Proxmox", "https://minipve.sysloggh.net"),
+        ("SearXNG", "http://192.168.68.7:8888"),
+        ("Firecrawl", "http://192.168.68.7:3002/health"),
+    ]
+    report["endpoints"] = []
+    for name, url in endpoints:
+        code = http_get(url)
+        report["endpoints"].append({"name": name, "code": code})
+    
+    # ── LiteLLM Specific Checks (from litellm-health prose contract) ──
+    report["litellm"] = {"checks": []}
+    
+    # Check 1: LiteLLM aggregate health endpoint (router binds to 127.0.0.1, check via SSH)
+    health_unified = ssh(LITELLM_BACKEND, "curl -sf http://127.0.0.1:9000/health/unified -o /dev/null -w '%{http_code}' 2>/dev/null")
+    report["litellm"]["health_unified"] = health_unified or "000"
+    report["litellm"]["checks"].append({"name": "unified-health", "status": "pass" if health_unified == "200" else "fail", "code": health_unified or "000"})
+    
+    # Check 2: Nginx-proxied internal endpoints
+    for path, name in [("/litellm/ui/", "nginx-ui"), ("/litellm/docs", "nginx-docs")]:
+        code = http_get(f"http://{LITELLM_BACKEND}{path}")
+        report["litellm"]["checks"].append({"name": name, "status": "pass" if code == "200" else "fail", "code": code})
+    
+    # Check 3: Docker container health for LiteLLM stack
+    expected_containers = ["harness-litellm", "harness-nginx", "harness-router",
+                          "harness-postgres", "harness-redis", "harness-dashboard"]
+    actual_names = [c["name"] for c in containers2]
+    report["litellm"]["expected_containers"] = expected_containers
+    report["litellm"]["missing_containers"] = [e for e in expected_containers if e not in actual_names]
+    report["litellm"]["checks"].append({
+        "name": "container-health",
+        "status": "pass" if not report["litellm"]["missing_containers"] else "fail",
+        "missing": report["litellm"]["missing_containers"],
+        "healthy": sum(1 for c in containers2 if "healthy" in c.get("status","")),
+        "total": len(expected_containers)
+    })
+    
+    # Check 4: OIDC Auth endpoint
+    auth_code = http_get(f"https://auth.sysloggh.net")
+    report["litellm"]["auth_status"] = auth_code
+    report["litellm"]["checks"].append({"name": "oidc-auth", "status": "pass" if auth_code in ("200","302") else "fail", "code": auth_code})
+    
+    # Check 5: Synthetic API call through LiteLLM
+    api_check = http_get(f"{LITELLM_PUBLIC}/v1/models", auth=SYNTHETIC_API_KEY)
+    report["litellm"]["api_models"] = api_check
+    report["litellm"]["checks"].append({"name": "api-endpoint", "status": "pass" if api_check == "200" else "fail", "code": api_check})
+    
+    # ── NFS Mounts ──
+    nfs = ssh("192.168.68.7", "df -h /media/storage /media/mediastore 2>/dev/null | tail -n +2")
+    report["nfs"] = []
+    for line in nfs.strip().split("\n"):
+        parts = line.split()
+        if len(parts) >= 6:
+            report["nfs"].append({"mount": parts[5], "size": parts[1], "used": parts[2], "pct": parts[4], "avail": parts[3]})
+    
+    # ── Zulip Extension Health (from zulip-health prose contract) ──
+    report["zulip_ext"] = {}
+    
+    # Phase 1: Health endpoint
+    health_body = http_get_body("http://localhost:9200/health")
+    zulip_health = {}
+    try:
+        zulip_health = json.loads(health_body) if health_body else {}
+    except:
+        zulip_health = {}
+    report["zulip_ext"]["connected"] = zulip_health.get("connected", False)
+    report["zulip_ext"]["queue_id"] = zulip_health.get("queue_id")
+    report["zulip_ext"]["last_error"] = zulip_health.get("last_error")
+    report["zulip_ext"]["messages_processed"] = zulip_health.get("messages_processed", 0)
+    report["zulip_ext"]["retry_count"] = zulip_health.get("retry_count", 0)
+    
+    # Phase 2: PM2 process check
+    pm2_raw = subprocess.check_output(
+        "pm2 show abiba-zulip --no-color 2>/dev/null | grep -E 'status|restarts|uptime'",
+        shell=True).decode().strip()
+    pm2 = {}
+    for line in pm2_raw.split("\n"):
+        if "│" in line:
+            parts = line.split("│")
+            if len(parts) >= 3:
+                key = parts[1].strip()
+                val = parts[2].strip()
+                pm2[key] = val
+    report["zulip_ext"]["pm2"] = pm2
+    report["zulip_ext"]["pm2_healthy"] = pm2.get("status") == "online"
+    
+    # Phase 3: Echo loop detection
+    skipped_count = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Skipped.*bot msgs", 15)
+    report["zulip_ext"]["bot_skipped_15min"] = skipped_count
+    
+    # Phase 4: Response delivery stats
+    finalized = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Finalized", 60)
+    failed = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Failed to finalize", 60)
+    report["zulip_ext"]["finalized_1h"] = finalized
+    report["zulip_ext"]["failed_finalize_1h"] = failed
+    report["zulip_ext"]["finalize_fail_pct"] = round(failed / (finalized + failed) * 100, 0) if (finalized + failed) > 0 else 0
+    
+    # Phase 5: Zulip server liveness
+    zulip_server_code = http_get(f"{ZULIP_SITE}/api/v1/server_settings", auth=ZULIP_AUTH)
+    report["zulip_ext"]["server_status"] = zulip_server_code
+    
+    # ── Agent Status ──
+    report["agents"] = {}
+    
+    # Abiba (pi)
+    report["agents"]["abiba"] = {
+        "platform": "pi", "ct": 100, "ip": "192.168.68.24",
+        "zulip_connected": zulip_health.get("connected", False),
+        "zulip_processed": zulip_health.get("messages_processed", 0),
+        "pm2_status": pm2.get("status", "unknown"),
+        "pm2_restarts": pm2.get("restarts", "?"),
+        "pm2_uptime": pm2.get("uptime", "?"),
+    }
+    
+    # Tanko (CT 122)
+    tanko_state = ssh_jerome("192.168.68.122", "cat ~/.hermes/gateway_state.json 2>/dev/null")
+    tanko_data = {}
+    try:
+        tanko_data = json.loads(tanko_state) if tanko_state else {}
+    except:
+        tanko_data = {}
+    platforms = tanko_data.get("platforms", {})
+    report["agents"]["tanko"] = {
+        "platform": "hermes", "ct": 112, "ip": "192.168.68.122",
+        "gateway_state": tanko_data.get("gateway_state", "unknown"),
+        "zulip_state": platforms.get("zulip", {}).get("state", "unknown"),
+        "telegram_state": platforms.get("telegram", {}).get("state", "unknown"),
+        "gateway_pid": tanko_data.get("pid"),
+        "updated_at": tanko_data.get("updated_at"),
+    }
+    
+    # Mumuni (CT 114, IP 192.168.68.123)
+    mumuni_state = ssh("192.168.68.123", "cat ~/.hermes/gateway_state.json 2>/dev/null")
+    mumuni_data = {}
+    try:
+        mumuni_data = json.loads(mumuni_state) if mumuni_state else {}
+    except:
+        mumuni_data = {}
+    mumuni_platforms = mumuni_data.get("platforms", {})
+    report["agents"]["mumuni"] = {
+        "platform": "hermes", "ct": 114, "ip": "192.168.68.123",
+        "gateway_state": mumuni_data.get("gateway_state", "unknown"),
+        "telegram_state": mumuni_platforms.get("telegram", {}).get("state", "unknown"),
+        "zulip_state": mumuni_platforms.get("zulip", {}).get("state", "not_installed"),
+        "email_state": mumuni_platforms.get("email", {}).get("state", "unknown"),
+        "hermes_version": "",
+    }
+    # Get Hermes version
+    ver = ssh("192.168.68.123", "hermes --version 2>/dev/null | head -1")
+    if ver:
+        report["agents"]["mumuni"]["hermes_version"] = ver.split("·")[0].replace("Hermes Agent ","").strip()
+    
+    return report
+
+
+# ── HTML Dashboard ──
+
+def build_html(r):
+    issues = []
+    
+    # Proxmox issues
+    if r["nodes_online"] < r["node_count"]:
+        issues.append(f"🔴 {r['node_count'] - r['nodes_online']} Proxmox node(s) offline")
+    if r["stopped_vms"]:
+        issues.append(f"🔴 {len(r['stopped_vms'])} VM(s)/CT(s) stopped — {', '.join(r['stopped_vms'][:3])}")
+    
+    # Endpoint issues
+    for ep in r["endpoints"]:
+        if ep["code"] in ("000", "timeout"):
+            issues.append(f"🔴 {ep['name']} — unreachable")
+        elif ep["code"] >= "500":
+            issues.append(f"🟡 {ep['name']} — HTTP {ep['code']}")
+    
+    # Docker issues
+    for c in r["docker_vm"].get("unhealthy", []):
+        issues.append(f"🟡 docker-vm: {c['name']} — unhealthy")
+    
+    # Small cluster mode warning
+    if r["docker_vm"].get("total", 0) == 0:
+        issues.append("🟡 docker-vm: no containers reported — host may be down")
+    
+    # LiteLLM issues
+    for check in r.get("litellm", {}).get("checks", []):
+        if check["status"] == "fail":
+            issues.append(f"🔴 LiteLLM: {check['name']} — {'missing: ' + ', '.join(check.get('missing',[])) if check.get('missing') else 'HTTP ' + check.get('code','?')}")
+    
+    # Zulip extension issues
+    z = r.get("zulip_ext", {})
+    if not z.get("connected"):
+        issues.append("🔴 Zulip extension: disconnected")
+    if z.get("last_error"):
+        issues.append(f"🟡 Zulip extension: {z['last_error'][:80]}")
+    if z.get("retry_count", 0) >= 3:
+        issues.append(f"🟡 Zulip extension: {z['retry_count']} retries")
+    if z.get("finalize_fail_pct", 0) > 50:
+        issues.append(f"🔴 Zulip extension: {z['finalize_fail_pct']:.0f}% edit failures")
+    if z.get("bot_skipped_15min", 0) > 100:
+        issues.append(f"🟡 Zulip echo loop: {z['bot_skipped_15min']} bot msgs skipped in 15min")
+    if z.get("server_status") and z["server_status"] not in ("200", "302"):
+        issues.append(f"🔴 Zulip server: HTTP {z['server_status']}")
+    
+    # Agent issues
+    def agent_gateway(agent):
+        return agent.get("gateway_state") or agent.get("pm2_status") or "unknown"
+    for name, agent in r.get("agents", {}).items():
+        gs = agent_gateway(agent)
+        if gs == "unknown":
+            issues.append(f"🟡 {name}: gateway state unknown (unreachable)")
+        elif gs != "running" and gs != "online":
+            issues.append(f"🔴 {name}: gateway {gs}")
+    
+    status = "🟢 All Healthy" if not issues else f"{'🔴' if sum(1 for i in issues if '🔴' in i) > 0 else '🟡'} {len(issues)} Issue(s)"
+    
+    # ── Build HTML ──
+    html = f"""<!DOCTYPE html>
+<html><head><meta charset="utf-8"><style>
+body {{ font-family: -apple-system, BlinkMacSystemFont, sans-serif; background: #0d1117; color: #c9d1d9; padding: 20px; }}
+h1 {{ color: #58a6ff; }}
+h2 {{ color: #8b949e; border-bottom: 1px solid #30363d; padding-bottom: 8px; margin-top: 24px; }}
+h3 {{ color: #c9d1d9; margin: 12px 0 6px 0; }}
+.card {{ background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; margin: 12px 0; }}
+.grid {{ display: flex; gap: 12px; flex-wrap: wrap; }}
+.stat {{ flex: 1; min-width: 140px; background: #21262d; border-radius: 6px; padding: 12px; text-align: center; }}
+.stat .num {{ font-size: 28px; font-weight: bold; color: #58a6ff; }}
+.stat .label {{ font-size: 12px; color: #8b949e; }}
+table {{ width: 100%; border-collapse: collapse; }}
+td, th {{ padding: 6px 12px; border-bottom: 1px solid #21262d; text-align: left; font-size: 13px; }}
+th {{ color: #8b949e; font-weight: normal; }}
+.green {{ color: #3fb950; }}
+.red {{ color: #f85149; }}
+.yellow {{ color: #d29922; }}
+.alert {{ border-radius: 8px; padding: 12px; margin: 12px 0; }}
+.good {{ background: #1f3d1f; border: 1px solid #3fb950; }}
+.bad {{ background: #3d1f1f; border: 1px solid #f85149; }}
+.warn {{ background: #3d3d1f; border: 1px solid #d29922; }}
+</style></head><body>
+<h1>🏗️ Syslog Infrastructure Report</h1>
+<p style="color:#8b949e">{TIME_STR}</p>
+
+<div class="alert {'good' if not issues else 'bad' if any('🔴' in i for i in issues) else 'warn'}">
+<p style="margin:0;font-size:16px"><b>{status}</b></p>
+<p style="margin:4px 0 0 0;font-size:13px">
+{r['node_count']} PVE nodes · {r['total_vms']} VMs/CTs · {r['running_vms']} running · 
+{r['docker_vm']['total'] + r['docker_syslog']['total'] + r['docker_netbird']['total']} containers · 
+{len(r['endpoints'])} endpoints · {len(r.get('agents',{}))} agents
+</p>
+</div>
+"""
+    
+    # Issues list
+    if issues:
+        html += '<div class="card"><h2>🚨 Issues ({})</h2>'.format(len(issues))
+        for i in issues:
+            html += f'<p style="margin:4px 0;font-size:13px">{i}</p>'
+        html += '</div>'
+    
+    # ── Quick Stats ──
+    html += '<div class="card"><h2>📊 Quick Stats</h2><div class="grid">'
+    stats = [
+        ("PVE Nodes", f"{r['nodes_online']}/{r['node_count']}", "green" if r['nodes_online'] == r['node_count'] else "red"),
+        ("VMs/CTs", f"{r['running_vms']}/{r['total_vms']}", "green" if r['running_vms'] == r['total_vms'] else "red"),
+        ("Containers", f"{r['docker_vm']['running']}/{r['docker_vm']['total']}", "green" if r['docker_vm']['running'] == r['docker_vm']['total'] else "yellow"),
+        ("LiteLLM Ctrs", f"{r['docker_syslog']['running']}/{r['docker_syslog']['total']}", "green" if r['docker_syslog']['running'] == r['docker_syslog']['total'] else "red"),
+        ("Netbird Ctrs", f"{r['docker_netbird']['running']}/{r['docker_netbird']['total']}", "green" if r['docker_netbird']['running'] == r['docker_netbird']['total'] else "red"),
+        ("Endpoints", f"{sum(1 for e in r['endpoints'] if e['code'] in ('200','302','401'))}/{len(r['endpoints'])}", "green",),
+        ("Zulip Ext", "✅ Connected" if r.get('zulip_ext',{}).get('connected') else "❌ Disconnected", "green" if r.get('zulip_ext',{}).get('connected') else "red"),
+    ]
+    for label, val, color in stats:
+        html += f'<div class="stat"><div class="num {color}">{val}</div><div class="label">{label}</div></div>'
+    
+    # VLM/GPU capacity
+    gpu_nodes = [v for vms in r["vms_by_node"].values() for v in vms if "llm" in v["name"].lower() or "gpu" in v["name"].lower() or "ocu" in v["name"].lower()]
+    gpu_running = sum(1 for g in gpu_nodes if g["status"] == "running")
+    html += f'<div class="stat"><div class="num {"green" if gpu_running == len(gpu_nodes) else "yellow"}">{gpu_running}/{len(gpu_nodes)}</div><div class="label">GPU VMs</div></div>'
+    html += '</div></div>'
+    
+    # ── LiteLLM Section ──
+    html += '<div class="card"><h2>⚡ LiteLLM Inference Stack</h2>'
+    
+    # Endpoint status table
+    html += '<table><tr><th>Check</th><th>Status</th></tr>'
+    for check in r.get("litellm", {}).get("checks", []):
+        color = "green" if check["status"] == "pass" else "red"
+        detail = check.get("code", "")
+        if check.get("missing"):
+            detail = f"missing: {', '.join(check['missing'])}"
+        elif check.get("healthy") is not None:
+            detail = f"{check['healthy']}/{check['total']} healthy"
+        html += f'<tr><td>{check["name"]}</td><td class="{color}">{detail}</td></tr>'
+    html += '</table>'
+    
+    # LiteLLM container detail
+    html += '<h3>Containers (CT 116)</h3><table><tr><th>Container</th><th>Status</th><th>Image</th></tr>'
+    for c in r["docker_syslog"]["containers"]:
+        color = "green" if "Up" in c["status"] and "healthy" in c["status"] else ("yellow" if "Up" in c["status"] else "red")
+        html += f'<tr><td>{c["name"]}</td><td class="{color}">{c["status"]}</td><td style="font-size:11px;color:#8b949e">{c["image"]}</td></tr>'
+    html += '</table>'
+    html += '</div>'
+    
+    # ── Proxmox Nodes ──
+    html += '<div class="card"><h2>📦 Proxmox Cluster</h2><div class="grid">'
+    for name, nd in sorted(r["nodes"].items()):
+        cpu_color = "green" if nd["cpu_pct"] < 50 else ("yellow" if nd["cpu_pct"] < 80 else "red")
+        ram_color = "green" if nd["ram_pct"] < 70 else ("yellow" if nd["ram_pct"] < 85 else "red")
+        disk_color = "green" if nd["disk_pct"] < 70 else ("yellow" if nd["disk_pct"] < 85 else "red")
+        html += f'<div class="stat"><div class="num {cpu_color}">{nd["cpu_pct"]}%</div><div class="label"><b>{name}</b><br>CPU: {nd["cpu_pct"]}%<br>RAM: <span class="{ram_color}">{nd["ram_pct"]:.0f}%</span><br>DISK: <span class="{disk_color}">{nd["disk_pct"]:.0f}%</span><br>⬆ {nd["uptime_h"]}h</div></div>'
+    html += '</div>'
+    if r["stopped_vms"]:
+        html += f'<p class="red" style="font-size:13px">Stopped: {", ".join(r["stopped_vms"])}</p>'
+    html += '</div>'
+    
+    # ── VM/CT Lists per Node ──
+    for node_name in ["amdpve", "minipve", "storepve", "acerpve", "ocupve"]:
+        node_vms = r["vms_by_node"].get(node_name, [])
+        if not node_vms: continue
+        stopped_count = sum(1 for v in node_vms if v["status"] != "running")
+        badge = f'<span class="green">{len(node_vms)-stopped_count} running</span>'
+        if stopped_count:
+            badge += f' <span class="red">{stopped_count} stopped</span>'
+        html += f'<div class="card"><h3>🔹 {node_name}</h3><p style="font-size:12px;color:#8b949e">{badge}</p>'
+        html += '<table><tr><th>ID</th><th>Type</th><th>Name</th><th>Status</th><th>CPU</th><th>RAM</th><th>Disk</th></tr>'
+        for v in node_vms:
+            color = "green" if v["status"] == "running" else "red"
+            html += f'<tr><td>{v["vmid"]}</td><td>{v["type"]}</td><td>{v["name"]}</td><td class="{color}">{v["status"]}</td><td>{v["cpu"]}</td><td>{v["ram"]}</td><td>{v["disk"]}</td></tr>'
+        html += '</table></div>'
+    
+    # ── Storage ──
+    html += '<div class="card"><h2>💾 Storage</h2><table><tr><th>Pool</th><th>Used</th><th>Total</th><th>%</th><th>Type</th></tr>'
+    for s in sorted(r["storage"], key=lambda x: -x["pct"]):
+        color = "green" if s["pct"] < 70 else ("yellow" if s["pct"] < 85 else "red")
+        html += f'<tr><td>{s["name"]}</td><td>{s["used"]}</td><td>{s["total"]}</td><td class="{color}">{s["pct"]:.0f}%</td><td>{s["type"]}</td></tr>'
+    html += '</table></div>'
+    
+    # ── Docker Ecosystems ──
+    html += '<div class="card"><h2>🐳 Docker Ecosystems</h2>'
+    for host, key, label in [
+        ("docker-vm (.7) — storepve", "docker_vm", f"Disk: {r['docker_vm'].get('disk_used','?')}"),
+        ("syslog-api (.116) — minipve (LiteLLM)", "docker_syslog", ""),
+        ("Netbird (72.61.0.17)", "docker_netbird", ""),
+    ]:
+        d = r[key]
+        html += f'<h3>{host} — {d["running"]}/{d["total"]} running {label}</h3>'
+        if d.get("reclaimable"):
+            html += f'<p style="font-size:12px;color:#8b949e">Reclaimable Docker space: {d["reclaimable"]}</p>'
+        html += '<table><tr><th>Container</th><th>Status</th></tr>'
+        for c in sorted(d["containers"], key=lambda x: x["name"]):
+            color = "green" if "Up" in c["status"] and "healthy" in c["status"] else ("yellow" if "Up" in c["status"] else "red")
+            html += f'<tr><td>{c["name"]}</td><td class="{color}">{c["status"][:50]}</td></tr>'
+        html += '</table>'
+    html += '</div>'
+    
+    # ── Agent Status ──
+    html += '<div class="card"><h2>🤖 Agent Status</h2><table><tr><th>Agent</th><th>Platform</th><th>CT</th><th>Gateway</th><th>Zulip</th><th>Processed</th></tr>'
+    for name, agent in sorted(r.get("agents", {}).items()):
+        if name == "abiba":
+            zulip_state = "✅" if agent.get("zulip_connected") else "❌"
+            gateway = f"pm2:{agent.get('pm2_status','?')} (↺{agent.get('pm2_restarts','?')})"
+            processed = f"{agent.get('zulip_processed', 0)} msgs"
+        elif name == "tanko":
+            zulip_state = "✅" if agent.get("zulip_state") == "connected" else ("❌" if agent.get("zulip_state") == "disconnected" else "⬜")
+            gateway = agent.get("gateway_state", "?")
+            processed = agent.get("updated_at", "")[:10]
+        elif name == "mumuni":
+            zulip_state = "⬜" if agent.get("zulip_state") == "not_installed" else ("✅" if agent.get("zulip_state") == "connected" else "⬜")
+            gateway = agent.get("gateway_state", "?")
+            tg = "✅" if agent.get("telegram_state") == "connected" else "❌"
+            ver = agent.get("hermes_version", "")
+            processed = f"TG:{tg} v{ver}"
+        else:
+            zulip_state = "⬜"
+            gateway = agent.get("gateway_state", "?")
+            processed = ""
+        html += f'<tr><td><b>{name}</b></td><td>{agent["platform"]}</td><td>{agent["ct"]}</td><td>{gateway}</td><td>{zulip_state}</td><td style="font-size:12px">{processed}</td></tr>'
+    html += '</table></div>'
+    
+    # ── Zulip Extension Health ──
+    html += '<div class="card"><h2>💬 Zulip Extension (Abiba)</h2><table><tr><th>Check</th><th>Status</th></tr>'
+    z = r.get("zulip_ext", {})
+    
+    checks = [
+        ("Connection", "✅ Connected" if z.get("connected") else "❌ Disconnected"),
+        ("Queue", f"✅ {z.get('queue_id','none')[:12]}..." if z.get("queue_id") else "❌ No queue"),
+        ("Messages", f"{z.get('messages_processed',0)} processed"),
+        ("PM2 Status", f"{z.get('pm2',{}).get('status','?')} (↺{z.get('pm2',{}).get('restarts','?')} restarts, up {z.get('pm2',{}).get('uptime','?')})"),
+        ("Bot Echo Loop", f"{z.get('bot_skipped_15min',0)} skipped in 15min" if z.get('bot_skipped_15min',0) > 0 else "✅ No echo loop"),
+        ("Edit Success", f"{z.get('finalized_1h',0)} ok / {z.get('failed_finalize_1h',0)} fail" if z.get('failed_finalize_1h',0) > 0 else f"✅ {z.get('finalized_1h',0)} ok"),
+        ("Zulip Server", f"✅ HTTP {z.get('server_status','?')}" if z.get('server_status') in ('200','302') else f"❌ HTTP {z.get('server_status','?')}"),
+    ]
+    for name, val in checks:
+        color = "green" if "✅" in val or "ok" in val.lower() else ("yellow" if "🟡" in val or "skip" in val.lower() else "red")
+        html += f'<tr><td>{name}</td><td class="{color}">{val}</td></tr>'
+    html += '</table></div>'
+    
+    # ── Network Endpoints ──
+    html += '<div class="card"><h2>🌐 Network Endpoints</h2><table><tr><th>Service</th><th>Status</th></tr>'
+    for ep in r["endpoints"]:
+        color = "green" if ep["code"] in ("200","302","401") else ("yellow" if ep["code"] >= "400" else "red")
+        html += f'<tr><td>{ep["name"]}</td><td class="{color}">HTTP {ep["code"]}</td></tr>'
+    html += '</table></div>'
+    
+    # ── NFS ──
+    html += '<div class="card"><h2>🗄️ NFS Mounts</h2><table><tr><th>Mount</th><th>Size</th><th>Used</th><th>Avail</th><th>%</th></tr>'
+    for m in r["nfs"]:
+        pct_str = m.get("pct","0%")
+        pct_num = int(pct_str.replace("%","")) if pct_str.replace("%","").strip().isdigit() else 0
+        color = "green" if pct_num < 80 else ("yellow" if pct_num < 90 else "red")
+        html += f'<tr><td>{m["mount"]}</td><td>{m["size"]}</td><td>{m["used"]}</td><td>{m.get("avail","?")}</td><td class="{color}">{pct_str}</td></tr>'
+    html += '</table></div>'
+    
+    # ── Suggestions ──
+    html += '<div class="card"><h2>💡 Suggestions</h2><ul style="font-size:13px;line-height:1.6">'
+    
+    added = False
+    
+    # Storage warnings
+    for s in r["storage"]:
+        if s["name"] == "mediastore" and s["pct"] > 70:
+            html += f'<li>📀 mediastore at {s["pct"]:.0f}% ({s["used"]}/{s["total"]}) — plan expansion or cleanup</li>'
+            added = True
+    
+    # Stopped VMs
+    if r["stopped_vms"]:
+        html += f'<li>🛑 Stopped VMs/CTs need attention: {", ".join(r["stopped_vms"][:5])}</li>'
+        added = True
+    
+    # Endpoint issues
+    for ep in r["endpoints"]:
+        if ep["code"] >= "500":
+            html += f'<li>🌐 {ep["name"]} returning HTTP {ep["code"]}</li>'
+            added = True
+    
+    # Zulip extension
+    if not z.get("connected"):
+        html += f'<li>💬 Zulip extension disconnected — run: pm2 restart abiba-zulip</li>'
+        added = True
+    if z.get("finalize_fail_pct", 0) > 50:
+        html += f'<li>💬 Zulip edit failures at {z["finalize_fail_pct"]:.0f}% — check editMessage API</li>'
+        added = True
+    
+    # LiteLLM
+    for check in r.get("litellm", {}).get("checks", []):
+        if check["status"] == "fail" and check.get("missing"):
+            html += f'<li>⚡ LiteLLM missing containers: {", ".join(check["missing"])}</li>'
+            added = True
+    
+    # Backup compliance
+    html += '<li>🗄️ Verify PBS backups completed within last 48h via Proxmox Backup Server (CT 107)</li>'
+    
+    # Heartbeat suggestion
+    html += '<li>💓 All agents now have Zulip heartbeat logging — silence detection active</li>'
+    
+    if not added:
+        html += '<li>✅ Nothing flagged — infrastructure is healthy</li>'
+    
+    html += '</ul></div>'
+    html += '<p style="color:#484f58;font-size:11px;text-align:center;margin-top:20px">Syslog Infrastructure Monitor v2.0 · Generated ' + TIME_STR + '</p>'
+    html += '</body></html>'
+    return html
+
+
+# ── Send Email ──
+
+def send_email(html_content, subject_prefix=""):
+    FROM = "abiba@sysloggh.com"
+    TO = "jerome@sysloggh.com"
+    SUBJECT = f"{subject_prefix}{'🏗️ Infrastructure Report — ' + DATE_STR}"
+    
+    msg = MIMEMultipart("alternative")
+    msg["From"] = FROM
+    msg["To"] = TO
+    msg["Subject"] = SUBJECT
+    msg.attach(MIMEText("Infrastructure report in HTML format — enable images to view.", "plain"))
+    msg.attach(MIMEText(html_content, "html"))
+    
+    try:
+        EMAIL_PASSWORD = "rgbuomwcydxwbszd"
+        GMAIL_EMAIL = "jtabiri@gmail.com"
+        
+        server = smtplib.SMTP("smtp.gmail.com", 587)
+        server.starttls()
+        server.login(GMAIL_EMAIL, EMAIL_PASSWORD)
+        server.sendmail(FROM, [TO], msg.as_string())
+        server.quit()
+        return True, "✅ Email sent to jerome@sysloggh.com"
+    except Exception as e:
+        return False, f"❌ Email failed: {e}"
+
+
+# ── Main ──
+
+if __name__ == "__main__":
+    is_test = "--test-email" in sys.argv
+    
+    print(f"{'🧪 TEST MODE' if is_test else '📊'} Collecting infrastructure data...")
+    report = collect()
+    
+    if "--json" in sys.argv:
+        print(json.dumps(report, indent=2, default=str))
+        sys.exit(0)
+    
+    print("   Building dashboard...")
+    html = build_html(report)
+    
+    if is_test:
+        prefix = "🧪 TEST — "
+        print("   Sending test email...")
+    else:
+        prefix = ""
+        print("   Sending email...")
+    
+    ok, msg = send_email(html, subject_prefix=prefix)
+    print(f"   {msg}")
+    
+    # Show summary
+    issues = sum(1 for i in ["red"] if report.get("zulip_ext", {}).get("connected") == False)
+    print(f"\n📋 Summary:")
+    print(f"   Proxmox: {report['nodes_online']}/{report['node_count']} nodes online")
+    print(f"   VMs/CTs: {report['running_vms']}/{report['total_vms']} running")
+    print(f"   Zulip Ext: {'✅' if report.get('zulip_ext',{}).get('connected') else '❌'}")
+    print(f"   LiteLLM: {sum(1 for c in report.get('litellm',{}).get('checks',[]) if c['status']=='pass')}/{len(report.get('litellm',{}).get('checks',[]))} checks pass")
+    agent_parts = []
+for k,v in report.get('agents',{}).items():
+    agent_parts.append(f"{k}:{v.get('gateway_state',v.get('pm2_status','?'))}")
+print(f"   Agents: {', '.join(agent_parts)}")

scripts/zulip-monitor.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+# /root/scripts/zulip-monitor.sh — Zulip Mesh Health Monitor
+# Implements zulip-health.prose.md v2
+# Runs every 15 min via cron. Alerts via Telegram.
+set -euo pipefail
+
+ZULIP_SITE="https://chat.sysloggh.net"
+ZULIP_EMAIL="abiba-bot@chat.sysloggh.net"
+ZULIP_KEY="cKTDMZAPW08dk3zl05sStzO7HRztzyn8"
+OWNER_ZULIP_ID="9"
+
+# Email config
+GMAIL_USER="jtabiri@gmail.com"
+GMAIL_PASS="rgbuomwcydxwbszd"
+EMAIL_TO="jerome@sysloggh.com"
+
+notify() {
+  local severity="$1" msg="$2"
+  echo "[$severity] $msg"
+  
+  # Zulip DM to owner
+  local content="${severity} Zulip Monitor: ${msg}"
+  local form="type=private&to=%5B${OWNER_ZULIP_ID}%5D&content=$(python3 -c "import urllib.parse; print(urllib.parse.quote('''${content}'''))")"
+  curl -sf -X POST "${ZULIP_SITE}/api/v1/messages" \
+    -u "${ZULIP_EMAIL}:${ZULIP_KEY}" \
+    -d "${form}" > /dev/null 2>&1 || true
+  
+  # Email alert
+  local subject="${severity} Zulip Monitor Alert"
+  python3 -c "
+import smtplib
+from email.mime.text import MIMEText
+m = MIMEText('''${msg}''')
+m['From'] = 'abiba@sysloggh.com'
+m['To'] = '${EMAIL_TO}'
+m['Subject'] = '${subject}'
+s = smtplib.SMTP('smtp.gmail.com', 587)
+s.starttls()
+s.login('${GMAIL_USER}', '${GMAIL_PASS}')
+s.sendmail('abiba@sysloggh.com', ['${EMAIL_TO}'], m.as_string())
+s.quit()
+" 2>/dev/null || true
+}
+
+TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
+ISSUES=0
+LOG="/root/zulip-health-monitor.log"
+
+echo "=== Zulip Health Check — $TIMESTAMP ===" >> "$LOG"
+
+# ── Global: Zulip Server ──
+SERVER_CODE=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 \
+  https://chat.sysloggh.net/api/v1/server_settings \
+  -u 'abiba-bot@chat.sysloggh.net:cKTDMZAPW08dk3zl05sStzO7HRztzyn8' 2>/dev/null || echo "000")
+if [ "$SERVER_CODE" != "200" ]; then
+  notify "🔴" "Zulip server returned HTTP $SERVER_CODE"
+  ISSUES=$((ISSUES + 1))
+else
+  echo "  Server: ✅ HTTP 200" >> "$LOG"
+fi
+
+# ── Platform A: pi (Abiba) ──
+PI_HEALTH=$(curl -sf --connect-timeout 5 http://localhost:9200/health 2>/dev/null || echo "{}")
+PI_CONNECTED=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('connected',False))" 2>/dev/null)
+PI_ERROR=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('last_error') or '')" 2>/dev/null)
+PI_RETRIES=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('retry_count',0))" 2>/dev/null)
+
+if [ "$PI_CONNECTED" != "True" ]; then
+  notify "🔴" "Abiba pi extension DISCONNECTED — restarting"
+  pm2 restart abiba-zulip 2>/dev/null || true
+  ISSUES=$((ISSUES + 1))
+  echo "  Abiba: ❌ Disconnected — restarted" >> "$LOG"
+elif [ -n "$PI_ERROR" ]; then
+  notify "🟡" "Abiba pi extension error: ${PI_ERROR:0:100}"
+  echo "  Abiba: 🟡 Error: ${PI_ERROR:0:100}" >> "$LOG"
+elif [ "$PI_RETRIES" -ge 3 ]; then
+  notify "🟡" "Abiba pi extension: $PI_RETRIES retries — restarting"
+  pm2 restart abiba-zulip 2>/dev/null || true
+  echo "  Abiba: 🟡 $PI_RETRIES retries — restarted" >> "$LOG"
+else
+  echo "  Abiba: ✅ Connected (processed=$(echo "$PI_HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('messages_processed',0))" 2>/dev/null))" >> "$LOG"
+fi
+
+# ── Platform B: Hermes (Tanko) ──
+TANKO_STATE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 jerome@192.168.68.122 \
+  "cat ~/.hermes/gateway_state.json 2>/dev/null" 2>/dev/null || echo "{}")
+TANKO_ZULIP=$(echo "$TANKO_STATE" | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+p=d.get('platforms',{}).get('zulip',{})
+print(p.get('state','unknown'))
+" 2>/dev/null)
+
+if [ "$TANKO_ZULIP" != "connected" ]; then
+  notify "🔴" "Tanko (Hermes) Zulip state: $TANKO_ZULIP — needs restart"
+  ISSUES=$((ISSUES + 1))
+  echo "  Tanko: ❌ state=$TANKO_ZULIP" >> "$LOG"
+else
+  echo "  Tanko: ✅ Zulip connected" >> "$LOG"
+fi
+
+# ── Platform B: Hermes (Mumuni) ──
+MUMUNI_STATE=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.123 \
+  "cat ~/.hermes/gateway_state.json 2>/dev/null" 2>/dev/null || echo "{}")
+MUMUNI_ZULIP=$(echo "$MUMUNI_STATE" | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+p=d.get('platforms',{}).get('zulip',{})
+print(p.get('state','unknown'))
+" 2>/dev/null)
+
+if [ "$MUMUNI_ZULIP" != "connected" ]; then
+  notify "🔴" "Mumuni (Hermes) Zulip state: $MUMUNI_ZULIP"
+  ISSUES=$((ISSUES + 1))
+  echo "  Mumuni: ❌ state=$MUMUNI_ZULIP" >> "$LOG"
+else
+  echo "  Mumuni: ✅ Zulip connected" >> "$LOG"
+fi
+
+# ── Platform C: Agent Zero (kagentz) ──
+AZ_A2A=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \
+  "docker exec agent-zero curl -s --connect-timeout 5 http://127.0.0.1:8001/.well-known/agent.json 2>/dev/null" 2>/dev/null || echo "")
+AZ_ALIVE=$(echo "$AZ_A2A" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('name',''))" 2>/dev/null)
+
+if [ "$AZ_ALIVE" != "kagentz" ]; then
+  notify "🔴" "kagentz A2A server DOWN — restarting"
+  ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \
+    "docker exec agent-zero bash -c 'pkill -9 -f a2a_agent; sleep 1; cd /a0 && /opt/venv-a0/bin/python3 -u /a0/usr/a2a_agent.py > /tmp/a2a.log 2>&1 &'" 2>/dev/null || true
+  ISSUES=$((ISSUES + 1))
+  echo "  kagentz: ❌ A2A down — restarted" >> "$LOG"
+else
+  echo "  kagentz: ✅ A2A alive" >> "$LOG"
+  
+  # Check adapter process
+  AZ_ADAPTER=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \
+    "docker exec agent-zero ps aux 2>/dev/null | grep adapter | grep -v grep | wc -l" 2>/dev/null || echo "0")
+  if [ "$AZ_ADAPTER" -lt 1 ]; then
+    notify "🔴" "kagentz Zulip adapter DOWN — restarting"
+    ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.68.14 \
+      "docker exec agent-zero bash -c 'cd /a0/usr/kagentz-zulip && ZULIP_SITE=https://chat.sysloggh.net ZULIP_EMAIL=kagentz-bot@chat.sysloggh.net ZULIP_API_KEY=E9q9PXJTxftPYBkb5pBDWupDO7KK21ty ZULIP_AGENT_NAME=kagentz A2A_URL=http://localhost:8001/a2a A2A_TOKEN=8zNgdOEXzYxjQvTl /opt/venv-a0/bin/python3 -u adapter.py > /tmp/zulip-adapter.log 2>&1 &'" 2>/dev/null || true
+    ISSUES=$((ISSUES + 1))
+    echo "  kagentz: ❌ Adapter down — restarted" >> "$LOG"
+  else
+    echo "  kagentz: ✅ Adapter running" >> "$LOG"
+  fi
+fi
+
+# ── Summary ──
+if [ "$ISSUES" -eq 0 ]; then
+  echo "  Result: ✅ All healthy" >> "$LOG"
+else
+  echo "  Result: 🔴 $ISSUES issue(s) found" >> "$LOG"
+  notify "🔴" "$ISSUES issue(s) found — check /root/zulip-health-monitor.log"
+fi
+
+echo "" >> "$LOG"