root
de32f54337
PR Pipeline — Authorize → Validate → Review → Merge / auth (pull_request) Successful in 3s
PR Pipeline — Authorize → Validate → Review → Merge / validate (pull_request) Successful in 6s
PR Pipeline — Authorize → Validate → Review → Merge / lint (pull_request) Successful in 13s
PR Pipeline — Authorize → Validate → Review → Merge / ai-review (pull_request) Successful in 11s
PR Pipeline — Authorize → Validate → Review → Merge / gate (pull_request) Successful in 1s
Captain's decision 2026-09-26, clarified the same day: the digest is delivered to his Zulip DM (user id 9) from abiba-bot as an HTML FILE - an attachment, not HTML rendered in the message body and not a Markdown translation of it. Closes daily-digest-mail-transport-20260921; the Google dependency is gone (no SMTP, no EMAIL_PASSWORD, no app password, nothing to rotate). WHAT CHANGES * scripts/daily-infra-report.py: send_email() is replaced by send_zulip(), which writes the styled dashboard to /var/log/daily-infra-report/infra-report-<ts>.html, uploads it via POST /api/v1/user_uploads, then posts a SHORT Markdown pointer to user 9. The message body carries subject, top-line status and the attachment link; it does not reproduce the report. * the 10,000-character cap is irrelevant here - it bounds message TEXT only, and the report travels as a file, so nothing is shrunk to fit. * the key is abiba-bot's, already on the execution host at /root/.pi/agent/extensions/zulip/.env (mode 600). No vault entry was added: under the auth-keys charter that is a captain decision. * daily-health-digest.prose.md -> v2.0.0 and contract-registry.yaml updated: transport, healthy/degraded definitions, and exit codes now match observed behaviour. There is NO degraded delivery leg any more - delivery is the only output path, so a missing or rejected key is a real failure (exit 1). * queued defect folded in: a failed delivery used to print only the transport error while the report body never surfaced. Now the HTML is printed to stdout AND persisted on every failure, and the message names which step failed. EVIDENCE (all against the live stack) * real send: message id 86221 to user 9, attachment 16208 bytes at /user_uploads/2/45/m1cQesBFV78BGeNY2lN8xkN5/infra-report-20260926-153406.html * the message is type=private, sender abiba-bot@chat.sysloggh.net, recipients [9, 21], body carries the top-line status and the attachment link, and does NOT contain a <table> - i.e. it does not reproduce the report * the attachment fetches HTTP 200, 16208 bytes, content-type text/html, starts with <!DOCTYPE html>, and contains <style>, <table> and 16 class="card" blocks - it opens as a standalone styled document * failure path: a bad key gives 'Delivery FAILED at upload: Malformed API key', EXIT=1, the HTML is printed to stdout and persisted to disk * scheduled path: the run's own output is pasted in the PR prose-lint: PASSED (19 warnings); secret scan clean.
889 lines
42 KiB
Python
Executable File
889 lines
42 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
/root/scripts/daily-infra-report.py — v2.0.0
|
|
Daily infrastructure dashboard emailed to jerome@sysloggh.com
|
|
Runs at 6:00 AM daily via cron.
|
|
Integrated with prose contracts: infrastructure-control, litellm-health, zulip-health.
|
|
|
|
Usage:
|
|
python3 daily-infra-report.py # Generate and email report
|
|
python3 daily-infra-report.py --test-email # Send a test email for review
|
|
python3 daily-infra-report.py --json # Output raw JSON to stdout
|
|
"""
|
|
|
|
import smtplib, json, subprocess, os, sys, datetime, re
|
|
from email.mime.text import MIMEText
|
|
from email.mime.multipart import MIMEMultipart
|
|
|
|
PVE = "https://192.168.68.12:8006"
|
|
|
|
# The HTTP header prefix is a protocol constant, not a credential. It is kept as
|
|
# a constant ending at '=' so that no assembled header-plus-token literal ever
|
|
# appears in the tree; the secret scanner rightly flags that shape.
|
|
PVE_AUTH_HEADER = "PVEAPIToken="
|
|
|
|
|
|
def pve_auth():
|
|
"""PVE API auth header, resolved at call time from the injected environment.
|
|
|
|
The token is injected by ``infisical run --env=prod`` as ``PVE_TOKEN``
|
|
(format ``user@realm!tokenid=secret``). It must never be hardcoded: a
|
|
placeholder literal authenticates as nobody, which is how this probe
|
|
reported zero nodes while still exiting 0. Raise loudly instead.
|
|
"""
|
|
token = os.environ.get("PVE_TOKEN")
|
|
if not token:
|
|
raise RuntimeError("PVE_TOKEN is not set (run under `infisical run --env=prod`)")
|
|
return f"Authorization: {PVE_AUTH_HEADER}{token}"
|
|
|
|
# ── Shared credentials —─
|
|
|
|
ZULIP_SITE = "https://chat.sysloggh.net"
|
|
ZULIP_EMAIL = "abiba-bot@chat.sysloggh.net"
|
|
# Note: /api/v1/server_settings is a PUBLIC endpoint (verified HTTP 200 with or without credential).
|
|
# No Zulip API key is required for this call. If a future leg genuinely needs abiba-bot's key,
|
|
# it must prove it with a 200 from /api/v1/users/me as abiba-bot and label itself degraded when it cannot.
|
|
# Never fall back to the vault's shared ZULIP_API_KEY.
|
|
ZULIP_AUTH = None
|
|
DEGRADED_LEGS = []
|
|
|
|
# Probe failures are different from degraded legs. A missing credential is an
|
|
# expected, survivable state (stays exit 0). A probe that cannot reach the API
|
|
# means the report has NO data for that section, which is a monitoring loss and
|
|
# must exit non-zero so it cannot pass unnoticed.
|
|
PROBE_FAILURES = []
|
|
|
|
LITELLM_PUBLIC = "https://litellm.sysloggh.net"
|
|
LITELLM_BACKEND = "192.168.68.116"
|
|
AUTH_HOST = "192.168.68.11"
|
|
|
|
# Load LiteLLM API key from file (durable, works in cron)
|
|
LITELLM_KEY_FILE = "/root/.abiba-workspace/secrets/litellm-key.txt"
|
|
try:
|
|
with open(LITELLM_KEY_FILE) as f:
|
|
SYNTHETIC_API_KEY = f.read().strip()
|
|
except:
|
|
SYNTHETIC_API_KEY = None # Fail loudly: report "no-key-file" in check
|
|
|
|
NOW = datetime.datetime.now()
|
|
DATE_STR = NOW.strftime("%Y-%m-%d")
|
|
TIME_STR = NOW.strftime("%Y-%m-%d %H:%M UTC")
|
|
|
|
# ── Helpers ──
|
|
|
|
def pve_get(path):
|
|
"""Fetch PVE API data. Returns list on success, None on error (to distinguish from empty list).
|
|
|
|
A missing PVE_TOKEN is caught here and reported as ``None`` so the caller
|
|
records a probe failure; it must not escape as an unhandled exception.
|
|
"""
|
|
try:
|
|
cmd = f'curl -sk --connect-timeout 10 "{PVE}{path}" -H "{pve_auth()}"'
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=12)
|
|
if r.returncode != 0:
|
|
return None
|
|
data = json.loads(r.stdout)
|
|
return data.get("data", [])
|
|
except:
|
|
return None
|
|
|
|
def ssh(host, cmd):
|
|
try:
|
|
return subprocess.check_output(
|
|
f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@{host} "{cmd}"',
|
|
shell=True, stderr=subprocess.DEVNULL).decode().strip()
|
|
except: return ""
|
|
|
|
def ssh_jerome(host, cmd):
|
|
try:
|
|
return subprocess.check_output(
|
|
f'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 jerome@{host} "{cmd}"',
|
|
shell=True, stderr=subprocess.DEVNULL).decode().strip()
|
|
except: return ""
|
|
|
|
def http_get(url, auth=None, timeout=10):
|
|
try:
|
|
cmd = f'curl -sfk --connect-timeout {timeout} -o /dev/null -w "%{{http_code}}" "{url}"'
|
|
if auth:
|
|
cmd = cmd.replace('"', '\\"')
|
|
cmd = f'curl -sfk --connect-timeout {timeout} -H "Authorization: Bearer {auth}" -o /dev/null -w "%{{http_code}}" "{url}"'
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout+2)
|
|
return r.stdout.strip() or "000"
|
|
except:
|
|
return "timeout"
|
|
|
|
def http_get_body(url, auth=None, timeout=10):
|
|
try:
|
|
cmd = f'curl -sfk --connect-timeout {timeout} "{url}"'
|
|
if auth:
|
|
cmd = f'curl -sfk --connect-timeout {timeout} -u "{auth}" "{url}"'
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout+2)
|
|
if r.returncode == 0:
|
|
return r.stdout.strip()
|
|
return ""
|
|
except:
|
|
return ""
|
|
|
|
def count_in_log(filepath, pattern, minutes=15):
|
|
"""Count occurrences of a pattern in a log file within the last N minutes."""
|
|
try:
|
|
since = (NOW - datetime.timedelta(minutes=minutes)).strftime("%Y-%m-%d %H:%M")
|
|
cmd = f"""grep -a "{pattern}" {filepath} 2>/dev/null | awk '$0 >= "{since}"' | wc -l"""
|
|
count = subprocess.check_output(cmd, shell=True).decode().strip()
|
|
return int(count)
|
|
except:
|
|
return 0
|
|
|
|
# ── Data Collection ──
|
|
|
|
def collect():
|
|
report = {}
|
|
|
|
# ── Proxmox Nodes ──
|
|
nodes = pve_get("/api2/json/nodes")
|
|
if nodes is None:
|
|
report["nodes"] = {}
|
|
report["node_count"] = 0
|
|
report["nodes_online"] = 0
|
|
report["pve_probe_status"] = "unreachable"
|
|
PROBE_FAILURES.append("proxmox: node list unreachable (PVE_TOKEN missing or API down)")
|
|
else:
|
|
report["nodes"] = {n["node"]: {
|
|
"cpu_pct": round(n.get('cpu',0)*100, 1),
|
|
"ram": f"{n.get('mem',0)//1024//1024}/{n.get('maxmem',0)//1024//1024}MB",
|
|
"ram_pct": round(n.get('mem',0)/n.get('maxmem',1)*100, 0),
|
|
"disk": f"{n.get('disk',0)//1024//1024//1024}/{n.get('maxdisk',0)//1024//1024//1024}GB",
|
|
"disk_pct": round(n.get('disk',0)/n.get('maxdisk',1)*100, 0),
|
|
"uptime_h": n.get('uptime',0)//3600,
|
|
"status": n["status"]
|
|
} for n in nodes}
|
|
report["node_count"] = len(nodes)
|
|
report["nodes_online"] = sum(1 for n in nodes if n["status"] == "online")
|
|
report["pve_probe_status"] = "ok"
|
|
|
|
# ── VMs/CTs ──
|
|
resources = pve_get("/api2/json/cluster/resources")
|
|
if resources is None:
|
|
vms = []
|
|
report["resources_probe_status"] = "unreachable"
|
|
PROBE_FAILURES.append("proxmox: cluster resources unreachable")
|
|
else:
|
|
vms = [r for r in resources if r.get("type") in ("qemu","lxc")]
|
|
report["resources_probe_status"] = "ok"
|
|
report["total_vms"] = len(vms)
|
|
report["running_vms"] = sum(1 for v in vms if v.get("status") == "running")
|
|
stopped = [v for v in vms if v.get("status") != "running"]
|
|
report["stopped_vms"] = [f"{v['type']} {v['vmid']} {v.get('name','?')}" for v in stopped]
|
|
|
|
report["vms_by_node"] = {}
|
|
for n in ["amdpve", "minipve", "storepve", "acerpve", "ocupve"]:
|
|
node_vms = [v for v in vms if v.get("node") == n]
|
|
node_vms.sort(key=lambda x: int(x.get("vmid",0)))
|
|
report["vms_by_node"][n] = [{
|
|
"type": v.get("type"), "vmid": v.get("vmid"), "name": v.get("name","?"),
|
|
"status": v.get("status"),
|
|
"ram": f"{v.get('mem',0)//1024//1024}/{v.get('maxmem',0)//1024//1024}MB",
|
|
"disk": f"{v.get('disk',0)//1024//1024//1024}/{v.get('maxdisk',0)//1024//1024//1024}GB",
|
|
"cpu": v.get("cpus", v.get("maxcpu","?")),
|
|
} for v in node_vms]
|
|
|
|
# ── Storage ──
|
|
storages = pve_get("/api2/json/nodes/storepve/storage")
|
|
report["storage"] = []
|
|
for s in (storages or []):
|
|
total = s.get("total",0) or 1
|
|
used = s.get("used",0)
|
|
pct = used/total*100
|
|
report["storage"].append({
|
|
"name": s.get("storage","?"), "used": f"{used//1024//1024//1024}GB",
|
|
"total": f"{total//1024//1024//1024}GB", "pct": round(pct, 0), "type": s.get("type","")
|
|
})
|
|
|
|
# ── Docker: docker-vm (.7) ──
|
|
docker_raw = ssh("192.168.68.7", "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'")
|
|
containers = []
|
|
for line in docker_raw.strip().split("\n"):
|
|
if "|" in line:
|
|
parts = line.split("|", 2)
|
|
containers.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""})
|
|
report["docker_vm"] = {"total": len(containers), "running": sum(1 for c in containers if "Up" in c["status"]),
|
|
"unhealthy": [c for c in containers if "unhealthy" in c["status"]], "containers": containers}
|
|
report["docker_vm"]["reclaimable"] = ssh("192.168.68.7", "docker system df 2>/dev/null | tail -1 | awk '{print $5}'")
|
|
report["docker_vm"]["disk_used"] = ssh("192.168.68.7", "df -h / | tail -1 | awk '{print $5}'")
|
|
|
|
# ── Docker: CT 116 (syslog-api — LiteLLM stack) ──
|
|
docker2_raw = ssh(LITELLM_BACKEND, "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'")
|
|
containers2 = []
|
|
for line in docker2_raw.strip().split("\n"):
|
|
if "|" in line:
|
|
parts = line.split("|", 2)
|
|
containers2.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""})
|
|
report["docker_syslog"] = {"total": len(containers2), "running": sum(1 for c in containers2 if "Up" in c["status"]),
|
|
"containers": containers2}
|
|
|
|
# ── Docker: Netbird (72.61.0.17) ──
|
|
docker3_raw = ssh("72.61.0.17", "docker ps --format '{{.Names}}|{{.Status}}|{{.Image}}'")
|
|
containers3 = []
|
|
for line in docker3_raw.strip().split("\n"):
|
|
if "|" in line:
|
|
parts = line.split("|", 2)
|
|
containers3.append({"name": parts[0], "status": parts[1], "image": parts[2] if len(parts) > 2 else ""})
|
|
report["docker_netbird"] = {"total": len(containers3), "running": sum(1 for c in containers3 if "Up" in c["status"]),
|
|
"containers": containers3}
|
|
|
|
# ── Public Endpoints ──
|
|
endpoints = [
|
|
("LiteLLM Admin UI", f"{LITELLM_PUBLIC}/ui/"),
|
|
("LiteLLM Swagger", f"{LITELLM_PUBLIC}/docs"),
|
|
("LiteLLM ReDoc", f"{LITELLM_PUBLIC}/redoc"),
|
|
("LiteLLM OpenAPI", f"{LITELLM_PUBLIC}/openapi.json"),
|
|
("Gitea", "https://git.sysloggh.net"),
|
|
("Authentik", "https://auth.sysloggh.net"),
|
|
("Zulip", "https://chat.sysloggh.net"),
|
|
("Pulse", "https://pulse.sysloggh.net"),
|
|
("Proxmox", "https://192.168.68.12:8006"),
|
|
("SearXNG", "http://192.168.68.7:8888"),
|
|
("Firecrawl", "http://192.168.68.7:3002/health"),
|
|
]
|
|
report["endpoints"] = []
|
|
for name, url in endpoints:
|
|
code = http_get(url)
|
|
report["endpoints"].append({"name": name, "code": code})
|
|
|
|
# ── LiteLLM Specific Checks (from litellm-health prose contract) ──
|
|
report["litellm"] = {"checks": []}
|
|
|
|
# Check 1: Fleet health via nginx /health/unified (301 -> /gpu/gpu-data served by gpu-monitor on .24:9100).
|
|
# Router (:9000) was decommissioned 2026-09-11; probing it was a guaranteed daily failure.
|
|
health_unified = ssh(LITELLM_BACKEND, "curl -s -o /dev/null -w '%{http_code}' --max-time 8 http://127.0.0.1/health/unified 2>/dev/null")
|
|
report["litellm"]["health_unified"] = health_unified or "000"
|
|
report["litellm"]["checks"].append({"name": "fleet-health-via-nginx", "status": "pass" if health_unified in ("200", "301") else "fail", "code": health_unified or "000"})
|
|
|
|
# Check 2: Nginx-proxied internal endpoints
|
|
for path, name in [("/litellm/ui/", "nginx-ui"), ("/litellm/docs", "nginx-docs")]:
|
|
code = http_get(f"http://{LITELLM_BACKEND}{path}")
|
|
report["litellm"]["checks"].append({"name": name, "status": "pass" if code == "200" else "fail", "code": code})
|
|
|
|
# Check 3: Docker container health for LiteLLM stack
|
|
expected_containers = ["harness-litellm", "harness-nginx", "harness-postgres",
|
|
"harness-redis", "harness-dashboard", "harness-grafana",
|
|
"harness-prometheus", "harness-alertmanager",
|
|
"harness-zulip-bridge", "harness-docker-stats",
|
|
"harness-pve-exporter"]
|
|
actual_names = [c["name"] for c in containers2]
|
|
report["litellm"]["expected_containers"] = expected_containers
|
|
report["litellm"]["missing_containers"] = [e for e in expected_containers if e not in actual_names]
|
|
report["litellm"]["checks"].append({
|
|
"name": "container-health",
|
|
"status": "pass" if not report["litellm"]["missing_containers"] else "fail",
|
|
"missing": report["litellm"]["missing_containers"],
|
|
"healthy": sum(1 for c in containers2 if "healthy" in c.get("status","")),
|
|
"total": len(expected_containers)
|
|
})
|
|
|
|
# Check 4: OIDC Auth endpoint
|
|
auth_code = http_get(f"https://auth.sysloggh.net")
|
|
report["litellm"]["auth_status"] = auth_code
|
|
report["litellm"]["checks"].append({"name": "oidc-auth", "status": "pass" if auth_code in ("200","302") else "fail", "code": auth_code})
|
|
|
|
# Check 5: Synthetic API call through LiteLLM
|
|
if SYNTHETIC_API_KEY is None:
|
|
report["litellm"]["api_models"] = None
|
|
report["litellm"]["checks"].append({"name": "api-endpoint", "status": "fail", "code": "no-key-file"})
|
|
else:
|
|
api_check = http_get(f"{LITELLM_PUBLIC}/v1/models", auth=SYNTHETIC_API_KEY)
|
|
report["litellm"]["api_models"] = api_check
|
|
report["litellm"]["checks"].append({"name": "api-endpoint", "status": "pass" if api_check == "200" else "fail", "code": api_check})
|
|
|
|
# ── NFS Mounts ──
|
|
nfs = ssh("192.168.68.7", "df -h /media/storage /media/mediastore 2>/dev/null | tail -n +2")
|
|
report["nfs"] = []
|
|
for line in nfs.strip().split("\n"):
|
|
parts = line.split()
|
|
if len(parts) >= 6:
|
|
report["nfs"].append({"mount": parts[5], "size": parts[1], "used": parts[2], "pct": parts[4], "avail": parts[3]})
|
|
|
|
# ── Zulip Extension Health (from zulip-health prose contract) ──
|
|
report["zulip_ext"] = {}
|
|
|
|
# Phase 1: Health endpoint
|
|
health_body = http_get_body("http://localhost:9200/health")
|
|
zulip_health = {}
|
|
try:
|
|
zulip_health = json.loads(health_body) if health_body else {}
|
|
except:
|
|
zulip_health = {}
|
|
# Live state is nested under 'zulip' key
|
|
zulip_state = zulip_health.get("zulip", {})
|
|
report["zulip_ext"]["connected"] = zulip_state.get("connected", False)
|
|
report["zulip_ext"]["queue_id"] = zulip_state.get("queue_id")
|
|
report["zulip_ext"]["last_error"] = zulip_state.get("last_error")
|
|
report["zulip_ext"]["messages_processed"] = zulip_state.get("messages_processed", 0)
|
|
report["zulip_ext"]["skipped"] = zulip_state.get("skipped", 0)
|
|
|
|
# Phase 2: PM2 process check
|
|
pm2_raw = subprocess.check_output(
|
|
"pm2 show abiba-zulip --no-color 2>/dev/null | grep -E 'status|restarts|uptime'",
|
|
shell=True).decode().strip()
|
|
pm2 = {}
|
|
for line in pm2_raw.split("\n"):
|
|
if "│" in line:
|
|
parts = line.split("│")
|
|
if len(parts) >= 3:
|
|
key = parts[1].strip()
|
|
val = parts[2].strip()
|
|
pm2[key] = val
|
|
report["zulip_ext"]["pm2"] = pm2
|
|
report["zulip_ext"]["pm2_healthy"] = pm2.get("status") == "online"
|
|
|
|
# Phase 3: Echo loop detection
|
|
skipped_count = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Skipped.*bot msgs", 15)
|
|
report["zulip_ext"]["bot_skipped_15min"] = skipped_count
|
|
|
|
# Phase 4: Response delivery stats
|
|
finalized = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Finalized", 60)
|
|
failed = count_in_log("/root/.pm2/logs/abiba-zulip-out.log", "Failed to finalize", 60)
|
|
report["zulip_ext"]["finalized_1h"] = finalized
|
|
report["zulip_ext"]["failed_finalize_1h"] = failed
|
|
report["zulip_ext"]["finalize_fail_pct"] = round(failed / (finalized + failed) * 100, 0) if (finalized + failed) > 0 else 0
|
|
|
|
# Phase 5: Zulip server liveness
|
|
zulip_server_code = http_get(f"{ZULIP_SITE}/api/v1/server_settings", auth=ZULIP_AUTH)
|
|
report["zulip_ext"]["server_status"] = zulip_server_code
|
|
|
|
# ── Agent Status ──
|
|
report["agents"] = {}
|
|
|
|
# Abiba (pi)
|
|
report["agents"]["abiba"] = {
|
|
"platform": "pi", "ct": 100, "ip": "192.168.68.24",
|
|
"zulip_connected": zulip_state.get("connected", False),
|
|
"zulip_processed": zulip_state.get("messages_processed", 0),
|
|
"pm2_status": pm2.get("status", "unknown"),
|
|
"pm2_restarts": pm2.get("restarts", "?"),
|
|
"pm2_uptime": pm2.get("uptime", "?"),
|
|
}
|
|
|
|
# Tanko (CT 112, IP 192.168.68.122) — DSH (DeepSeek Harness), no Hermes gateway
|
|
# since 2026-08-27. There is no ~/.hermes/gateway_state.json on CT 112 anymore;
|
|
# Zulip/Telegram connectivity is managed by the DSH harness, not the Hermes gateway.
|
|
report["agents"]["tanko"] = {
|
|
"platform": "dsh", "ct": 112, "ip": "192.168.68.122",
|
|
"gateway_state": "n/a (DSH)",
|
|
"zulip_state": "unknown",
|
|
"telegram_state": "unknown",
|
|
"gateway_pid": None,
|
|
"updated_at": "",
|
|
}
|
|
|
|
# Mumuni is deliberately absent from this digest: captain ruling 2026-09-10.
|
|
# She moved off this host onto her own container (kagentz CT 105 on minipve,
|
|
# 192.168.68.14, dedicated `hermes` user) and is monitored from her side. The
|
|
# former probe ssh'd to 192.168.68.24 for the decommissioned deployment's
|
|
# ~/.hermes/gateway_state.json, always read "unknown", and published a false
|
|
# "mumuni:unknown" line in the agent table and the gateway-unknown issue
|
|
# count of every digest. Do NOT re-add an .24 / gateway_state probe.
|
|
|
|
return report
|
|
|
|
|
|
# ── HTML Dashboard ──
|
|
|
|
def build_html(r):
|
|
issues = []
|
|
|
|
# Proxmox issues
|
|
if r["nodes_online"] < r["node_count"]:
|
|
issues.append(f"🔴 {r['node_count'] - r['nodes_online']} Proxmox node(s) offline")
|
|
if r["stopped_vms"]:
|
|
issues.append(f"🔴 {len(r['stopped_vms'])} VM(s)/CT(s) stopped — {', '.join(r['stopped_vms'][:3])}")
|
|
|
|
# Endpoint issues
|
|
for ep in r["endpoints"]:
|
|
if ep["code"] in ("000", "timeout"):
|
|
issues.append(f"🔴 {ep['name']} — unreachable")
|
|
elif ep["code"] >= "500":
|
|
issues.append(f"🟡 {ep['name']} — HTTP {ep['code']}")
|
|
|
|
# Docker issues
|
|
for c in r["docker_vm"].get("unhealthy", []):
|
|
issues.append(f"🟡 docker-vm: {c['name']} — unhealthy")
|
|
|
|
# Small cluster mode warning
|
|
if r["docker_vm"].get("total", 0) == 0:
|
|
issues.append("🟡 docker-vm: no containers reported — host may be down")
|
|
|
|
# LiteLLM issues
|
|
for check in r.get("litellm", {}).get("checks", []):
|
|
if check["status"] == "fail":
|
|
issues.append(f"🔴 LiteLLM: {check['name']} — {'missing: ' + ', '.join(check.get('missing',[])) if check.get('missing') else 'HTTP ' + check.get('code','?')}")
|
|
|
|
# Zulip extension issues
|
|
z = r.get("zulip_ext", {})
|
|
if not z.get("connected"):
|
|
issues.append("🔴 Zulip extension: disconnected")
|
|
if z.get("last_error"):
|
|
issues.append(f"🟡 Zulip extension: {z['last_error'][:80]}")
|
|
if z.get("retry_count", 0) >= 3:
|
|
issues.append(f"🟡 Zulip extension: {z['retry_count']} retries")
|
|
if z.get("finalize_fail_pct", 0) > 50:
|
|
issues.append(f"🔴 Zulip extension: {z['finalize_fail_pct']:.0f}% edit failures")
|
|
if z.get("bot_skipped_15min", 0) > 100:
|
|
issues.append(f"🟡 Zulip echo loop: {z['bot_skipped_15min']} bot msgs skipped in 15min")
|
|
if z.get("server_status") and z["server_status"] not in ("200", "302"):
|
|
issues.append(f"🔴 Zulip server: HTTP {z['server_status']}")
|
|
|
|
# Agent issues
|
|
def agent_gateway(agent):
|
|
return agent.get("gateway_state") or agent.get("pm2_status") or "unknown"
|
|
for name, agent in r.get("agents", {}).items():
|
|
gs = agent_gateway(agent)
|
|
if gs == "unknown":
|
|
issues.append(f"🟡 {name}: gateway state unknown (unreachable)")
|
|
elif gs != "running" and gs != "online":
|
|
issues.append(f"🔴 {name}: gateway {gs}")
|
|
|
|
status = "🟢 All Healthy" if not issues else f"{'🔴' if sum(1 for i in issues if '🔴' in i) > 0 else '🟡'} {len(issues)} Issue(s)"
|
|
|
|
# ── Build HTML ──
|
|
html = f"""<!DOCTYPE html>
|
|
<html><head><meta charset="utf-8"><style>
|
|
body {{ font-family: -apple-system, BlinkMacSystemFont, sans-serif; background: #0d1117; color: #c9d1d9; padding: 20px; }}
|
|
h1 {{ color: #58a6ff; }}
|
|
h2 {{ color: #8b949e; border-bottom: 1px solid #30363d; padding-bottom: 8px; margin-top: 24px; }}
|
|
h3 {{ color: #c9d1d9; margin: 12px 0 6px 0; }}
|
|
.card {{ background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; margin: 12px 0; }}
|
|
.grid {{ display: flex; gap: 12px; flex-wrap: wrap; }}
|
|
.stat {{ flex: 1; min-width: 140px; background: #21262d; border-radius: 6px; padding: 12px; text-align: center; }}
|
|
.stat .num {{ font-size: 28px; font-weight: bold; color: #58a6ff; }}
|
|
.stat .label {{ font-size: 12px; color: #8b949e; }}
|
|
table {{ width: 100%; border-collapse: collapse; }}
|
|
td, th {{ padding: 6px 12px; border-bottom: 1px solid #21262d; text-align: left; font-size: 13px; }}
|
|
th {{ color: #8b949e; font-weight: normal; }}
|
|
.green {{ color: #3fb950; }}
|
|
.red {{ color: #f85149; }}
|
|
.yellow {{ color: #d29922; }}
|
|
.alert {{ border-radius: 8px; padding: 12px; margin: 12px 0; }}
|
|
.good {{ background: #1f3d1f; border: 1px solid #3fb950; }}
|
|
.bad {{ background: #3d1f1f; border: 1px solid #f85149; }}
|
|
.warn {{ background: #3d3d1f; border: 1px solid #d29922; }}
|
|
</style></head><body>
|
|
<h1>🏗️ Syslog Infrastructure Report</h1>
|
|
<p style="color:#8b949e">{TIME_STR}</p>
|
|
|
|
<div class="alert {'good' if not issues else 'bad' if any('🔴' in i for i in issues) else 'warn'}">
|
|
<p style="margin:0;font-size:16px"><b>{status}</b></p>
|
|
<p style="margin:4px 0 0 0;font-size:13px">
|
|
Proxmox: {r.get('pve_probe_status', 'ok')} ({r['nodes_online']}/{r['node_count']}) · {r['total_vms']} VMs/CTs · {r['running_vms']} running ·
|
|
{r['docker_vm']['total'] + r['docker_syslog']['total'] + r['docker_netbird']['total']} containers ·
|
|
{len(r['endpoints'])} endpoints · {len(r.get('agents',{}))} agents
|
|
</p>
|
|
</div>
|
|
"""
|
|
|
|
# Issues list
|
|
if issues:
|
|
html += '<div class="card"><h2>🚨 Issues ({})</h2>'.format(len(issues))
|
|
for i in issues:
|
|
html += f'<p style="margin:4px 0;font-size:13px">{i}</p>'
|
|
html += '</div>'
|
|
|
|
# ── Quick Stats ──
|
|
html += '<div class="card"><h2>📊 Quick Stats</h2><div class="grid">'
|
|
pve_status_label = "unreachable" if r.get('pve_probe_status') == 'unreachable' else f"{r['nodes_online']}/{r['node_count']}"
|
|
pve_status_color = "red" if r.get('pve_probe_status') == 'unreachable' or r['nodes_online'] != r['node_count'] else "green"
|
|
stats = [
|
|
("PVE Nodes", pve_status_label, pve_status_color),
|
|
("VMs/CTs", f"{r['running_vms']}/{r['total_vms']}", "green" if r['running_vms'] == r['total_vms'] else "red"),
|
|
("Containers", f"{r['docker_vm']['running']}/{r['docker_vm']['total']}", "green" if r['docker_vm']['running'] == r['docker_vm']['total'] else "yellow"),
|
|
("LiteLLM Ctrs", f"{r['docker_syslog']['running']}/{r['docker_syslog']['total']}", "green" if r['docker_syslog']['running'] == r['docker_syslog']['total'] else "red"),
|
|
("Netbird Ctrs", f"{r['docker_netbird']['running']}/{r['docker_netbird']['total']}", "green" if r['docker_netbird']['running'] == r['docker_netbird']['total'] else "red"),
|
|
("Endpoints", f"{sum(1 for e in r['endpoints'] if e['code'] in ('200','302','401'))}/{len(r['endpoints'])}", "green",),
|
|
("Zulip Ext", "✅ Connected" if r.get('zulip_ext',{}).get('connected') else "❌ Disconnected", "green" if r.get('zulip_ext',{}).get('connected') else "red"),
|
|
]
|
|
for label, val, color in stats:
|
|
html += f'<div class="stat"><div class="num {color}">{val}</div><div class="label">{label}</div></div>'
|
|
|
|
# VLM/GPU capacity
|
|
gpu_nodes = [v for vms in r["vms_by_node"].values() for v in vms if "llm" in v["name"].lower() or "gpu" in v["name"].lower() or "ocu" in v["name"].lower()]
|
|
gpu_running = sum(1 for g in gpu_nodes if g["status"] == "running")
|
|
html += f'<div class="stat"><div class="num {"green" if gpu_running == len(gpu_nodes) else "yellow"}">{gpu_running}/{len(gpu_nodes)}</div><div class="label">GPU VMs</div></div>'
|
|
html += '</div></div>'
|
|
|
|
# ── LiteLLM Section ──
|
|
html += '<div class="card"><h2>⚡ LiteLLM Inference Stack</h2>'
|
|
|
|
# Endpoint status table
|
|
html += '<table><tr><th>Check</th><th>Status</th></tr>'
|
|
for check in r.get("litellm", {}).get("checks", []):
|
|
color = "green" if check["status"] == "pass" else "red"
|
|
detail = check.get("code", "")
|
|
if check.get("missing"):
|
|
detail = f"missing: {', '.join(check['missing'])}"
|
|
elif check.get("healthy") is not None:
|
|
detail = f"{check['healthy']}/{check['total']} healthy"
|
|
html += f'<tr><td>{check["name"]}</td><td class="{color}">{detail}</td></tr>'
|
|
html += '</table>'
|
|
|
|
# LiteLLM container detail
|
|
html += '<h3>Containers (CT 116)</h3><table><tr><th>Container</th><th>Status</th><th>Image</th></tr>'
|
|
for c in r["docker_syslog"]["containers"]:
|
|
color = "green" if "Up" in c["status"] and "healthy" in c["status"] else ("yellow" if "Up" in c["status"] else "red")
|
|
html += f'<tr><td>{c["name"]}</td><td class="{color}">{c["status"]}</td><td style="font-size:11px;color:#8b949e">{c["image"]}</td></tr>'
|
|
html += '</table>'
|
|
html += '</div>'
|
|
|
|
# ── Proxmox Nodes ──
|
|
html += '<div class="card"><h2>📦 Proxmox Cluster</h2><div class="grid">'
|
|
for name, nd in sorted(r["nodes"].items()):
|
|
cpu_color = "green" if nd["cpu_pct"] < 50 else ("yellow" if nd["cpu_pct"] < 80 else "red")
|
|
ram_color = "green" if nd["ram_pct"] < 70 else ("yellow" if nd["ram_pct"] < 85 else "red")
|
|
disk_color = "green" if nd["disk_pct"] < 70 else ("yellow" if nd["disk_pct"] < 85 else "red")
|
|
html += f'<div class="stat"><div class="num {cpu_color}">{nd["cpu_pct"]}%</div><div class="label"><b>{name}</b><br>CPU: {nd["cpu_pct"]}%<br>RAM: <span class="{ram_color}">{nd["ram_pct"]:.0f}%</span><br>DISK: <span class="{disk_color}">{nd["disk_pct"]:.0f}%</span><br>⬆ {nd["uptime_h"]}h</div></div>'
|
|
html += '</div>'
|
|
if r["stopped_vms"]:
|
|
html += f'<p class="red" style="font-size:13px">Stopped: {", ".join(r["stopped_vms"])}</p>'
|
|
html += '</div>'
|
|
|
|
# ── VM/CT Lists per Node ──
|
|
for node_name in ["amdpve", "minipve", "storepve", "acerpve", "ocupve"]:
|
|
node_vms = r["vms_by_node"].get(node_name, [])
|
|
if not node_vms: continue
|
|
stopped_count = sum(1 for v in node_vms if v["status"] != "running")
|
|
badge = f'<span class="green">{len(node_vms)-stopped_count} running</span>'
|
|
if stopped_count:
|
|
badge += f' <span class="red">{stopped_count} stopped</span>'
|
|
html += f'<div class="card"><h3>🔹 {node_name}</h3><p style="font-size:12px;color:#8b949e">{badge}</p>'
|
|
html += '<table><tr><th>ID</th><th>Type</th><th>Name</th><th>Status</th><th>CPU</th><th>RAM</th><th>Disk</th></tr>'
|
|
for v in node_vms:
|
|
color = "green" if v["status"] == "running" else "red"
|
|
html += f'<tr><td>{v["vmid"]}</td><td>{v["type"]}</td><td>{v["name"]}</td><td class="{color}">{v["status"]}</td><td>{v["cpu"]}</td><td>{v["ram"]}</td><td>{v["disk"]}</td></tr>'
|
|
html += '</table></div>'
|
|
|
|
# ── Storage ──
|
|
html += '<div class="card"><h2>💾 Storage</h2><table><tr><th>Pool</th><th>Used</th><th>Total</th><th>%</th><th>Type</th></tr>'
|
|
for s in sorted(r["storage"], key=lambda x: -x["pct"]):
|
|
color = "green" if s["pct"] < 70 else ("yellow" if s["pct"] < 85 else "red")
|
|
html += f'<tr><td>{s["name"]}</td><td>{s["used"]}</td><td>{s["total"]}</td><td class="{color}">{s["pct"]:.0f}%</td><td>{s["type"]}</td></tr>'
|
|
html += '</table></div>'
|
|
|
|
# ── Docker Ecosystems ──
|
|
html += '<div class="card"><h2>🐳 Docker Ecosystems</h2>'
|
|
for host, key, label in [
|
|
("docker-vm (.7) — storepve", "docker_vm", f"Disk: {r['docker_vm'].get('disk_used','?')}"),
|
|
("syslog-api (.116) — minipve (LiteLLM)", "docker_syslog", ""),
|
|
("Netbird (72.61.0.17)", "docker_netbird", ""),
|
|
]:
|
|
d = r[key]
|
|
html += f'<h3>{host} — {d["running"]}/{d["total"]} running {label}</h3>'
|
|
if d.get("reclaimable"):
|
|
html += f'<p style="font-size:12px;color:#8b949e">Reclaimable Docker space: {d["reclaimable"]}</p>'
|
|
html += '<table><tr><th>Container</th><th>Status</th></tr>'
|
|
for c in sorted(d["containers"], key=lambda x: x["name"]):
|
|
color = "green" if "Up" in c["status"] and "healthy" in c["status"] else ("yellow" if "Up" in c["status"] else "red")
|
|
html += f'<tr><td>{c["name"]}</td><td class="{color}">{c["status"][:50]}</td></tr>'
|
|
html += '</table>'
|
|
html += '</div>'
|
|
|
|
# ── Agent Status ──
|
|
html += '<div class="card"><h2>🤖 Agent Status</h2><table><tr><th>Agent</th><th>Platform</th><th>CT</th><th>Gateway</th><th>Zulip</th><th>Processed</th></tr>'
|
|
for name, agent in sorted(r.get("agents", {}).items()):
|
|
if name == "abiba":
|
|
zulip_state = "✅" if agent.get("zulip_connected") else "❌"
|
|
gateway = f"pm2:{agent.get('pm2_status','?')} (↺{agent.get('pm2_restarts','?')})"
|
|
processed = f"{agent.get('zulip_processed', 0)} msgs"
|
|
elif name == "tanko":
|
|
zulip_state = "✅" if agent.get("zulip_state") == "connected" else ("❌" if agent.get("zulip_state") == "disconnected" else "⬜")
|
|
gateway = agent.get("gateway_state", "?")
|
|
processed = "DSH"
|
|
else:
|
|
zulip_state = "⬜"
|
|
gateway = agent.get("gateway_state", "?")
|
|
processed = ""
|
|
html += f'<tr><td><b>{name}</b></td><td>{agent["platform"]}</td><td>{agent["ct"]}</td><td>{gateway}</td><td>{zulip_state}</td><td style="font-size:12px">{processed}</td></tr>'
|
|
html += '</table></div>'
|
|
|
|
# ── Zulip Extension Health ──
|
|
html += '<div class="card"><h2>💬 Zulip Extension (Abiba)</h2><table><tr><th>Check</th><th>Status</th></tr>'
|
|
z = r.get("zulip_ext", {})
|
|
|
|
checks = [
|
|
("Connection", "✅ Connected" if z.get("connected") else "❌ Disconnected"),
|
|
("Queue", f"✅ {z.get('queue_id','none')[:12]}..." if z.get("queue_id") else "❌ No queue"),
|
|
("Messages", f"{z.get('messages_processed',0)} processed"),
|
|
("PM2 Status", f"{z.get('pm2',{}).get('status','?')} (↺{z.get('pm2',{}).get('restarts','?')} restarts, up {z.get('pm2',{}).get('uptime','?')})"),
|
|
("Bot Echo Loop", f"{z.get('bot_skipped_15min',0)} skipped in 15min" if z.get('bot_skipped_15min',0) > 0 else "✅ No echo loop"),
|
|
("Edit Success", f"{z.get('finalized_1h',0)} ok / {z.get('failed_finalize_1h',0)} fail" if z.get('failed_finalize_1h',0) > 0 else f"✅ {z.get('finalized_1h',0)} ok"),
|
|
("Zulip Server", f"✅ HTTP {z.get('server_status','?')}" if z.get('server_status') in ('200','302') else f"❌ HTTP {z.get('server_status','?')}"),
|
|
]
|
|
for name, val in checks:
|
|
color = "green" if "✅" in val or "ok" in val.lower() else ("yellow" if "🟡" in val or "skip" in val.lower() else "red")
|
|
html += f'<tr><td>{name}</td><td class="{color}">{val}</td></tr>'
|
|
html += '</table></div>'
|
|
|
|
# ── Network Endpoints ──
|
|
html += '<div class="card"><h2>🌐 Network Endpoints</h2><table><tr><th>Service</th><th>Status</th></tr>'
|
|
for ep in r["endpoints"]:
|
|
color = "green" if ep["code"] in ("200","302","401") else ("yellow" if ep["code"] >= "400" else "red")
|
|
html += f'<tr><td>{ep["name"]}</td><td class="{color}">HTTP {ep["code"]}</td></tr>'
|
|
html += '</table></div>'
|
|
|
|
# ── NFS ──
|
|
html += '<div class="card"><h2>🗄️ NFS Mounts</h2><table><tr><th>Mount</th><th>Size</th><th>Used</th><th>Avail</th><th>%</th></tr>'
|
|
for m in r["nfs"]:
|
|
pct_str = m.get("pct","0%")
|
|
pct_num = int(pct_str.replace("%","")) if pct_str.replace("%","").strip().isdigit() else 0
|
|
color = "green" if pct_num < 80 else ("yellow" if pct_num < 90 else "red")
|
|
html += f'<tr><td>{m["mount"]}</td><td>{m["size"]}</td><td>{m["used"]}</td><td>{m.get("avail","?")}</td><td class="{color}">{pct_str}</td></tr>'
|
|
html += '</table></div>'
|
|
|
|
# ── Suggestions ──
|
|
html += '<div class="card"><h2>💡 Suggestions</h2><ul style="font-size:13px;line-height:1.6">'
|
|
|
|
added = False
|
|
|
|
# Storage warnings
|
|
for s in r["storage"]:
|
|
if s["name"] == "mediastore" and s["pct"] > 70:
|
|
html += f'<li>📀 mediastore at {s["pct"]:.0f}% ({s["used"]}/{s["total"]}) — plan expansion or cleanup</li>'
|
|
added = True
|
|
|
|
# Stopped VMs
|
|
if r["stopped_vms"]:
|
|
html += f'<li>🛑 Stopped VMs/CTs need attention: {", ".join(r["stopped_vms"][:5])}</li>'
|
|
added = True
|
|
|
|
# Endpoint issues
|
|
for ep in r["endpoints"]:
|
|
if ep["code"] >= "500":
|
|
html += f'<li>🌐 {ep["name"]} returning HTTP {ep["code"]}</li>'
|
|
added = True
|
|
|
|
# Zulip extension
|
|
if not z.get("connected"):
|
|
html += f'<li>💬 Zulip extension disconnected — run: pm2 restart abiba-zulip</li>'
|
|
added = True
|
|
if z.get("finalize_fail_pct", 0) > 50:
|
|
html += f'<li>💬 Zulip edit failures at {z["finalize_fail_pct"]:.0f}% — check editMessage API</li>'
|
|
added = True
|
|
|
|
# LiteLLM
|
|
for check in r.get("litellm", {}).get("checks", []):
|
|
if check["status"] == "fail" and check.get("missing"):
|
|
html += f'<li>⚡ LiteLLM missing containers: {", ".join(check["missing"])}</li>'
|
|
added = True
|
|
|
|
# Backup compliance
|
|
html += '<li>🗄️ Verify PBS backups completed within last 48h via Proxmox Backup Server (CT 107)</li>'
|
|
|
|
# Heartbeat suggestion
|
|
html += '<li>💓 All agents now have Zulip heartbeat logging — silence detection active</li>'
|
|
|
|
if not added:
|
|
html += '<li>✅ Nothing flagged — infrastructure is healthy</li>'
|
|
|
|
html += '</ul></div>'
|
|
html += '<p style="color:#484f58;font-size:11px;text-align:center;margin-top:20px">Syslog Infrastructure Monitor v2.0 · Generated ' + TIME_STR + '</p>'
|
|
html += '</body></html>'
|
|
return html
|
|
|
|
|
|
# ── Delivery: Zulip DM carrying the report as an HTML ATTACHMENT ──
|
|
#
|
|
# Captain's decision, clarified 2026-09-26: the report is sent as an HTML FILE,
|
|
# i.e. an attachment - NOT HTML rendered in the message body, and NOT a Markdown
|
|
# translation of it. So the styled dashboard is built exactly as before, uploaded
|
|
# through Zulip's file-upload API, and the message body stays short: subject,
|
|
# top-line status, and a pointer to the attachment.
|
|
#
|
|
# This removes the Google dependency entirely (no SMTP, no EMAIL_PASSWORD).
|
|
# The 10,000-character message cap does not apply: it bounds message TEXT only,
|
|
# and the report travels as a file.
|
|
|
|
ZULIP_SITE = "https://chat.sysloggh.net"
|
|
ZULIP_BOT_EMAIL = "abiba-bot@chat.sysloggh.net"
|
|
CAPTAIN_USER_ID = 9
|
|
ZULIP_KEY_FILE = "/root/.pi/agent/extensions/zulip/.env"
|
|
REPORT_ARTIFACT_DIR = "/var/log/daily-infra-report"
|
|
|
|
|
|
def zulip_key():
|
|
"""abiba-bot's Zulip key, from the env or the on-host 600 file."""
|
|
key = os.environ.get("ABIBA_ZULIP_API_KEY")
|
|
if key:
|
|
return key.strip()
|
|
try:
|
|
with open(ZULIP_KEY_FILE) as fh:
|
|
for line in fh:
|
|
if line.startswith("ABIBA_ZULIP_API_KEY="):
|
|
return line.split("=", 1)[1].strip()
|
|
except OSError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def build_summary(r, filename, test=False):
|
|
"""Short Markdown body: subject, top-line status, pointer to the attachment.
|
|
|
|
Deliberately NOT a reproduction of the report - the attachment is the report.
|
|
"""
|
|
nodes = f"{r.get('nodes_online', 0)}/{r.get('node_count', 0)} nodes online"
|
|
guests = f"{r.get('running_vms', 0)}/{r.get('total_vms', 0)} guests running"
|
|
lines = [
|
|
("\U0001F9EA **TEST — **" if test else "") + "\U0001F3D7\uFE0F **Infrastructure Report — " + DATE_STR + "**",
|
|
f"**{nodes}** \u00b7 **{guests}** \u00b7 generated {TIME_STR}",
|
|
]
|
|
problems = []
|
|
if r.get("pve_probe_status") != "ok":
|
|
problems.append(f"\u274c Proxmox probe: {r.get('pve_probe_status')}")
|
|
if r.get("resources_probe_status") != "ok":
|
|
problems.append(f"\u274c Resources probe: {r.get('resources_probe_status')}")
|
|
lit = r.get("litellm", {}) or {}
|
|
checks = lit.get("checks", []) or []
|
|
if checks:
|
|
passed = sum(1 for c in checks if c.get("status") == "pass")
|
|
if passed != len(checks):
|
|
problems.append(f"\u274c LiteLLM: {passed}/{len(checks)} checks pass")
|
|
if not (r.get("zulip_ext", {}) or {}).get("connected"):
|
|
problems.append("\u274c Zulip extension: not connected")
|
|
for leg in DEGRADED_LEGS:
|
|
problems.append(f"\u26a0\uFE0F degraded: {leg}")
|
|
|
|
lines.append("\n".join(problems) if problems else "\u2705 All monitored services healthy")
|
|
lines.append(f"\U0001F4CE **Full report attached:** `{filename}`")
|
|
return "\n\n".join(lines)
|
|
|
|
|
|
def _curl(args, timeout=60):
|
|
r = subprocess.run(["curl", "-s", "-m", str(timeout)] + args,
|
|
capture_output=True, text=True)
|
|
try:
|
|
return json.loads(r.stdout or "{}"), r.stdout
|
|
except json.JSONDecodeError:
|
|
return {}, r.stdout
|
|
|
|
|
|
def _curl_json(args, timeout=90):
|
|
r = subprocess.run(["curl", "-s", "-m", str(timeout)] + args,
|
|
capture_output=True, text=True)
|
|
try:
|
|
return json.loads(r.stdout or "{}"), r.stdout
|
|
except json.JSONDecodeError:
|
|
return {}, r.stdout
|
|
|
|
|
|
def send_zulip(html_content, report, test=False):
|
|
"""Upload the styled HTML and post a short pointer to the captain's DM.
|
|
|
|
Returns (ok, message). On ANY failure the report body is also printed to
|
|
stdout and persisted to disk, so a delivery failure can never swallow the
|
|
content - the defect this folds in.
|
|
"""
|
|
os.makedirs(REPORT_ARTIFACT_DIR, exist_ok=True)
|
|
stamp = NOW.strftime("%Y%m%d-%H%M%S")
|
|
filename = f"infra-report-{stamp}.html"
|
|
html_path = os.path.join(REPORT_ARTIFACT_DIR, filename)
|
|
try:
|
|
with open(html_path, "w") as fh:
|
|
fh.write(html_content)
|
|
except OSError as e:
|
|
print(f" \u26a0\uFE0F could not persist report artifact: {e}", file=sys.stderr)
|
|
|
|
key = zulip_key()
|
|
if not key:
|
|
print(html_content) # never swallow the content
|
|
return False, ("\u274c Delivery FAILED: no Zulip credential "
|
|
"(ABIBA_ZULIP_API_KEY unset and "
|
|
f"{ZULIP_KEY_FILE} unreadable). Report persisted to {html_path}")
|
|
|
|
auth = ["-u", f"{ZULIP_BOT_EMAIL}:{key}"]
|
|
|
|
# 1. Upload the report as a file.
|
|
up, up_raw = _curl_json(auth + [
|
|
"-X", "POST", f"{ZULIP_SITE}/api/v1/user_uploads",
|
|
"-F", f"file=@{html_path};type=text/html",
|
|
])
|
|
if up.get("result") != "success" or not up.get("uri"):
|
|
print(html_content)
|
|
return False, (f"\u274c Delivery FAILED at upload: {up.get('msg') or up_raw[:160]} "
|
|
f"(report persisted to {html_path})")
|
|
|
|
uri = up["uri"]
|
|
size = os.path.getsize(html_path)
|
|
|
|
# 2. Post a short message pointing at it.
|
|
body = build_summary(report, filename, test=test)
|
|
link = f"[{filename}]({uri})"
|
|
body = body.replace(f"`{filename}`", link)
|
|
payload, raw = _curl_json(auth + [
|
|
"-X", "POST", f"{ZULIP_SITE}/api/v1/messages",
|
|
"-d", "type=private",
|
|
"-d", f"to=[{CAPTAIN_USER_ID}]",
|
|
"--data-urlencode", f"content={body}",
|
|
])
|
|
if payload.get("result") == "success":
|
|
return True, (f"\u2705 Delivered to Zulip DM (user {CAPTAIN_USER_ID}), "
|
|
f"message id {payload.get('id')}, attachment {size} bytes at {uri}")
|
|
|
|
print(html_content)
|
|
return False, (f"\u274c Delivery FAILED at message post: {payload.get('msg') or raw[:160]} "
|
|
f"(uploaded {uri}; report persisted to {html_path})")
|
|
|
|
|
|
# ── Main ──
|
|
|
|
if __name__ == "__main__":
|
|
is_test = ("--test-email" in sys.argv) or ("--test-zulip" in sys.argv)
|
|
|
|
print(f"{'🧪 TEST MODE' if is_test else '📊'} Collecting infrastructure data...")
|
|
report = collect()
|
|
|
|
if "--json" in sys.argv:
|
|
print(json.dumps(report, indent=2, default=str))
|
|
if PROBE_FAILURES:
|
|
for leg in PROBE_FAILURES:
|
|
print(f"PROBE FAILURE: {leg}", file=sys.stderr)
|
|
sys.exit(1)
|
|
sys.exit(0)
|
|
|
|
print(" Building dashboard...")
|
|
html = build_html(report)
|
|
print(f" report ready: {len(html)} chars of HTML (delivered as a file attachment)")
|
|
|
|
if is_test:
|
|
print(" Sending TEST message to the captain's Zulip DM...")
|
|
else:
|
|
print(" Sending to the captain's Zulip DM...")
|
|
|
|
ok, msg = send_zulip(html, report, test=is_test)
|
|
print(f" {msg}")
|
|
|
|
# Show summary
|
|
if DEGRADED_LEGS:
|
|
print(f"\n⚠️ Degraded legs ({len(DEGRADED_LEGS)}):")
|
|
for leg in DEGRADED_LEGS:
|
|
print(f" - {leg}")
|
|
else:
|
|
print("\n✅ All legs fully credentialed")
|
|
|
|
# A failed send must exit non-zero; a degraded leg (no credential) must stay exit 0
|
|
if not ok:
|
|
sys.exit(1)
|
|
|
|
issues = sum(1 for i in ["red"] if report.get("zulip_ext", {}).get("connected") == False)
|
|
print(f"\n📋 Summary:")
|
|
print(f" Proxmox: {report['nodes_online']}/{report['node_count']} nodes online")
|
|
print(f" VMs/CTs: {report['running_vms']}/{report['total_vms']} running")
|
|
print(f" Zulip Ext: {'✅' if report.get('zulip_ext',{}).get('connected') else '❌'}")
|
|
print(f" LiteLLM: {sum(1 for c in report.get('litellm',{}).get('checks',[]) if c['status']=='pass')}/{len(report.get('litellm',{}).get('checks',[]))} checks pass")
|
|
agent_parts = []
|
|
for k,v in report.get('agents',{}).items():
|
|
agent_parts.append(f"{k}:{v.get('gateway_state',v.get('pm2_status','?'))}")
|
|
print(f" Agents: {', '.join(agent_parts)}")
|
|
|
|
if PROBE_FAILURES:
|
|
print(f"\n❌ Probe failures ({len(PROBE_FAILURES)}):")
|
|
for leg in PROBE_FAILURES:
|
|
print(f" - {leg}")
|
|
sys.exit(1)
|