From 38d2f7d9567a6659c35032fc37bdac35e4b94c05 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 1 Jul 2026 22:25:26 +0000 Subject: [PATCH] feat: update contracts with GPU optimizations, Strix Halo/ROCm status, alert migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gpu-fleet: documented parallel=2/batch=4096 optimizations for RTX 3090/5070 - gpu-fleet: added Strix Halo ROCm status (installed, blocked by HSA runtime on Debian 13) - gpu-fleet: documented alert migration from DM → #agent-hub stream topics - gpu-monitor: added Alert Delivery section with stream topic conventions - All GPU alerts now go to #agent-hub for cross-agent visibility --- gpu-fleet.prose.md | 211 +++++++++++++++++++++++++++++-------------- gpu-monitor.prose.md | 146 ++++++++++++++++++++++-------- 2 files changed, 253 insertions(+), 104 deletions(-) diff --git a/gpu-fleet.prose.md b/gpu-fleet.prose.md index cbc0e4d..c0e2770 100644 --- a/gpu-fleet.prose.md +++ b/gpu-fleet.prose.md @@ -3,109 +3,186 @@ kind: responsibility name: gpu-fleet description: > Manages the GPU inference fleet across all hosts. Handles model deployment, - registration, hot-reload, health checks, LiteLLM sync, agent key management, - and stale model cleanup. Replaces hardcoded router.py config with declarative - gpu_roster.yaml. + registration, health checks, LiteLLM sync, agent key management, GPU + saturation watchdog, Prometheus/Grafana monitoring, and self-healing. + Current as of 2026-06-30 post-migration: nginx routes /v1 → LiteLLM, + router runs behind LiteLLM on :9000 (internal-only). agent: abiba triggers: - on model add/remove - on GPU health degradation - on agent key rotation + - on router restart (roster must be loaded) --- ## Maintains - gpu_roster: { models: map, hosts: map } — Single source of truth for all GPU models -- router: { status: "healthy", models: array } -- litellm: { status: "healthy", models: array } +- router: { status: "healthy", roster_loaded: bool, models: array } +- litellm: { status: "healthy", keys: array, models: array } - agent_keys: { agent: api_key } — All agent API keys registered in LiteLLM DB -- health: { gpus: array, circuit_breakers: array } +- health: { gpus: array, circuit_breakers: array } — Fleet-wide health state +- monitor: { status: "running", version: "2.0.0" } — GPU monitor server on pi (:9100) +- watchdog: { status: "running" } — GPU saturation watchdog (restarts stuck llama-server) +- benchmarks: { tok_per_sec: map, baseline: map, history: array } — Inference speed benchmarks tracked over time +- grafana: { status: "running", dashboards: ["gpu-fleet"] } — Grafana on CT 116 (:3001) +- prometheus: { status: "running", targets: 5 } — Scrapes GPU :9400 exporters + LiteLLM + +## Fleet Topology (Current — June 2026) + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ CT 116 (192.168.68.116) — Inference Harness Host │ +│ │ +│ nginx:80 (entrypoint) │ +│ ├─ /v1/* → harness-litellm:4000 (API requests) │ +│ ├─ /admin/* → harness-litellm:4000 (admin endpoints) │ +│ ├─ /dashboard/ → harness-dashboard:3000 (harness UI) │ +│ ├─ /litellm/* → harness-litellm:4000 (LiteLLM UI + API) │ +│ ├─ /grafana/* → harness-grafana:3000 (dashboards) │ +│ ├─ /health/* → harness-litellm:4000 (health probes) │ +│ └─ /gpu/* → 192.168.68.24:9100 (fleet monitor) │ +│ │ +│ Containers: │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ LiteLLM │ │ Router │ │Dashboard │ │ Grafana │ │ +│ │ :4000 │─▶│ :9000 │ │ :3000 │ │ :3000 │ │ +│ │ keys+sync│ │internal │ │ harness │ │ Prometheus│ │ +│ │ fallback │ │only! │ │ UI │ │ data src │ │ +│ └──────────┘ └───┬──────┘ └──────────┘ └──────────┘ │ +│ │ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │PostgreSQL│ │ Redis │ │Prometheus│ │ +│ │ :5432 │ │ :6379 │ │ :9090 │ │ +│ └──────────┘ └──────────┘ └──────────┘ │ +└────────────────────┼─────────────────────────────────────────────┘ + │ + ┌───────────────┼───────────────┬──────────────────┐ + │ │ │ │ +┌────▼─────┐ ┌──────▼──────┐ ┌────▼──────┐ ┌───────▼──────┐ +│ CT 8 │ │ CT 110 │ │ CT 15 │ │ pi (.24) │ +│ RTX 3090 │ │ RTX 5070 │ │ Strix Halo│ │ GPU Monitor │ +│ 24GB │ │ 12GB │ │ 64GB UMA │ │ :9100 │ +│ │ │ │ │ llama-srv │ │ Watchdog │ +│ qwen3.6 │ │ gemma-4-12b │ │ ornith35B │ │ Prometheus │ +│ 27B-code │ │ :8090 │ │ :8080 │ │ exporter │ +│ :8090 │ │ :9400 (exp) │ │ :9400(exp)│ │ :9401 │ +│ :9400 │ └─────────────┘ └───────────┘ └──────────────┘ +└──────────┘ +``` + +## Current Model Assignments + +| Model | GPU | Host | VRAM | Status | +|-------|-----|------|------|--------| +| qwen3.6-27B-code | RTX 3090 | .8 (llm-gpu) | 23.4/24GB | ✅ healthy | +| gemma-4-12b | RTX 5070 | .110 (ocu-llm) | 11.2/12.2GB | ✅ healthy | +| ornith-1.0-35b | Strix Halo CPU | .15 (amdpve) | 28.2/64GB | ✅ healthy | ## Operations ### add-model 1. Download model file from Hugging Face or source 2. Check disk space + GPU VRAM compatibility -3. Build/update llama.cpp if needed (cmake, Vulkan/CUDA flags) -4. Start llama-server via systemd service -5. Add to `gpu_roster.yaml` -6. Hot-reload router: `POST /admin/roster/reload` -7. Sync LiteLLM config: add model to model_list + fallbacks -8. Generate/update agent keys for new model access +3. Start llama-server via systemd service on GPU host +4. Add to `/opt/inference-harness/gpu_roster.yaml` (or router env vars) +5. Hot-reload or restart router +6. Add model to LiteLLM config (`model_list` + `fallbacks`) +7. Generate agent keys for new model access via `/key/generate` +8. Add Prometheus scrape target for the new GPU exporter 9. Verify end-to-end: LiteLLM → Router → Model ### remove-model 1. Drain active requests (wait for active=0) -2. Update `gpu_roster.yaml` — remove entry -3. Hot-reload router -4. Remove from LiteLLM config -5. Stop llama-server (systemd) -6. Cleanup model files (optional — keep if model might return) +2. Remove from LiteLLM config +3. Remove from router config +4. Stop llama-server (systemd) +5. Remove Prometheus scrape target +6. Cleanup model files (optional) ### heal -1. Check all GPUs via router `/health/unified` -2. Reset stuck circuit breakers if idle -3. Restart dead llama-server instances -4. Flush Redis counters if stale -5. Sync LiteLLM if configs diverge +1. Check all GPUs via router internal `:9000/health/unified` +2. Check LiteLLM health via nginx `:80/litellm/health/liveliness` +3. Reset stuck circuit breakers if idle (Redis) +4. Restart dead llama-server instances via SSH +5. Flush Redis active counters if stale +6. Verify GPU monitor server is running on pi (:9100) +7. Verify watchdog is running on pi +8. Restart router if roster not loaded (check logs for STARTUP ROSTER) +9. Reload roster via `POST :9000/admin/roster/reload` if available ### sync-keys -1. List all agent keys in LiteLLM DB via `/key/list` -2. Compare against expected agent list -3. Generate missing keys via `/key/generate` -4. Update agent configs to use `api_key_env: LITELLM_API_KEY` -5. Set `LITELLM_API_KEY` in system-wide env + startup wrapper -6. Restart gateways +1. List all agent keys in LiteLLM DB via `GET /key/list` +2. Compare against expected agent list: [tanko, mumuni, abiba, koby, koonimo, kagenz0] +3. Generate missing keys via `POST /key/generate` with unlimited budget +4. Update agent configs — `/etc/environment` LITELLM_API_KEY +5. Send Zulip DM to agents that can't be reached via SSH +6. Verify each key with test request through full chain +7. Document keys in knowledge graph ### list -Show full fleet status: GPUs, models, VRAM, active requests, circuit breakers +Show full fleet status: GPUs, models, VRAM, active requests, circuit breakers, keys + +## Agent Keys (LiteLLM DB — Current 2026-06-30) + +| Agent | Key | Host | Access | +|-------|-----|------|--------| +| Tanko | `sk-3ZsdWJbbNSo9zSnJN2OsJw` | .122 | SSH jerome | +| Mumuni | `sk-XY2aUfvy2BIs6kp1ZPh6VA` | .123 | SSH root | +| Abiba | `sk-kxbPgbnV2Zkn6TKQbAEcEg` | .24 | local | +| Koby | `sk-wb86PvcZXWkjShAIBxrpDg` | ? | Zulip DM | +| Koonimo | `sk-Prx_vRXYb2VEzDmhPCbNeg` | .114 (baggy) | no SSH | +| Kagenz0 | `sk-Dh4CDkaHebMLEp8qqq20qA` | ? | no SSH | + +**Key update procedure**: Update `/etc/environment` → `LITELLM_API_KEY=sk-...` → restart Hermes. +If no SSH access, send Zulip DM via abiba-bot. ## Configuration Files | File | Host | Purpose | |------|------|---------| -| `gpu_roster.yaml` | CT 116 (`/opt/inference-harness/`) | Model-to-GPU mapping | -| `litellm_config.yaml` | CT 116 | LiteLLM proxy config | -| `roster_loader.py` | CT 116 (router container) | Hot-reloadable roster parser | -| `/etc/systemd/system/moe-server.service` | .15 | Ornith-1.0-35B server | -| Agent configs | .122, .123, .114 | `api_key_env: LITELLM_API_KEY` | +| `/opt/inference-harness/docker-compose.yml` | CT 116 | All containers (router, litellm, nginx, postgres, redis, dashboard) | +| `/opt/inference-harness/litellm_config.yaml` | CT 116 | LiteLLM proxy config (models, fallbacks, timeouts) | +| `/opt/inference-harness/router/router.py` | CT 116 | Router source (builds via compose) | +| `/etc/nginx/nginx.conf` | CT 116 (nginx container) | Routes /v1→LiteLLM, /dashboard/, /litellm/, /grafana/, /health | +| `/opt/monitoring/prometheus.yml` | CT 116 | Prometheus scrape config (5 targets) | +| `/root/scripts/gpu-monitor-server.py` | pi (.24) | GPU fleet monitor v2.1.0 (with benchmarks) | +| `/root/scripts/gpu_benchmark.py` | pi (.24) | GPU inference benchmark module (tok/s tracking) | +| `/root/scripts/gpu-saturation-watchdog.py` | pi (.24) | Auto-restart stuck llama-server | +| `/root/dashboard/gpu-fleet.html` | pi (.24) | Live HTML dashboard | +| `/etc/systemd/system/llama-server.service` | .8, .110, .15 | llama-server daemons | -## GPU Fleet Topology +## Prometheus & Grafana -``` -┌──────────────────────────────────────────────────┐ -│ LiteLLM (:4001) │ -│ Config: /opt/inference-harness/litellm_config.yaml │ -└──────────────┬───────────────────────────────────┘ - │ -┌──────────────▼───────────────────────────────────┐ -│ Router (:9000) │ -│ Roster: /app/gpu_roster.yaml (hot-reload) │ -│ Redis: slot booking + circuit breakers │ -└──────┬──────────┬──────────┬─────────────────────┘ - │ │ │ - ┌────▼───┐ ┌───▼────┐ ┌──▼──────┐ - │ gemma │ │ qwen │ │ ornith │ - │ 4-12b │ │ 27B │ │ 1.0-35B │ - │ .110 │ │ .8 │ │ .15 │ - │ 5070 │ │ 3090 │ │ Strix │ - └────────┘ └────────┘ └─────────┘ -``` +| Component | URL | Details | +|-----------|-----|---------| +| Grafana | `http://192.168.68.116:3001/` | admin / syslog-grafana-2026 | +| GPU Dashboard | `http://192.168.68.116:3001/d/gpu-fleet` | Gauges + time series | +| Prometheus | `http://192.168.68.116:9090/` (internal) | 5 scrape targets | +| GPU Exporters | `:9400/metrics` on .8, .110, .15 | NVIDIA/AMD GPU metrics | +| Router Exporter | `:9401/metrics` on .24 | Router + LiteLLM metrics | -## Agent Key Architecture +## Known Issues & Watch Points -All agents use `api_key_env: LITELLM_API_KEY` in Hermes configs: +- **Router startup race**: Compose router.py doesn't call load_roster(). Reload thread sleeps 30s first. + Fix: trigger roster reload via SSH after restart, or rebuild image with startup load_roster(). +- **LiteLLM /metrics**: Requires auth. Prometheus uses `/health/liveliness` as workaround. +- **VRAM**: RTX 3090 at ~9.6/24GB (60% free), RTX 5070 at ~6.5/12GB (46% free). Healthy. +- **RTX 3090 optimization**: `--parallel 2 --batch-size 4096` — doubled concurrent throughput. +- **RTX 5070 optimization**: Removed draft model freed ~2GB VRAM for vision tasks. `--parallel 2`. +- **Strix Halo GPU**: ROCm 6.3.1 installed, HIP compiler works via clang++ + device libs. **Blocked**: HSA runtime driver needs `rocm-llvm` → `libstdc++-11-dev` not on Debian 13. Fixes: Docker with ROCm, or reinstall to Ubuntu 24.04. +- **Alert migration**: All alerts now go to `#agent-hub` topics (`alerts-gpu`, `alerts-pm2`, `alerts-infra`) instead of DMs. Cross-agent visibility enabled. +- **tok/s benchmarks**: Measured every 5 min via LiteLLM proxy. Baselines tracked with 30%/50% degradation thresholds. +- **NetBird 502**: Tanko routes through NetBird for litellm.sysloggh.net. Use direct IP if NetBird down. -| Agent | Host | Config | Env Source | -|-------|------|--------|------------| -| Mumuni | .123 | 7 refs + 6 subagent profiles | `/etc/environment` + wrapper | -| Tanko | .122 | 4 refs | `/etc/environment` + wrapper | -| Koonimo | .114 | 5 refs | `/etc/environment` + wrapper | +## GPU Inference Benchmarks (Current) -To rotate all keys: update `LITELLM_API_KEY` value, restart gateways. +| GPU | Model | Gen tok/s | Prompt tok/s | Baseline | Samples | +|-----|-------|-----------|--------------|----------|---------| +| RTX 3090 (.8) | gemma-4-12b | 75 | 305 | 74 | 6 | +| RTX 5070 (.110) | qwen3.6-27B-code | 75 | 323 | 75 | 6 | +| Strix Halo (.15) | ornith-1.0-35b | 70 | 209 | 70 | 6 | -## Sync Protocol - -1. Edit `gpu_roster.yaml` → `POST /admin/roster/reload` (hot, no restart) -2. Edit `litellm_config.yaml` → `docker restart harness-litellm` -3. Agent key changes → generate via `POST /key/generate` with master key +Benchmarks run through LiteLLM proxy (192.168.68.116:4001) every 5 minutes. +Degradation alerts fire at 30% (warning) and 50% (critical) below baseline. +History stored at `/root/data/toks-history.json` with 7-day rolling window. diff --git a/gpu-monitor.prose.md b/gpu-monitor.prose.md index 88f13f7..80fbfbd 100644 --- a/gpu-monitor.prose.md +++ b/gpu-monitor.prose.md @@ -2,58 +2,130 @@ kind: responsibility name: gpu-monitor description: > - Monitors GPU fleet health, utilization, and model routing in - real-time. Polls sidecar metrics every 30s, logs to knowledge graph, - generates a live HTML dashboard, and alerts on anomalies. + Comprehensive GPU fleet monitor — polls every subsystem (sidecars, router, + LiteLLM, Strix Halo, dashboard) every 15s, renders a live HTML dashboard, + checks alert thresholds, and exposes a JSON API for downstream consumers. +agent: abiba --- ## Maintains -- dense-gpu: { host: "192.168.68.8", name: "RTX 3090", vram: "24GB", models: ["gemma-4-12b"] } -- moe-gpu: { host: "192.168.68.110", name: "RTX 5070", vram: "12GB", models: ["qwen3.6-27B-code"] } -- strix-halo: { host: "192.168.68.15", name: "AMD Strix Halo", models: ["ornith-1.0-35b"], engine: "llama-server CPU" } -- router-health: { model_availability: array, circuit_breakers: array } +- gpu-fleet-data: { gpus: array, strix: object, router: object, litellm: object, summary: object, alerts: array } — Full fleet snapshot refreshed every 15s +- dashboard: live HTML at `/root/dashboard/gpu-fleet.html`, served on port 9100 +- gpu-data-api: JSON at `GET /gpu-data` on port 9100 +- health-endpoint: `GET /health` on port 9100 → 200 if cache populated, 503 if warming up -## Dashboard (live HTML) +## GPU Fleet Topology -Generated at `/root/dashboard/gpu-fleet.html` — responsive, mobile-friendly, auto-refreshes every 30s. - -### Layout ``` -┌─────────────────────────────────────────────────────┐ -│ 🖥️ GPU Fleet Dashboard [live] │ -├──────────────────┬──────────────────┬────────────────┤ -│ Dense (RTX 3090)│ MoE (RTX 5070) │ Strix Halo │ -│ ─────────────── │ ─────────────── │ ───────────── │ -│ Temp: 41°C │ Temp: 39°C │ CPU: 16t │ -│ VRAM: 23.4/24.6 │ VRAM: 11.2/12.2│ Context: 262K │ -│ Power: 12W │ Power: 5W │ Model: 35B │ -│ Model: gemma │ Model: qwen27B │ │ -├──────────────────┴──────────────────┴────────────────┤ -│ Router Health: ✅ All models available │ -│ Circuit Breakers: gemma=0 qwen=0 ornith=0 │ -│ Last updated: 2026-06-29 22:30 UTC │ -└─────────────────────────────────────────────────────┘ +┌─────────────────────────────────────────────────────────┐ +│ GPU Monitor Server (:9100) — 192.168.68.65 │ +│ Polls every 15s, serves dashboard + JSON API │ +└──┬──────────┬──────────┬──────────┬─────────────────────┘ + │ │ │ │ + ▼ ▼ ▼ ▼ +┌──────┐ ┌──────┐ ┌────────┐ ┌──────────┐ +│.8:8090│ │.110 │ │.116:80 │ │.15 (SSH) │ +│RTX3090│ │:8090 │ │nginx │ │Strix Halo│ +│gemma │ │RTX5070│ │router │ │ornith35B │ +└──────┘ │qwen27B│ │LiteLLM │ └──────────┘ + └──────┘ │dashboard│ + └────────┘ ``` +### Subsystems Polled + +| Subsystem | Endpoint | Frequency | Metrics | +|-----------|----------|-----------|---------| +| GPU Sidecar (RTX 3090) | `http://192.168.68.8:8090/health` | 15s | temp, VRAM, util, power, fan | +| GPU Sidecar (RTX 5070) | `http://192.168.68.110:8090/health` | 15s | temp, VRAM, util, power, fan | +| Router (unified) | `http://192.168.68.116/health/unified` | 15s | models, CB, scores, GPU status | +| Router (basic) | `http://192.168.68.116/health` | 15s | basic aliveness | +| LiteLLM | `http://192.168.68.116/litellm/health` | 15s | proxy health, model count | +| Strix Halo | SSH `192.168.68.15` + `:8080/health` | 15s | process status, CPU load, llama health | +| Dashboard | `http://192.168.68.116/dashboard/` | 15s | harness-dashboard aliveness | + +### Alert Delivery + +All alerts are sent to `#agent-hub` stream topics: +- `alerts-gpu` — GPU fleet alerts (temp, VRAM, utilization) +- `alerts-pm2` — PM2 process health (restarts, crashes) +- `alerts-infra` — Infrastructure health (LiteLLM, Zulip, containers) + +This replaces the previous DM-only delivery. All agents on the mesh can see and respond to alerts. + ## Alert Thresholds | Metric | Warning | Critical | |--------|---------|----------| | GPU Temp | >80°C | >90°C | | VRAM Usage | >90% | >95% | -| GPU Util | >95% for 5min | >95% for 15min | -| Model Down | — | No response to test prompt | -| Circuit Breaker Open | 1 model | >1 model | +| GPU Util | >95% | >98% | +| Sidecar Unreachable | — | critical (connectivity) | +| Model Down | — | critical (circuit breaker open) | + +### JSON API Response Schema (/gpu-data) + +```json +{ + "gpus": [{ "name", "gpu_name", "temp_c", "vram_used_mb", "vram_total_mb", + "gpu_util_pct", "power_w", "power_limit_w", "fan_pct", + "models", "hostname", "active_requests", "health_score", + "circuit_open" }], + "strix": { "status", "cpu_load", "llama_health" }, + "router": { "available_models", "circuit_breaker", "gpus", "scores", + "status", "redis", "_basic" }, + "litellm": { ... }, + "dashboard": { "reachable": bool }, + "summary": { "fleet_status", "models_available", "models_total", + "circuit_breakers_open", "gpu_count", "gpu_errors", + "strix_running", "router_reachable", "litellm_reachable" }, + "alerts": [{ "gpu", "metric", "level", "value", "threshold" }], + "updated": "ISO8601", + "monitor_version": "2.0.0" +} +``` + +## Operations + +### list +`curl http://localhost:9100/gpu-data | jq` — Full fleet status + +### check-health +`curl http://localhost:9100/health` — Monitor self-check + +### view-dashboard +Open `http://localhost:9100/` in browser — Live HTML dashboard + +### restart +```bash +pkill -f gpu-monitor-server.py +python3 /root/scripts/gpu-monitor-server.py & +``` +Or via PM2: `pm2 restart gpu-monitor` + +### check-router +The router health is accessed through nginx on port 80 (NOT port 9000 directly). +`curl http://192.168.68.116/health/unified` — Router unified health via nginx proxy +`curl http://192.168.68.116:9000/health/unified` — ❌ WILL FAIL (port bound to 127.0.0.1 only) + +## Configuration Files + +| File | Host | Purpose | +|------|------|---------| +| `/root/scripts/gpu-monitor-server.py` | pi (.65) | Monitor server v2.0.0 | +| `/root/dashboard/gpu-fleet.html` | pi (.65) | Live HTML dashboard | +| `/etc/nginx/nginx.conf` | CT 116 | Routes /health/* → router | ## Execution -1. **Poll sidecars** (every 30s): - - GET http://192.168.68.8:8090/health → RTX 3090 metrics - - GET http://192.168.68.110:8090/health → RTX 5070 metrics - - SSH 192.168.68.15 → llama-server process stats -2. **Poll router**: GET /health/unified for model health + circuit breakers -3. **Render dashboard**: Generate HTML at /root/dashboard/gpu-fleet.html -4. **Check thresholds**: Alert via Zulip DM if any metric out of bounds -5. **Log to KG**: Create [LEARN] node hourly with aggregated metrics -6. **Repeat** every 30 seconds +1. **Poll sidecars** (every 15s): GET .8:8090/health, .110:8090/health +2. **Poll router** (every 15s): GET .116/health/unified + /health via nginx:80 +3. **Poll LiteLLM** (every 15s): GET .116/litellm/health via nginx:80 +4. **Poll Strix** (every 15s): SSH .15 for process check + GET .15:8080/health +5. **Poll dashboard** (every 15s): GET .116/dashboard/ +6. **Check alerts**: Compare metrics against thresholds +7. **Compute summary**: Fleet-wide health aggregation +8. **Render dashboard**: Generate HTML at /root/dashboard/gpu-fleet.html +9. **Serve API**: HTTP server on port 9100 +10. **Repeat** every 15 seconds