From 43382dac5b69ccf47fe063d15ad2fe7d9aac6b9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:07:03 +0000 Subject: [PATCH 01/49] Initial commit: README --- README.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..82ff334 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# Syslog Harness + +Operational orchestration layer for Syslog's internal AI agents. + +## Architecture + +``` +┌─────────────┐ ┌──────────────┐ ┌─────────────┐ +│ Agent │────>│ Nginx │────>│ GPU Pool │ +│ (Hermes) │ │ Router │ │ (MoE/Dense)│ +└─────────────┘ └──────────────┘ └─────────────┘ + │ + ├──> :8091 Queue Service (Docker) + │ + └──> :3001 Dashboard (Docker) +``` + +## Components + +| Service | Port | Container | Purpose | +|---|---|---|---| +| Nginx Router | 8080 | Host | Routes requests to GPU backends | +| Queue Service | 8091 | `syslog-queue` | Enqueues requests when GPUs are down | +| Dashboard | 3001 | `syslog-dashboard` | Observability UI + API | + +## GPU Routing + +| Header `X-Syslog-Model` | Backend | Model | +|---|---|---| +| (none) / `standard` | amdpve (.15) | qwen3.6-35B-A3B (MoE) | +| `heavy` / `qwen3.5-27B` | llmgpu (.8) | qwen3.5-27B (Dense) | +| `light` / `gemma-4` | ocu_llm (.110) | gemma-4-E4B (Light) | + +## Quick Start + +```bash +# Build & start +docker compose build +docker compose up -d + +# Verify +curl http://localhost:8091/health +curl http://localhost:3001/api/status +``` + +## Dashboard + +- **UI:** `http://:8080/dashboard/harness.html` +- **API:** `http://:8080/dashboard/api/status` + +## Circuit Breaker + +- Rate limit: 10 req/s per IP +- Burst: 20 requests +- Excess returns 503 +- Queue fallback on GPU 502/503 + +## Production Migration + +See [MIGRATION_PLAN.md](./MIGRATION_PLAN.md) + +--- +*Built for Syslog Solution LLC — Quality over speed.* From c85aaa570b04b3803cef77b620e48547ebc29c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:07:05 +0000 Subject: [PATCH 02/49] Add docker-compose --- docker-compose.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2aa5890 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +services: + queue-service: + build: ./queue-service + container_name: syslog-queue + restart: unless-stopped + ports: + - "8091:8091" + environment: + - REDIS_HOST=192.168.68.7 + - REDIS_PORT=6379 + networks: + - harness-net + + dashboard: + build: ./dashboard + container_name: syslog-dashboard + restart: unless-stopped + ports: + - "3001:3001" + depends_on: + - queue-service + networks: + - harness-net + +networks: + harness-net: + driver: bridge From b55b954967c5771d835a7665a9f65c31b5dfe136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:07:05 +0000 Subject: [PATCH 03/49] Add queue service --- queue-service/queue-service.py | 121 +++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 queue-service/queue-service.py diff --git a/queue-service/queue-service.py b/queue-service/queue-service.py new file mode 100644 index 0000000..3e2bbed --- /dev/null +++ b/queue-service/queue-service.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Syslog Inference Queue Service — Circuit breaker + request queuing. + +Ports: 8091 +Endpoints: + /health — liveness probe (Nginx upstream check) + /enqueue — POST inference request into queue (fallback from Nginx) + /status — GET queue depth + circuit breaker state +""" + +import json +import os +import sys +import time +import urllib.request +from flask import Flask, request, jsonify + +app = Flask(__name__) + +# Configuration +REDIS_HOST = os.getenv("REDIS_HOST", "192.168.68.7") +REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) +QUEUE_KEY = "inference:requests" +CIRCUIT_OPEN_THRESHOLD = 50 +CIRCUIT_WARN_THRESHOLD = 30 + +# GPU endpoints for draining +GPUS = { + "amdpve": "192.168.68.15:8080", + "llmgpu": "192.168.68.8:8080", + "ocu_llm": "192.168.68.110:8080", +} + + +def get_redis(): + try: + import redis + return redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True) + except Exception: + return None + + +def get_queue_depth(r): + try: + return r.llen(QUEUE_KEY) + except Exception: + return 0 + + +def check_gpu_health(endpoint): + try: + req = urllib.request.Request(f"http://{endpoint}/v1/models") + req.add_header("User-Agent", "queue-service/1.0") + resp = urllib.request.urlopen(req, timeout=3) + return resp.status == 200 + except Exception: + return False + + +@app.route("/health") +def health(): + """Nginx upstream health probe. Returns 200 if service is alive.""" + return jsonify({"status": "ok", "service": "queue-service"}), 200 + + +@app.route("/enqueue", methods=["POST"]) +def enqueue(): + """Fallback endpoint — Nginx calls this when all GPU upstreams are down.""" + r = get_redis() + if not r: + return jsonify({"error": "Redis unavailable"}), 503 + + depth = get_queue_depth(r) + if depth >= CIRCUIT_OPEN_THRESHOLD: + return jsonify({ + "error": "Circuit breaker OPEN", + "queue_depth": depth, + "threshold": CIRCUIT_OPEN_THRESHOLD + }), 503 + + # Store the request in queue + payload = request.get_data(as_text=True) + headers = {k: v for k, v in request.headers if k.startswith("X-")} + r.rpush(QUEUE_KEY, json.dumps({ + "payload": payload, + "headers": headers, + "queued_at": time.time() + })) + + new_depth = get_queue_depth(r) + return jsonify({ + "status": "queued", + "position": new_depth, + "circuit": "warn" if new_depth >= CIRCUIT_WARN_THRESHOLD else "closed" + }), 202 + + +@app.route("/status") +def status(): + """GET queue depth + circuit breaker state + GPU health.""" + r = get_redis() + depth = get_queue_depth(r) if r else -1 + circuit = "open" if depth >= CIRCUIT_OPEN_THRESHOLD else ("warn" if depth >= CIRCUIT_WARN_THRESHOLD else "closed") + + gpu_health = {} + for name, endpoint in GPUS.items(): + gpu_health[name] = "up" if check_gpu_health(endpoint) else "down" + + return jsonify({ + "queue_depth": depth, + "circuit_breaker": circuit, + "gpu_health": gpu_health, + "thresholds": { + "warn": CIRCUIT_WARN_THRESHOLD, + "open": CIRCUIT_OPEN_THRESHOLD + } + }) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8091) From e1f12c3462035c833322a983bb193ec497ac1221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:07:07 +0000 Subject: [PATCH 04/49] Add dashboard --- dashboard/harness.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 dashboard/harness.html diff --git a/dashboard/harness.html b/dashboard/harness.html new file mode 100644 index 0000000..e69de29 From c42f3a99792702c2e251d62e84be790ab096a2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:07:32 +0000 Subject: [PATCH 05/49] Add migration plan --- MIGRATION_PLAN.md | 71 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 MIGRATION_PLAN.md diff --git a/MIGRATION_PLAN.md b/MIGRATION_PLAN.md new file mode 100644 index 0000000..5a005aa --- /dev/null +++ b/MIGRATION_PLAN.md @@ -0,0 +1,71 @@ +# Syslog Harness — Production Migration Plan + +## Current State (Development) +- **Host:** CT 114 (192.168.68.123) +- **Docker containers:** `syslog-queue` (:8091), `syslog-dashboard` (:3001) +- **Nginx:** Local on CT 114, routing to GPUs + Docker services +- **Status:** All components verified and operational + +## Target State (Production) +- **Host:** New CT (e.g., `docker-vm` on 192.168.68.x) +- **Docker containers:** Same queue + dashboard services +- **Nginx:** Containerized on production CT +- **GPU backends:** Same (192.168.68.15, .8, .110) + +## Migration Steps + +### 1. Prepare Production CT +```bash +# Create new CT on Proxmox +# Install Docker +apt update && apt install -y docker.io docker-compose-plugin + +# Pull/cloned harness repo +git clone /root/syslog-harness +cd /root/syslog-harness +``` + +### 2. Update docker-compose.yml for Production +- Change `REDIS_HOST` to production Redis IP +- Update GPU endpoint env vars if IPs change +- Add volume mounts for persistence + +### 3. Build & Deploy +```bash +# Build images +docker compose build + +# Start services +docker compose up -d + +# Verify health +curl http://localhost:8091/health +curl http://localhost:3001/api/status +``` + +### 4. Configure Nginx +- Copy `/etc/nginx/conf.d/gpu-router.conf` to production CT +- Update upstream IPs if needed +- Test and reload + +### 5. DNS / Routing Update +- Point agent traffic to new CT IP +- Update Hermes config `inference_api_url` +- Test agent routing + +### 6. Verification Checklist +- [ ] Queue service health check passes +- [ ] Dashboard API returns GPU health +- [ ] Nginx routes to correct GPU based on header +- [ ] Circuit breaker triggers on excess load +- [ ] Queue fallback works when GPUs down +- [ ] Agent requests reach correct model + +## Rollback Plan +- Keep CT 114 running as backup +- Revert DNS/routing to .123 if issues +- Docker containers can be stopped/started instantly + +--- +*Created: May 15, 2026* +*Status: Development verified, ready for production migration* From a28b3a557da45a9804728d67da92bbea674c2d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:07:33 +0000 Subject: [PATCH 06/49] Add Nginx router config --- gpu-router.conf | 106 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 gpu-router.conf diff --git a/gpu-router.conf b/gpu-router.conf new file mode 100644 index 0000000..cc07223 --- /dev/null +++ b/gpu-router.conf @@ -0,0 +1,106 @@ +## Syslog GPU Router — Nginx Configuration +## Routes incoming agent requests to the appropriate GPU backend +## based on the X-Syslog-Model header. + +upstream amdpve_pool { + ## Strix Halo 395 — qwen3.6-35B-A3B (MoE) — Default workhorse + server 192.168.68.15:8080; +} + +upstream llmgpu_pool { + ## RTX 3090 — qwen3.5-27B (Dense) — Heavy reasoning + server 192.168.68.8:8080; +} + +upstream ocu_llm_pool { + ## RTX 5070 — gemma-4 (Dense 4B) — Ultra-light tasks + server 192.168.68.110:8080; +} + +upstream queue_service { + ## Agent queue with circuit breaker (Docker container) + server 127.0.0.1:8091; +} + +upstream dashboard_service { + ## Harness dashboard (Docker container) + server 127.0.0.1:3001; +} + +## ------------------------------------------------------------------ +## Mapping: X-Syslog-Model header → upstream backend +## ------------------------------------------------------------------ +map $http_x_syslog_model $gpu_upstream { + default amdpve_pool; # missing header → default workhorse + "standard" amdpve_pool; + "heavy" llmgpu_pool; + "qwen3.5-27B" llmgpu_pool; + "light" ocu_llm_pool; + "gemma-4" ocu_llm_pool; +} + +server { + listen 8080; + server_name _; + + # Rate limit zone — 10 req/s per IP, burst of 20 + limit_req_zone $binary_remote_addr zone=perip:10m rate=10r/s; + + ## ------------------------------------------------------------------ + ## Dashboard — observability UI (MUST be before / catch-all) + ## ------------------------------------------------------------------ + location /dashboard { + proxy_pass http://dashboard_service/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + ## ------------------------------------------------------------------ + ## Main location — proxy to selected upstream + ## ------------------------------------------------------------------ + location / { + limit_req zone=perip burst=20 nodelay; + limit_req_status 503; + proxy_pass http://$gpu_upstream; + + ## Preserve original host and headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + ## Pass through the model header so backends can log it + proxy_pass_header X-Syslog-Model; + + ## Streaming support (SSE for LLM responses) + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + + ## Basic failover — retry on error or timeout + proxy_next_upstream error timeout http_502 http_503; + proxy_next_upstream_tries 2; + + ## Add a response header for observability + add_header X-Routed-To $gpu_upstream always; + + ## Fallback to queue when all GPU upstreams are down + error_page 502 503 504 = @queue_fallback; + } + + ## ------------------------------------------------------------------ + ## Queue fallback — enqueue when GPUs are unavailable + ## ------------------------------------------------------------------ + location @queue_fallback { + rewrite ^ /enqueue break; + proxy_pass http://queue_service; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Content-Type $content_type; + proxy_pass_request_body on; + } +} From 37f7c95b05c5db2164f1b44fe822d55fd7bb1ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:07:34 +0000 Subject: [PATCH 07/49] Add env example --- .env.example | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..528aa95 --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# Syslog Harness Environment +REDIS_HOST=192.168.68.8 +REDIS_PORT=6379 +AMDPVE_ENDPOINT=http://192.168.68.15:8080 +LLMGPU_ENDPOINT=http://192.168.68.8:8080 +OCU_LLM_ENDPOINT=http://192.168.68.110:8080 +CIRCUIT_BREAKER_THRESHOLD=5 +CIRCUIT_BREAKER_TIMEOUT=30 From 7d00bbec0e3cce31bfd295fcb0936b7af72150f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:34:49 +0000 Subject: [PATCH 08/49] Add Dockerfile.queue --- Dockerfile.queue | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Dockerfile.queue diff --git a/Dockerfile.queue b/Dockerfile.queue new file mode 100644 index 0000000..e69de29 From cf7f61650fdaf558098ea83e5bc0f4e545d95278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:34:52 +0000 Subject: [PATCH 09/49] Add Dockerfile.dashboard --- Dockerfile.dashboard | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Dockerfile.dashboard diff --git a/Dockerfile.dashboard b/Dockerfile.dashboard new file mode 100644 index 0000000..e69de29 From b65ea22765fcff6cd96028eedc6c781be1db6125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mumuni=20=F0=9F=A6=85=20=28Syslog=20Falcon=29?= Date: Fri, 15 May 2026 21:35:13 +0000 Subject: [PATCH 10/49] Update Nginx Docker config --- gpu-router-docker.conf | 106 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 gpu-router-docker.conf diff --git a/gpu-router-docker.conf b/gpu-router-docker.conf new file mode 100644 index 0000000..10e930f --- /dev/null +++ b/gpu-router-docker.conf @@ -0,0 +1,106 @@ +## Syslog GPU Router — Nginx Configuration (Docker-internal) +## Routes incoming agent requests to the appropriate GPU backend +## based on the X-Syslog-Model header. + +upstream amdpve_pool { + ## Strix Halo 395 — qwen3.6-35B-A3B (MoE) — Default workhorse + server 192.168.68.15:8080; +} + +upstream llmgpu_pool { + ## RTX 3090 — qwen3.5-27B (Dense) — Heavy reasoning + server 192.168.68.8:8080; +} + +upstream ocu_llm_pool { + ## RTX 5070 — gemma-4 (Dense 4B) — Ultra-light tasks + server 192.168.68.110:8080; +} + +upstream queue_service { + ## Agent queue with circuit breaker (Docker container) + server queue-service:8091; +} + +upstream dashboard_service { + ## Harness dashboard (Docker container) + server dashboard:3001; +} + +## ------------------------------------------------------------------ +## Mapping: X-Syslog-Model header → upstream backend +## ------------------------------------------------------------------ +map $http_x_syslog_model $gpu_upstream { + default amdpve_pool; + "standard" amdpve_pool; + "heavy" llmgpu_pool; + "qwen3.5-27B" llmgpu_pool; + "light" ocu_llm_pool; + "gemma-4" ocu_llm_pool; +} + +## Rate limit zone — 10 req/s per IP, burst of 20 +limit_req_zone $binary_remote_addr zone=perip:10m rate=10r/s; + +server { + listen 80; + server_name _; + + ## ------------------------------------------------------------------ + ## Dashboard — observability UI (MUST be before / catch-all) + ## ------------------------------------------------------------------ + location /dashboard { + proxy_pass http://dashboard_service/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + ## ------------------------------------------------------------------ + ## Main location — proxy to selected upstream + ## ------------------------------------------------------------------ + location / { + limit_req zone=perip burst=20 nodelay; + limit_req_status 503; + proxy_pass http://$gpu_upstream; + + ## Preserve original host and headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + ## Pass through the model header so backends can log it + proxy_pass_header X-Syslog-Model; + + ## Streaming support (SSE for LLM responses) + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + + ## Basic failover — retry on error or timeout + proxy_next_upstream error timeout http_502 http_503; + proxy_next_upstream_tries 2; + + ## Add a response header for observability + add_header X-Routed-To $gpu_upstream always; + + ## Fallback to queue when all GPU upstreams are down + error_page 502 503 504 = @queue_fallback; + } + + ## ------------------------------------------------------------------ + ## Queue fallback — enqueue when GPUs are unavailable + ## ------------------------------------------------------------------ + location @queue_fallback { + rewrite ^ /enqueue break; + proxy_pass http://queue_service; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Content-Type $content_type; + proxy_pass_request_body on; + } +} From 7b6c6aabe1794e88e36631cb766f09473bbc3bb4 Mon Sep 17 00:00:00 2001 From: "Abiba (pi)" Date: Sat, 16 May 2026 18:51:50 +0000 Subject: [PATCH 11/49] =?UTF-8?q?Initial=20commit:=20CT=20116=20inference?= =?UTF-8?q?=20harness=20=E2=80=94=20nginx,=20LiteLLM,=20router,=20dashboar?= =?UTF-8?q?d,=20Redis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Complexity-based routing (MoE default, Dense heavy, Gemma light) - Per-agent API keys with metrics tracking - Time-series usage graphs (24h/7d/30d) - Streaming support (SSE passthrough) - Unicode cleanup (ASCII-only output) - Vision support (gemma-4-E4B) - Tier enforcement (starter/professional/enterprise) - GPU health monitoring via sidecar polling - Unified dashboard with line graph --- .gitignore | 5 + README.md | 39 +++++ dashboard/Dockerfile | 7 + dashboard/dashboard.py | 290 +++++++++++++++++++++++++++++++++++++ dashboard/requirements.txt | 2 + docker-compose.yml | 77 ++++++++++ litellm_config.yaml | 25 ++++ nginx/nginx.conf | 79 ++++++++++ router/Dockerfile | 9 ++ router/requirements.txt | 3 + router/router.py | 213 +++++++++++++++++++++++++++ 11 files changed, 749 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 dashboard/Dockerfile create mode 100644 dashboard/dashboard.py create mode 100644 dashboard/requirements.txt create mode 100644 docker-compose.yml create mode 100644 litellm_config.yaml create mode 100644 nginx/nginx.conf create mode 100644 router/Dockerfile create mode 100644 router/requirements.txt create mode 100644 router/router.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1edc785 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +.env +redis-data/ +ssl/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..4381487 --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# syslog-harness — Inference API Harness + +CT 116 Docker stack for routing local GPU models through a unified OpenAI-compatible API. + +## Architecture + +``` +nginx :80 → router :9000 → GPU backends + ├─ qwen3.6-35B-A3B (MoE) @ 192.168.68.15:8080 + ├─ qwen3.6-27B-code (Dense) @ 192.168.68.8:8080 + └─ gemma-4-E4B (Light) @ 192.168.68.110:8080 + +LiteLLM :8081 (fallback) | Dashboard :3000 | Redis :6379 (local) +``` + +## Deploy + +```bash +cd /opt/inference-harness +docker compose up -d +``` + +## Endpoints + +| URL | Purpose | +|-----|---------| +| `/v1/chat/completions` | Inference API (OpenAI-compatible) | +| `/v1/models` | Available models | +| `/` | Dashboard (GPU health, routing, agents, timeseries) | + +## Agent API Keys + +| Agent | Key | +|-------|-----| +| Abiba | `sk-syslog-abiba` | +| Mumuni | `sk-syslog-mumuni` | +| Tanko | `sk-syslog-tanko` | +| Koby | `sk-syslog-koby` | +| Kagenz0 | `sk-syslog-kagenz0` | diff --git a/dashboard/Dockerfile b/dashboard/Dockerfile new file mode 100644 index 0000000..2e207e3 --- /dev/null +++ b/dashboard/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY dashboard.py . +EXPOSE 3000 +CMD ["python", "dashboard.py"] diff --git a/dashboard/dashboard.py b/dashboard/dashboard.py new file mode 100644 index 0000000..79b5ff3 --- /dev/null +++ b/dashboard/dashboard.py @@ -0,0 +1,290 @@ +"""Harness Dashboard.""" +import os, json, time, queue, threading +import requests +from flask import Flask, request, render_template_string, Response, stream_with_context + +ROUTER_METRICS = os.environ.get("ROUTER_METRICS_URL", "http://router:9000/metrics") + +app = Flask(__name__) +sse_subscribers = [] +sse_lock = threading.Lock() + +def fetch_state(): + try: + r = requests.get(ROUTER_METRICS, timeout=5) + if r.status_code == 200: return r.json() + except Exception: pass + return {"gpus":[],"route_counts":{},"agent_counts":{},"recent":[],"timestamp":time.time()} + +def broadcast_loop(): + while True: + time.sleep(3) + data = fetch_state() + payload = json.dumps(data) + with sse_lock: + dead = [] + for q in sse_subscribers: + try: q.put(payload) + except Exception: dead.append(q) + for q in dead: sse_subscribers.remove(q) + +threading.Thread(target=broadcast_loop, daemon=True).start() + +DASHBOARD_HTML = r""" + + + + +Inference Harness - Syslog Solution LLC + + + +
+

⚡ Inference Harness

+
+ + connecting... + + 0 requests +
+
+
+
+
GPU Health
+
Loading...
+
+
+
Model Distribution
+
-
+
+
+
+ Usage Over Time +
+ + + +
+
+
+
Loading...
+
+
+
+
+
Agent Activity
+
-
+
+
+
Live Request Stream
+
+ + + +
TimeAgentModelReasonTier
Waiting for data...
+
+
+
+ + +""" + +@app.route("/") +def dashboard(): + return render_template_string(DASHBOARD_HTML) + +@app.route("/api/state") +def api_state(): + return fetch_state() + +@app.route("/api/timeseries") +def api_timeseries(): + period = request.args.get("period", "day") + try: + r = requests.get("http://router:9000/metrics/timeseries?period=" + period, timeout=5) + if r.status_code == 200: return r.json() + except Exception: pass + return {"models": {}, "labels": []} + +@app.route("/api/stream") +def api_stream(): + def event_stream(): + q = queue.Queue() + with sse_lock: sse_subscribers.append(q) + try: + data = fetch_state() + yield "data: " + json.dumps(data) + "\n\n" + while True: + try: msg = q.get(timeout=3); yield "data: " + msg + "\n\n" + except queue.Empty: + data = fetch_state() + yield "data: " + json.dumps(data) + "\n\n" + except GeneratorExit: pass + finally: + with sse_lock: + if q in sse_subscribers: sse_subscribers.remove(q) + return Response(stream_with_context(event_stream()), mimetype="text/event-stream", + headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) + +@app.route("/health") +def health(): + return {"status":"healthy","service":"harness-dashboard"} + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=3000, debug=False) diff --git a/dashboard/requirements.txt b/dashboard/requirements.txt new file mode 100644 index 0000000..b1d9136 --- /dev/null +++ b/dashboard/requirements.txt @@ -0,0 +1,2 @@ +flask==3.1.* +requests==2.32.* diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8d999af --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,77 @@ +version: '3.8' + +services: + redis: + image: redis:7-alpine + container_name: harness-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + router: + build: ./router + container_name: harness-router + restart: unless-stopped + ports: + - "9000:9000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_MOE_URL=http://192.168.68.15:8080/v1 + - GPU_DENSE_URL=http://192.168.68.8:8080/v1 + - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 + depends_on: + redis: + condition: service_healthy + + litellm: + image: ghcr.io/berriai/litellm:main-stable + command: ["--config", "/app/config.yaml", "--port", "4000"] + container_name: harness-litellm + restart: unless-stopped + ports: + - "8081:4000" + volumes: + - ./litellm_config.yaml:/app/config.yaml + environment: + - LITELLM_MASTER_KEY=sk-syslog-local-master-key + depends_on: + redis: + condition: service_healthy + + nginx: + image: nginx:alpine + container_name: harness-nginx + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - litellm + - dashboard + + dashboard: + build: ./dashboard + container_name: harness-dashboard + restart: unless-stopped + ports: + - "3000:3000" + environment: + - REDIS_URL=redis://redis:6379 + - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 + depends_on: + - redis + +volumes: + redis-data: + +# LiteLLM command override to load config +# (appended to fix config loading issue) diff --git a/litellm_config.yaml b/litellm_config.yaml new file mode 100644 index 0000000..0677909 --- /dev/null +++ b/litellm_config.yaml @@ -0,0 +1,25 @@ +model_list: + - model_name: qwen3.6-35B-A3B + litellm_params: + model: openai/qwen3.6-35B-A3B + api_base: http://192.168.68.15:8080/v1 + api_key: "not-needed" + + - model_name: qwen3.6-27B-code + litellm_params: + model: openai/qwen3.6-27B-code-text + api_base: http://192.168.68.8:8080/v1 + api_key: "not-needed" + + - model_name: gemma-4-E4B + litellm_params: + model: openai/gemma-4-E4B + api_base: http://192.168.68.110:8080/v1 + api_key: "not-needed" + +general_settings: + master_key: sk-syslog-local-master-key + +litellm_settings: + drop_params: true + request_timeout: 120 diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..fc101ee --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,79 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { worker_connections 1024; } + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main launching rt=; + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log; + sendfile on; + keepalive_timeout 65; + + upstream router_api { server router:9000; } + upstream dashboard_ui { server dashboard:3000; } + upstream litellm_backend { server litellm:4000; } + + server { + listen 80; + + # Disable buffering for SSE streams + proxy_buffering off; + + # API — through router + location /v1/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_read_timeout 300s; + proxy_buffering off; + } + + # SSE streaming endpoint + location /stream { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + proxy_buffering off; + chunked_transfer_encoding off; + } + + # Dashboard API proxy for SSE + location /api/ { + proxy_pass http://dashboard_ui; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_buffering off; + } + + # LiteLLM debug + location /litellm/ { + rewrite ^/litellm/(.*) /$1 break; + proxy_pass http://litellm_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Authorization $http_authorization; + } + + # Dashboard + location / { + proxy_pass http://dashboard_ui; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_buffering off; + } + + location /health { + return 200 "{\"status\":\"healthy\"}"; + add_header Content-Type application/json; + } + } +} diff --git a/router/Dockerfile b/router/Dockerfile new file mode 100644 index 0000000..b5d85f7 --- /dev/null +++ b/router/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12-slim + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY router.py . + +EXPOSE 9000 +CMD ["python", "router.py"] diff --git a/router/requirements.txt b/router/requirements.txt new file mode 100644 index 0000000..2ec289b --- /dev/null +++ b/router/requirements.txt @@ -0,0 +1,3 @@ +flask==3.1.* +redis==5.2.* +requests==2.32.* diff --git a/router/router.py b/router/router.py new file mode 100644 index 0000000..34a2dd7 --- /dev/null +++ b/router/router.py @@ -0,0 +1,213 @@ +import os, json, time, logging, traceback, threading, queue +import requests, redis +from flask import Flask, request, jsonify, Response, stream_with_context + +REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379") +GPU_MOE_URL = os.environ.get("GPU_MOE_URL", "http://192.168.68.15:8080/v1") +GPU_DENSE_URL = os.environ.get("GPU_DENSE_URL", "http://192.168.68.8:8080/v1") +GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1") + +GPU_SIDECARS = { + "qwen3.6-35B-A3B": "http://192.168.68.15:8090", + "qwen3.6-27B-code": "http://192.168.68.8:8090", + "gemma-4-E4B": "http://192.168.68.110:8090", +} +GPU_URLS = { + "qwen3.6-35B-A3B": GPU_MOE_URL, + "qwen3.6-27B-code": GPU_DENSE_URL, + "gemma-4-E4B": GPU_LIGHT_URL, +} +TIER_MODELS = { + "starter": ["gemma-4-E4B"], + "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-E4B"], + "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-E4B"], +} +API_KEYS = { + "sk-syslog-local-master-key": {"tier": "enterprise", "agent": "admin"}, + "sk-syslog-abiba": {"tier": "enterprise", "agent": "Abiba"}, + "sk-syslog-mumuni": {"tier": "enterprise", "agent": "Mumuni"}, + "sk-syslog-tanko": {"tier": "enterprise", "agent": "Tanko"}, + "sk-syslog-koby": {"tier": "enterprise", "agent": "Koby"}, + "sk-syslog-kagenz0": {"tier": "enterprise", "agent": "Kagenz0"}, + "sk-starter-abc123": {"tier": "starter", "agent": "test-starter"}, + "sk-professional-xyz789": {"tier": "professional", "agent": "test-pro"}, +} + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s") +log = logging.getLogger("router") +try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping() +except Exception: r = None + +app = Flask(__name__) +sse_subscribers = []; sse_lock = threading.Lock() + +def check_gpu_health(model): + url = GPU_SIDECARS.get(model) + if not url: return {"status": "unknown"} + try: + resp = requests.get(url, timeout=5) + if resp.status_code == 200: + d = resp.json() + pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 + return {"status": "healthy" if pct < 90 else "saturated", "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} + except Exception: pass + return {"status": "down"} + +def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")] + +def estimate_tokens(msgs): return sum(len(str(m.get("content",""))) for m in msgs) // 4 + +def route(rd, tier): + msgs = rd.get("messages",[]); t = estimate_tokens(msgs) + sys = any(m.get("role")=="system" for m in msgs) + turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) + hints = rd.get("routing_hints",{}) + allowed = TIER_MODELS.get(tier, ["gemma-4-E4B"]) + avail = [m for m in available_models() if m in allowed] + if not avail: return {"model": allowed[0], "reason": "all_saturated"} + req = rd.get("model","auto") + if req != "auto": return {"model": req if req in avail else avail[0], "reason": "explicit"} + if hints: + if hints.get("priority")=="speed" and "gemma-4-E4B" in avail: return {"model":"gemma-4-E4B","reason":"hint_speed"} + if hints.get("priority")=="quality" and "qwen3.6-27B-code" in avail: return {"model":"qwen3.6-27B-code","reason":"hint_quality"} + if t > 4000 or sys or turns > 6: + for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","gemma-4-E4B"]: + if m in avail: return {"model":m,"reason":"heavy_reasoning"} + first_msg = msgs[0].get("content","") if msgs else "" + words = len(first_msg.split()) if isinstance(first_msg, str) else 99 + if words <= 3 and turns <= 1 and not sys and "gemma-4-E4B" in avail: + return {"model":"gemma-4-E4B","reason":"ultra_light"} + if "qwen3.6-35B-A3B" in avail: return {"model":"qwen3.6-35B-A3B","reason":"default_moe"} + return {"model":avail[0],"reason":"fallback"} + +def clean_unicode(text): + if not isinstance(text, str): return text + return text.replace("\u2014","-").replace("\u2013","-").replace("\u2018",").replace(u2019,").replace("\u201c",').replace(u201d,').replace("\u2026","...").replace("\u00a0"," ") + +def clean_response(d): + if isinstance(d, dict): return {k: clean_response(v) for k,v in d.items()} + if isinstance(d, list): return [clean_response(v) for v in d] + if isinstance(d, str): return clean_unicode(d) + return d + +def get_metrics(): + d = {"gpus":[],"route_counts":{},"agent_counts":{},"tier_counts":{},"recent":[],"timestamp":time.time()} + for m in GPU_URLS: + h = check_gpu_health(m) + d["gpus"].append({"id":m,"gpu_name":h.get("gpu_name",m),"status":h.get("status"),"vram_used_mb":h.get("vram_used_mb"),"vram_total_mb":h.get("vram_total_mb"),"vram_pct":h.get("vram_pct"),"temp_c":h.get("temp_c"),"gpu_util_pct":h.get("gpu_util_pct"),"power_w":h.get("power_w"),"power_limit_w":h.get("power_limit_w")}) + if r: + try: + for m in GPU_URLS: d["route_counts"][m] = int(r.get("routes:"+m) or 0) + for k,v in API_KEYS.items(): + c = int(r.get("routes:agent:"+v["agent"]) or 0) + if c>0: d["agent_counts"][v["agent"]] = c + for t in TIER_MODELS: d["tier_counts"][t] = int(r.get("routes:tier:"+t) or 0) + raw = r.lrange("routes:recent",0,49) + d["recent"] = [json.loads(x) for x in raw] if raw else [] + except Exception: pass + return d + +def bcast(): + data = get_metrics(); payload = json.dumps(data) + with sse_lock: + dead = [] + for q in sse_subscribers: + try: q.put(payload) + except Exception: dead.append(q) + for q in dead: sse_subscribers.remove(q) + +@app.route("/v1/chat/completions", methods=["POST"]) +def chat(): + try: + rd = request.get_json(force=True) + ak = request.headers.get("Authorization","").replace("Bearer ","") + ki = API_KEYS.get(ak, {"tier":"starter","agent":"unknown"}) + tier, agent = ki["tier"], ki["agent"] + d = route(rd, tier); model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] + is_stream = rd.get("stream", False) + log.info("ROUTE: %s -> %s (%s) stream=%s", agent, model, reason, is_stream) + if r: + try: + r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent) + r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H")) + r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent})) + r.ltrim("routes:recent",0,999) + except Exception: pass + start = time.time() + resp = requests.post(url+"/chat/completions", json=rd, + headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=120, stream=is_stream) + lat = int((time.time()-start)*1000) + if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 + if is_stream: + def gen(): + for raw in resp.iter_content(chunk_size=None, decode_unicode=True): + if raw: yield clean_unicode(raw) + bcast() + return Response(stream_with_context(gen()), mimetype="text/event-stream") + data = clean_response(resp.json()) + for c in data.get("choices",[]): + msg = c.get("message",{}) + if not msg.get("content") and msg.get("reasoning_content"): + msg["content"] = msg["reasoning_content"] + data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat} + bcast() + return jsonify(data) + except requests.Timeout: return jsonify({"error":"timeout"}), 504 + except Exception as e: + log.error("Error: %s\n%s", e, traceback.format_exc()) + return jsonify({"error":str(e)}), 500 + +@app.route("/v1/models") +def models(): return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":check_gpu_health(m).get("status"),"gpu":check_gpu_health(m).get("gpu_name")} for m in GPU_URLS]}) + +@app.route("/health") +def health(): return jsonify({"status":"healthy","redis":"connected" if r else "down","gpus":{m:check_gpu_health(m) for m in GPU_URLS},"available_models":available_models()}) + +@app.route("/metrics") +def metrics(): return jsonify(get_metrics()) + +@app.route("/metrics/timeseries") +def metrics_timeseries(): + period = request.args.get("period", "day"); models_list = list(GPU_URLS.keys()) + data = {"models": {}, "labels": []} + if period == "day": + buckets = [time.strftime("%Y%m%d%H", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] + data["labels"] = [time.strftime("%H:00", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)] + elif period == "week": + buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] + data["labels"] = [time.strftime("%a", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)] + else: + buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] + data["labels"] = [time.strftime("%m/%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)] + if r: + for model in models_list: + counts = [] + for bucket in buckets: + total = 0 + if period in ("week","month"): + for hh in range(24): total += int(r.get("ts:"+model+":"+bucket+"{:02d}".format(hh)) or 0) + else: total = int(r.get("ts:"+model+":"+bucket) or 0) + counts.append(total) + data["models"][model] = counts + return jsonify(data) + +@app.route("/stream") +def stream(): + def ev(): + q = queue.Queue() + with sse_lock: sse_subscribers.append(q) + try: + yield "data: "+json.dumps(get_metrics())+"\n\n" + while True: + try: yield "data: "+q.get(timeout=3)+"\n\n" + except queue.Empty: yield "data: "+json.dumps(get_metrics())+"\n\n" + except GeneratorExit: pass + finally: + with sse_lock: + if q in sse_subscribers: sse_subscribers.remove(q) + return Response(stream_with_context(ev()), mimetype="text/event-stream", + headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) + +if __name__ == "__main__": + log.info("Router on :9000") + app.run(host="0.0.0.0", port=9000, debug=False) From ec0f9fac634e4518baca86953af3b84ea597773e Mon Sep 17 00:00:00 2001 From: "Abiba (pi)" Date: Sat, 16 May 2026 19:12:58 +0000 Subject: [PATCH 12/49] Fix: clean_unicode now uses chr()-based replacements + ASCII strip to prevent bash heredoc corruption. Emoji and all non-ASCII now fully stripped. --- router/router.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/router/router.py b/router/router.py index 34a2dd7..f47e0de 100644 --- a/router/router.py +++ b/router/router.py @@ -81,8 +81,19 @@ def route(rd, tier): return {"model":avail[0],"reason":"fallback"} def clean_unicode(text): - if not isinstance(text, str): return text - return text.replace("\u2014","-").replace("\u2013","-").replace("\u2018",").replace(u2019,").replace("\u201c",').replace(u201d,').replace("\u2026","...").replace("\u00a0"," ") + if not isinstance(text, str): + return text + # Replace common Unicode punctuation with ASCII equivalents + text = text.replace(chr(0x2014), "-") + text = text.replace(chr(0x2013), "-") + text = text.replace(chr(0x2018), "'") + text = text.replace(chr(0x2019), "'") + text = text.replace(chr(0x201C), '"') + text = text.replace(chr(0x201D), '"') + text = text.replace(chr(0x2026), "...") + text = text.replace(chr(0x00A0), " ") + # Strip ALL remaining non-ASCII (emoji, symbols) + return text.encode("ascii", "ignore").decode("ascii") def clean_response(d): if isinstance(d, dict): return {k: clean_response(v) for k,v in d.items()} From 2db2796e533c5fcc765b7c11e6f6245a6d455d4f Mon Sep 17 00:00:00 2001 From: "Abiba (pi)" Date: Sat, 16 May 2026 19:26:46 +0000 Subject: [PATCH 13/49] Dashboard: rename to SyslogAI Harness, GPU bar now shows utilization instead of VRAM --- dashboard/dashboard.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboard/dashboard.py b/dashboard/dashboard.py index 79b5ff3..9b4e064 100644 --- a/dashboard/dashboard.py +++ b/dashboard/dashboard.py @@ -35,7 +35,7 @@ DASHBOARD_HTML = r""" -Inference Harness - Syslog Solution LLC +SyslogAI Harness - Syslog Solution LLC -
-

⚡ SyslogAI Harness

-
- - connecting... - - 0 requests + + +
+
+
⚡ SyslogAI Harness
+
+ + live · +
+
+
+
0
Requests
+
0
Active
+
0
Agents
-
- -
-
GPU Health
-
Loading...
-
- -
-
Queue Status
-
-
Loading...
+ +
+ +
Loading...
+
Loading...
+
Loading...
+ + +
Queue Status
+
Model Distribution
+
Agent Activity
+ + +
+ Usage Over Time +
+ + +
-
-
-
Model Distribution
-
-
-
-
-
Agent Activity
-
-
-
- -
-
- Usage Over Time -
- - - -
-
-
-
Loading...
-
-
-
- -
-
Live Request Stream
-
- - - -
TimeAgentModelReasonTier
Waiting for data...
-
-
+
+
GPU Metrics
+ + +
Live Stream
+
+ + +
TimeAgentModelReasonTier
+
+ """ -@app.route("/") +@app.route("/") def dashboard(): return render_template_string(DASHBOARD_HTML) @app.route("/api/state") diff --git a/nginx/nginx.conf b/nginx/nginx.conf index fc101ee..c5363ff 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -32,7 +32,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header Authorization $http_authorization; proxy_connect_timeout 10s; - proxy_read_timeout 300s; + proxy_read_timeout 600s; proxy_buffering off; } diff --git a/router/router.py b/router/router.py index 2af7acd..ffdceb8 100644 --- a/router/router.py +++ b/router/router.py @@ -210,7 +210,7 @@ def chat(): except Exception: pass start = time.time() resp = requests.post(url+"/chat/completions", json=rd, - headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=120, stream=is_stream) + headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream) lat = int((time.time()-start)*1000) gpu_decr(model) # Release slot @@ -229,8 +229,9 @@ def chat(): data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model)} bcast() return jsonify(data) - except requests.Timeout: - gpu_decr(model if 'model' in dir() else "unknown") + except requests.Timeout: + try: gpu_decr(model) + except: pass return jsonify({"error":"timeout"}), 504 except Exception as e: log.error("Error: %s\n%s", e, traceback.format_exc()) From 8f3b0c664709ad2849c0e3e1134f0f4bdb0b9be6 Mon Sep 17 00:00:00 2001 From: "Abiba (pi)" Date: Sun, 17 May 2026 01:52:28 +0000 Subject: [PATCH 18/49] Router: health check verifies actual llama.cpp endpoint, gpu_decr negative guard, AMD sidecar fixed (sysfs fallback) --- router/router.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/router/router.py b/router/router.py index ffdceb8..fba7836 100644 --- a/router/router.py +++ b/router/router.py @@ -59,7 +59,10 @@ def gpu_incr(model): if r: r.incr("active:" + model) def gpu_decr(model): - if r: r.decr("active:" + model) + if r: + v = r.decr("active:" + model) + if v and int(v) < 0: + r.set("active:" + model, 0) # never go negative def check_gpu_health(model): url = GPU_SIDECARS.get(model) @@ -69,7 +72,16 @@ def check_gpu_health(model): if resp.status_code == 200: d = resp.json() pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 - return {"status": "healthy" if pct < 90 else "saturated", "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} + status = "healthy" if pct < 90 else "saturated" + # Also check if llama.cpp endpoint is actually responding + gpu_url = GPU_URLS.get(model, "") + try: + hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=3) + if hr.status_code != 200: + status = "down" + except Exception: + status = "down" + return {"status": status, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} except Exception: pass return {"status": "down"} @@ -230,10 +242,7 @@ def chat(): bcast() return jsonify(data) except requests.Timeout: - try: gpu_decr(model) - except: pass return jsonify({"error":"timeout"}), 504 - except Exception as e: log.error("Error: %s\n%s", e, traceback.format_exc()) return jsonify({"error":str(e)}), 500 From 4f032b035c3532f2559f08ac05c7c7b3c9d8944f Mon Sep 17 00:00:00 2001 From: "Abiba (pi)" Date: Sun, 17 May 2026 09:05:27 +0000 Subject: [PATCH 19/49] Mumuni review action items: health checks for all containers, version pinning, 503+Retry-After on all-GPU saturation --- docker-compose.yml | 20 ++++++++++++++++++++ router/router.py | 41 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8d999af..48671b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,11 @@ services: - GPU_MOE_URL=http://192.168.68.15:8080/v1 - GPU_DENSE_URL=http://192.168.68.8:8080/v1 - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] + interval: 15s + timeout: 5s + retries: 3 depends_on: redis: condition: service_healthy @@ -42,6 +47,11 @@ services: - ./litellm_config.yaml:/app/config.yaml environment: - LITELLM_MASTER_KEY=sk-syslog-local-master-key + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] + interval: 15s + timeout: 5s + retries: 3 depends_on: redis: condition: service_healthy @@ -54,6 +64,11 @@ services: - "80:80" volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] + interval: 15s + timeout: 5s + retries: 3 depends_on: - litellm - dashboard @@ -67,6 +82,11 @@ services: environment: - REDIS_URL=redis://redis:6379 - GPU_SIDECARS=192.168.68.15:8090,192.168.68.8:8090,192.168.68.110:8090 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3000/health')"] + interval: 15s + timeout: 5s + retries: 3 depends_on: - redis diff --git a/router/router.py b/router/router.py index fba7836..902c670 100644 --- a/router/router.py +++ b/router/router.py @@ -46,6 +46,29 @@ log = logging.getLogger("router") try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping() except Exception: r = None + +def counter_audit_loop(): + """Every 30s, check GPU slots and reset counters if all slots idle.""" + while True: + time.sleep(30) + if not r: continue + for model, url in GPU_URLS.items(): + try: + resp = requests.get(url.replace("/v1","") + "/slots", + headers={"Authorization": "Bearer not-needed"}, timeout=5) + if resp.status_code == 200: + slots = resp.json() + all_idle = all(not s.get("is_processing", False) for s in slots) + if all_idle: + current = int(r.get("active:" + model) or 0) + if current > 0: + r.set("active:" + model, 0) + log.info("AUDIT: Reset stuck counter for %s (was %d)", model, current) + except Exception: + pass + +threading.Thread(target=counter_audit_loop, daemon=True).start() + app = Flask(__name__) sse_subscribers = []; sse_lock = threading.Lock() @@ -118,7 +141,7 @@ def route(rd, tier): hints = rd.get("routing_hints",{}) allowed = TIER_MODELS.get(tier, ["gemma-4-E4B"]) avail = [m for m in available_models() if m in allowed] - if not avail: return {"model": allowed[0], "reason": "all_saturated"} + if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} req = rd.get("model","auto") if req != "auto": @@ -207,10 +230,16 @@ def chat(): ak = request.headers.get("Authorization","").replace("Bearer ","") ki = API_KEYS.get(ak, {"tier":"starter","agent":"unknown"}) tier, agent = ki["tier"], ki["agent"] - d = route(rd, tier); model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] + d = route(rd, tier) + if d.get("saturated"): + resp = jsonify({"error": "All GPUs saturated", "retry_after_s": 5}) + resp.headers["Retry-After"] = "5" + return resp, 503 + model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] is_stream = rd.get("stream", False) - gpu_incr(model) # Track active request + gpu_incr(model) + decremented = False log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1)) if r: @@ -224,7 +253,8 @@ def chat(): resp = requests.post(url+"/chat/completions", json=rd, headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream) lat = int((time.time()-start)*1000) - gpu_decr(model) # Release slot + gpu_decr(model) + decremented = True # Release slot if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 if is_stream: @@ -241,6 +271,9 @@ def chat(): data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model)} bcast() return jsonify(data) + if not decremented: + try: gpu_decr(model) + except: pass except requests.Timeout: return jsonify({"error":"timeout"}), 504 log.error("Error: %s\n%s", e, traceback.format_exc()) From 9c31b5d62274bbcea1bc19bb3c100b23000ca547 Mon Sep 17 00:00:00 2001 From: Abiba Date: Tue, 19 May 2026 15:03:34 +0000 Subject: [PATCH 20/49] May 19, 2026: Full harness update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Model migration: gemma-4-E4B → qwen3.5-9b-vlm - Dashboard reorder: Usage Over Time + GPU Metrics to top - Router counter leak fix (gpu_decr in except handler) - VLM slot upgrade 1→2 - Redis stale key cleanup - Automated maintenance cron job - LiteLLM config update - GPU router config update - README update --- .gitignore | 4 +--- README.md | 2 +- dashboard/dashboard.py | 34 +++++++++++++++++----------------- gpu-router-docker.conf | 4 ++-- gpu-router.conf | 4 ++-- litellm_config.yaml | 4 ++-- router/router.py | 37 ++++++++++++++++++------------------- 7 files changed, 43 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index 1edc785..eeb7737 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ +.git __pycache__/ *.pyc -.env -redis-data/ -ssl/ diff --git a/README.md b/README.md index 4381487..04e9659 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ CT 116 Docker stack for routing local GPU models through a unified OpenAI-compat nginx :80 → router :9000 → GPU backends ├─ qwen3.6-35B-A3B (MoE) @ 192.168.68.15:8080 ├─ qwen3.6-27B-code (Dense) @ 192.168.68.8:8080 - └─ gemma-4-E4B (Light) @ 192.168.68.110:8080 + └─ qwen3.5-9b-vlm (VLM) @ 192.168.68.110:8080 LiteLLM :8081 (fallback) | Dashboard :3000 | Redis :6379 (local) ``` diff --git a/dashboard/dashboard.py b/dashboard/dashboard.py index a1e5ee6..ecda580 100644 --- a/dashboard/dashboard.py +++ b/dashboard/dashboard.py @@ -80,17 +80,7 @@ body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMac
- -
Loading...
-
Loading...
-
Loading...
- - -
Queue Status
-
Model Distribution
-
Agent Activity
- - +
Usage Over Time
@@ -101,6 +91,16 @@ body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMac
GPU Metrics
+ +
Loading...
+
Loading...
+
Loading...
+ + +
Queue Status
+
Model Distribution
+
Agent Activity
+
Live Stream
@@ -111,9 +111,9 @@ body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMac """ @@ -205,6 +314,26 @@ def dashboard(): return render_template_string(DASHBOARD_HTML) @app.route("/api/state") def api_state(): return fetch_state() +@app.route("/api/scatter") +def api_scatter(): + window = request.args.get("window", "24") + model = request.args.get("model", "all") + try: + r = requests.get(f"http://router:9000/metrics/scatter?window={window}&model={model}", timeout=10) + if r.status_code == 200: return r.json() + except Exception: pass + return {"points": [], "count": 0} + +@app.route("/api/performance") +def api_performance(): + window = request.args.get("window", "24") + model = request.args.get("model", "all") + try: + r = requests.get(f"http://router:9000/metrics/performance?window={window}&model={model}", timeout=10) + if r.status_code == 200: return r.json() + except Exception: pass + return {"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}} + @app.route("/api/timeseries") def api_timeseries(): period = request.args.get("period", "day") diff --git a/phase0-dual-keys.json b/phase0-dual-keys.json new file mode 100644 index 0000000..6114325 --- /dev/null +++ b/phase0-dual-keys.json @@ -0,0 +1,20 @@ +{ + "sk-syslog-local-master-key": {"tier": "enterprise", "agent": "admin", "deprecated": true}, + "sk-33a0d6e6-a6da12fb483770d6f63f543b7e16a742": {"tier": "enterprise", "agent": "admin"}, + "sk-syslog-abiba": {"tier": "enterprise", "agent": "Abiba", "deprecated": true}, + "sk-6a6e49d0-e35e27c524ce0ba2de8f862a0e966b0c": {"tier": "enterprise", "agent": "Abiba"}, + "sk-syslog-mumuni": {"tier": "enterprise", "agent": "Mumuni", "deprecated": true}, + "sk-ba249c99-29e3b163a8d8ab7abb8663cae0dceb37": {"tier": "enterprise", "agent": "Mumuni"}, + "sk-syslog-tanko": {"tier": "enterprise", "agent": "Tanko", "deprecated": true}, + "sk-4a67112a-d9b62caf5a5881a15837df506f782708": {"tier": "enterprise", "agent": "Tanko"}, + "sk-syslog-koby": {"tier": "enterprise", "agent": "Koby", "deprecated": true}, + "sk-9dd66bba-316f616ec40cb51f3b489a0146bb986f": {"tier": "enterprise", "agent": "Koby"}, + "sk-syslog-kagenz0": {"tier": "enterprise", "agent": "Kagenz0", "deprecated": true}, + "sk-5a05020f-a984a8a20f731c5a65dfc07d51495bb3": {"tier": "enterprise", "agent": "Kagenz0"}, + "sk-syslog-koonimo": {"tier": "enterprise", "agent": "Koonimo", "deprecated": true}, + "sk-b2f0e1b2-70b7cf1c5a5e402a49b5469e88d9e335": {"tier": "enterprise", "agent": "Koonimo"}, + "sk-starter-abc123": {"tier": "starter", "agent": "test-starter", "deprecated": true}, + "sk-a8307a00-cd76daa07dd61c41aeb4545656f8b8c4": {"tier": "starter", "agent": "test-starter"}, + "sk-professional-xyz789": {"tier": "professional", "agent": "test-pro", "deprecated": true}, + "sk-b519a91e-4d2774c6c7da3fc31326e69ccc426781": {"tier": "professional", "agent": "test-pro"} +} diff --git a/router/router.py b/router/router.py index 8d1a754..7abbc0b 100644 --- a/router/router.py +++ b/router/router.py @@ -1,4 +1,4 @@ -import os, json, time, logging, traceback, threading, queue +import os, json, time, logging, traceback, threading, queue, statistics, math import requests, redis from flask import Flask, request, jsonify, Response, stream_with_context @@ -19,16 +19,16 @@ GPU_URLS = { } # Max concurrent requests per GPU (based on llama.cpp --parallel) GPU_MAX_CONCURRENT = { - "qwen3.6-35B-A3B": 2, # 2 slots + "qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating) "qwen3.6-27B-code": 2, # 2 slots - "gemma-4-12b": 2, # 2 slots (7.1GB VRAM) + "gemma-4-12b": 2, # 2 slots (12GB VRAM, 4GB headroom) } # Context window sizes (tokens) — used for compaction signals GPU_CONTEXT = { - "qwen3.6-35B-A3B": 131072, + "qwen3.6-35B-A3B": 262144, "qwen3.6-27B-code": 262144, - "gemma-4-12b": 262144, # 262K context (gemma-4-12b) + "gemma-4-12b": 262144, } TIER_MODELS = { @@ -36,18 +36,45 @@ TIER_MODELS = { "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], } -API_KEYS = { - "sk-syslog-local-master-key": {"tier": "enterprise", "agent": "admin"}, - "sk-syslog-abiba": {"tier": "enterprise", "agent": "Abiba"}, - "sk-syslog-mumuni": {"tier": "enterprise", "agent": "Mumuni"}, - "sk-syslog-tanko": {"tier": "enterprise", "agent": "Tanko"}, - "sk-syslog-koby": {"tier": "enterprise", "agent": "Koby"}, - "sk-syslog-kagenz0": {"tier": "enterprise", "agent": "Kagenz0"}, - "sk-syslog-koonimo": {"tier": "enterprise", "agent": "Koonimo"}, - "sk-starter-abc123": {"tier": "starter", "agent": "test-starter"}, - "sk-professional-xyz789": {"tier": "professional", "agent": "test-pro"}, +# ── PHASE 0: Dual-Key API Key System ── +# API_KEYS env var is REQUIRED (JSON string). No hardcoded fallback. +# Format: {"sk-xxx": {"tier": "enterprise", "agent": "Name"}} +# Deprecated keys get {"deprecated": true} — still accepted, logged with warning. +_raw_keys = os.environ.get("API_KEYS") +if not _raw_keys: + raise RuntimeError("FATAL: API_KEYS environment variable is required. " + "Set it in docker-compose.yml or .env file. " + "No hardcoded keys fallback — this is a security feature.") +API_KEYS = json.loads(_raw_keys) +log.info("Loaded %d API keys from env var (%d deprecated)", + len(API_KEYS), sum(1 for v in API_KEYS.values() if v.get("deprecated"))) +# Rate limits: requests per minute per API key tier +RATE_LIMIT_RPM = { + "enterprise": 120, + "professional": 60, + "starter": 20, } +def check_rate_limit(api_key, tier): + """Token bucket rate limiter using Redis. Returns (allowed, retry_after_or_remaining, reset_seconds).""" + if not r: + return True, 999, 60 + limit = RATE_LIMIT_RPM.get(tier, 30) + key = f"ratelimit:{api_key}" + current = int(r.get(key) or 0) + if current >= limit: + ttl = r.ttl(key) + retry = max(ttl, 1) if ttl and ttl > 0 else 60 + return False, retry, 0 + pipe = r.pipeline() + pipe.incr(key) + pipe.expire(key, 60) # 1-minute sliding window + pipe.execute() + remaining = limit - (current + 1) + reset_seconds = r.ttl(key) or 60 + return True, remaining, reset_seconds + + logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s") log = logging.getLogger("router") try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping() @@ -94,24 +121,25 @@ def gpu_decr(model): if v and int(v) < 0: r.set("active:" + model, 0) # never go negative -def check_gpu_health(model): +def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3): url = GPU_SIDECARS.get(model) if not url: return {"status": "unknown"} try: - resp = requests.get(url, timeout=5) + resp = requests.get(url, timeout=sidecar_timeout) if resp.status_code == 200: d = resp.json() pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 - status = "healthy" if pct < 90 else "saturated" + status = "healthy" # VRAM usage != saturation; busy slots handled by is_gpu_busy() + vram_warning = pct >= 95 # Also check if llama.cpp endpoint is actually responding gpu_url = GPU_URLS.get(model, "") try: - hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=3) + hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout) if hr.status_code != 200: status = "down" except Exception: status = "down" - return {"status": status, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} + return {"status": status, "vram_warning": vram_warning, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")} except Exception: pass return {"status": "down"} @@ -121,14 +149,65 @@ def estimate_tokens(msgs): """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text).""" return len(json.dumps(msgs, default=str)) // 3.5 +def store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, stream): + """Store detailed performance record in Redis for analytics.""" + if not r: return + try: + total_ms = queue_ms + inference_ms + tps = completion_tokens / (inference_ms / 1000) if inference_ms > 0 and completion_tokens > 0 else 0 + rec = json.dumps({ + "ts": time.time(), + "model": model, "agent": agent, "tier": tier, "reason": reason, + "queue_ms": round(queue_ms, 1), + "inference_ms": round(inference_ms, 1), + "total_ms": round(total_ms, 1), + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "tokens_per_sec": round(tps, 1), + "stream": stream + }) + # Global recent list (last 500) + r.lpush("perf:recent", rec) + r.ltrim("perf:recent", 0, 499) + # Per-model list (last 200) + r.lpush("perf:model:" + model, rec) + r.ltrim("perf:model:" + model, 0, 199) + # Per-reason list (last 200) + r.lpush("perf:reason:" + reason, rec) + r.ltrim("perf:reason:" + reason, 0, 199) + # Per-agent list (last 200) + r.lpush("perf:agent:" + agent, rec) + r.ltrim("perf:agent:" + agent, 0, 199) + except Exception: + pass + def is_gpu_busy(model): """Check if GPU is at or near max concurrent capacity.""" active = gpu_active_count(model) max_c = GPU_MAX_CONCURRENT.get(model, 1) return active >= max_c -def select_best_gpu(candidates, reason): - """Pick the best GPU from candidates IN ORDER — first non-busy one wins.""" +def select_best_gpu(candidates, reason, agent=""): + """Pick best GPU, spreading agents across GPUs to prevent hotspots.""" + # Count how many distinct agents are on each GPU + gpu_agent_counts = {} + if r: + for m in GPU_URLS: + count = 0 + for ak in API_KEYS.values(): + if r.get("agent_gpu:" + ak["agent"] + ":" + m): + count += 1 + gpu_agent_counts[m] = count + # First pass: prefer GPUs with 0 other agents (fresh GPU for this agent) + for m in candidates: + if not is_gpu_busy(m) and gpu_agent_counts.get(m, 0) == 0: + return {"model": m, "reason": reason} + # Second pass: prefer GPU this agent is NOT already on (skip own GPU) + if agent: + for m in candidates: + if not is_gpu_busy(m) and not r.get("agent_gpu:" + agent + ":" + m): + return {"model": m, "reason": reason} + # Third pass: any non-busy GPU for m in candidates: if not is_gpu_busy(m): return {"model": m, "reason": reason} @@ -144,7 +223,7 @@ def select_best_gpu(candidates, reason): return {"model": best, "reason": "load_balanced_" + reason} return None -def route(rd, tier): +def route(rd, tier, agent=""): msgs = rd.get("messages",[]); t = estimate_tokens(msgs) sys = any(m.get("role")=="system" for m in msgs) turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) @@ -163,53 +242,52 @@ def route(rd, tier): if is_gpu_busy(target) and req in allowed: alts = [m for m in avail if m != target and m in allowed] if alts: - alt = select_best_gpu(alts, "explicit") + alt = select_best_gpu(alts, "explicit", agent) if alt: return alt return {"model": target, "reason": "explicit"} if hints: if hints.get("priority")=="speed" and "gemma-4-12b" in avail: - return select_best_gpu(["gemma-4-12b"], "hint_speed") or {"model":"gemma-4-12b","reason":"hint_speed"} - if hints.get("priority")=="quality" and "qwen3.6-27B-code" in avail: - return select_best_gpu(["qwen3.6-27B-code"], "hint_quality") or {"model":"qwen3.6-27B-code","reason":"hint_quality"} + return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} + if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: + return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} first_msg = msgs[0].get("content","") if msgs else "" words = len(first_msg.split()) if isinstance(first_msg, str) else 99 - # TIER 1: Lightweight — single-turn short queries → VLM first - if not sys and turns <= 1 and words <= 100 and "gemma-4-12b" in avail: + # TIER 1: Lightweight — single-turn short queries → VLM (fastest) + if not sys and turns <= 1 and t <= 500 and words <= 100 and "gemma-4-12b" in avail: if not is_gpu_busy("gemma-4-12b"): return {"model":"gemma-4-12b","reason":"lightweight"} - # VLM busy — fall back to Dense, then MoE - fallback = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code"] if m in avail] - result = select_best_gpu(fallback, "lightweight_fallback") + # VLM busy — Dense is faster for short queries than MoE + fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(fallback, "lightweight_fallback", agent) if result: return result - # TIER 2: Simple conversations — short context, any prompt → VLM preferred - if t <= 1000 and turns <= 4 and "gemma-4-12b" in avail: + # TIER 2: Simple conversations — VLM primary (up to 15K tok), fastest for moderate chat + if t <= 15000 and turns <= 12 and "gemma-4-12b" in avail: if not is_gpu_busy("gemma-4-12b"): return {"model":"gemma-4-12b","reason":"simple_conv"} - # VLM busy — try Dense - if "qwen3.6-27B-code" in avail and not is_gpu_busy("qwen3.6-27B-code"): - return {"model":"qwen3.6-27B-code","reason":"simple_conv_fallback"} + # VLM busy — fall back to Dense, then MoE + fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(fallback, "simple_conv_fallback", agent) + if result: return result - # TIER 3: Heavy reasoning — extremely large context or very long conversations - if t > 50000 or turns > 25: - # MoE first (131K context handles heavy sessions), then Dense (98K reasoning), then Light (131K fallback) + # TIER 3: Medium complexity — Dense primary, VLM fallback (quality + speed balance) + if t <= 25000: + candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "medium", agent) + if result: return result + + # TIER 4: Heavy reasoning — MoE primary (workhorse), Dense fallback + if t > 25000: candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"] if m in avail] - result = select_best_gpu(candidates, "heavy_reasoning") + result = select_best_gpu(candidates, "heavy_reasoning", agent) if result: return result - # TIER 4: Default — MoE first, VLM helps, Dense last (slow) - if t <= 50000: - candidates = [m for m in ["qwen3.6-35B-A3B","gemma-4-12b","qwen3.6-27B-code"] if m in avail] - result = select_best_gpu(candidates, "default") - if result: return result - - # Fallback — best available - if "qwen3.6-35B-A3B" in avail and not is_gpu_busy("qwen3.6-35B-A3B"): - return {"model":"qwen3.6-35B-A3B","reason":"default_moe"} - result = select_best_gpu([m for m in avail], "fallback") + # TIER 5: Default — Dense primary, MoE fallback + candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "default", agent) if result: return result return {"model":avail[0],"reason":"last_resort"} @@ -266,6 +344,27 @@ def chat(): return jsonify({"error": "Unauthorized — valid API key required"}), 401 ki = API_KEYS[ak] tier, agent = ki["tier"], ki["agent"] + # Phase 0: dual-key transition — log deprecated key usage + if ki.get("deprecated"): + new_key = next((k for k, v in API_KEYS.items() + if v.get("agent") == agent and not v.get("deprecated")), None) + log.warning("DEPRECATED_KEY: agent=%s using old key %s...%s — switch to %s...%s", + agent, ak[:12], ak[-8:], + new_key[:12] if new_key else "N/A", + new_key[-8:] if new_key else "N/A") + if r: + r.incr("deprecated_usage:" + agent) + + # Rate limit check + allowed, rl_val, reset_sec = check_rate_limit(ak, tier) + if not allowed: + resp = jsonify({"error": "Rate limit exceeded", "retry_after_s": rl_val}) + resp.headers["Retry-After"] = str(rl_val) + resp.headers["X-RateLimit-Limit"] = str(RATE_LIMIT_RPM.get(tier, 30)) + resp.headers["X-RateLimit-Remaining"] = "0" + resp.headers["X-RateLimit-Reset"] = str(int(time.time() + rl_val)) + log.warning("RATE_LIMIT: %s (%s) exceeded limit", agent, ak[-8:]) + return resp, 429 # Allow agent to override queue timeout via header q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT))) @@ -281,7 +380,7 @@ def chat(): r.set("session:" + session_id, session_tokens, ex=86400) # TTL 24h except Exception: pass - d = route(rd, tier) + d = route(rd, tier, agent) queue_start = time.time() # Queue loop: wait for a GPU slot instead of immediate 503 @@ -293,22 +392,31 @@ def chat(): log.warning("QUEUE_TIMEOUT: %s waited %.1fs, all GPUs saturated", agent, elapsed) return resp, 503 time.sleep(0.5) # poll every 500ms - d = route(rd, tier) + d = route(rd, tier, agent) - waited = time.time() - queue_start - if waited > 0.5: - log.info("QUEUED: %s waited %.1fs before slot opened", agent, waited) + queue_ms = (time.time() - queue_start) * 1000 + if queue_ms > 500: + log.info("QUEUED: %s waited %.0fms before slot opened", agent, queue_ms) model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]] + + # Stash rate limit values for response headers + _rl_remaining = rl_val + _rl_limit = RATE_LIMIT_RPM.get(tier, 30) + _rl_reset = reset_sec is_stream = rd.get("stream", False) gpu_incr(model) log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1)) + # Track which GPU this agent is using (TTL 120s covers typical request) + if r and agent: + try: r.setex("agent_gpu:" + agent + ":" + model, 120, "1") + except: pass if r: try: r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent) r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H")) - r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent})) + r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent,"queue_ms": round(queue_ms,1)})) r.ltrim("routes:recent",0,999) except Exception: pass start = time.time() @@ -319,14 +427,47 @@ def chat(): if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502 if is_stream: + # Buffer SSE chunks, handle split lines for large responses + chunks = [] + stream_timings = {} + buf = "" # accumulate partial lines + for raw in resp.iter_content(chunk_size=None, decode_unicode=True): + if raw: + cleaned = clean_unicode(raw) + chunks.append(cleaned) + buf += cleaned + # Process complete lines from buffer + while "\n" in buf: + line, buf = buf.split("\n", 1) + line = line.strip() + if line.startswith("data: ") and not stream_timings: + js = line[6:].strip() + if js.startswith("{") and "timings" in js and "predicted_n" in js: + try: + tj = json.loads(js).get("timings", {}) + if tj: + stream_timings = tj + except: pass + # Store perf record with real token counts from stream + if stream_timings: + pt = stream_timings.get("prompt_n", 0) + ct = stream_timings.get("predicted_n", 0) + tps = stream_timings.get("predicted_per_second", 0) + gen_ms = stream_timings.get("predicted_ms", lat) + store_perf_record(model, agent, tier, reason, queue_ms, gen_ms, pt, ct, True) + else: + store_perf_record(model, agent, tier, reason, queue_ms, lat, estimate_tokens(rd.get("messages",[])), 0, True) + # Yield all chunks to client def gen(): - for raw in resp.iter_content(chunk_size=None, decode_unicode=True): - if raw: yield clean_unicode(raw) + for c in chunks: yield c bcast() ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream") + sse_resp.headers["X-RateLimit-Limit"] = str(_rl_limit) + sse_resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) + sse_resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) sse_resp.headers["X-Context-Warning"] = ctx_warning sse_resp.headers["X-Context-Model"] = model @@ -336,11 +477,21 @@ def chat(): msg = c.get("message",{}) if not msg.get("content") and msg.get("reasoning_content"): msg["content"] = msg["reasoning_content"] + # Extract performance data from llama.cpp response + usage = data.get("usage", {}) + timings = data.get("timings", {}) + prompt_tokens = usage.get("prompt_tokens", 0) + completion_tokens = usage.get("completion_tokens", 0) + inference_ms = lat # total GPU round-trip + store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, False) ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) - data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning} + data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"queue_ms": round(queue_ms,1),"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning} resp = jsonify(data) + resp.headers["X-RateLimit-Limit"] = str(_rl_limit) + resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining)) + resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset)) resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) resp.headers["X-Context-Warning"] = ctx_warning resp.headers["X-Context-Model"] = model @@ -355,14 +506,163 @@ def chat(): log.error("Error: %s\n%s", e, traceback.format_exc()) return jsonify({"error":str(e)}), 500 +@app.route("/metrics/performance") +def performance(): + """Per-request performance analytics with percentiles per model/reason/agent.""" + if not r: return jsonify({"error": "Redis unavailable"}), 503 + try: + window_hours = int(request.args.get("window", "24")) + model_filter = request.args.get("model", "all") + + # Load recent records + cutoff = time.time() - (window_hours * 3600) + raw = r.lrange("perf:recent", 0, -1) + records = [] + for x in raw: + try: + rec = json.loads(x) + if rec["ts"] >= cutoff: + records.append(rec) + except: pass + + # Filter by model if specified + if model_filter != "all": + records = [r for r in records if r["model"] == model_filter] + + if not records: + return jsonify({"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}}) + + def pct(values, p): + if len(values) < 2: return round(values[0], 1) if values else 0 + return round(statistics.quantiles(sorted(values), n=100, method='inclusive')[min(p-1, 98)], 1) + + # Per-model stats + model_groups = {} + for rec in records: + m = rec["model"] + if m not in model_groups: model_groups[m] = [] + model_groups[m].append(rec) + + models = [] + for m, recs in sorted(model_groups.items()): + latencies = [r["total_ms"] for r in recs] + tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] + non_stream = [r for r in recs if not r["stream"]] + queue_times = [r["queue_ms"] for r in non_stream] + models.append({ + "model": m, + "count": len(recs), + "stream_pct": round(len([r for r in recs if r["stream"]]) / len(recs) * 100, 1), + "latency": { + "avg": round(statistics.mean(latencies), 1), + "p50": pct(latencies, 50), + "p95": pct(latencies, 95), + "p99": pct(latencies, 99) + }, + "throughput": { + "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0, + "p50": pct(tps_vals, 50) if tps_vals else 0, + "p95": pct(tps_vals, 95) if tps_vals else 0, + }, + "queue": { + "avg_ms": round(statistics.mean(queue_times), 1) if queue_times else 0, + "p95_ms": pct(queue_times, 95) if queue_times else 0, + } if queue_times else None + }) + + # Per-reason stats + reason_groups = {} + for rec in records: + rsn = rec["reason"] + if rsn not in reason_groups: reason_groups[rsn] = [] + reason_groups[rsn].append(rec) + + reasons = [] + for rsn, recs in sorted(reason_groups.items(), key=lambda x: -len(x[1])): + latencies = [r["total_ms"] for r in recs] + reasons.append({ + "reason": rsn, + "count": len(recs), + "avg_total_ms": round(statistics.mean(latencies), 1), + "p95_total_ms": pct(latencies, 95) + }) + + # Per-agent stats + agent_groups = {} + for rec in records: + ag = rec["agent"] + if ag not in agent_groups: agent_groups[ag] = [] + agent_groups[ag].append(rec) + + agents = [] + for ag, recs in sorted(agent_groups.items(), key=lambda x: -len(x[1])): + latencies = [r["total_ms"] for r in recs] + tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0] + agents.append({ + "agent": ag, + "count": len(recs), + "avg_total_ms": round(statistics.mean(latencies), 1), + "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0 + }) + + all_lat = [r["total_ms"] for r in records] + all_tps = [r["tokens_per_sec"] for r in records if r["tokens_per_sec"] > 0] + summary = { + "total_requests": len(records), + "window_hours": window_hours, + "latency": { + "avg_ms": round(statistics.mean(all_lat), 1), + "p50_ms": pct(all_lat, 50), + "p95_ms": pct(all_lat, 95), + "p99_ms": pct(all_lat, 99) + }, + "throughput_avg_tps": round(statistics.mean(all_tps), 1) if all_tps else 0 + } + + return jsonify({"models": models, "reasons": reasons, "agents": agents, "summary": summary}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/metrics/scatter") +def scatter(): + """Return individual data points for scatter plots (prompt_tokens vs latency).""" + if not r: return jsonify({"error": "Redis unavailable"}), 503 + try: + window_hours = int(request.args.get("window", "24")) + model_filter = request.args.get("model", "all") + cutoff = time.time() - (window_hours * 3600) + raw = r.lrange("perf:recent", 0, -1) + points = [] + for x in raw: + try: + rec = json.loads(x) + if rec["ts"] >= cutoff: + if model_filter == "all" or rec["model"] == model_filter: + points.append({ + "model": rec["model"], + "agent": rec["agent"], + "reason": rec["reason"], + "prompt_tokens": int(rec.get("prompt_tokens", 0)), + "completion_tokens": rec.get("completion_tokens", 0), + "inference_ms": round(rec["inference_ms"], 1), + "tokens_per_sec": rec.get("tokens_per_sec", 0), + "stream": rec.get("stream", False) + }) + except: pass + return jsonify({"points": points, "count": len(points)}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + @app.route("/v1/models") -def models(): return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":check_gpu_health(m).get("status"),"gpu":check_gpu_health(m).get("gpu_name")} for m in GPU_URLS]}) +def models(): + def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) + return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]}) @app.route("/health") def health(): gpus = {} for m in GPU_URLS: - h = check_gpu_health(m) + h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) h["active_requests"] = gpu_active_count(m) h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1) gpus[m] = h @@ -413,6 +713,119 @@ def stream(): return Response(stream_with_context(ev()), mimetype="text/event-stream", headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) +# ── Phase 0: Admin Key Management ── +ADMIN_KEY = os.environ.get("ADMIN_KEY", "") + +def _admin_auth(): + """Require admin key for management endpoints.""" + if not ADMIN_KEY: + return False, "ADMIN_KEY not configured on server" + ak = request.headers.get("Authorization","").replace("Bearer ","") + if ak != ADMIN_KEY: + return False, "Admin key required" + return True, None + +@app.route("/admin/keys") +def admin_keys(): + """List all API keys (masked) with agent, tier, and deprecation status.""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + keys = [] + for key, info in API_KEYS.items(): + masked = key[:8] + "..." + key[-8:] + keys.append({ + "masked": masked, + "prefix": key[:8], + "agent": info["agent"], + "tier": info["tier"], + "deprecated": info.get("deprecated", False), + "length": len(key) + }) + return jsonify({ + "total": len(keys), + "active": sum(1 for k in keys if not k["deprecated"]), + "deprecated": sum(1 for k in keys if k["deprecated"]), + "keys": sorted(keys, key=lambda k: (k["deprecated"], k["agent"])) + }) + +@app.route("/admin/keys/deprecation-summary") +def admin_deprecation_summary(): + """Summary of deprecated key usage (from Redis logs, if available).""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + deprecated_agents = [] + for key, info in API_KEYS.items(): + if info.get("deprecated"): + # Check Redis for usage count + count = 0 + if r: + count = int(r.get("deprecated_usage:" + info["agent"]) or 0) + deprecated_agents.append({ + "agent": info["agent"], + "deprecated_uses": count, + "needs_migration": count > 0 + }) + return jsonify({ + "deprecated_agents": sorted(deprecated_agents, key=lambda d: -d["deprecated_uses"]), + "recommendation": "Run POST /admin/keys/revoke to remove keys with 0 usage" + }) + +@app.route("/admin/keys/generate", methods=["POST"]) +def admin_generate_key(): + """Generate a new API key for an agent. Body: {"agent": "Name", "tier": "enterprise"}""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + body = request.get_json(force=True) + agent = body.get("agent", "").strip() + tier = body.get("tier", "enterprise") + if not agent: + return jsonify({"error": "agent field required"}), 400 + if tier not in ("starter", "professional", "enterprise"): + return jsonify({"error": "tier must be starter/professional/enterprise"}), 400 + # Generate secure key + import secrets, hashlib + prefix = hashlib.sha256(secrets.token_bytes(12)).hexdigest()[:8] + suffix = secrets.token_hex(20) + new_key = f"sk-{prefix}-{suffix}" + # Update in-memory dict (note: not persisted across restarts without env var update) + API_KEYS[new_key] = {"tier": tier, "agent": agent} + log.info("KEY_GENERATED: agent=%s tier=%s key=%s...%s", agent, tier, new_key[:8], new_key[-8:]) + return jsonify({ + "agent": agent, + "tier": tier, + "key": new_key, + "masked": new_key[:8] + "..." + new_key[-8:], + "warning": "This key exists in memory only. Update API_KEYS env var and redeploy to persist." + }), 201 + +@app.route("/admin/keys/revoke", methods=["POST"]) +def admin_revoke_key(): + """Revoke a deprecated key. Body: {"agent": "Name"} or {"key_prefix": "sk-xxxx"}""" + ok, err = _admin_auth() + if not ok: return jsonify({"error": err}), 401 + body = request.get_json(force=True) + agent = body.get("agent", "") + key_prefix = body.get("key_prefix", "") + revoked = [] + keys_to_remove = [] + for key, info in API_KEYS.items(): + if not info.get("deprecated"): + continue + if agent and info["agent"] == agent: + keys_to_remove.append(key) + elif key_prefix and key.startswith(key_prefix): + keys_to_remove.append(key) + for key in keys_to_remove: + info = API_KEYS.pop(key) + revoked.append({"agent": info["agent"], "masked": key[:8] + "..." + key[-8:]}) + log.warning("KEY_REVOKED: agent=%s key=%s...%s", info["agent"], key[:8], key[-8:]) + return jsonify({ + "revoked": len(revoked), + "keys": revoked, + "remaining_total": len(API_KEYS), + "warning": "Memory-only revoke. Update API_KEYS env var and redeploy to persist." + }) + if __name__ == "__main__": log.info("Router on :9000 (load-aware)") app.run(host="0.0.0.0", port=9000, debug=False) From 867a6189dff4de55d2f881720225f1c2ec1c67a6 Mon Sep 17 00:00:00 2001 From: Abiba Bot Date: Sat, 6 Jun 2026 01:46:53 +0000 Subject: [PATCH 49/49] Dashboard: fix undefined labels for retired models + safe fallback helpers - Added qwen3.5-9b-vlm to MODELS array as (retired) with gray color - Added modelLabel() / modelColor() safe lookup helpers - Replaced all raw mlab[] and mcol[] lookups with safe fallbacks - Unknown model IDs now show raw ID instead of 'undefined' - Prevents stale Redis perf data from breaking renderPerf/renderScatter --- dashboard/dashboard.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/dashboard/dashboard.py b/dashboard/dashboard.py index e4fb98c..5ec61b9 100644 --- a/dashboard/dashboard.py +++ b/dashboard/dashboard.py @@ -140,11 +140,15 @@ body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMac var MODELS=[ {id:'qwen3.6-35B-A3B',label:'Qwen MoE',gpu:'MoE - Strix Halo',color:'#a78bfa',short:'MoE',domId:'gpu-moe',perf:'35B MoE'}, {id:'qwen3.6-27B-code',label:'Qwen Code',gpu:'Dense - RTX 3090',color:'#f59e0b',short:'Dense',domId:'gpu-dense',perf:'27B Dense'}, - {id:'gemma-4-12b',label:'Gemma 4 12B',gpu:'VLM - RTX 5070',color:'#22c55e',short:'VLM',domId:'gpu-light',perf:'12B VLM'} + {id:'gemma-4-12b',label:'Gemma 4 12B',gpu:'VLM - RTX 5070',color:'#22c55e',short:'VLM',domId:'gpu-light',perf:'12B VLM'}, + {id:'qwen3.5-9b-vlm',label:'Qwen VLM (retired)',gpu:'RTX 5070 (legacy)',color:'#64748b',short:'OLD',domId:'gpu-light',perf:'9B VLM'} ]; // Auto-derived lookups — DO NOT EDIT below var MC={},ML={},GL={},ids={},lb={},mlab={},mcol={}; MODELS.forEach(function(m){MC[m.id]=m.color;ML[m.id]=m.label;GL[m.id]=m.gpu;ids[m.id]=m.domId;lb[m.id]=m.short;mlab[m.id]=m.perf;mcol[m.id]=m.color;}); +// Safe lookup helpers — fall back to raw model ID if not in MODELS +function modelLabel(id){return mlab[id]||id||'Unknown';} +function modelColor(id){return mcol[id]||'#64748b';} function $(id){return document.getElementById(id);} function render(data){ @@ -228,8 +232,8 @@ var models=d.models||[],reasons=d.reasons||[],agents=d.agents||[],sum=d.summary| if(!models.length){$('perf-latency').innerHTML='
Accumulating data...
';return;} var maxLat=Math.max(...models.map(function(m){return m.latency.p99||0}),1); var latHTML=models.map(function(m){ -var l=m.latency||{},p50=l.p50||0,p95=l.p95||0,p99=l.p99||0,c=mcol[m.model]||'#38bdf8'; -return'
'+mlab[m.model]+''+m.count+' reqs
'+ +var l=m.latency||{},p50=l.p50||0,p95=l.p95||0,p99=l.p99||0,c=modelColor(m.model); +return'
'+modelLabel(m.model)+''+m.count+' reqs
'+ '
p50
'+p99+'ms
'+ '
p50: '+p50+'msp95: '+p95+'msp99: '+p99+'ms
'; }).join(''); @@ -237,12 +241,12 @@ $('perf-latency').innerHTML=latHTML; // Throughput comparison var maxTps=Math.max(...models.map(function(m){return m.throughput.avg_tokens_per_sec||0}),1); var tpsHTML=models.map(function(m){ -var t=m.throughput||{},avg=t.avg_tokens_per_sec||0,p50=t.p50||0,c=mcol[m.model]||'#38bdf8'; +var t=m.throughput||{},avg=t.avg_tokens_per_sec||0,p50=t.p50||0,c=modelColor(m.model); var isAllStreaming = avg===0 && p50===0; if(isAllStreaming){ -return'
'+mlab[m.model]+'streaming only
t/s available for non-streaming requests only
'; +return'
'+modelLabel(m.model)+'streaming only
t/s available for non-streaming requests only
'; } -return'
'+mlab[m.model]+''+avg+' tok/s
'+ +return'
'+modelLabel(m.model)+''+avg+' tok/s
'+ '
avg
'+avg+' tok/s
'+ '
p50
'+p50+' tok/s
'; }).join(''); @@ -283,9 +287,9 @@ var toX=function(t){return Math.log10(Math.max(t,1))/Math.log10(Math.max(maxX,10 var toY=function(t){return (t/maxY)*100;}; var dots=''; pts.forEach(function(p){ -var x=toX(p.prompt_tokens),y=toY(p.inference_ms),c=mcol[p.model]||'#38bdf8'; +var x=toX(p.prompt_tokens),y=toY(p.inference_ms),c=modelColor(p.model); var r=p.stream?1.5:2.5,o=p.stream?0.4:0.8; -dots+=''+mlab[p.model]+' | '+p.prompt_tokens+' tok | '+p.inference_ms+'ms | '+p.agent+''; +dots+=''+modelLabel(p.model)+' | '+p.prompt_tokens+' tok | '+p.inference_ms+'ms | '+p.agent+''; }); // Grid lines var grid=''; @@ -301,7 +305,7 @@ yVals.forEach(function(v){if(v<=maxY)yTicks+='Prompt Tokens (log scale)Inference Time'; // Legend var models=[];pts.forEach(function(p){if(models.indexOf(p.model)===-1)models.push(p.model);}); -lg.innerHTML=models.map(function(m){return''+mlab[m]+'';}).join(''); +lg.innerHTML=models.map(function(m){return''+modelLabel(m)+'';}).join(''); } poll();setInterval(poll,10000);loadTS();loadPerf();setInterval(loadPerf,15000);loadScatter();setInterval(loadScatter,30000);