diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..528aa95 --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# Syslog Harness Environment +REDIS_HOST=192.168.68.8 +REDIS_PORT=6379 +AMDPVE_ENDPOINT=http://192.168.68.15:8080 +LLMGPU_ENDPOINT=http://192.168.68.8:8080 +OCU_LLM_ENDPOINT=http://192.168.68.110:8080 +CIRCUIT_BREAKER_THRESHOLD=5 +CIRCUIT_BREAKER_TIMEOUT=30 diff --git a/Dockerfile.dashboard b/Dockerfile.dashboard new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile.queue b/Dockerfile.queue new file mode 100644 index 0000000..e69de29 diff --git a/MIGRATION_PLAN.md b/MIGRATION_PLAN.md new file mode 100644 index 0000000..5a005aa --- /dev/null +++ b/MIGRATION_PLAN.md @@ -0,0 +1,71 @@ +# Syslog Harness — Production Migration Plan + +## Current State (Development) +- **Host:** CT 114 (192.168.68.123) +- **Docker containers:** `syslog-queue` (:8091), `syslog-dashboard` (:3001) +- **Nginx:** Local on CT 114, routing to GPUs + Docker services +- **Status:** All components verified and operational + +## Target State (Production) +- **Host:** New CT (e.g., `docker-vm` on 192.168.68.x) +- **Docker containers:** Same queue + dashboard services +- **Nginx:** Containerized on production CT +- **GPU backends:** Same (192.168.68.15, .8, .110) + +## Migration Steps + +### 1. Prepare Production CT +```bash +# Create new CT on Proxmox +# Install Docker +apt update && apt install -y docker.io docker-compose-plugin + +# Pull/cloned harness repo +git clone /root/syslog-harness +cd /root/syslog-harness +``` + +### 2. Update docker-compose.yml for Production +- Change `REDIS_HOST` to production Redis IP +- Update GPU endpoint env vars if IPs change +- Add volume mounts for persistence + +### 3. Build & Deploy +```bash +# Build images +docker compose build + +# Start services +docker compose up -d + +# Verify health +curl http://localhost:8091/health +curl http://localhost:3001/api/status +``` + +### 4. Configure Nginx +- Copy `/etc/nginx/conf.d/gpu-router.conf` to production CT +- Update upstream IPs if needed +- Test and reload + +### 5. DNS / Routing Update +- Point agent traffic to new CT IP +- Update Hermes config `inference_api_url` +- Test agent routing + +### 6. Verification Checklist +- [ ] Queue service health check passes +- [ ] Dashboard API returns GPU health +- [ ] Nginx routes to correct GPU based on header +- [ ] Circuit breaker triggers on excess load +- [ ] Queue fallback works when GPUs down +- [ ] Agent requests reach correct model + +## Rollback Plan +- Keep CT 114 running as backup +- Revert DNS/routing to .123 if issues +- Docker containers can be stopped/started instantly + +--- +*Created: May 15, 2026* +*Status: Development verified, ready for production migration* diff --git a/README.md b/README.md new file mode 100644 index 0000000..c5f42d9 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ +# syslog-harness — Inference API Harness + +CT 116 Docker stack for routing local GPU models through a unified OpenAI-compatible API. + +## Architecture + +``` +nginx :80 → router :9000 → GPU backends + ├─ qwen3.6-35B-A3B (MoE) @ 192.168.68.15:8080 [2 slots, 262K ctx] + ├─ qwen3.6-27B-code (Dense) @ 192.168.68.8:8080 [2 slots, 262K ctx] + └─ gemma-4-12b (VLM) @ 192.168.68.110:8080 [2 slots, 262K ctx] + Total: 6 concurrent slots + +LiteLLM :8081 (fallback) | Dashboard :3000 | Redis :6379 (local) +``` + +## Deploy + +```bash +cd /opt/inference-harness +docker compose up -d +``` + +## Endpoints + +| URL | Purpose | +|-----|---------| +| `/v1/chat/completions` | Inference API (OpenAI-compatible) — **API key required** | +| `/v1/models` | Available models | +| `/` | Dashboard (GPU health, routing, agents, timeseries) | + +## Authentication + +**All `/v1/chat/completions` requests require a valid API key** via `Authorization: Bearer `. Missing or invalid keys return **401 Unauthorized**. + +## Agent API Keys + +| Agent | Key | +|-------|-----| +| Abiba | `sk-syslog-abiba` | +| Mumuni | `sk-syslog-mumuni` | +| Tanko | `sk-syslog-tanko` | +| Koby | `sk-syslog-koby` | +| Kagenz0 | `sk-syslog-kagenz0` | +| Koonimo | `sk-syslog-koonimo` | + +## Routing Tiers + +| Tier | Trigger | Priority | +|------|---------|----------| +| Lightweight | No system prompt, ≤1 turn, ≤100 words | VLM → MoE → Dense | +| Simple Conv | ≤1000 tokens, ≤4 turns | VLM → MoE → Dense | +| Heavy | >4000 tokens OR >8 turns | Dense → MoE → VLM | +| Default | Everything else | MoE → VLM → Dense | + +## Queue + +When all GPUs are saturated, requests enter a polling queue (500ms intervals) instead of returning 503 immediately. Timeout: 30s (configurable via `QUEUE_TIMEOUT` env or `X-Queue-Timeout` header). + +## Models + +| GPU | Model | VRAM | Slots | Context | Best For | +|-----|-------|------|-------| +| Strix Halo | qwen3.6-35B-A3B (MoE) | 65GB | 2 | 262K | General quality | +| RTX 3090 | qwen3.6-27B-code (Dense) | 24GB | 2 | 262K | Code, reasoning | +| RTX 5070 | gemma-4-12b (VLM) | 12GB | 2 | 262K | Speed, vision | + +## Maintenance + +Automated cron job runs daily at 3:00 AM UTC (`/opt/inference-harness/maintenance.sh`): +- Cleans Redis timeseries keys >60 days +- Prunes Docker build cache >7 days +- Logs container health and Redis memory + +Logs: `/var/log/harness-maintenance.log` diff --git a/dashboard/harness.html b/dashboard/harness.html new file mode 100644 index 0000000..e69de29 diff --git a/gpu-router-docker.conf b/gpu-router-docker.conf new file mode 100644 index 0000000..c17c2f5 --- /dev/null +++ b/gpu-router-docker.conf @@ -0,0 +1,106 @@ +## Syslog GPU Router — Nginx Configuration (Docker-internal) +## Routes incoming agent requests to the appropriate GPU backend +## based on the X-Syslog-Model header. + +upstream amdpve_pool { + ## Strix Halo 395 — qwen3.6-35B-A3B (MoE) — Default workhorse + server 192.168.68.15:8080; +} + +upstream llmgpu_pool { + ## RTX 3090 — qwen3.5-27B (Dense) — Heavy reasoning + server 192.168.68.8:8080; +} + +upstream ocu_llm_pool { + ## New Backend — gemma-4-12b (VLM) — Vision + light tasks + server 192.168.68.110:8080; +} + +upstream queue_service { + ## Agent queue with circuit breaker (Docker container) + server queue-service:8091; +} + +upstream dashboard_service { + ## Harness dashboard (Docker container) + server dashboard:3001; +} + +## ------------------------------------------------------------------ +## Mapping: X-Syslog-Model header → upstream backend +## ------------------------------------------------------------------ +map $http_x_syslog_model $gpu_upstream { + default amdpve_pool; + "standard" amdpve_pool; + "heavy" llmgpu_pool; + "qwen3.5-27B" llmgpu_pool; + "light" ocu_llm_pool; + "gemma-4-12b" ocu_llm_pool; +} + +## Rate limit zone — 10 req/s per IP, burst of 20 +limit_req_zone $binary_remote_addr zone=perip:10m rate=10r/s; + +server { + listen 80; + server_name _; + + ## ------------------------------------------------------------------ + ## Dashboard — observability UI (MUST be before / catch-all) + ## ------------------------------------------------------------------ + location /dashboard { + proxy_pass http://dashboard_service/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + ## ------------------------------------------------------------------ + ## Main location — proxy to selected upstream + ## ------------------------------------------------------------------ + location / { + limit_req zone=perip burst=20 nodelay; + limit_req_status 503; + proxy_pass http://$gpu_upstream; + + ## Preserve original host and headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + ## Pass through the model header so backends can log it + proxy_pass_header X-Syslog-Model; + + ## Streaming support (SSE for LLM responses) + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + + ## Basic failover — retry on error or timeout + proxy_next_upstream error timeout http_502 http_503; + proxy_next_upstream_tries 2; + + ## Add a response header for observability + add_header X-Routed-To $gpu_upstream always; + + ## Fallback to queue when all GPU upstreams are down + error_page 502 503 504 = @queue_fallback; + } + + ## ------------------------------------------------------------------ + ## Queue fallback — enqueue when GPUs are unavailable + ## ------------------------------------------------------------------ + location @queue_fallback { + rewrite ^ /enqueue break; + proxy_pass http://queue_service; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Content-Type $content_type; + proxy_pass_request_body on; + } +} diff --git a/gpu-router.conf b/gpu-router.conf new file mode 100644 index 0000000..d0e9712 --- /dev/null +++ b/gpu-router.conf @@ -0,0 +1,106 @@ +## Syslog GPU Router — Nginx Configuration +## Routes incoming agent requests to the appropriate GPU backend +## based on the X-Syslog-Model header. + +upstream amdpve_pool { + ## Strix Halo 395 — qwen3.6-35B-A3B (MoE) — Default workhorse + server 192.168.68.15:8080; +} + +upstream llmgpu_pool { + ## RTX 3090 — qwen3.5-27B (Dense) — Heavy reasoning + server 192.168.68.8:8080; +} + +upstream ocu_llm_pool { + ## New Backend — gemma-4-12b (VLM) — Vision + light tasks + server 192.168.68.110:8080; +} + +upstream queue_service { + ## Agent queue with circuit breaker (Docker container) + server 127.0.0.1:8091; +} + +upstream dashboard_service { + ## Harness dashboard (Docker container) + server 127.0.0.1:3001; +} + +## ------------------------------------------------------------------ +## Mapping: X-Syslog-Model header → upstream backend +## ------------------------------------------------------------------ +map $http_x_syslog_model $gpu_upstream { + default amdpve_pool; # missing header → default workhorse + "standard" amdpve_pool; + "heavy" llmgpu_pool; + "qwen3.5-27B" llmgpu_pool; + "light" ocu_llm_pool; + "gemma-4-12b" ocu_llm_pool; +} + +server { + listen 8080; + server_name _; + + # Rate limit zone — 10 req/s per IP, burst of 20 + limit_req_zone $binary_remote_addr zone=perip:10m rate=10r/s; + + ## ------------------------------------------------------------------ + ## Dashboard — observability UI (MUST be before / catch-all) + ## ------------------------------------------------------------------ + location /dashboard { + proxy_pass http://dashboard_service/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + ## ------------------------------------------------------------------ + ## Main location — proxy to selected upstream + ## ------------------------------------------------------------------ + location / { + limit_req zone=perip burst=20 nodelay; + limit_req_status 503; + proxy_pass http://$gpu_upstream; + + ## Preserve original host and headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + ## Pass through the model header so backends can log it + proxy_pass_header X-Syslog-Model; + + ## Streaming support (SSE for LLM responses) + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + + ## Basic failover — retry on error or timeout + proxy_next_upstream error timeout http_502 http_503; + proxy_next_upstream_tries 2; + + ## Add a response header for observability + add_header X-Routed-To $gpu_upstream always; + + ## Fallback to queue when all GPU upstreams are down + error_page 502 503 504 = @queue_fallback; + } + + ## ------------------------------------------------------------------ + ## Queue fallback — enqueue when GPUs are unavailable + ## ------------------------------------------------------------------ + location @queue_fallback { + rewrite ^ /enqueue break; + proxy_pass http://queue_service; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Content-Type $content_type; + proxy_pass_request_body on; + } +} diff --git a/phase0-dual-keys.json b/phase0-dual-keys.json new file mode 100644 index 0000000..6114325 --- /dev/null +++ b/phase0-dual-keys.json @@ -0,0 +1,20 @@ +{ + "sk-syslog-local-master-key": {"tier": "enterprise", "agent": "admin", "deprecated": true}, + "sk-33a0d6e6-a6da12fb483770d6f63f543b7e16a742": {"tier": "enterprise", "agent": "admin"}, + "sk-syslog-abiba": {"tier": "enterprise", "agent": "Abiba", "deprecated": true}, + "sk-6a6e49d0-e35e27c524ce0ba2de8f862a0e966b0c": {"tier": "enterprise", "agent": "Abiba"}, + "sk-syslog-mumuni": {"tier": "enterprise", "agent": "Mumuni", "deprecated": true}, + "sk-ba249c99-29e3b163a8d8ab7abb8663cae0dceb37": {"tier": "enterprise", "agent": "Mumuni"}, + "sk-syslog-tanko": {"tier": "enterprise", "agent": "Tanko", "deprecated": true}, + "sk-4a67112a-d9b62caf5a5881a15837df506f782708": {"tier": "enterprise", "agent": "Tanko"}, + "sk-syslog-koby": {"tier": "enterprise", "agent": "Koby", "deprecated": true}, + "sk-9dd66bba-316f616ec40cb51f3b489a0146bb986f": {"tier": "enterprise", "agent": "Koby"}, + "sk-syslog-kagenz0": {"tier": "enterprise", "agent": "Kagenz0", "deprecated": true}, + "sk-5a05020f-a984a8a20f731c5a65dfc07d51495bb3": {"tier": "enterprise", "agent": "Kagenz0"}, + "sk-syslog-koonimo": {"tier": "enterprise", "agent": "Koonimo", "deprecated": true}, + "sk-b2f0e1b2-70b7cf1c5a5e402a49b5469e88d9e335": {"tier": "enterprise", "agent": "Koonimo"}, + "sk-starter-abc123": {"tier": "starter", "agent": "test-starter", "deprecated": true}, + "sk-a8307a00-cd76daa07dd61c41aeb4545656f8b8c4": {"tier": "starter", "agent": "test-starter"}, + "sk-professional-xyz789": {"tier": "professional", "agent": "test-pro", "deprecated": true}, + "sk-b519a91e-4d2774c6c7da3fc31326e69ccc426781": {"tier": "professional", "agent": "test-pro"} +} diff --git a/queue-service/queue-service.py b/queue-service/queue-service.py new file mode 100644 index 0000000..3e2bbed --- /dev/null +++ b/queue-service/queue-service.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Syslog Inference Queue Service — Circuit breaker + request queuing. + +Ports: 8091 +Endpoints: + /health — liveness probe (Nginx upstream check) + /enqueue — POST inference request into queue (fallback from Nginx) + /status — GET queue depth + circuit breaker state +""" + +import json +import os +import sys +import time +import urllib.request +from flask import Flask, request, jsonify + +app = Flask(__name__) + +# Configuration +REDIS_HOST = os.getenv("REDIS_HOST", "192.168.68.7") +REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) +QUEUE_KEY = "inference:requests" +CIRCUIT_OPEN_THRESHOLD = 50 +CIRCUIT_WARN_THRESHOLD = 30 + +# GPU endpoints for draining +GPUS = { + "amdpve": "192.168.68.15:8080", + "llmgpu": "192.168.68.8:8080", + "ocu_llm": "192.168.68.110:8080", +} + + +def get_redis(): + try: + import redis + return redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True) + except Exception: + return None + + +def get_queue_depth(r): + try: + return r.llen(QUEUE_KEY) + except Exception: + return 0 + + +def check_gpu_health(endpoint): + try: + req = urllib.request.Request(f"http://{endpoint}/v1/models") + req.add_header("User-Agent", "queue-service/1.0") + resp = urllib.request.urlopen(req, timeout=3) + return resp.status == 200 + except Exception: + return False + + +@app.route("/health") +def health(): + """Nginx upstream health probe. Returns 200 if service is alive.""" + return jsonify({"status": "ok", "service": "queue-service"}), 200 + + +@app.route("/enqueue", methods=["POST"]) +def enqueue(): + """Fallback endpoint — Nginx calls this when all GPU upstreams are down.""" + r = get_redis() + if not r: + return jsonify({"error": "Redis unavailable"}), 503 + + depth = get_queue_depth(r) + if depth >= CIRCUIT_OPEN_THRESHOLD: + return jsonify({ + "error": "Circuit breaker OPEN", + "queue_depth": depth, + "threshold": CIRCUIT_OPEN_THRESHOLD + }), 503 + + # Store the request in queue + payload = request.get_data(as_text=True) + headers = {k: v for k, v in request.headers if k.startswith("X-")} + r.rpush(QUEUE_KEY, json.dumps({ + "payload": payload, + "headers": headers, + "queued_at": time.time() + })) + + new_depth = get_queue_depth(r) + return jsonify({ + "status": "queued", + "position": new_depth, + "circuit": "warn" if new_depth >= CIRCUIT_WARN_THRESHOLD else "closed" + }), 202 + + +@app.route("/status") +def status(): + """GET queue depth + circuit breaker state + GPU health.""" + r = get_redis() + depth = get_queue_depth(r) if r else -1 + circuit = "open" if depth >= CIRCUIT_OPEN_THRESHOLD else ("warn" if depth >= CIRCUIT_WARN_THRESHOLD else "closed") + + gpu_health = {} + for name, endpoint in GPUS.items(): + gpu_health[name] = "up" if check_gpu_health(endpoint) else "down" + + return jsonify({ + "queue_depth": depth, + "circuit_breaker": circuit, + "gpu_health": gpu_health, + "thresholds": { + "warn": CIRCUIT_WARN_THRESHOLD, + "open": CIRCUIT_OPEN_THRESHOLD + } + }) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8091) diff --git a/router/route_v2.py b/router/route_v2.py new file mode 100644 index 0000000..0ee8e47 --- /dev/null +++ b/router/route_v2.py @@ -0,0 +1,77 @@ +def route(rd, tier, agent=""): + msgs = rd.get("messages",[]); t = estimate_tokens(msgs) + sys = any(m.get("role")=="system" for m in msgs) + turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) + hints = rd.get("routing_hints",{}) + allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) + avail = [m for m in available_models() if m in allowed] + if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} + if all(is_gpu_busy(m) for m in avail): + return {"model": avail[0], "reason": "all_saturated", "saturated": True} + + # GUARD: multimodal -> VLM only (sole vision model) + has_image = any( + isinstance(m.get("content"), list) and + any(p.get("type") == "image_url" for p in m["content"] if isinstance(p, dict)) + for m in msgs + ) + if has_image: + if "gemma-4-12b" in avail and not is_gpu_busy("gemma-4-12b"): + return {"model": "gemma-4-12b", "reason": "vision"} + elif "gemma-4-12b" in avail: + return {"model": "gemma-4-12b", "reason": "vision_saturated", "saturated": True} + else: + return {"model": allowed[0], "reason": "vision_unavailable"} + + req = rd.get("model","auto") + if req != "auto": + target = req if req in avail else avail[0] + if is_gpu_busy(target) and req in allowed: + alts = [m for m in avail if m != target and m in allowed] + if alts: + alt = select_best_gpu(alts, "explicit", agent) + if alt: return alt + return {"model": target, "reason": "explicit"} + + if hints: + if hints.get("priority")=="speed" and "gemma-4-12b" in avail: + return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} + if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: + return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} + if hints.get("priority")=="code" and "qwen3.6-27B-code" in avail: + return select_best_gpu(["qwen3.6-27B-code"], "hint_code", agent) or {"model":"qwen3.6-27B-code","reason":"hint_code"} + + first_msg = msgs[0].get("content","") if msgs else "" + words = len(first_msg.split()) if isinstance(first_msg, str) else 99 + + # TIER 1: Tiny - single-turn micro queries -> VLM (fastest) + if not sys and turns <= 1 and t <= 300 and words <= 100 and "gemma-4-12b" in avail: + if not is_gpu_busy("gemma-4-12b"): + return {"model":"gemma-4-12b","reason":"tiny"} + fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(fallback, "tiny_fallback", agent) + if result: return result + + # TIER 2: Light - moderate chat -> Dense primary, saves VLM for vision/speed + if t <= 5000 and turns <= 4: + candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "light", agent) + if result: return result + + # TIER 3: Medium - quality matters -> MoE primary, Dense fallback + if t <= 30000: + candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"] if m in avail] + result = select_best_gpu(candidates, "medium", agent) + if result: return result + + # TIER 4: Heavy - big context needs big model -> MoE->Dense->VLM + if t > 30000: + candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"] if m in avail] + result = select_best_gpu(candidates, "heavy", agent) + if result: return result + + # TIER 5: Default - best quality first + candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"] if m in avail] + result = select_best_gpu(candidates, "default", agent) + if result: return result + return {"model":avail[0],"reason":"last_resort"} diff --git a/router/route_v3.py b/router/route_v3.py new file mode 100644 index 0000000..68eaf00 --- /dev/null +++ b/router/route_v3.py @@ -0,0 +1,92 @@ +def route(rd, tier, agent=""): + msgs = rd.get("messages",[]); t = estimate_tokens(msgs) + sys = any(m.get("role")=="system" for m in msgs) + turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) + hints = rd.get("routing_hints",{}) + allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) + avail = [m for m in available_models() if m in allowed] + if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} + if all(is_gpu_busy(m) for m in avail): + return {"model": avail[0], "reason": "all_saturated", "saturated": True} + + # GUARD: multimodal -> VLM only (sole vision model) + has_image = any( + isinstance(m.get("content"), list) and + any(p.get("type") == "image_url" for p in m["content"] if isinstance(p, dict)) + for m in msgs + ) + if has_image: + if "gemma-4-12b" in avail and not is_gpu_busy("gemma-4-12b"): + return {"model": "gemma-4-12b", "reason": "vision"} + elif "gemma-4-12b" in avail: + return {"model": "gemma-4-12b", "reason": "vision_saturated", "saturated": True} + else: + return {"model": allowed[0], "reason": "vision_unavailable"} + + req = rd.get("model","auto") + if req != "auto": + target = req if req in avail else avail[0] + if is_gpu_busy(target) and req in allowed: + alts = [m for m in avail if m != target and m in allowed] + if alts: + alt = select_best_gpu(alts, "explicit", agent) + if alt: return alt + return {"model": target, "reason": "explicit"} + + if hints: + if hints.get("priority")=="speed" and "gemma-4-12b" in avail: + return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} + if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: + return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} + if hints.get("priority")=="code" and "qwen3.6-27B-code" in avail: + return select_best_gpu(["qwen3.6-27B-code"], "hint_code", agent) or {"model":"qwen3.6-27B-code","reason":"hint_code"} + + first_msg = msgs[0].get("content","") if msgs else "" + words = len(first_msg.split()) if isinstance(first_msg, str) else 99 + + # TIER 1: Tiny - single-turn micro queries -> VLM (fastest) + if not sys and turns <= 1 and t <= 300 and words <= 100 and "gemma-4-12b" in avail: + if not is_gpu_busy("gemma-4-12b"): + return {"model":"gemma-4-12b","reason":"tiny"} + fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(fallback, "tiny_fallback", agent) + if result: return result + + # TIER 2: Light - moderate chat -> Dense primary, saves VLM for vision/speed + if t <= 5000 and turns <= 4: + candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "light", agent) + if result: return result + + # TIER 3: Medium - quality matters -> MoE primary (60%), Dense spillover (40%) + if t <= 30000: + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) + result = select_best_gpu(candidates, "medium", agent) + if result: return result + + # TIER 4: Heavy - big context -> MoE primary (60%), Dense spillover (40%) + if t > 30000: + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) + result = select_best_gpu(candidates, "heavy", agent) + if result: return result + + # TIER 5: Default - MoE primary (60%), Dense spillover (40%) + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) + result = select_best_gpu(candidates, "default", agent) + if result: return result + return {"model":avail[0],"reason":"last_resort"} + + +def moe_spillover(avail, default_order): + """Spill 40% of MoE-first traffic to Dense to prevent Strix Halo overheating. + Only applies when MoE is first candidate, available, and not busy.""" + import random + if (default_order[0] == "qwen3.6-35B-A3B" + and "qwen3.6-35B-A3B" in avail + and not is_gpu_busy("qwen3.6-35B-A3B") + and "qwen3.6-27B-code" in avail + and not is_gpu_busy("qwen3.6-27B-code") + and random.random() < 0.4): + # Swap: Dense first, MoE second + return ["qwen3.6-27B-code","qwen3.6-35B-A3B"] + [m for m in default_order[2:] if m in avail and m not in ("qwen3.6-27B-code","qwen3.6-35B-A3B")] + return [m for m in default_order if m in avail]