From 24f0928ea1c40a90d747cbc843c2f72f1709069e Mon Sep 17 00:00:00 2001 From: Abiba Date: Sun, 7 Jun 2026 22:49:46 +0000 Subject: [PATCH] =?UTF-8?q?Phase=200:=20model=20migration=20(qwen3.5-9b-vl?= =?UTF-8?q?m=20=E2=86=92=20gemma-4-12b),=20context=20alignment=20(all=2026?= =?UTF-8?q?2K),=20routing=20tiers=20+=20MoE=20spillover,=20fix=20dashboard?= =?UTF-8?q?=20window=3D1h=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- router/router.py | 116 +++++++++++++++++++++++++++++------------------ 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/router/router.py b/router/router.py index 16b2b5d..fd34c01 100644 --- a/router/router.py +++ b/router/router.py @@ -10,31 +10,31 @@ GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1") GPU_SIDECARS = { "qwen3.6-35B-A3B": "http://192.168.68.15:8090", "qwen3.6-27B-code": "http://192.168.68.8:8090", - "qwen3.5-9b-vlm": "http://192.168.68.110:8090", + "gemma-4-12b": "http://192.168.68.110:8090", } GPU_URLS = { "qwen3.6-35B-A3B": GPU_MOE_URL, "qwen3.6-27B-code": GPU_DENSE_URL, - "qwen3.5-9b-vlm": GPU_LIGHT_URL, + "gemma-4-12b": GPU_LIGHT_URL, } # Max concurrent requests per GPU (based on llama.cpp --parallel) GPU_MAX_CONCURRENT = { "qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating) "qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM) - "qwen3.5-9b-vlm": 2, # 2 slots (12GB VRAM, 4GB headroom) + "gemma-4-12b": 2, # 2 slots (7.1GB VRAM) } # Context window sizes (tokens) — used for compaction signals GPU_CONTEXT = { "qwen3.6-35B-A3B": 262144, - "qwen3.6-27B-code": 131072, - "qwen3.5-9b-vlm": 262144, + "qwen3.6-27B-code": 262144, + "gemma-4-12b": 262144, } TIER_MODELS = { - "starter": ["qwen3.5-9b-vlm"], - "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "qwen3.5-9b-vlm"], - "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "qwen3.5-9b-vlm"], + "starter": ["gemma-4-12b"], + "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], + "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"], } # API keys loaded from API_KEYS env var (set in docker-compose.yml) # Fallback is dev-only — production MUST set API_KEYS env var @@ -216,74 +216,100 @@ def select_best_gpu(candidates, reason, agent=""): return {"model": best, "reason": "load_balanced_" + reason} return None + + def route(rd, tier, agent=""): msgs = rd.get("messages",[]); t = estimate_tokens(msgs) sys = any(m.get("role")=="system" for m in msgs) turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) hints = rd.get("routing_hints",{}) - allowed = TIER_MODELS.get(tier, ["qwen3.5-9b-vlm"]) + allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) avail = [m for m in available_models() if m in allowed] if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} - # Check if all available GPUs are at max capacity if all(is_gpu_busy(m) for m in avail): return {"model": avail[0], "reason": "all_saturated", "saturated": True} - + + # GUARD: multimodal -> VLM only (sole vision model) + has_image = any( + isinstance(m.get("content"), list) and + any(p.get("type") == "image_url" for p in m["content"] if isinstance(p, dict)) + for m in msgs + ) + if has_image: + if "gemma-4-12b" in avail and not is_gpu_busy("gemma-4-12b"): + return {"model": "gemma-4-12b", "reason": "vision"} + elif "gemma-4-12b" in avail: + return {"model": "gemma-4-12b", "reason": "vision_saturated", "saturated": True} + else: + return {"model": allowed[0], "reason": "vision_unavailable"} + req = rd.get("model","auto") if req != "auto": target = req if req in avail else avail[0] - # If explicit model is busy, check if another can take it if is_gpu_busy(target) and req in allowed: alts = [m for m in avail if m != target and m in allowed] if alts: alt = select_best_gpu(alts, "explicit", agent) if alt: return alt return {"model": target, "reason": "explicit"} - + if hints: - if hints.get("priority")=="speed" and "qwen3.5-9b-vlm" in avail: - return select_best_gpu(["qwen3.5-9b-vlm"], "hint_speed", agent) or {"model":"qwen3.5-9b-vlm","reason":"hint_speed"} + if hints.get("priority")=="speed" and "gemma-4-12b" in avail: + return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} - + if hints.get("priority")=="code" and "qwen3.6-27B-code" in avail: + return select_best_gpu(["qwen3.6-27B-code"], "hint_code", agent) or {"model":"qwen3.6-27B-code","reason":"hint_code"} + first_msg = msgs[0].get("content","") if msgs else "" words = len(first_msg.split()) if isinstance(first_msg, str) else 99 - - # TIER 1: Lightweight — single-turn short queries → VLM (fastest) - if not sys and turns <= 1 and t <= 500 and words <= 100 and "qwen3.5-9b-vlm" in avail: - if not is_gpu_busy("qwen3.5-9b-vlm"): - return {"model":"qwen3.5-9b-vlm","reason":"lightweight"} - # VLM busy — Dense is faster for short queries than MoE + + # TIER 1: Tiny - single-turn micro queries -> VLM (fastest) + if not sys and turns <= 1 and t <= 300 and words <= 100 and "gemma-4-12b" in avail: + if not is_gpu_busy("gemma-4-12b"): + return {"model":"gemma-4-12b","reason":"tiny"} fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] - result = select_best_gpu(fallback, "lightweight_fallback", agent) + result = select_best_gpu(fallback, "tiny_fallback", agent) if result: return result - - # TIER 2: Simple conversations — VLM primary (up to 15K tok), fastest for moderate chat - if t <= 15000 and turns <= 12 and "qwen3.5-9b-vlm" in avail: - if not is_gpu_busy("qwen3.5-9b-vlm"): - return {"model":"qwen3.5-9b-vlm","reason":"simple_conv"} - # VLM busy — fall back to Dense, then MoE - fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] - result = select_best_gpu(fallback, "simple_conv_fallback", agent) + + # TIER 2: Light - moderate chat -> VLM first (fastest), Dense fallback + if t <= 5000 and turns <= 4: + candidates = [m for m in ["gemma-4-12b","qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "light", agent) if result: return result - - # TIER 3: Medium complexity — Dense primary, VLM fallback (quality + speed balance) - if t <= 25000: - candidates = [m for m in ["qwen3.6-27B-code","qwen3.5-9b-vlm","qwen3.6-35B-A3B"] if m in avail] + + # TIER 3: Medium - quality matters -> MoE primary (60%), Dense spillover (40%) + if t <= 30000: + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) result = select_best_gpu(candidates, "medium", agent) if result: return result - - # TIER 4: Heavy reasoning — MoE primary (workhorse), Dense fallback - if t > 25000: - candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","qwen3.5-9b-vlm"] if m in avail] - result = select_best_gpu(candidates, "heavy_reasoning", agent) + + # TIER 4: Heavy - quality first -> Dense primary, MoE fallback + if t > 30000: + candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","gemma-4-12b"] if m in avail] + result = select_best_gpu(candidates, "heavy", agent) if result: return result - - # TIER 5: Default — Dense primary, MoE fallback - candidates = [m for m in ["qwen3.6-27B-code","qwen3.5-9b-vlm","qwen3.6-35B-A3B"] if m in avail] + + # TIER 5: Default - MoE primary (60%), Dense spillover (40%) + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) result = select_best_gpu(candidates, "default", agent) if result: return result return {"model":avail[0],"reason":"last_resort"} + +def moe_spillover(avail, default_order): + """Spill 40% of MoE-first traffic to Dense to prevent Strix Halo overheating. + Only applies when MoE is first candidate, available, and not busy.""" + import random + if (default_order[0] == "qwen3.6-35B-A3B" + and "qwen3.6-35B-A3B" in avail + and not is_gpu_busy("qwen3.6-35B-A3B") + and "qwen3.6-27B-code" in avail + and not is_gpu_busy("qwen3.6-27B-code") + and random.random() < 0.4): + # Swap: Dense first, MoE second + return ["qwen3.6-27B-code","qwen3.6-35B-A3B"] + [m for m in default_order[2:] if m in avail and m not in ("qwen3.6-27B-code","qwen3.6-35B-A3B")] + return [m for m in default_order if m in avail] def clean_unicode(text): if not isinstance(text, str): return text text = text.replace(chr(0x2014), "-"); text = text.replace(chr(0x2013), "-") @@ -494,7 +520,7 @@ def performance(): """Per-request performance analytics with percentiles per model/reason/agent.""" if not r: return jsonify({"error": "Redis unavailable"}), 503 try: - window_hours = int(request.args.get("window", "24")) + window_hours = int(request.args.get("window", "24").replace("h","")) model_filter = request.args.get("model", "all") # Load recent records @@ -611,7 +637,7 @@ def scatter(): """Return individual data points for scatter plots (prompt_tokens vs latency).""" if not r: return jsonify({"error": "Redis unavailable"}), 503 try: - window_hours = int(request.args.get("window", "24")) + window_hours = int(request.args.get("window", "24").replace("h","")) model_filter = request.args.get("model", "all") cutoff = time.time() - (window_hours * 3600) raw = r.lrange("perf:recent", 0, -1)