From e6a6c302115536b1ba5cd351c1b20b211b8a79a6 Mon Sep 17 00:00:00 2001 From: Abiba Bot Date: Fri, 5 Jun 2026 23:48:45 +0000 Subject: [PATCH] =?UTF-8?q?router:=20v3=20=E2=80=94=205-tier=20routing=20w?= =?UTF-8?q?ith=20vision=20guard,=20MoE=20spillover,=20code=20hint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GUARD: multimodal -> VLM only TIER 1 Tiny: VLM->Dense->MoE TIER 2 Light: VLM->Dense->MoE TIER 3 Medium: MoE<->Dense (40% spill)->VLM TIER 4 Heavy: Dense->MoE->VLM (quality first) TIER 5 Default: MoE<->Dense (40% spill)->VLM HINTS: speed->VLM, quality->MoE, code->Dense + moe_spillover(): 60/40 MoE/Dense load distribution --- router/route_v3.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 router/route_v3.py diff --git a/router/route_v3.py b/router/route_v3.py new file mode 100644 index 0000000..68eaf00 --- /dev/null +++ b/router/route_v3.py @@ -0,0 +1,92 @@ +def route(rd, tier, agent=""): + msgs = rd.get("messages",[]); t = estimate_tokens(msgs) + sys = any(m.get("role")=="system" for m in msgs) + turns = len([m for m in msgs if m.get("role") in ("user","assistant")]) + hints = rd.get("routing_hints",{}) + allowed = TIER_MODELS.get(tier, ["gemma-4-12b"]) + avail = [m for m in available_models() if m in allowed] + if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} + if all(is_gpu_busy(m) for m in avail): + return {"model": avail[0], "reason": "all_saturated", "saturated": True} + + # GUARD: multimodal -> VLM only (sole vision model) + has_image = any( + isinstance(m.get("content"), list) and + any(p.get("type") == "image_url" for p in m["content"] if isinstance(p, dict)) + for m in msgs + ) + if has_image: + if "gemma-4-12b" in avail and not is_gpu_busy("gemma-4-12b"): + return {"model": "gemma-4-12b", "reason": "vision"} + elif "gemma-4-12b" in avail: + return {"model": "gemma-4-12b", "reason": "vision_saturated", "saturated": True} + else: + return {"model": allowed[0], "reason": "vision_unavailable"} + + req = rd.get("model","auto") + if req != "auto": + target = req if req in avail else avail[0] + if is_gpu_busy(target) and req in allowed: + alts = [m for m in avail if m != target and m in allowed] + if alts: + alt = select_best_gpu(alts, "explicit", agent) + if alt: return alt + return {"model": target, "reason": "explicit"} + + if hints: + if hints.get("priority")=="speed" and "gemma-4-12b" in avail: + return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"} + if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail: + return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"} + if hints.get("priority")=="code" and "qwen3.6-27B-code" in avail: + return select_best_gpu(["qwen3.6-27B-code"], "hint_code", agent) or {"model":"qwen3.6-27B-code","reason":"hint_code"} + + first_msg = msgs[0].get("content","") if msgs else "" + words = len(first_msg.split()) if isinstance(first_msg, str) else 99 + + # TIER 1: Tiny - single-turn micro queries -> VLM (fastest) + if not sys and turns <= 1 and t <= 300 and words <= 100 and "gemma-4-12b" in avail: + if not is_gpu_busy("gemma-4-12b"): + return {"model":"gemma-4-12b","reason":"tiny"} + fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(fallback, "tiny_fallback", agent) + if result: return result + + # TIER 2: Light - moderate chat -> Dense primary, saves VLM for vision/speed + if t <= 5000 and turns <= 4: + candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail] + result = select_best_gpu(candidates, "light", agent) + if result: return result + + # TIER 3: Medium - quality matters -> MoE primary (60%), Dense spillover (40%) + if t <= 30000: + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) + result = select_best_gpu(candidates, "medium", agent) + if result: return result + + # TIER 4: Heavy - big context -> MoE primary (60%), Dense spillover (40%) + if t > 30000: + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) + result = select_best_gpu(candidates, "heavy", agent) + if result: return result + + # TIER 5: Default - MoE primary (60%), Dense spillover (40%) + candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"]) + result = select_best_gpu(candidates, "default", agent) + if result: return result + return {"model":avail[0],"reason":"last_resort"} + + +def moe_spillover(avail, default_order): + """Spill 40% of MoE-first traffic to Dense to prevent Strix Halo overheating. + Only applies when MoE is first candidate, available, and not busy.""" + import random + if (default_order[0] == "qwen3.6-35B-A3B" + and "qwen3.6-35B-A3B" in avail + and not is_gpu_busy("qwen3.6-35B-A3B") + and "qwen3.6-27B-code" in avail + and not is_gpu_busy("qwen3.6-27B-code") + and random.random() < 0.4): + # Swap: Dense first, MoE second + return ["qwen3.6-27B-code","qwen3.6-35B-A3B"] + [m for m in default_order[2:] if m in avail and m not in ("qwen3.6-27B-code","qwen3.6-35B-A3B")] + return [m for m in default_order if m in avail]