Abiba Bot
e6a6c30211
GUARD: multimodal -> VLM only TIER 1 Tiny: VLM->Dense->MoE TIER 2 Light: VLM->Dense->MoE TIER 3 Medium: MoE<->Dense (40% spill)->VLM TIER 4 Heavy: Dense->MoE->VLM (quality first) TIER 5 Default: MoE<->Dense (40% spill)->VLM HINTS: speed->VLM, quality->MoE, code->Dense + moe_spillover(): 60/40 MoE/Dense load distribution
93 lines
4.6 KiB
Python
93 lines
4.6 KiB
Python
def route(rd, tier, agent=""):
|
|
msgs = rd.get("messages",[]); t = estimate_tokens(msgs)
|
|
sys = any(m.get("role")=="system" for m in msgs)
|
|
turns = len([m for m in msgs if m.get("role") in ("user","assistant")])
|
|
hints = rd.get("routing_hints",{})
|
|
allowed = TIER_MODELS.get(tier, ["gemma-4-12b"])
|
|
avail = [m for m in available_models() if m in allowed]
|
|
if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True}
|
|
if all(is_gpu_busy(m) for m in avail):
|
|
return {"model": avail[0], "reason": "all_saturated", "saturated": True}
|
|
|
|
# GUARD: multimodal -> VLM only (sole vision model)
|
|
has_image = any(
|
|
isinstance(m.get("content"), list) and
|
|
any(p.get("type") == "image_url" for p in m["content"] if isinstance(p, dict))
|
|
for m in msgs
|
|
)
|
|
if has_image:
|
|
if "gemma-4-12b" in avail and not is_gpu_busy("gemma-4-12b"):
|
|
return {"model": "gemma-4-12b", "reason": "vision"}
|
|
elif "gemma-4-12b" in avail:
|
|
return {"model": "gemma-4-12b", "reason": "vision_saturated", "saturated": True}
|
|
else:
|
|
return {"model": allowed[0], "reason": "vision_unavailable"}
|
|
|
|
req = rd.get("model","auto")
|
|
if req != "auto":
|
|
target = req if req in avail else avail[0]
|
|
if is_gpu_busy(target) and req in allowed:
|
|
alts = [m for m in avail if m != target and m in allowed]
|
|
if alts:
|
|
alt = select_best_gpu(alts, "explicit", agent)
|
|
if alt: return alt
|
|
return {"model": target, "reason": "explicit"}
|
|
|
|
if hints:
|
|
if hints.get("priority")=="speed" and "gemma-4-12b" in avail:
|
|
return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"}
|
|
if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail:
|
|
return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"}
|
|
if hints.get("priority")=="code" and "qwen3.6-27B-code" in avail:
|
|
return select_best_gpu(["qwen3.6-27B-code"], "hint_code", agent) or {"model":"qwen3.6-27B-code","reason":"hint_code"}
|
|
|
|
first_msg = msgs[0].get("content","") if msgs else ""
|
|
words = len(first_msg.split()) if isinstance(first_msg, str) else 99
|
|
|
|
# TIER 1: Tiny - single-turn micro queries -> VLM (fastest)
|
|
if not sys and turns <= 1 and t <= 300 and words <= 100 and "gemma-4-12b" in avail:
|
|
if not is_gpu_busy("gemma-4-12b"):
|
|
return {"model":"gemma-4-12b","reason":"tiny"}
|
|
fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail]
|
|
result = select_best_gpu(fallback, "tiny_fallback", agent)
|
|
if result: return result
|
|
|
|
# TIER 2: Light - moderate chat -> Dense primary, saves VLM for vision/speed
|
|
if t <= 5000 and turns <= 4:
|
|
candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail]
|
|
result = select_best_gpu(candidates, "light", agent)
|
|
if result: return result
|
|
|
|
# TIER 3: Medium - quality matters -> MoE primary (60%), Dense spillover (40%)
|
|
if t <= 30000:
|
|
candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"])
|
|
result = select_best_gpu(candidates, "medium", agent)
|
|
if result: return result
|
|
|
|
# TIER 4: Heavy - big context -> MoE primary (60%), Dense spillover (40%)
|
|
if t > 30000:
|
|
candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"])
|
|
result = select_best_gpu(candidates, "heavy", agent)
|
|
if result: return result
|
|
|
|
# TIER 5: Default - MoE primary (60%), Dense spillover (40%)
|
|
candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"])
|
|
result = select_best_gpu(candidates, "default", agent)
|
|
if result: return result
|
|
return {"model":avail[0],"reason":"last_resort"}
|
|
|
|
|
|
def moe_spillover(avail, default_order):
|
|
"""Spill 40% of MoE-first traffic to Dense to prevent Strix Halo overheating.
|
|
Only applies when MoE is first candidate, available, and not busy."""
|
|
import random
|
|
if (default_order[0] == "qwen3.6-35B-A3B"
|
|
and "qwen3.6-35B-A3B" in avail
|
|
and not is_gpu_busy("qwen3.6-35B-A3B")
|
|
and "qwen3.6-27B-code" in avail
|
|
and not is_gpu_busy("qwen3.6-27B-code")
|
|
and random.random() < 0.4):
|
|
# Swap: Dense first, MoE second
|
|
return ["qwen3.6-27B-code","qwen3.6-35B-A3B"] + [m for m in default_order[2:] if m in avail and m not in ("qwen3.6-27B-code","qwen3.6-35B-A3B")]
|
|
return [m for m in default_order if m in avail]
|