abiba-bot/inference-harness
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Phase 0: model migration (qwen3.5-9b-vlm → gemma-4-12b), context alignment (all 262K), routing tiers + MoE spillover, fix dashboard window=1h parsing

Browse Source
This commit is contained in:
Abiba
2026-06-07 22:49:46 +00:00
parent 0cb4597b0e
commit 24f0928ea1
1 changed files with 71 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
router/router.py
+63 -37
View File
@@ -10,31 +10,31 @@ GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1")
GPU_SIDECARS = {
"qwen3.6-35B-A3B": "http://192.168.68.15:8090",
"qwen3.6-27B-code": "http://192.168.68.8:8090",
"qwen3.5-9b-vlm": "http://192.168.68.110:8090",
"gemma-4-12b": "http://192.168.68.110:8090",
}
GPU_URLS = {
"qwen3.6-35B-A3B": GPU_MOE_URL,
"qwen3.6-27B-code": GPU_DENSE_URL,
"qwen3.5-9b-vlm": GPU_LIGHT_URL,
"gemma-4-12b": GPU_LIGHT_URL,
}
# Max concurrent requests per GPU (based on llama.cpp --parallel)
GPU_MAX_CONCURRENT = {
"qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating)
"qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM)
"qwen3.5-9b-vlm": 2, # 2 slots (12GB VRAM, 4GB headroom)
"gemma-4-12b": 2, # 2 slots (7.1GB VRAM)
}
# Context window sizes (tokens) — used for compaction signals
GPU_CONTEXT = {
"qwen3.6-35B-A3B": 262144,
"qwen3.6-27B-code": 131072,
"qwen3.5-9b-vlm": 262144,
"qwen3.6-27B-code": 262144,
"gemma-4-12b": 262144,
}
TIER_MODELS = {
"starter": ["qwen3.5-9b-vlm"],
"professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "qwen3.5-9b-vlm"],
"enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "qwen3.5-9b-vlm"],
"starter": ["gemma-4-12b"],
"professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"],
"enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"],
}
# API keys loaded from API_KEYS env var (set in docker-compose.yml)
# Fallback is dev-only — production MUST set API_KEYS env var
@@ -216,22 +216,36 @@ def select_best_gpu(candidates, reason, agent=""):
return {"model": best, "reason": "load_balanced_" + reason}
return None
def route(rd, tier, agent=""):
msgs = rd.get("messages",[]); t = estimate_tokens(msgs)
sys = any(m.get("role")=="system" for m in msgs)
turns = len([m for m in msgs if m.get("role") in ("user","assistant")])
hints = rd.get("routing_hints",{})
allowed = TIER_MODELS.get(tier, ["qwen3.5-9b-vlm"])
allowed = TIER_MODELS.get(tier, ["gemma-4-12b"])
avail = [m for m in available_models() if m in allowed]
if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True}
# Check if all available GPUs are at max capacity
if all(is_gpu_busy(m) for m in avail):
return {"model": avail[0], "reason": "all_saturated", "saturated": True}
# GUARD: multimodal -> VLM only (sole vision model)
has_image = any(
isinstance(m.get("content"), list) and
any(p.get("type") == "image_url" for p in m["content"] if isinstance(p, dict))
for m in msgs
)
if has_image:
if "gemma-4-12b" in avail and not is_gpu_busy("gemma-4-12b"):
return {"model": "gemma-4-12b", "reason": "vision"}
elif "gemma-4-12b" in avail:
return {"model": "gemma-4-12b", "reason": "vision_saturated", "saturated": True}
else:
return {"model": allowed[0], "reason": "vision_unavailable"}
req = rd.get("model","auto")
if req != "auto":
target = req if req in avail else avail[0]
# If explicit model is busy, check if another can take it
if is_gpu_busy(target) and req in allowed:
alts = [m for m in avail if m != target and m in allowed]
if alts:
@@ -240,50 +254,62 @@ def route(rd, tier, agent=""):
return {"model": target, "reason": "explicit"}
if hints:
if hints.get("priority")=="speed" and "qwen3.5-9b-vlm" in avail:
return select_best_gpu(["qwen3.5-9b-vlm"], "hint_speed", agent) or {"model":"qwen3.5-9b-vlm","reason":"hint_speed"}
if hints.get("priority")=="speed" and "gemma-4-12b" in avail:
return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"}
if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail:
return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"}
if hints.get("priority")=="code" and "qwen3.6-27B-code" in avail:
return select_best_gpu(["qwen3.6-27B-code"], "hint_code", agent) or {"model":"qwen3.6-27B-code","reason":"hint_code"}
first_msg = msgs[0].get("content","") if msgs else ""
words = len(first_msg.split()) if isinstance(first_msg, str) else 99
# TIER 1: Lightweight — single-turn short queries → VLM (fastest)
if not sys and turns <= 1 and t <= 500 and words <= 100 and "qwen3.5-9b-vlm" in avail:
if not is_gpu_busy("qwen3.5-9b-vlm"):
return {"model":"qwen3.5-9b-vlm","reason":"lightweight"}
# VLM busy — Dense is faster for short queries than MoE
# TIER 1: Tiny - single-turn micro queries -> VLM (fastest)
if not sys and turns <= 1 and t <= 300 and words <= 100 and "gemma-4-12b" in avail:
if not is_gpu_busy("gemma-4-12b"):
return {"model":"gemma-4-12b","reason":"tiny"}
fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail]
result = select_best_gpu(fallback, "lightweight_fallback", agent)
result = select_best_gpu(fallback, "tiny_fallback", agent)
if result: return result
# TIER 2: Simple conversations — VLM primary (up to 15K tok), fastest for moderate chat
if t <= 15000 and turns <= 12 and "qwen3.5-9b-vlm" in avail:
if not is_gpu_busy("qwen3.5-9b-vlm"):
return {"model":"qwen3.5-9b-vlm","reason":"simple_conv"}
# VLM busy — fall back to Dense, then MoE
fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail]
result = select_best_gpu(fallback, "simple_conv_fallback", agent)
# TIER 2: Light - moderate chat -> VLM first (fastest), Dense fallback
if t <= 5000 and turns <= 4:
candidates = [m for m in ["gemma-4-12b","qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail]
result = select_best_gpu(candidates, "light", agent)
if result: return result
# TIER 3: Medium complexity — Dense primary, VLM fallback (quality + speed balance)
if t <= 25000:
candidates = [m for m in ["qwen3.6-27B-code","qwen3.5-9b-vlm","qwen3.6-35B-A3B"] if m in avail]
# TIER 3: Medium - quality matters -> MoE primary (60%), Dense spillover (40%)
if t <= 30000:
candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"])
result = select_best_gpu(candidates, "medium", agent)
if result: return result
# TIER 4: Heavy reasoning — MoE primary (workhorse), Dense fallback
if t > 25000:
candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","qwen3.5-9b-vlm"] if m in avail]
result = select_best_gpu(candidates, "heavy_reasoning", agent)
# TIER 4: Heavy - quality first -> Dense primary, MoE fallback
if t > 30000:
candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","gemma-4-12b"] if m in avail]
result = select_best_gpu(candidates, "heavy", agent)
if result: return result
# TIER 5: Default — Dense primary, MoE fallback
candidates = [m for m in ["qwen3.6-27B-code","qwen3.5-9b-vlm","qwen3.6-35B-A3B"] if m in avail]
# TIER 5: Default - MoE primary (60%), Dense spillover (40%)
candidates = moe_spillover(avail, ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"])
result = select_best_gpu(candidates, "default", agent)
if result: return result
return {"model":avail[0],"reason":"last_resort"}
def moe_spillover(avail, default_order):
"""Spill 40% of MoE-first traffic to Dense to prevent Strix Halo overheating.
Only applies when MoE is first candidate, available, and not busy."""
import random
if (default_order[0] == "qwen3.6-35B-A3B"
and "qwen3.6-35B-A3B" in avail
and not is_gpu_busy("qwen3.6-35B-A3B")
and "qwen3.6-27B-code" in avail
and not is_gpu_busy("qwen3.6-27B-code")
and random.random() < 0.4):
# Swap: Dense first, MoE second
return ["qwen3.6-27B-code","qwen3.6-35B-A3B"] + [m for m in default_order[2:] if m in avail and m not in ("qwen3.6-27B-code","qwen3.6-35B-A3B")]
return [m for m in default_order if m in avail]
def clean_unicode(text):
if not isinstance(text, str): return text
text = text.replace(chr(0x2014), "-"); text = text.replace(chr(0x2013), "-")
@@ -494,7 +520,7 @@ def performance():
"""Per-request performance analytics with percentiles per model/reason/agent."""
if not r: return jsonify({"error": "Redis unavailable"}), 503
try:
window_hours = int(request.args.get("window", "24"))
window_hours = int(request.args.get("window", "24").replace("h",""))
model_filter = request.args.get("model", "all")
# Load recent records
@@ -611,7 +637,7 @@ def scatter():
"""Return individual data points for scatter plots (prompt_tokens vs latency)."""
if not r: return jsonify({"error": "Redis unavailable"}), 503
try:
window_hours = int(request.args.get("window", "24"))
window_hours = int(request.args.get("window", "24").replace("h",""))
model_filter = request.args.get("model", "all")
cutoff = time.time() - (window_hours * 3600)
raw = r.lrange("perf:recent", 0, -1)