fix: heavy tier Dense→MoE→VLM
This commit is contained in:
+7
-7
@@ -27,7 +27,7 @@ GPU_MAX_CONCURRENT = {
|
|||||||
# Context window sizes (tokens) — used for compaction signals
|
# Context window sizes (tokens) — used for compaction signals
|
||||||
GPU_CONTEXT = {
|
GPU_CONTEXT = {
|
||||||
"qwen3.6-35B-A3B": 131072,
|
"qwen3.6-35B-A3B": 131072,
|
||||||
"qwen3.6-27B-code": 65536,
|
"qwen3.6-27B-code": 98304,
|
||||||
"qwen3.5-9b-vlm": 131072,
|
"qwen3.5-9b-vlm": 131072,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -190,8 +190,8 @@ def route(rd, tier):
|
|||||||
|
|
||||||
# TIER 3: Heavy reasoning — extremely large context or very long conversations
|
# TIER 3: Heavy reasoning — extremely large context or very long conversations
|
||||||
if t > 50000 or turns > 25:
|
if t > 50000 or turns > 25:
|
||||||
# Prefer models with larger context windows (MoE/VLM at 131K, Dense at 65K)
|
# Dense first (98K, purpose-built for reasoning), then MoE/VLM 131K
|
||||||
candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.5-9b-vlm","qwen3.6-27B-code"] if m in avail]
|
candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","qwen3.5-9b-vlm"] if m in avail]
|
||||||
result = select_best_gpu(candidates, "heavy_reasoning")
|
result = select_best_gpu(candidates, "heavy_reasoning")
|
||||||
if result: return result
|
if result: return result
|
||||||
|
|
||||||
@@ -308,10 +308,10 @@ def chat():
|
|||||||
if raw: yield clean_unicode(raw)
|
if raw: yield clean_unicode(raw)
|
||||||
bcast()
|
bcast()
|
||||||
ctx_remaining = GPU_CONTEXT.get(model, 65536) - estimate_tokens(rd.get("messages",[]))
|
ctx_remaining = GPU_CONTEXT.get(model, 65536) - estimate_tokens(rd.get("messages",[]))
|
||||||
r = Response(stream_with_context(gen()), mimetype="text/event-stream")
|
sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream")
|
||||||
r.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
|
sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
|
||||||
r.headers["X-Context-Model"] = model
|
sse_resp.headers["X-Context-Model"] = model
|
||||||
return r
|
return sse_resp
|
||||||
data = clean_response(resp.json())
|
data = clean_response(resp.json())
|
||||||
for c in data.get("choices",[]):
|
for c in data.get("choices",[]):
|
||||||
msg = c.get("message",{})
|
msg = c.get("message",{})
|
||||||
|
|||||||
Reference in New Issue
Block a user