feat: Dense 128K context + 2 slots, VLM second in Heavy tier
- Dense GPU_CONTEXT: 192K→128K (131072) to free VRAM - Dense max_concurrent: 1→2 (VRAM now sufficient) - Heavy tier: Dense → VLM → MoE (VLM handles 262K context) - Total slots: 6 (2 Dense + 2 MoE + 2 VLM) Distribution target: Dense 50%, VLM 30%, MoE 20% NOTE: Requires llama.cpp restart on 192.168.68.8 with --ctx-size 131072
This commit is contained in:
+4
-4
@@ -20,14 +20,14 @@ GPU_URLS = {
|
|||||||
# Max concurrent requests per GPU (based on llama.cpp --parallel)
|
# Max concurrent requests per GPU (based on llama.cpp --parallel)
|
||||||
GPU_MAX_CONCURRENT = {
|
GPU_MAX_CONCURRENT = {
|
||||||
"qwen3.6-35B-A3B": 2, # 2 slots (Dense-first routing reduces thermal load)
|
"qwen3.6-35B-A3B": 2, # 2 slots (Dense-first routing reduces thermal load)
|
||||||
"qwen3.6-27B-code": 1, # 1 slot (24GB VRAM saturated at 256K ctx)
|
"qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM)
|
||||||
"qwen3.5-9b-vlm": 2, # 2 slots (12GB VRAM, 4GB headroom)
|
"qwen3.5-9b-vlm": 2, # 2 slots (12GB VRAM, 4GB headroom)
|
||||||
}
|
}
|
||||||
|
|
||||||
# Context window sizes (tokens) — used for compaction signals
|
# Context window sizes (tokens) — used for compaction signals
|
||||||
GPU_CONTEXT = {
|
GPU_CONTEXT = {
|
||||||
"qwen3.6-35B-A3B": 262144,
|
"qwen3.6-35B-A3B": 262144,
|
||||||
"qwen3.6-27B-code": 196608,
|
"qwen3.6-27B-code": 131072,
|
||||||
"qwen3.5-9b-vlm": 262144,
|
"qwen3.5-9b-vlm": 262144,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -233,9 +233,9 @@ def route(rd, tier):
|
|||||||
result = select_best_gpu(candidates, "medium")
|
result = select_best_gpu(candidates, "medium")
|
||||||
if result: return result
|
if result: return result
|
||||||
|
|
||||||
# TIER 4: Heavy reasoning — Dense first (thermal), MoE fallback (192K/262K context)
|
# TIER 4: Heavy reasoning — Dense first, VLM second (262K ctx), MoE last (thermal)
|
||||||
if t > 25000 or turns > 15:
|
if t > 25000 or turns > 15:
|
||||||
candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","qwen3.5-9b-vlm"] if m in avail]
|
candidates = [m for m in ["qwen3.6-27B-code","qwen3.5-9b-vlm","qwen3.6-35B-A3B"] if m in avail]
|
||||||
result = select_best_gpu(candidates, "heavy_reasoning")
|
result = select_best_gpu(candidates, "heavy_reasoning")
|
||||||
if result: return result
|
if result: return result
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user