From 9a0d69ce8dfcfb2e11bc2847699843914d68defc Mon Sep 17 00:00:00 2001 From: Abiba Date: Wed, 27 May 2026 07:15:58 +0000 Subject: [PATCH] feat: Dense 128K context + 2 slots, VLM second in Heavy tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dense GPU_CONTEXT: 192K→128K (131072) to free VRAM - Dense max_concurrent: 1→2 (VRAM now sufficient) - Heavy tier: Dense → VLM → MoE (VLM handles 262K context) - Total slots: 6 (2 Dense + 2 MoE + 2 VLM) Distribution target: Dense 50%, VLM 30%, MoE 20% NOTE: Requires llama.cpp restart on 192.168.68.8 with --ctx-size 131072 --- router/router.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/router/router.py b/router/router.py index de0d340..799c083 100644 --- a/router/router.py +++ b/router/router.py @@ -20,14 +20,14 @@ GPU_URLS = { # Max concurrent requests per GPU (based on llama.cpp --parallel) GPU_MAX_CONCURRENT = { "qwen3.6-35B-A3B": 2, # 2 slots (Dense-first routing reduces thermal load) - "qwen3.6-27B-code": 1, # 1 slot (24GB VRAM saturated at 256K ctx) + "qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM) "qwen3.5-9b-vlm": 2, # 2 slots (12GB VRAM, 4GB headroom) } # Context window sizes (tokens) — used for compaction signals GPU_CONTEXT = { "qwen3.6-35B-A3B": 262144, - "qwen3.6-27B-code": 196608, + "qwen3.6-27B-code": 131072, "qwen3.5-9b-vlm": 262144, } @@ -233,9 +233,9 @@ def route(rd, tier): result = select_best_gpu(candidates, "medium") if result: return result - # TIER 4: Heavy reasoning — Dense first (thermal), MoE fallback (192K/262K context) + # TIER 4: Heavy reasoning — Dense first, VLM second (262K ctx), MoE last (thermal) if t > 25000 or turns > 15: - candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","qwen3.5-9b-vlm"] if m in avail] + candidates = [m for m in ["qwen3.6-27B-code","qwen3.5-9b-vlm","qwen3.6-35B-A3B"] if m in avail] result = select_best_gpu(candidates, "heavy_reasoning") if result: return result