fix: non-blocking GPU health checks + 256K turboquant context upgrade

router/router.py:
- check_gpu_health() now accepts configurable timeouts (sidecar_timeout, gpu_timeout)
- /health and /v1/models endpoints use fast 1.5s/1s timeouts (non-blocking)
- /v1/models now calls check_gpu_health once per model instead of twice
- GPU_CONTEXT updated to 262144 across all models (turboquant upgrade)
- 27B max_concurrent reduced 2→1 (24GB VRAM saturated at 256K context)

docker-compose.yml:
- Router healthcheck timeout 5s→15s, interval 15s→30s
- Nginx healthcheck timeout 5s→15s, interval 15s→30s

Fixes dashboard hang when any GPU is unreachable.

docker-compose.yml

@@ -29,8 +29,8 @@ services:
       - GPU_LIGHT_URL=http://192.168.68.110:8080/v1
     healthcheck:
       test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"]
-      interval: 15s
+      interval: 30s
-      timeout: 5s
+      timeout: 15s
       retries: 3
     depends_on:
       redis:
@@ -68,8 +68,8 @@ services:
       - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
     healthcheck:
       test: ["CMD", "curl", "-f", "http://127.0.0.1/health"]
-      interval: 15s
+      interval: 30s
-      timeout: 5s
+      timeout: 15s
       retries: 3
     depends_on:
       - litellm

router/router.py

@@ -20,15 +20,15 @@ GPU_URLS = {
 # Max concurrent requests per GPU (based on llama.cpp --parallel)
 GPU_MAX_CONCURRENT = {
     "qwen3.6-35B-A3B": 2,   # 2 slots
-    "qwen3.6-27B-code": 2,  # 2 slots
+    "qwen3.6-27B-code": 1,  # 1 slot (24GB VRAM saturated at 256K ctx)
     "qwen3.5-9b-vlm": 2,       # 2 slots (12GB VRAM, 4GB headroom)
 }
 
 # Context window sizes (tokens) — used for compaction signals
 GPU_CONTEXT = {
-    "qwen3.6-35B-A3B": 131072,
+    "qwen3.6-35B-A3B": 262144,
-    "qwen3.6-27B-code": 98304,
+    "qwen3.6-27B-code": 262144,
-    "qwen3.5-9b-vlm": 131072,
+    "qwen3.5-9b-vlm": 262144,
 }
 
 TIER_MODELS = {
@@ -94,11 +94,11 @@ def gpu_decr(model):
         if v and int(v) < 0:
             r.set("active:" + model, 0)  # never go negative
 
-def check_gpu_health(model):
+def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3):
     url = GPU_SIDECARS.get(model)
     if not url: return {"status": "unknown"}
     try:
-        resp = requests.get(url, timeout=5)
+        resp = requests.get(url, timeout=sidecar_timeout)
         if resp.status_code == 200:
             d = resp.json()
             pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100
@@ -106,7 +106,7 @@ def check_gpu_health(model):
             # Also check if llama.cpp endpoint is actually responding
             gpu_url = GPU_URLS.get(model, "")
             try:
-                hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=3)
+                hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout)
                 if hr.status_code != 200:
                     status = "down"
             except Exception:
@@ -117,7 +117,9 @@ def check_gpu_health(model):
 
 def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")]
 
-def estimate_tokens(msgs): return sum(len(str(m.get("content",""))) for m in msgs) // 4
+def estimate_tokens(msgs):
+    """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text)."""
+    return len(json.dumps(msgs, default=str)) // 3.5
 
 def is_gpu_busy(model):
     """Check if GPU is at or near max concurrent capacity."""
@@ -150,6 +152,9 @@ def route(rd, tier):
     allowed = TIER_MODELS.get(tier, ["qwen3.5-9b-vlm"])
     avail = [m for m in available_models() if m in allowed]
     if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True}
+    # Check if all available GPUs are at max capacity
+    if all(is_gpu_busy(m) for m in avail):
+        return {"model": avail[0], "reason": "all_saturated", "saturated": True}
     
     req = rd.get("model","auto")
     if req != "auto":
@@ -190,8 +195,8 @@ def route(rd, tier):
     
     # TIER 3: Heavy reasoning — extremely large context or very long conversations
     if t > 50000 or turns > 25:
-        # Dense first (98K, purpose-built for reasoning), then MoE/VLM 131K
+        # MoE first (131K context handles heavy sessions), then Dense (98K reasoning), then Light (131K fallback)
-        candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","qwen3.5-9b-vlm"] if m in avail]
+        candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","qwen3.5-9b-vlm"] if m in avail]
         result = select_best_gpu(candidates, "heavy_reasoning")
         if result: return result
     
@@ -265,6 +270,17 @@ def chat():
         # Allow agent to override queue timeout via header
         q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT)))
         
+        # Cross-turn context tracking: accumulate tokens per session
+        session_id = request.headers.get("X-Session-Id", "")
+        session_tokens = 0
+        if session_id and r:
+            try:
+                prev = int(r.get("session:" + session_id) or 0)
+                current = estimate_tokens(rd.get("messages",[]))
+                session_tokens = max(prev, current)  # context only grows
+                r.set("session:" + session_id, session_tokens, ex=86400)  # TTL 24h
+            except Exception: pass
+        
         d = route(rd, tier)
         queue_start = time.time()
         
@@ -307,9 +323,12 @@ def chat():
                 for raw in resp.iter_content(chunk_size=None, decode_unicode=True):
                     if raw: yield clean_unicode(raw)
             bcast()
-            ctx_remaining = GPU_CONTEXT.get(model, 65536) - estimate_tokens(rd.get("messages",[]))
+            ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[])))
+            ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100
+            ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok"))
             sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream")
             sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
+            sse_resp.headers["X-Context-Warning"] = ctx_warning
             sse_resp.headers["X-Context-Model"] = model
             return sse_resp
         data = clean_response(resp.json())
@@ -317,10 +336,13 @@ def chat():
             msg = c.get("message",{})
             if not msg.get("content") and msg.get("reasoning_content"):
                 msg["content"] = msg["reasoning_content"]
-        ctx_remaining = GPU_CONTEXT.get(model, 65536) - estimate_tokens(rd.get("messages",[]))
+        ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[])))
-        data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining)}
+        ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100
+        ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok"))
+        data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning}
         resp = jsonify(data)
         resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
+        resp.headers["X-Context-Warning"] = ctx_warning
         resp.headers["X-Context-Model"] = model
         bcast()
         return resp
@@ -334,13 +356,15 @@ def chat():
         return jsonify({"error":str(e)}), 500
 
 @app.route("/v1/models")
-def models(): return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":check_gpu_health(m).get("status"),"gpu":check_gpu_health(m).get("gpu_name")} for m in GPU_URLS]})
+def models():
+    def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1)
+    return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]})
 
 @app.route("/health")
 def health():
     gpus = {}
     for m in GPU_URLS:
-        h = check_gpu_health(m)
+        h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1)
         h["active_requests"] = gpu_active_count(m)
         h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1)
         gpus[m] = h