diff --git a/docker-compose.yml b/docker-compose.yml index 7ded8b3..7fd7541 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,8 +29,8 @@ services: - GPU_LIGHT_URL=http://192.168.68.110:8080/v1 healthcheck: test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/health')"] - interval: 15s - timeout: 5s + interval: 30s + timeout: 15s retries: 3 depends_on: redis: @@ -68,8 +68,8 @@ services: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] - interval: 15s - timeout: 5s + interval: 30s + timeout: 15s retries: 3 depends_on: - litellm diff --git a/router/router.py b/router/router.py index bfd8d7a..12bd57a 100644 --- a/router/router.py +++ b/router/router.py @@ -20,15 +20,15 @@ GPU_URLS = { # Max concurrent requests per GPU (based on llama.cpp --parallel) GPU_MAX_CONCURRENT = { "qwen3.6-35B-A3B": 2, # 2 slots - "qwen3.6-27B-code": 2, # 2 slots + "qwen3.6-27B-code": 1, # 1 slot (24GB VRAM saturated at 256K ctx) "qwen3.5-9b-vlm": 2, # 2 slots (12GB VRAM, 4GB headroom) } # Context window sizes (tokens) — used for compaction signals GPU_CONTEXT = { - "qwen3.6-35B-A3B": 131072, - "qwen3.6-27B-code": 98304, - "qwen3.5-9b-vlm": 131072, + "qwen3.6-35B-A3B": 262144, + "qwen3.6-27B-code": 262144, + "qwen3.5-9b-vlm": 262144, } TIER_MODELS = { @@ -94,11 +94,11 @@ def gpu_decr(model): if v and int(v) < 0: r.set("active:" + model, 0) # never go negative -def check_gpu_health(model): +def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3): url = GPU_SIDECARS.get(model) if not url: return {"status": "unknown"} try: - resp = requests.get(url, timeout=5) + resp = requests.get(url, timeout=sidecar_timeout) if resp.status_code == 200: d = resp.json() pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100 @@ -106,7 +106,7 @@ def check_gpu_health(model): # Also check if llama.cpp endpoint is actually responding gpu_url = GPU_URLS.get(model, "") try: - hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=3) + hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout) if hr.status_code != 200: status = "down" except Exception: @@ -117,7 +117,9 @@ def check_gpu_health(model): def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")] -def estimate_tokens(msgs): return sum(len(str(m.get("content",""))) for m in msgs) // 4 +def estimate_tokens(msgs): + """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text).""" + return len(json.dumps(msgs, default=str)) // 3.5 def is_gpu_busy(model): """Check if GPU is at or near max concurrent capacity.""" @@ -150,6 +152,9 @@ def route(rd, tier): allowed = TIER_MODELS.get(tier, ["qwen3.5-9b-vlm"]) avail = [m for m in available_models() if m in allowed] if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True} + # Check if all available GPUs are at max capacity + if all(is_gpu_busy(m) for m in avail): + return {"model": avail[0], "reason": "all_saturated", "saturated": True} req = rd.get("model","auto") if req != "auto": @@ -190,8 +195,8 @@ def route(rd, tier): # TIER 3: Heavy reasoning — extremely large context or very long conversations if t > 50000 or turns > 25: - # Dense first (98K, purpose-built for reasoning), then MoE/VLM 131K - candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","qwen3.5-9b-vlm"] if m in avail] + # MoE first (131K context handles heavy sessions), then Dense (98K reasoning), then Light (131K fallback) + candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","qwen3.5-9b-vlm"] if m in avail] result = select_best_gpu(candidates, "heavy_reasoning") if result: return result @@ -265,6 +270,17 @@ def chat(): # Allow agent to override queue timeout via header q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT))) + # Cross-turn context tracking: accumulate tokens per session + session_id = request.headers.get("X-Session-Id", "") + session_tokens = 0 + if session_id and r: + try: + prev = int(r.get("session:" + session_id) or 0) + current = estimate_tokens(rd.get("messages",[])) + session_tokens = max(prev, current) # context only grows + r.set("session:" + session_id, session_tokens, ex=86400) # TTL 24h + except Exception: pass + d = route(rd, tier) queue_start = time.time() @@ -307,9 +323,12 @@ def chat(): for raw in resp.iter_content(chunk_size=None, decode_unicode=True): if raw: yield clean_unicode(raw) bcast() - ctx_remaining = GPU_CONTEXT.get(model, 65536) - estimate_tokens(rd.get("messages",[])) + ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) + ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 + ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream") sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) + sse_resp.headers["X-Context-Warning"] = ctx_warning sse_resp.headers["X-Context-Model"] = model return sse_resp data = clean_response(resp.json()) @@ -317,10 +336,13 @@ def chat(): msg = c.get("message",{}) if not msg.get("content") and msg.get("reasoning_content"): msg["content"] = msg["reasoning_content"] - ctx_remaining = GPU_CONTEXT.get(model, 65536) - estimate_tokens(rd.get("messages",[])) - data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining)} + ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[]))) + ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100 + ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok")) + data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning} resp = jsonify(data) resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining)) + resp.headers["X-Context-Warning"] = ctx_warning resp.headers["X-Context-Model"] = model bcast() return resp @@ -334,13 +356,15 @@ def chat(): return jsonify({"error":str(e)}), 500 @app.route("/v1/models") -def models(): return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":check_gpu_health(m).get("status"),"gpu":check_gpu_health(m).get("gpu_name")} for m in GPU_URLS]}) +def models(): + def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) + return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]}) @app.route("/health") def health(): gpus = {} for m in GPU_URLS: - h = check_gpu_health(m) + h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1) h["active_requests"] = gpu_active_count(m) h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1) gpus[m] = h