abiba-bot/inference-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: non-blocking GPU health checks + 256K turboquant context upgrade

Browse Source 
router/router.py:
- check_gpu_health() now accepts configurable timeouts (sidecar_timeout, gpu_timeout)
- /health and /v1/models endpoints use fast 1.5s/1s timeouts (non-blocking)
- /v1/models now calls check_gpu_health once per model instead of twice
- GPU_CONTEXT updated to 262144 across all models (turboquant upgrade)
- 27B max_concurrent reduced 2→1 (24GB VRAM saturated at 256K context)

docker-compose.yml:
- Router healthcheck timeout 5s→15s, interval 15s→30s
- Nginx healthcheck timeout 5s→15s, interval 15s→30s

Fixes dashboard hang when any GPU is unreachable.
This commit is contained in:
Abiba
2026-05-23 05:57:13 +00:00
parent 0983337fdb
commit 41939104c7
2 changed files with 43 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
router/router.py
+39 -15
View File
@@ -20,15 +20,15 @@ GPU_URLS = {
# Max concurrent requests per GPU (based on llama.cpp --parallel)
GPU_MAX_CONCURRENT = {
"qwen3.6-35B-A3B": 2, # 2 slots
"qwen3.6-27B-code": 2, # 2 slots
"qwen3.6-27B-code": 1, # 1 slot (24GB VRAM saturated at 256K ctx)
"qwen3.5-9b-vlm": 2, # 2 slots (12GB VRAM, 4GB headroom)
}
# Context window sizes (tokens) — used for compaction signals
GPU_CONTEXT = {
"qwen3.6-35B-A3B": 131072,
"qwen3.6-27B-code": 98304,
"qwen3.5-9b-vlm": 131072,
"qwen3.6-35B-A3B": 262144,
"qwen3.6-27B-code": 262144,
"qwen3.5-9b-vlm": 262144,
}
TIER_MODELS = {
@@ -94,11 +94,11 @@ def gpu_decr(model):
if v and int(v) < 0:
r.set("active:" + model, 0) # never go negative
def check_gpu_health(model):
def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3):
url = GPU_SIDECARS.get(model)
if not url: return {"status": "unknown"}
try:
resp = requests.get(url, timeout=5)
resp = requests.get(url, timeout=sidecar_timeout)
if resp.status_code == 200:
d = resp.json()
pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100
@@ -106,7 +106,7 @@ def check_gpu_health(model):
# Also check if llama.cpp endpoint is actually responding
gpu_url = GPU_URLS.get(model, "")
try:
hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=3)
hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout)
if hr.status_code != 200:
status = "down"
except Exception:
@@ -117,7 +117,9 @@ def check_gpu_health(model):
def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")]
def estimate_tokens(msgs): return sum(len(str(m.get("content",""))) for m in msgs) // 4
def estimate_tokens(msgs):
"""Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text)."""
return len(json.dumps(msgs, default=str)) // 3.5
def is_gpu_busy(model):
"""Check if GPU is at or near max concurrent capacity."""
@@ -150,6 +152,9 @@ def route(rd, tier):
allowed = TIER_MODELS.get(tier, ["qwen3.5-9b-vlm"])
avail = [m for m in available_models() if m in allowed]
if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True}
# Check if all available GPUs are at max capacity
if all(is_gpu_busy(m) for m in avail):
return {"model": avail[0], "reason": "all_saturated", "saturated": True}
req = rd.get("model","auto")
if req != "auto":
@@ -190,8 +195,8 @@ def route(rd, tier):
# TIER 3: Heavy reasoning — extremely large context or very long conversations
if t > 50000 or turns > 25:
# Dense first (98K, purpose-built for reasoning), then MoE/VLM 131K
candidates = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B","qwen3.5-9b-vlm"] if m in avail]
# MoE first (131K context handles heavy sessions), then Dense (98K reasoning), then Light (131K fallback)
candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","qwen3.5-9b-vlm"] if m in avail]
result = select_best_gpu(candidates, "heavy_reasoning")
if result: return result
@@ -265,6 +270,17 @@ def chat():
# Allow agent to override queue timeout via header
q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT)))
# Cross-turn context tracking: accumulate tokens per session
session_id = request.headers.get("X-Session-Id", "")
session_tokens = 0
if session_id and r:
try:
prev = int(r.get("session:" + session_id) or 0)
current = estimate_tokens(rd.get("messages",[]))
session_tokens = max(prev, current) # context only grows
r.set("session:" + session_id, session_tokens, ex=86400) # TTL 24h
except Exception: pass
d = route(rd, tier)
queue_start = time.time()
@@ -307,9 +323,12 @@ def chat():
for raw in resp.iter_content(chunk_size=None, decode_unicode=True):
if raw: yield clean_unicode(raw)
bcast()
ctx_remaining = GPU_CONTEXT.get(model, 65536) - estimate_tokens(rd.get("messages",[]))
ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[])))
ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100
ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok"))
sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream")
sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
sse_resp.headers["X-Context-Warning"] = ctx_warning
sse_resp.headers["X-Context-Model"] = model
return sse_resp
data = clean_response(resp.json())
@@ -317,10 +336,13 @@ def chat():
msg = c.get("message",{})
if not msg.get("content") and msg.get("reasoning_content"):
msg["content"] = msg["reasoning_content"]
ctx_remaining = GPU_CONTEXT.get(model, 65536) - estimate_tokens(rd.get("messages",[]))
data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining)}
ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[])))
ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100
ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok"))
data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning}
resp = jsonify(data)
resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
resp.headers["X-Context-Warning"] = ctx_warning
resp.headers["X-Context-Model"] = model
bcast()
return resp
@@ -334,13 +356,15 @@ def chat():
return jsonify({"error":str(e)}), 500
@app.route("/v1/models")
def models(): return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":check_gpu_health(m).get("status"),"gpu":check_gpu_health(m).get("gpu_name")} for m in GPU_URLS]})
def models():
def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1)
return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]})
@app.route("/health")
def health():
gpus = {}
for m in GPU_URLS:
h = check_gpu_health(m)
h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1)
h["active_requests"] = gpu_active_count(m)
h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1)
gpus[m] = h