abiba-bot/inference-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: decouple VRAM usage from saturation status

Browse Source 
VRAM percentage no longer marks GPU as saturated.
Saturation is about slot availability (handled by is_gpu_busy()),
not memory usage. Added vram_warning boolean flag (≥95% threshold)
for informational monitoring without affecting routing decisions.

27B Dense now correctly shows healthy at 91% VRAM.
This commit is contained in:
Abiba
2026-05-23 06:00:37 +00:00
parent 41939104c7
commit ddde6646de
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
router/router.py
+3 -2
View File
@@ -102,7 +102,8 @@ def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3):
if resp.status_code == 200:
d = resp.json()
pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100
status = "healthy" if pct < 90 else "saturated"
status = "healthy" # VRAM usage != saturation; busy slots handled by is_gpu_busy()
vram_warning = pct >= 95
# Also check if llama.cpp endpoint is actually responding
gpu_url = GPU_URLS.get(model, "")
try:
@@ -111,7 +112,7 @@ def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3):
status = "down"
except Exception:
status = "down"
return {"status": status, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")}
return {"status": status, "vram_warning": vram_warning, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")}
except Exception: pass
return {"status": "down"}