diff --git a/router/router.py b/router/router.py index 2dbeceb..b14fca8 100644 --- a/router/router.py +++ b/router/router.py @@ -211,6 +211,23 @@ def is_gpu_busy(model): max_c = GPU_MAX_CONCURRENT.get(model, 1) return active >= max_c + + +# Phase 3: Dynamic GPU Weighting (Health Score) +def gpu_health_score(model): + """Score a GPU based on VRAM, temperature, and load. Lower = better.""" + h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1) + if h.get("status") == "down": + return 999 # never pick down GPUs + vram_pct = h.get("vram_pct", 50) + temp_c = h.get("temp_c", 50) + active = gpu_active_count(model) + max_c = GPU_MAX_CONCURRENT.get(model, 1) + load_pct = (active / max_c) * 100 + # Score: lower = better (more headroom, cooler, less loaded) + score = vram_pct * 0.4 + max(temp_c - 30, 0) * 0.3 + load_pct * 0.3 + return score + def select_best_gpu(candidates, reason, agent=""): """Pick best GPU, spreading agents across GPUs to prevent hotspots.""" # Count how many distinct agents are on each GPU @@ -222,17 +239,19 @@ def select_best_gpu(candidates, reason, agent=""): if r.get("agent_gpu:" + ak["agent"] + ":" + m): count += 1 gpu_agent_counts[m] = count + # Phase 3: Sort candidates by health score before selection + sorted_candidates = sorted(candidates, key=gpu_health_score) # First pass: prefer GPUs with 0 other agents (fresh GPU for this agent) - for m in candidates: + for m in sorted_candidates: if not is_gpu_busy(m) and gpu_agent_counts.get(m, 0) == 0: return {"model": m, "reason": reason} # Second pass: prefer GPU this agent is NOT already on (skip own GPU) if agent: - for m in candidates: + for m in sorted_candidates: if not is_gpu_busy(m) and not r.get("agent_gpu:" + agent + ":" + m): return {"model": m, "reason": reason} # Third pass: any non-busy GPU - for m in candidates: + for m in sorted_candidates: if not is_gpu_busy(m): return {"model": m, "reason": reason} # All busy — pick least loaded