feat(dashboard): live GPU health scoring + real KPIs
- Added /metrics/gpu-health endpoint with live health scores (VRAM 40%, temp 30%, load 30%) - Added /metrics/latency endpoint for dashboard KPIs - Added GPU_LABELS for human-readable model names - Dashboard v2: rewired to real data endpoints - KPI cards: GPUs online, circuit trips, avg latency, req/min, active requests - Health scores from actual gpu_health_score() function - Rolling 60-sample history chart (real data, no simulation) - Status: green/yellow/red based on tripped circuits - No CDN dependency (pure CSS) - Auto-refresh every 15s - nginx: /dashboard/ serves static files with cache headers - docker-compose: dashboard volume mount Co-authored-by: Abiba <abiba@sysloggh.com>
This commit is contained in:
@@ -31,6 +31,13 @@ GPU_URLS = {
|
||||
"gemma-4-12b": GPU_LIGHT_URL,
|
||||
}
|
||||
# Max concurrent requests per GPU (based on llama.cpp --parallel)
|
||||
|
||||
GPU_LABELS = {
|
||||
"qwen3.6-35B-A3B": "Qwen3.6 35B (Strix Halo)",
|
||||
"qwen3.6-27B-code": "Qwen3.6 27B Code (RTX 3090)",
|
||||
"gemma-4-12b": "Gemma-4 12B (RTX 5070)",
|
||||
}
|
||||
|
||||
GPU_MAX_CONCURRENT = {
|
||||
"qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating)
|
||||
"qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM)
|
||||
@@ -815,6 +822,55 @@ def metrics_circuit_breaker():
|
||||
}
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@app.route("/metrics/gpu-health")
|
||||
def metrics_gpu_health():
|
||||
"""Live GPU health scores + circuit breaker + KPIs."""
|
||||
result = {"gpus": [], "ts": time.time()}
|
||||
for model in GPU_URLS:
|
||||
h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1)
|
||||
score = gpu_health_score(model)
|
||||
active = gpu_active_count(model)
|
||||
max_c = GPU_MAX_CONCURRENT.get(model, 1)
|
||||
cb_tripped = bool(r and r.exists("circuit:" + model + ":open"))
|
||||
cb_count = int(r.get("circuit:" + model + ":count") or 0) if r else 0
|
||||
result["gpus"].append({
|
||||
"id": model,
|
||||
"label": GPU_LABELS.get(model, model),
|
||||
"status": h.get("status", "unknown"),
|
||||
"vram_pct": h.get("vram_pct", 0),
|
||||
"temp_c": h.get("temp_c", 0),
|
||||
"vram_used_mb": h.get("vram_used_mb", 0),
|
||||
"vram_total_mb": h.get("vram_total_mb", 0),
|
||||
"gpu_name": h.get("gpu_name", model),
|
||||
"health_score": round(score, 1),
|
||||
"active_requests": active,
|
||||
"max_concurrent": max_c,
|
||||
"circuit_tripped": cb_tripped,
|
||||
"circuit_trip_count": cb_count
|
||||
})
|
||||
online = sum(1 for g in result["gpus"] if g["status"] in ("healthy", "saturated"))
|
||||
trips = sum(g["circuit_trip_count"] for g in result["gpus"])
|
||||
result["kpi"] = {"gpus_online": online, "total_trips": trips, "total_gpus": len(GPU_URLS)}
|
||||
return jsonify(result)
|
||||
|
||||
@app.route("/metrics/latency")
|
||||
def metrics_latency():
|
||||
"""Lightweight latency summary for dashboard KPIs."""
|
||||
if not r: return jsonify({"avg_ms": 0, "requests_per_min": 0})
|
||||
recent = []
|
||||
for x in (r.lrange("routes:recent", 0, 49) or []):
|
||||
try: recent.append(json.loads(x))
|
||||
except: pass
|
||||
if not recent: return jsonify({"avg_ms": 0, "requests_per_min": 0, "count": 0})
|
||||
now = time.time()
|
||||
last_min = [x for x in recent if now - x.get("ts", 0) < 60]
|
||||
latencies = [x.get("queue_ms", 0) + x.get("inference_ms", 0) for x in last_min if "inference_ms" in x]
|
||||
return jsonify({
|
||||
"avg_ms": round(sum(latencies) / len(latencies), 1) if latencies else 0,
|
||||
"requests_per_min": len(last_min),
|
||||
"count": len(recent)
|
||||
})
|
||||
@app.route("/stream")
|
||||
def stream():
|
||||
def ev():
|
||||
|
||||
Reference in New Issue
Block a user