feat(dashboard): live GPU health scoring + real KPIs

- Added /metrics/gpu-health endpoint with live health scores (VRAM 40%, temp 30%, load 30%)
- Added /metrics/latency endpoint for dashboard KPIs
- Added GPU_LABELS for human-readable model names
- Dashboard v2: rewired to real data endpoints
  - KPI cards: GPUs online, circuit trips, avg latency, req/min, active requests
  - Health scores from actual gpu_health_score() function
  - Rolling 60-sample history chart (real data, no simulation)
  - Status: green/yellow/red based on tripped circuits
  - No CDN dependency (pure CSS)
  - Auto-refresh every 15s
- nginx: /dashboard/ serves static files with cache headers
- docker-compose: dashboard volume mount

Co-authored-by: Abiba <abiba@sysloggh.com>
This commit is contained in:
Abiba
2026-06-12 17:57:46 +00:00
parent a860a8fd0f
commit ad9881f141
4 changed files with 220 additions and 3 deletions
+56
View File
@@ -31,6 +31,13 @@ GPU_URLS = {
"gemma-4-12b": GPU_LIGHT_URL,
}
# Max concurrent requests per GPU (based on llama.cpp --parallel)
GPU_LABELS = {
"qwen3.6-35B-A3B": "Qwen3.6 35B (Strix Halo)",
"qwen3.6-27B-code": "Qwen3.6 27B Code (RTX 3090)",
"gemma-4-12b": "Gemma-4 12B (RTX 5070)",
}
GPU_MAX_CONCURRENT = {
"qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating)
"qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM)
@@ -815,6 +822,55 @@ def metrics_circuit_breaker():
}
return jsonify(result)
@app.route("/metrics/gpu-health")
def metrics_gpu_health():
"""Live GPU health scores + circuit breaker + KPIs."""
result = {"gpus": [], "ts": time.time()}
for model in GPU_URLS:
h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1)
score = gpu_health_score(model)
active = gpu_active_count(model)
max_c = GPU_MAX_CONCURRENT.get(model, 1)
cb_tripped = bool(r and r.exists("circuit:" + model + ":open"))
cb_count = int(r.get("circuit:" + model + ":count") or 0) if r else 0
result["gpus"].append({
"id": model,
"label": GPU_LABELS.get(model, model),
"status": h.get("status", "unknown"),
"vram_pct": h.get("vram_pct", 0),
"temp_c": h.get("temp_c", 0),
"vram_used_mb": h.get("vram_used_mb", 0),
"vram_total_mb": h.get("vram_total_mb", 0),
"gpu_name": h.get("gpu_name", model),
"health_score": round(score, 1),
"active_requests": active,
"max_concurrent": max_c,
"circuit_tripped": cb_tripped,
"circuit_trip_count": cb_count
})
online = sum(1 for g in result["gpus"] if g["status"] in ("healthy", "saturated"))
trips = sum(g["circuit_trip_count"] for g in result["gpus"])
result["kpi"] = {"gpus_online": online, "total_trips": trips, "total_gpus": len(GPU_URLS)}
return jsonify(result)
@app.route("/metrics/latency")
def metrics_latency():
"""Lightweight latency summary for dashboard KPIs."""
if not r: return jsonify({"avg_ms": 0, "requests_per_min": 0})
recent = []
for x in (r.lrange("routes:recent", 0, 49) or []):
try: recent.append(json.loads(x))
except: pass
if not recent: return jsonify({"avg_ms": 0, "requests_per_min": 0, "count": 0})
now = time.time()
last_min = [x for x in recent if now - x.get("ts", 0) < 60]
latencies = [x.get("queue_ms", 0) + x.get("inference_ms", 0) for x in last_min if "inference_ms" in x]
return jsonify({
"avg_ms": round(sum(latencies) / len(latencies), 1) if latencies else 0,
"requests_per_min": len(last_min),
"count": len(recent)
})
@app.route("/stream")
def stream():
def ev():