diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html new file mode 100644 index 0000000..f743140 --- /dev/null +++ b/dashboard/dashboard.html @@ -0,0 +1,145 @@ + + + + +Inference Harness - Dashboard + + + +
+ +
+

Inference Harness

Syslog Solution LLC · Real-time Monitoring

+
+ healthy + +
+
+ +
+

-

GPUs Online

+

-

Circuit Trips

+

-

Avg Latency

+

-

Requests/min

+

-

Active Requests

+
+ +

GPU Health Scoring Live: VRAM 40% · Temp 30% · Load 30%

+
+

Health Score History (60s rolling)

+ +

Inference Harness Dashboard · Syslog Solution LLC · Auto-refresh 15s

+
+ + + + diff --git a/docker-compose.yml b/docker-compose.yml index 7bf0ac2..5823ca4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,6 +68,7 @@ services: - "80:80" volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./dashboard:/opt/inference-harness/dashboard:ro healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1/health"] interval: 30s diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 5b953c8..8565840 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -31,7 +31,7 @@ http { # Disable buffering for SSE streams proxy_buffering off; - # API — through router + # API through router location /v1/ { proxy_pass http://router_api; proxy_http_version 1.1; @@ -42,7 +42,14 @@ http { proxy_read_timeout 600s; proxy_buffering off; } -location /admin/ { proxy_pass http://router_api; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Authorization $http_authorization; } + + location /admin/ { + proxy_pass http://router_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Authorization $http_authorization; + } # SSE streaming endpoint location /stream { @@ -71,7 +78,15 @@ location /admin/ { proxy_pass http://router_api; proxy_htt proxy_set_header Authorization $http_authorization; } - # Dashboard + # Professional Dashboard (Phase 1-3) - Static HTML served via Nginx + location /dashboard/ { + alias /opt/inference-harness/dashboard/; + index dashboard.html; + add_header Cache-Control "public, max-age=3600"; + add_header X-Content-Type-Options nosniff; + } + + # Legacy Dashboard (root) - Proxy to Flask app location / { proxy_pass http://dashboard_ui; proxy_http_version 1.1; diff --git a/router/router.py b/router/router.py index b14fca8..efa44fa 100644 --- a/router/router.py +++ b/router/router.py @@ -31,6 +31,13 @@ GPU_URLS = { "gemma-4-12b": GPU_LIGHT_URL, } # Max concurrent requests per GPU (based on llama.cpp --parallel) + +GPU_LABELS = { + "qwen3.6-35B-A3B": "Qwen3.6 35B (Strix Halo)", + "qwen3.6-27B-code": "Qwen3.6 27B Code (RTX 3090)", + "gemma-4-12b": "Gemma-4 12B (RTX 5070)", +} + GPU_MAX_CONCURRENT = { "qwen3.6-35B-A3B": 2, # 2 slots (cross-agent spread prevents overheating) "qwen3.6-27B-code": 2, # 2 slots (128K context frees VRAM) @@ -815,6 +822,55 @@ def metrics_circuit_breaker(): } return jsonify(result) + +@app.route("/metrics/gpu-health") +def metrics_gpu_health(): + """Live GPU health scores + circuit breaker + KPIs.""" + result = {"gpus": [], "ts": time.time()} + for model in GPU_URLS: + h = check_gpu_health(model, sidecar_timeout=1.5, gpu_timeout=1) + score = gpu_health_score(model) + active = gpu_active_count(model) + max_c = GPU_MAX_CONCURRENT.get(model, 1) + cb_tripped = bool(r and r.exists("circuit:" + model + ":open")) + cb_count = int(r.get("circuit:" + model + ":count") or 0) if r else 0 + result["gpus"].append({ + "id": model, + "label": GPU_LABELS.get(model, model), + "status": h.get("status", "unknown"), + "vram_pct": h.get("vram_pct", 0), + "temp_c": h.get("temp_c", 0), + "vram_used_mb": h.get("vram_used_mb", 0), + "vram_total_mb": h.get("vram_total_mb", 0), + "gpu_name": h.get("gpu_name", model), + "health_score": round(score, 1), + "active_requests": active, + "max_concurrent": max_c, + "circuit_tripped": cb_tripped, + "circuit_trip_count": cb_count + }) + online = sum(1 for g in result["gpus"] if g["status"] in ("healthy", "saturated")) + trips = sum(g["circuit_trip_count"] for g in result["gpus"]) + result["kpi"] = {"gpus_online": online, "total_trips": trips, "total_gpus": len(GPU_URLS)} + return jsonify(result) + +@app.route("/metrics/latency") +def metrics_latency(): + """Lightweight latency summary for dashboard KPIs.""" + if not r: return jsonify({"avg_ms": 0, "requests_per_min": 0}) + recent = [] + for x in (r.lrange("routes:recent", 0, 49) or []): + try: recent.append(json.loads(x)) + except: pass + if not recent: return jsonify({"avg_ms": 0, "requests_per_min": 0, "count": 0}) + now = time.time() + last_min = [x for x in recent if now - x.get("ts", 0) < 60] + latencies = [x.get("queue_ms", 0) + x.get("inference_ms", 0) for x in last_min if "inference_ms" in x] + return jsonify({ + "avg_ms": round(sum(latencies) / len(latencies), 1) if latencies else 0, + "requests_per_min": len(last_min), + "count": len(recent) + }) @app.route("/stream") def stream(): def ev():