Add GPU dashboard container + Nginx routing

2026-05-15 22:25:56 +00:00
commit e95475f431
5 changed files with 488 additions and 0 deletions
@@ -0,0 +1,183 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>GPU Monitor</title>
+<style>
+* { margin: 0; padding: 0; box-sizing: border-box; }
+body { background: #0d1117; color: #c9d1d9; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; padding: 20px; }
+h1 { font-size: 1.3em; margin-bottom: 4px; }
+.topbar { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; padding-bottom: 12px; border-bottom: 1px solid #21262d; }
+.topbar .status { font-size: 0.85em; color: #8b949e; }
+.topbar .status .dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; }
+.dot.green { background: #3fb950; }
+.dot.yellow { background: #d2991d; }
+.dot.red { background: #f85149; }
+.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 16px; }
+.card { background: #161b22; border: 1px solid #21262d; border-radius: 8px; padding: 16px; }
+.card.stale { opacity: 0.5; }
+.card.dead { opacity: 0.3; border-color: #f85149; }
+.card-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 12px; }
+.card-header .name { font-weight: 600; font-size: 1.05em; }
+.card-header .host { font-size: 0.8em; color: #8b949e; }
+.card-header .state { font-size: 0.75em; padding: 2px 8px; border-radius: 10px; font-weight: 600; }
+.state.idle { background: #1b3826; color: #3fb950; }
+.state.busy { background: #3d1f1a; color: #f85149; }
+.state.unknown { background: #21262d; color: #8b949e; }
+.metric { margin-bottom: 10px; }
+.metric-label { display: flex; justify-content: space-between; font-size: 0.82em; color: #8b949e; margin-bottom: 2px; }
+.metric-label .val { color: #c9d1d9; font-weight: 500; }
+.bar { height: 6px; border-radius: 3px; background: #21262d; overflow: hidden; }
+.bar-fill { height: 100%; border-radius: 3px; transition: width 0.5s ease; }
+.bar-fill.temp-cool { background: #3fb950; }
+.bar-fill.temp-warm { background: #d2991d; }
+.bar-fill.temp-hot { background: #f85149; }
+.bar-fill.util { background: #58a6ff; }
+.bar-fill.vram { background: #bc8cff; }
+.bar-fill.power { background: #f0883e; }
+.model-line { font-size: 0.82em; color: #8b949e; margin-top: 8px; padding-top: 8px; border-top: 1px solid #21262d; }
+.model-line span { color: #c9d1d9; }
+.error { color: #f85149; font-size: 0.85em; }
+</style>
+</head>
+<body>
+<div class="topbar">
+  <div>
+    <h1><a href="/" style="color:#58a6ff;text-decoration:none;">← Workspace</a> · GPU Monitor</h1>
+    <span class="status"><span class="dot green" id="status-dot"></span><span id="status-text">Loading...</span></span>
+  </div>
+  <div class="status" id="age">—</div>
+</div>
+<div class="cards" id="cards"></div>
+
+<script>
+const INTERVAL = 5000;
+let lastFetchTime = null;
+
+function updateClock() {
+  const el = document.getElementById('age');
+  if (!lastFetchTime) { el.textContent = '—'; return; }
+  const age = Math.round((Date.now() / 1000) - lastFetchTime);
+  el.textContent = age <= 60 ? `updated ${age}s ago` : `stale ${age}s ago`;
+}
+setInterval(updateClock, 1000);
+
+const TEMP_WARN = 70, TEMP_HOT = 82;
+const VRAM_WARN = 80, VRAM_HOT = 92;
+
+function tempClass(c) { return c > TEMP_HOT ? 'temp-hot' : c > TEMP_WARN ? 'temp-warm' : 'temp-cool'; }
+function vramClass(pct) { return pct > VRAM_HOT ? 'temp-hot' : pct > VRAM_WARN ? 'temp-warm' : 'temp-cool'; }
+function pct(val, max) { return max ? Math.round(val / max * 100) : 0; }
+function mbToGB(mb) { return mb ? (mb / 1024).toFixed(1) : '—'; }
+
+function renderCard(g) {
+  const hw = g.hardware || {};
+  const inf = g.inference || {};
+  const online = g.online !== false;
+  const stale = g.stale === true;
+  let cardClass = '';
+  if (!online) cardClass = 'dead';
+  else if (stale) cardClass = 'stale';
+
+  let stateClass = inf.state || 'unknown';
+  let stateLabel = inf.state ? inf.state.toUpperCase() : 'UNKNOWN';
+  if (!online) { stateClass = 'unknown'; stateLabel = 'OFFLINE'; }
+
+  const temp = hw.temp_c;
+  const util = hw.gpu_util_pct;
+  const vramUsed = hw.vram_used_mb;
+  const vramTotal = hw.vram_total_mb;
+  const power = hw.power_w;
+  const powerLimit = hw.power_limit_w;
+  const fan = hw.fan_pct;
+  const vendor = hw.vendor;
+
+  let html = `<div class="card ${cardClass}">`;
+  html += `<div class="card-header">`;
+  html += `<div><div class="name">${g.gpu_name}</div><div class="host">${g.host}</div></div>`;
+  html += `<div class="state ${stateClass}">${stateLabel}</div>`;
+  html += `</div>`;
+
+  if (!online) {
+    html += `<div class="error">Unreachable</div>`;
+  } else if (hw.error) {
+    html += `<div class="error">${hw.error}</div>`;
+  } else {
+    // Temperature
+    if (temp != null) {
+      html += `<div class="metric"><div class="metric-label"><span>Temperature</span><span class="val">${temp}°C</span></div>`;
+      html += `<div class="bar"><div class="bar-fill ${tempClass(temp)}" style="width:${Math.min(temp,100)}%"></div></div></div>`;
+    }
+    // Utilization
+    if (util != null) {
+      html += `<div class="metric"><div class="metric-label"><span>GPU Utilization</span><span class="val">${util}%</span></div>`;
+      html += `<div class="bar"><div class="bar-fill util" style="width:${util}%"></div></div></div>`;
+    }
+    // VRAM
+    if (vramUsed != null && vramTotal != null) {
+      const vramPct = pct(vramUsed, vramTotal);
+      html += `<div class="metric"><div class="metric-label"><span>VRAM</span><span class="val">${mbToGB(vramUsed)} / ${mbToGB(vramTotal)} GB</span></div>`;
+      html += `<div class="bar"><div class="bar-fill ${vramClass(vramPct)}" style="width:${vramPct}%"></div></div></div>`;
+    }
+    // Power
+    if (power != null) {
+      const powerPct = powerLimit ? pct(power, powerLimit) : 0;
+      const powerText = powerLimit ? `${power}W / ${powerLimit}W` : `${power}W`;
+      html += `<div class="metric"><div class="metric-label"><span>Power</span><span class="val">${powerText}</span></div>`;
+      if (powerLimit) html += `<div class="bar"><div class="bar-fill power" style="width:${powerPct}%"></div></div>`;
+      html += `</div>`;
+    }
+    // Fan (NVIDIA only)
+    if (fan != null) {
+      html += `<div class="metric"><div class="metric-label"><span>Fan Speed</span><span class="val">${fan}%</span></div>`;
+      html += `<div class="bar"><div class="bar-fill util" style="width:${fan}%"></div></div></div>`;
+    }
+  }
+
+  // Model loaded
+  html += `<div class="model-line">Model: <span>${inf.model || '—'}</span></div>`;
+  html += `</div>`;
+  return html;
+}
+
+async function refresh() {
+  try {
+    const resp = await fetch('gpu_metrics.json?t=' + Date.now());
+    const data = await resp.json();
+    const gpus = data.gpus || [];
+
+    document.getElementById('cards').innerHTML = gpus.map(renderCard).join('');
+
+    // Top bar status
+    const online = gpus.filter(g => g.online !== false).length;
+    const total = gpus.length;
+    const dot = document.getElementById('status-dot');
+    const txt = document.getElementById('status-text');
+    if (online === total) { dot.className = 'dot green'; txt.textContent = `${online}/${total} online`; }
+    else if (online > 0) { dot.className = 'dot yellow'; txt.textContent = `${online}/${total} online`; }
+    else { dot.className = 'dot red'; txt.textContent = 'All offline'; }
+
+    // Capture fetch time for live clock
+    lastFetchTime = Date.now() / 1000;
+  } catch(e) {
+    document.getElementById('status-dot').className = 'dot red';
+    document.getElementById('status-text').textContent = 'Collector down';
+  }
+}
+
+// Render skeletons instantly
+const SKELETONS = [
+  {host:'amdpve', gpu_name:'AMD Strix Halo', hardware:{}, inference:{}, online:true},
+  {host:'llmgpu', gpu_name:'RTX 3090', hardware:{}, inference:{}, online:true},
+  {host:'ocu-llm', gpu_name:'RTX 5070', hardware:{}, inference:{}, online:true},
+];
+document.getElementById('cards').innerHTML = SKELETONS.map(g =>
+  `<div class="card"><div class="card-header"><div><div class="name">${g.gpu_name}</div><div class="host">${g.host}</div></div><div class="state unknown">···</div></div><div class="model-line" style="color:#8b949e;">Loading metrics...</div></div>`
+).join('');
+
+refresh();
+setInterval(refresh, INTERVAL);
+</script>
+</body>
+</html>
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""GPU metrics collector — polls sidecars + llama.cpp every 10s, writes to Workspace."""
+
+import urllib.request, json, time, os
+
+HOSTS = [
+    {"name": "amdpve", "host": "192.168.68.15", "gpu": "AMD Strix Halo", "llama_port": 8080},
+    {"name": "llmgpu", "host": "192.168.68.8", "gpu": "RTX 3090", "llama_port": 8080},
+    {"name": "ocu-llm", "host": "192.168.68.110", "gpu": "RTX 5070", "llama_port": 8080},
+]
+OUTPUT = "/app/public/gpu_metrics.json"
+INTERVAL = 10
+STALE_THRESHOLD = 30  # seconds before marking stale
+DEAD_THRESHOLD = 60   # seconds before marking unreachable
+
+last_seen = {}
+
+
+def fetch_json(url, timeout=3):
+    try:
+        req = urllib.request.Request(url)
+        resp = urllib.request.urlopen(req, timeout=timeout)
+        return json.loads(resp.read().decode())
+    except Exception:
+        return None
+
+
+def collect_one(h):
+    """Collect GPU hardware + llama.cpp inference state for one host."""
+    name = h["name"]
+    host = h["host"]
+    now = time.time()
+
+    # GPU hardware from sidecar
+    gpu = fetch_json(f"http://{host}:8090/")
+
+    # llama.cpp inference state
+    llamacpp_health = fetch_json(f"http://{host}:{h['llama_port']}/health")
+    llamacpp_models = fetch_json(f"http://{host}:{h['llama_port']}/v1/models")
+
+    # Determine inference state
+    model_name = None
+    inference_state = "unknown"
+    if llamacpp_models:
+        models = llamacpp_models.get("data", [])
+        if models:
+            model_name = models[0].get("id")
+
+    if llamacpp_health:
+        status = llamacpp_health.get("status", "")
+        if status == "ok":
+            idle = llamacpp_health.get("slots_idle", 0)
+            processing = llamacpp_health.get("slots_processing", 0)
+            if idle and not processing:
+                inference_state = "idle"
+            elif processing:
+                inference_state = "busy"
+            else:
+                inference_state = "idle"
+
+    # Check for /slots endpoint for is_processing detail
+    slots = fetch_json(f"http://{host}:{h['llama_port']}/slots")
+    if slots and isinstance(slots, list) and len(slots) > 0:
+        if slots[0].get("is_processing"):
+            inference_state = "busy"
+
+    result = {
+        "host": name,
+        "gpu_name": h["gpu"],
+        "inference": {
+            "state": inference_state,
+            "model": model_name,
+        },
+        "hardware": gpu if gpu else None,
+        "online": gpu is not None,
+        "timestamp": now,
+    }
+
+    if gpu is not None:
+        last_seen[name] = now
+
+    if name in last_seen:
+        age = now - last_seen[name]
+        if age > DEAD_THRESHOLD:
+            result["online"] = False
+        elif age > STALE_THRESHOLD:
+            result["stale"] = True
+
+    return result
+
+
+def main():
+    print(f"GPU collector starting, output={OUTPUT}, interval={INTERVAL}s")
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+
+    while True:
+        start = time.time()
+        results = [collect_one(h) for h in HOSTS]
+
+        payload = {
+            "updated": start,
+            "gpus": results,
+        }
+
+        with open(OUTPUT + ".tmp", "w") as f:
+            json.dump(payload, f)
+        os.rename(OUTPUT + ".tmp", OUTPUT)
+
+        elapsed = time.time() - start
+        sleep_for = max(0, INTERVAL - elapsed)
+        time.sleep(sleep_for)
+
+
+if __name__ == "__main__":
+    main()