diff --git a/dashboard/dashboard.py b/dashboard/dashboard.py index 9b4e064..b8f218f 100644 --- a/dashboard/dashboard.py +++ b/dashboard/dashboard.py @@ -1,4 +1,4 @@ -"""Harness Dashboard.""" +"""SyslogAI Harness Dashboard.""" import os, json, time, queue, threading import requests from flask import Flask, request, render_template_string, Response, stream_with_context @@ -6,8 +6,7 @@ from flask import Flask, request, render_template_string, Response, stream_with_ ROUTER_METRICS = os.environ.get("ROUTER_METRICS_URL", "http://router:9000/metrics") app = Flask(__name__) -sse_subscribers = [] -sse_lock = threading.Lock() +sse_subscribers = []; sse_lock = threading.Lock() def fetch_state(): try: @@ -19,8 +18,7 @@ def fetch_state(): def broadcast_loop(): while True: time.sleep(3) - data = fetch_state() - payload = json.dumps(data) + data = fetch_state(); payload = json.dumps(data) with sse_lock: dead = [] for q in sse_subscribers: @@ -48,10 +46,7 @@ body { background: var(--bg); color: var(--text); min-height: 100vh; padding: clamp(12px, 3vw, 32px); } -.header { - display: flex; align-items: center; justify-content: space-between; - flex-wrap: wrap; gap: 12px; margin-bottom: 24px; -} +.header { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 12px; margin-bottom: 24px; } .header h1 { font-size: clamp(18px, 4vw, 26px); font-weight: 700; color: #fff; } .header h1 span { color: var(--accent); } .status-bar { display: flex; gap: 16px; align-items: center; flex-wrap: wrap; font-size: 13px; color: var(--dim); } @@ -59,23 +54,12 @@ body { .status-dot.live { background: var(--green); animation: pulse 2s infinite; } @keyframes pulse { 0%,100%{opacity:1} 50%{opacity:0.3} } .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 340px), 1fr)); gap: 16px; } -.card { - background: var(--card); border: 1px solid var(--border); - border-radius: 12px; padding: clamp(12px, 3vw, 20px); -} -.card-title { - font-size: 13px; font-weight: 600; text-transform: uppercase; - letter-spacing: 0.5px; color: var(--dim); margin-bottom: 14px; -} -.gpu-row { - display: flex; align-items: center; gap: 14px; padding: 10px 0; - border-bottom: 1px solid rgba(255,255,255,0.04); -} +.card { background: var(--card); border: 1px solid var(--border); border-radius: 12px; padding: clamp(12px, 3vw, 20px); } +.card-title { font-size: 13px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; color: var(--dim); margin-bottom: 14px; } +.full { grid-column: 1 / -1; } +.gpu-row { display: flex; align-items: center; gap: 14px; padding: 10px 0; border-bottom: 1px solid rgba(255,255,255,0.04); } .gpu-row:last-child { border-bottom: none; } -.gpu-icon { - width: 40px; height: 40px; border-radius: 10px; display: flex; - align-items: center; justify-content: center; font-size: 18px; flex-shrink: 0; -} +.gpu-icon { width: 40px; height: 40px; border-radius: 10px; display: flex; align-items: center; justify-content: center; font-size: 18px; flex-shrink: 0; } .gpu-icon.green { background: rgba(127,217,98,0.12); color: var(--green); } .gpu-icon.yellow { background: rgba(255,180,84,0.12); color: var(--yellow); } .gpu-icon.red { background: rgba(242,109,120,0.12); color: var(--red); } @@ -90,6 +74,10 @@ body { .vram-fill.green { background: var(--green); } .vram-fill.yellow { background: var(--yellow); } .vram-fill.red { background: var(--red); } +.slot-row { display: flex; gap: 3px; margin-top: 6px; } +.slot-dot { flex: 1; height: 6px; border-radius: 2px; background: rgba(255,255,255,0.06); } +.slot-dot.active { background: var(--accent); } +.slot-label { font-size: 10px; margin-top: 2px; } .bar-row { margin-bottom: 10px; } .bar-label { display: flex; justify-content: space-between; font-size: 12px; margin-bottom: 4px; } .bar-label .name { color: #e6edf3; } @@ -106,16 +94,13 @@ body { .agent-tanko { background: rgba(255,180,84,0.15); color: var(--yellow); } .agent-koby { background: rgba(89,194,255,0.15); color: var(--blue); } .agent-kagenz0 { background: rgba(127,217,98,0.15); color: var(--green); } +.agent-koonimo { background: rgba(255,180,84,0.15); color: #ff8c42; } .agent-unknown { background: rgba(255,255,255,0.06); color: var(--dim); } -.agent-admin { background: rgba(255,255,255,0.08); color: #e6edf3; } -.full { grid-column: 1 / -1; } -.period-btn { - background: var(--card); border: 1px solid var(--border); color: var(--dim); - padding: 4px 12px; border-radius: 6px; font-size: 12px; cursor: pointer; - font-family: inherit; transition: all 0.2s; -} +.period-btn { background: var(--card); border: 1px solid var(--border); color: var(--dim); padding: 4px 12px; border-radius: 6px; font-size: 12px; cursor: pointer; font-family: inherit; transition: all 0.2s; } .period-btn.active { background: var(--accent); color: #000; border-color: var(--accent); } .period-btn:hover { border-color: var(--accent); color: #e6edf3; } +.queue-ring { position: relative; display: inline-block; margin-bottom: 8px; } +.queue-center { position: absolute; top: 50%; left: 50%; transform: translate(-50%,-50%); text-align: center; } @media (max-width: 600px) { .gpu-metrics { gap: 10px; } .route-table { font-size: 11px; } @@ -134,15 +119,28 @@ body {
+
GPU Health
Loading...
+ +
+
Queue Status
+
+
Loading...
+
+
Model Distribution
-
-
+
+
Agent Activity
+
-
+
+ +
Usage Over Time
@@ -156,10 +154,7 @@ body {
-
-
Agent Activity
-
-
-
+
Live Request Stream
@@ -175,83 +170,93 @@ const MODEL_COLORS = {'gemma-4-E4B':'#7fd962','qwen3.6-27B-code':'#ffb454','qwen const MODEL_LABELS = {'gemma-4-E4B':'Gemma 4B','qwen3.6-27B-code':'Qwen Code 27B','qwen3.6-35B-A3B':'Qwen MoE 35B'}; const GPU_LABELS = {'NVIDIA GeForce RTX 5070':'RTX 5070 - Gemma 4B','NVIDIA GeForce RTX 3090':'RTX 3090 - Qwen Code 27B','AMD Radeon (Strix Halo)':'Strix Halo - Qwen MoE 35B'}; -function statusIcon(status) { - if (status === 'healthy') return '●'; - if (status === 'saturated') return '◉'; - return '○'; -} -function vramClass(pct) { if(pct>90)return'red';if(pct>75)return'yellow';return'green'; } +function statusIcon(s) { return s==='healthy'?'●':s==='saturated'?'◉':'○'; } +function vramClass(p) { return p>90?'red':p>75?'yellow':'green'; } function render(data) { if(!data||!data.gpus)return; - const total = Object.values(data.route_counts||{}).reduce((a,b)=>a+b,0); + var total = Object.values(data.route_counts||{}).reduce(function(a,b){return a+b;},0); document.getElementById('total-requests').textContent = total + ' requests'; document.getElementById('update-time').textContent = new Date().toLocaleTimeString(); + var gpus = data.gpus||[]; + document.getElementById('gpu-container').innerHTML = gpus.map(function(g){ + var active = g.active_requests||0, max = g.max_concurrent||1; + var slots = ''; + for(var i=0;i
'; + var slotLabel = '
'+active+'/'+max+' active
'; + return '
'+statusIcon(g.status)+'
'+(GPU_LABELS[g.gpu_name]||g.gpu_name||g.id||'?')+'
VRAM '+(g.vram_used_mb||'?')+'/'+(g.vram_total_mb||'?')+' MB
Temp '+(g.temp_c||'?')+'C
Util '+(g.gpu_util_pct||0)+'%
'+(g.power_w!=null?'
Power '+g.power_w+'W
':'')+'
'+slots+'
'+slotLabel+'
'+(g.gpu_util_pct||0)+'%
'; + }).join(''); - const gpus = data.gpus||[]; - document.getElementById('gpu-container').innerHTML = gpus.map(g => '
'+statusIcon(g.status)+'
'+(GPU_LABELS[g.gpu_name]||g.gpu_name||g.id||'?')+'
VRAM '+(g.vram_used_mb||'?')+'/'+(g.vram_total_mb||'?')+' MB
Temp '+(g.temp_c||'?')+'C
Util '+(g.gpu_util_pct||0)+'%
'+(g.power_w!=null?'
Power '+g.power_w+'W
':'')+'
'+(g.vram_pct||0)+'%
').join(''); + var rc = data.route_counts||{}, maxR = Math.max(1,rc.gemma||0,rc.qwen||0); + document.getElementById('route-bars').innerHTML = Object.entries(rc).length ? Object.entries(rc).sort(function(a,b){return b[1]-a[1];}).map(function(e){var m=e[0],c=e[1];return '
'+(MODEL_LABELS[m]||m)+''+c+' ('+(total?Math.round(c/total*100):0)+'%)
';}).join('') : '
No data yet
'; - const rc = data.route_counts||{}; - const maxR = Math.max(1,...Object.values(rc)); - document.getElementById('route-bars').innerHTML = Object.entries(rc).length ? Object.entries(rc).sort((a,b)=>b[1]-a[1]).map(([m,c])=>'
'+(MODEL_LABELS[m]||m)+''+c+' ('+(total?Math.round(c/total*100):0)+'%)
').join('') : '
No data yet
'; + var ac = data.agent_counts||{}, maxA = Math.max(1,ac.Tanko||0,ac.Koby||0,ac.Koonimo||0); + document.getElementById('agent-bars').innerHTML = Object.entries(ac).length ? Object.entries(ac).sort(function(a,b){return b[1]-a[1];}).map(function(e){var a=e[0],c=e[1];var cl='agent-'+a.toLowerCase().replace(/[^a-z]/g,'');return '
'+a+''+c+' reqs
';}).join('') : '
No activity yet
'; - const ac = data.agent_counts||{}; - const maxA = Math.max(1,...Object.values(ac)); - document.getElementById('agent-bars').innerHTML = Object.entries(ac).length ? Object.entries(ac).sort((a,b)=>b[1]-a[1]).map(([a,c])=>'
'+a+''+c+' reqs
').join('') : '
No agent activity yet
'; + var recent = data.recent||[]; + document.getElementById('route-tbody').innerHTML = recent.length ? recent.slice(0,25).map(function(r){var d=new Date(r.ts*1000),a=r.agent||'?',cl='agent-'+a.toLowerCase().replace(/[^a-z0-9]/g,'');return''+d.toLocaleTimeString()+''+a+''+(MODEL_LABELS[r.model]||r.model)+''+(r.reason||'')+''+(r.tier||'')+'';}).join('') : 'Waiting...'; - const recent = data.recent||[]; - document.getElementById('route-tbody').innerHTML = recent.length ? recent.slice(0,25).map(r=>{const d=new Date(r.ts*1000);const a=r.agent||'?';const cl='agent-'+a.toLowerCase().replace(/[^a-z0-9]/g,'');return''+d.toLocaleTimeString()+''+a+''+(MODEL_LABELS[r.model]||r.model)+''+(r.reason||'')+''+(r.tier||'')+'';}).join('') : 'Waiting for requests...'; + renderQueue(data); } -let currentPeriod = 'day'; -async function switchPeriod(p) { - currentPeriod = p; - document.querySelectorAll('.period-btn').forEach(b => b.classList.remove('active')); - document.querySelectorAll('.period-btn').forEach(b => { if(b.textContent.trim().startsWith(p==='day'?'24h':p==='week'?'7d':'30d')) b.classList.add('active'); }); - await loadTimeseries(); -} -async function loadTimeseries() { - try { const r = await fetch('/api/timeseries?period='+currentPeriod); renderTimeseries(await r.json()); } catch(e) {} +function renderQueue(data) { + var c = document.getElementById('queue-viz'); if(!c) return; + var gpus = data.gpus||[]; + var totA=0, totM=0; + gpus.forEach(function(g){ totA+=(g.active_requests||0); totM+=(g.max_concurrent||1); }); + var pct = totM>0?Math.round(totA/totM*100):0; + var status = pct>=100?'SATURATED':pct>=50?'BUSY':'IDLE'; + var sc = pct>=100?'var(--red)':pct>=50?'var(--yellow)':'var(--green)'; + var circ = 2*Math.PI*32, dash = (pct/100)*circ; + var h = '
'+totA+'
/ '+totM+'
'; + h += '
'+status+'
'; + var labels = {'qwen3.6-35B-A3B':'MoE','qwen3.6-27B-code':'Dense','gemma-4-E4B':'Gemma'}; + gpus.forEach(function(g){ + var a=g.active_requests||0,mx=g.max_concurrent||1,gp=mx>0?Math.round(a/mx*100):0; + var gc=gp>=100?'var(--red)':gp>=50?'var(--yellow)':'var(--green)'; + h+='
'; + h+=''+(labels[g.id]||g.id)+''; + h+='
'; + h+=''+a+'/'+mx+'
'; + }); + c.innerHTML = h; } + +var currentPeriod = 'day'; +function switchPeriod(p) { currentPeriod=p; document.querySelectorAll('.period-btn').forEach(function(b){b.classList.remove('active');}); document.querySelectorAll('.period-btn').forEach(function(b){if(b.textContent.trim().startsWith(p==='day'?'24h':p==='week'?'7d':'30d'))b.classList.add('active');}); loadTimeseries(); } +function loadTimeseries() { fetch('/api/timeseries?period='+currentPeriod).then(function(r){return r.json();}).then(renderTimeseries).catch(function(){}); } function renderTimeseries(d) { - const models = d.models||{}, labels = d.labels||[]; + var models=d.models||{},labels=d.labels||[]; if(!labels.length)return; - const container = document.getElementById('timeseries-chart'); - const legend = document.getElementById('timeseries-legend'); - const modelNames = Object.keys(models); - if(!modelNames.length){container.innerHTML='
No data yet
';return;} - const colors = {'gemma-4-E4B':'#7fd962','qwen3.6-27B-code':'#ffb454','qwen3.6-35B-A3B':'#d2a6ff'}; - const shortNames = {'gemma-4-E4B':'Gemma','qwen3.6-27B-code':'Qwen Code','qwen3.6-35B-A3B':'Qwen MoE'}; - let maxVal = 1; - for(const m in models) for(const v of models[m]) if(v>maxVal) maxVal=v; - maxVal = Math.ceil(maxVal*1.15)||1; - const W = labels.length>1?100/(labels.length-1):100, H=130; - let paths=''; - for(const m of modelNames){const vals=models[m]||[];let d='';for(let i=0;i';} - let grid=''; - for(let g=0;g<=4;g++){const y=(g/4)*H;grid+='';} - const svg=''+grid+paths+''; - const step=Math.max(1,Math.floor(labels.length/8)); - let lh='
'; - for(let i=0;i'; - lh+='
'; - container.innerHTML=svg+lh; - legend.innerHTML=modelNames.map(m=>''+shortNames[m]+'').join(''); + var c=document.getElementById('timeseries-chart'),l=document.getElementById('timeseries-legend'); + var mn=Object.keys(models); + if(!mn.length){c.innerHTML='
No data yet
';return;} + var colors={'gemma-4-E4B':'#7fd962','qwen3.6-27B-code':'#ffb454','qwen3.6-35B-A3B':'#d2a6ff'}; + var sn={'gemma-4-E4B':'Gemma','qwen3.6-27B-code':'Qwen Code','qwen3.6-35B-A3B':'Qwen MoE'}; + var mv=1; for(var m in models) for(var i=0;imv)mv=models[m][i]; + mv=Math.ceil(mv*1.15)||1; + var W=labels.length>1?100/(labels.length-1):100,H=130; + var paths=''; + for(var mi=0;mi';} + var grid='';for(var g=0;g<=4;g++){var y=(g/4)*H;grid+='';} + var svg=''+grid+paths+''; + var step=Math.max(1,Math.floor(labels.length/8)),lh='
'; + for(var i=0;i'+labels[i]+'
'; + lh+='
'; c.innerHTML=svg+lh; + l.innerHTML=mn.map(function(m){return''+sn[m]+'';}).join(''); } -function poll(){fetch('/api/state').then(r=>r.json()).then(data=>{render(data);document.getElementById('connection-status').textContent='live';document.getElementById('live-dot').className='status-dot live';}).catch(()=>{document.getElementById('connection-status').textContent='reconnecting...';document.getElementById('live-dot').className='status-dot';});} +function poll(){fetch('/api/state').then(function(r){return r.json();}).then(function(data){render(data);document.getElementById('connection-status').textContent='live';document.getElementById('live-dot').className='status-dot live';}).catch(function(){document.getElementById('connection-status').textContent='reconnecting...';document.getElementById('live-dot').className='status-dot';});} poll();setInterval(poll,3000);loadTimeseries(); """ @app.route("/") -def dashboard(): - return render_template_string(DASHBOARD_HTML) +def dashboard(): return render_template_string(DASHBOARD_HTML) @app.route("/api/state") -def api_state(): - return fetch_state() +def api_state(): return fetch_state() @app.route("/api/timeseries") def api_timeseries(): @@ -264,27 +269,22 @@ def api_timeseries(): @app.route("/api/stream") def api_stream(): - def event_stream(): + def ev(): q = queue.Queue() with sse_lock: sse_subscribers.append(q) try: - data = fetch_state() - yield "data: " + json.dumps(data) + "\n\n" + yield "data: "+json.dumps(fetch_state())+"\n\n" while True: - try: msg = q.get(timeout=3); yield "data: " + msg + "\n\n" - except queue.Empty: - data = fetch_state() - yield "data: " + json.dumps(data) + "\n\n" + try: msg = q.get(timeout=3); yield "data: "+msg+"\n\n" + except queue.Empty: yield "data: "+json.dumps(fetch_state())+"\n\n" except GeneratorExit: pass finally: with sse_lock: if q in sse_subscribers: sse_subscribers.remove(q) - return Response(stream_with_context(event_stream()), mimetype="text/event-stream", - headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) + return Response(stream_with_context(ev()), mimetype="text/event-stream", headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"}) @app.route("/health") -def health(): - return {"status":"healthy","service":"harness-dashboard"} +def health(): return {"status":"healthy","service":"harness-dashboard"} if __name__ == "__main__": app.run(host="0.0.0.0", port=3000, debug=False)