abiba-bot/inference-harness
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Phase 0: Dual-key router — no hardcoded keys, gemma sync, 262K context fix

Browse Source 
- Removed all hardcoded API_KEYS from source (env var now REQUIRED)
- Added dual-key transition support (old keys deprecated, logged)
- Model sync: qwen3.5-9b-vlm → gemma-4-12b everywhere
- Dense context: 131K → 262K (all 3 models at 262K)
- Added /admin/keys endpoints for key lifecycle management
- Added ADMIN_KEY env var for admin auth
- Dashboard already synced to gemma-4-12b labels

Deployment: set API_KEYS from phase0-dual-keys.json in docker-compose
Transition: 7 days dual-key, then POST /admin/keys/revoke
This commit is contained in:
Abiba Bot
2026-06-06 00:51:29 +00:00
parent 1e24bc3b9b
commit 39cf297a8f
3 changed files with 634 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dashboard/dashboard.py
+139 -10
View File
@@ -16,7 +16,7 @@ def fetch_state():
def broadcast_loop():
while True:
time.sleep(3)
time.sleep(5)
data = fetch_state(); payload = json.dumps(data)
with sse_lock:
dead = [q for q in sse_subscribers if not q.put(payload)]
@@ -101,7 +101,32 @@ body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMac
<div class="col-md-4"><div class="chart-card"><div class="title">Model Distribution</div><div id="route-bars"></div></div></div>
<div class="col-md-4"><div class="chart-card"><div class="title">Agent Activity</div><div id="agent-bars"></div></div></div>
<!-- ROW 4: Live Stream -->
<!-- ROW 4: Performance Analytics -->
<div class="col-12 mb-2"><div class="d-flex align-items-center gap-2"><span class="fw-bold text-white" style="font-size:14px">&#x1F4CA; Performance Analytics</span>
<div class="d-flex gap-1 ms-auto">
<button class="btn-sm-period active" onclick="switchPerfWindow('1')">1h</button>
<button class="btn-sm-period" onclick="switchPerfWindow('24')">24h</button>
</div>
</div></div>
<div class="col-md-6"><div class="chart-card"><div class="title">Latency — P50 / P95 / P99 (ms)</div><div id="perf-latency"></div></div></div>
<div class="col-md-6"><div class="chart-card"><div class="title">Throughput — Tokens / sec</div><div id="perf-throughput"></div></div></div>
<div class="col-md-6"><div class="chart-card"><div class="title">Routing Effectiveness — by Reason</div><div id="perf-reasons"></div></div></div>
<div class="col-md-6"><div class="chart-card"><div class="title">Agent Performance</div><div id="perf-agents"></div></div></div>
<!-- ROW 5: Latency vs Context Scatter -->
<div class="col-12"><div class="chart-card"><div class="title d-flex justify-content-between align-items-center">
<span>Latency vs Prompt Size — by Model</span>
<div class="d-flex gap-2">
<select id="scatter-model" onchange="loadScatter()" style="font-size:10px;background:#1e293b;color:#94a3b8;border:1px solid #334155;border-radius:4px;padding:2px 6px">
<option value="all">All Models</option>
<option value="gemma-4-12b">12B VLM</option>
<option value="qwen3.6-27B-code">27B Dense</option>
<option value="qwen3.6-35B-A3B">35B MoE</option>
</select>
</div>
</div><div id="scatter-plot" style="height:200px;position:relative"></div><div id="scatter-legend" class="d-flex justify-content-center gap-3 mt-2 flex-wrap small"></div></div></div>
<!-- ROW 6: Live Stream -->
<div class="col-12"><div class="chart-card"><div class="title">Live Stream</div>
<div class="table-responsive"><table class="table table-custom mb-0">
<thead><tr><th>Time</th><th>Agent</th><th>Model</th><th>Reason</th><th>Tier</th></tr></thead>
@@ -113,13 +138,13 @@ body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMac
<script>
// ═══ MODEL CONFIG — Update ONLY this array for model migrations ═══
var MODELS=[
{id:'qwen3.6-35B-A3B',label:'Qwen MoE',gpu:'MoE - Strix Halo',color:'#a78bfa',short:'MoE',domId:'gpu-moe'},
{id:'qwen3.6-27B-code',label:'Qwen Code',gpu:'Dense - RTX 3090',color:'#f59e0b',short:'Dense',domId:'gpu-dense'},
{id:'gemma-4-12b',label:'Gemma 4 12B',gpu:'VLM - RTX 5070',color:'#22c55e',short:'VLM',domId:'gpu-light'}
{id:'qwen3.6-35B-A3B',label:'Qwen MoE',gpu:'MoE - Strix Halo',color:'#a78bfa',short:'MoE',domId:'gpu-moe',perf:'35B MoE'},
{id:'qwen3.6-27B-code',label:'Qwen Code',gpu:'Dense - RTX 3090',color:'#f59e0b',short:'Dense',domId:'gpu-dense',perf:'27B Dense'},
{id:'gemma-4-12b',label:'Gemma 4 12B',gpu:'VLM - RTX 5070',color:'#22c55e',short:'VLM',domId:'gpu-light',perf:'12B VLM'}
];
// Auto-derived lookups — DO NOT EDIT below
var MC={},ML={},GL={},ids={},lb={};
MODELS.forEach(function(m){MC[m.id]=m.color;ML[m.id]=m.label;GL[m.id]=m.gpu;ids[m.id]=m.domId;lb[m.id]=m.short;});
var MC={},ML={},GL={},ids={},lb={},mlab={},mcol={};
MODELS.forEach(function(m){MC[m.id]=m.color;ML[m.id]=m.label;GL[m.id]=m.gpu;ids[m.id]=m.domId;lb[m.id]=m.short;mlab[m.id]=m.perf;mcol[m.id]=m.color;});
function $(id){return document.getElementById(id);}
function render(data){
@@ -166,7 +191,7 @@ el.innerHTML=h;
function renderGPUMetrics(data){
var el=$('gpu-metrics-card');if(!el)return;
var lb={'qwen3.6-35B-A3B':'MoE','qwen3.6-27B-code':'Dense','gemma-4-12b':'VLM'};
var h='';data.gpus.forEach(function(g){
var nm=lb[g.id]||g.id,tp=g.temp_c||0,ut=g.gpu_util_pct||0,pw=g.power_w||0,pl=g.power_limit_w||0;
var tc=tp>85?'#ef4444':tp>70?'#f59e0b':'#22c55e',uc=ut>90?'#ef4444':ut>70?'#f59e0b':'#22c55e';
@@ -193,8 +218,92 @@ var grid='';for(var g=0;g<=4;g++){var y=(g/4)*H;grid+='<line x1=\"0\" y1=\"'+y.t
cn.innerHTML='<svg viewBox=\"0 0 100 '+(H+16)+'\" style=\"width:100%;height:'+(H+20)+'px;display:block\" preserveAspectRatio=\"none\">'+grid+paths+'</svg>';
lg.innerHTML=mn.map(function(m){return'<span class=\"d-flex align-items-center gap-1\"><svg width=\"14\" height=\"8\"><line x1=\"0\" y1=\"4\" x2=\"14\" y2=\"4\" stroke=\"'+(MC[m]||'#38bdf8')+'\" stroke-width=\"2\"/></svg>'+(ML[m]||m)+'</span>';}).join('');
}
function poll(){fetch('/api/state').then(function(r){return r.json()}).then(function(data){render(data);$('connection-status').textContent='live';}).catch(function(){$('connection-status').textContent='reconnecting';});}
poll();setInterval(poll,3000);loadTS();
var perfWindow='24';
function switchPerfWindow(w){perfWindow=w;document.querySelectorAll('.btn-sm-period').forEach(function(b,i){if(i>=4)b.classList.toggle('active',b.textContent.trim().replace('h','')===w)});loadPerf();}
function loadPerf(){fetch('/api/performance?window='+perfWindow).then(function(r){return r.json()}).then(renderPerf).catch(function(){})}
function renderPerf(d){
var models=d.models||[],reasons=d.reasons||[],agents=d.agents||[],sum=d.summary||{};
// Latency bars: p50/p95/p99 per model
// mlab/mcol auto-derived from MODELS above
if(!models.length){$('perf-latency').innerHTML='<div class="text-secondary small text-center py-4">Accumulating data...</div>';return;}
var maxLat=Math.max(...models.map(function(m){return m.latency.p99||0}),1);
var latHTML=models.map(function(m){
var l=m.latency||{},p50=l.p50||0,p95=l.p95||0,p99=l.p99||0,c=mcol[m.model]||'#38bdf8';
return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span class="text-secondary">'+m.count+' reqs</span></div>'+
'<div class="d-flex align-items-center gap-2 mb-1"><span class="text-secondary" style="min-width:28px">p50</span><div class="flex-grow-1" style="height:14px;background:#1e293b;border-radius:4px;overflow:hidden;position:relative"><div style="position:absolute;left:0;top:0;height:100%;width:'+(p50/maxLat*100)+'%;background:'+c+';opacity:0.3;border-radius:4px"></div><div style="position:absolute;left:0;top:0;height:100%;width:'+(p95/maxLat*100)+'%;background:'+c+';opacity:0.5;border-radius:4px"></div><div style="position:absolute;left:0;top:0;height:100%;width:'+(p99/maxLat*100)+'%;background:'+c+';border-radius:4px"></div></div><span style="color:'+c+';min-width:48px;text-align:right;font-variant-numeric:tabular-nums">'+p99+'ms</span></div>'+
'<div class="d-flex gap-3" style="font-size:10px;color:#64748b;padding-left:32px"><span>p50: '+p50+'ms</span><span>p95: '+p95+'ms</span><span>p99: '+p99+'ms</span></div></div>';
}).join('');
$('perf-latency').innerHTML=latHTML;
// Throughput comparison
var maxTps=Math.max(...models.map(function(m){return m.throughput.avg_tokens_per_sec||0}),1);
var tpsHTML=models.map(function(m){
var t=m.throughput||{},avg=t.avg_tokens_per_sec||0,p50=t.p50||0,c=mcol[m.model]||'#38bdf8';
var isAllStreaming = avg===0 && p50===0;
if(isAllStreaming){
return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span style="color:#64748b;font-style:italic">streaming only</span></div><div class="text-secondary" style="font-size:10px">t/s available for non-streaming requests only</div></div>';
}
return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span style="color:'+c+'" class="fw-bold">'+avg+' tok/s</span></div>'+
'<div class="d-flex align-items-center gap-2"><span class="text-secondary" style="min-width:28px">avg</span><div class="flex-grow-1" style="height:6px;background:#1e293b;border-radius:3px;overflow:hidden"><div style="height:100%;width:'+(Math.max(avg/maxTps*100,6))+'%;background:'+c+';border-radius:3px"></div></div><span class="small" style="color:'+c+';min-width:54px;text-align:right">'+avg+' tok/s</span></div>'+
'<div class="d-flex align-items-center gap-2 mt-1"><span class="text-secondary" style="min-width:28px;font-size:10px">p50</span><div class="flex-grow-1" style="height:4px;background:#1e293b;border-radius:2px;overflow:hidden"><div style="height:100%;width:'+(Math.max(p50/maxTps*100,4))+'%;background:'+c+';opacity:0.5;border-radius:2px"></div></div><span style="font-size:10px;color:#64748b">'+p50+' tok/s</span></div></div>';
}).join('');
$('perf-throughput').innerHTML=tpsHTML;
// Routing reasons table
if(reasons.length){
var rHTML='<table class="table table-custom mb-0"><thead><tr><th>Reason</th><th>Count</th><th>Avg Lat</th><th>P95 Lat</th></tr></thead><tbody>';
reasons.forEach(function(r){rHTML+='<tr><td>'+r.reason+'</td><td>'+r.count+'</td><td>'+r.avg_total_ms+'ms</td><td>'+r.p95_total_ms+'ms</td></tr>';});
rHTML+='</tbody></table>';$('perf-reasons').innerHTML=rHTML;
}else{$('perf-reasons').innerHTML='<div class="text-secondary small text-center py-3">-</div>';}
// Agent performance
if(agents.length){
var maxAc=Math.max(...agents.map(function(a){return a.count||0}),1);
var aHTML=agents.map(function(a){return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+a.agent+'</span><span class="text-secondary">'+a.count+' reqs</span></div><div class="d-flex align-items-center gap-2"><div class="flex-grow-1" style="height:4px;background:#1e293b;border-radius:2px;overflow:hidden"><div style="height:100%;width:'+(a.count/maxAc*100)+'%;background:#38bdf8;border-radius:2px"></div></div><span class="small" style="color:#38bdf8;min-width:60px;text-align:right">'+a.avg_total_ms+'ms avg</span></div></div>';}).join('');
$('perf-agents').innerHTML=aHTML;
}else{$('perf-agents').innerHTML='<div class="text-secondary small text-center py-3">-</div>';}
}
var sseConnected = false;
function poll(){if(sseConnected)return;fetch('/api/state').then(function(r){return r.json()}).then(function(data){render(data);$('connection-status').textContent='live';}).catch(function(){$('connection-status').textContent='reconnecting';});}
// SSE connection tracking
var es = new EventSource('/api/stream');
es.onopen = function(){sseConnected = true; $('connection-status').textContent = 'live (SSE)';};
es.onmessage = function(e){try{render(JSON.parse(e.data));}catch(ex){}};
es.onerror = function(){sseConnected = false; $('connection-status').textContent = 'polling';};
function loadScatter(){
var m=$('scatter-model').value;
fetch('/api/scatter?window=24&model='+m).then(function(r){return r.json()}).then(renderScatter).catch(function(){});
}
function renderScatter(d){
var pts=d.points||[],el=$('scatter-plot'),lg=$('scatter-legend');
if(!pts.length){el.innerHTML='<div class="text-secondary small text-center py-5">No data yet</div>';return;}
var mcol=Object.assign({unknown:'#38bdf8'},mcol);
// mlab auto-derived from MODELS above
var maxX=Math.max.apply(null,pts.map(function(p){return p.prompt_tokens||0}))||1000;
var maxY=Math.max.apply(null,pts.map(function(p){return p.inference_ms||0}))||5000;
// Log scale for X axis (prompt tokens vary widely)
var toX=function(t){return Math.log10(Math.max(t,1))/Math.log10(Math.max(maxX,10))*100;};
var toY=function(t){return (t/maxY)*100;};
var dots='';
pts.forEach(function(p){
var x=toX(p.prompt_tokens),y=toY(p.inference_ms),c=mcol[p.model]||'#38bdf8';
var r=p.stream?1.5:2.5,o=p.stream?0.4:0.8;
dots+='<circle cx="'+x+'" cy="'+(100-y)+'" r="'+r+'" fill="'+c+'" opacity="'+o+'"><title>'+mlab[p.model]+' | '+p.prompt_tokens+' tok | '+p.inference_ms+'ms | '+p.agent+'</title></circle>';
});
// Grid lines
var grid='';
for(var i=1;i<=4;i++){grid+='<line x1="0" y1="'+(i*20)+'" x2="100" y2="'+(i*20)+'" stroke="#1e293b" stroke-width="0.5"/>';}
for(var i=1;i<=4;i++){grid+='<line x1="'+(i*20)+'" y1="0" x2="'+(i*20)+'" y2="100" stroke="#1e293b" stroke-width="0.5"/>';}
// Axis labels
var xTicks='';
var xVals=[10,100,1000,10000,100000];
xVals.forEach(function(v){if(v<=maxX)xTicks+='<text x="'+toX(v)+'" y="103" text-anchor="middle" font-size="8" fill="#64748b">'+(v>=1000?(v/1000)+'k':v)+'</text>';});
var yTicks='';
var yVals=[500,1000,5000,10000,50000,100000];
yVals.forEach(function(v){if(v<=maxY)yTicks+='<text x="-2" y="'+(97-toY(v))+'" text-anchor="end" font-size="8" fill="#64748b">'+(v>=1000?(v/1000)+'s':v+'ms')+'</text>';});
el.innerHTML='<svg viewBox="-35 0 140 115" style="width:100%;height:200px">'+grid+dots+xTicks+yTicks+'<text x="50" y="112" text-anchor="middle" font-size="9" fill="#475569">Prompt Tokens (log scale)</text><text x="-38" y="50" text-anchor="middle" font-size="9" fill="#475569" transform="rotate(-90,-38,50)">Inference Time</text></svg>';
// Legend
var models=[];pts.forEach(function(p){if(models.indexOf(p.model)===-1)models.push(p.model);});
lg.innerHTML=models.map(function(m){return'<span class="d-flex align-items-center gap-1 small"><svg width="10" height="10"><circle cx="5" cy="5" r="3.5" fill="'+(mcol[m]||'#38bdf8')+'"/></svg>'+mlab[m]+'</span>';}).join('');
}
poll();setInterval(poll,10000);loadTS();loadPerf();setInterval(loadPerf,15000);loadScatter();setInterval(loadScatter,30000);
</script>
</body>
</html>"""
@@ -205,6 +314,26 @@ def dashboard(): return render_template_string(DASHBOARD_HTML)
@app.route("/api/state")
def api_state(): return fetch_state()
@app.route("/api/scatter")
def api_scatter():
window = request.args.get("window", "24")
model = request.args.get("model", "all")
try:
r = requests.get(f"http://router:9000/metrics/scatter?window={window}&model={model}", timeout=10)
if r.status_code == 200: return r.json()
except Exception: pass
return {"points": [], "count": 0}
@app.route("/api/performance")
def api_performance():
window = request.args.get("window", "24")
model = request.args.get("model", "all")
try:
r = requests.get(f"http://router:9000/metrics/performance?window={window}&model={model}", timeout=10)
if r.status_code == 200: return r.json()
except Exception: pass
return {"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}}
@app.route("/api/timeseries")
def api_timeseries():
period = request.args.get("period", "day")