security: move API keys to env var, strip from source code fallback

- Added API_KEYS env var to router service in docker-compose.yml - Replaced hardcoded agent keys in router.py fallback with dev-only placeholder - Production now loads keys from environment, not source code - Resolves the final remaining item from CT116 security deep-dive Co-authored-by: Abiba via Kwame
2026-06-03 12:40:04 +00:00
parent 9a633583ab
commit 0cb4597b0e
5 changed files with 1099 additions and 11 deletions
@@ -0,0 +1,356 @@
+"""SyslogAI Harness Dashboard — Modern Design."""
+import os, json, time, queue, threading
+import requests
+from flask import Flask, request, render_template_string, Response, stream_with_context
+
+ROUTER_METRICS = os.environ.get("ROUTER_METRICS_URL", "http://router:9000/metrics")
+app = Flask(__name__)
+sse_subscribers = []; sse_lock = threading.Lock()
+
+def fetch_state():
+    try:
+        r = requests.get(ROUTER_METRICS, timeout=5)
+        if r.status_code == 200: return r.json()
+    except Exception: pass
+    return {"gpus":[],"route_counts":{},"agent_counts":{},"recent":[],"timestamp":time.time()}
+
+def broadcast_loop():
+    while True:
+        time.sleep(3)
+        data = fetch_state(); payload = json.dumps(data)
+        with sse_lock:
+            dead = [q for q in sse_subscribers if not q.put(payload)]
+            for q in dead: sse_subscribers.remove(q)
+threading.Thread(target=broadcast_loop, daemon=True).start()
+
+DASHBOARD_HTML = r"""<!DOCTYPE html>
+<html lang="en" data-bs-theme="dark">
+<head>
+<meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>SyslogAI Harness</title>
+<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
+<style>
+body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif; padding: 20px 24px; }
+.card { background: #111827; border: 1px solid #1e293b; border-radius: 10px; height: 100%; }
+.stat-card { background: #111827; border: 1px solid #1e293b; border-radius: 10px; padding: 18px 20px; text-align: center; }
+.stat-value { font-size: 28px; font-weight: 700; line-height: 1.1; }
+.stat-label { font-size: 11px; text-transform: uppercase; letter-spacing: 0.6px; color: #64748b; margin-top: 4px; }
+.gpu-card { background: #111827; border: 1px solid #1e293b; border-radius: 10px; padding: 16px 18px; height: 100%; }
+.gpu-card .title { font-size: 13px; font-weight: 600; color: #e2e8f0; margin-bottom: 12px; display: flex; align-items: center; gap: 8px; }
+.gpu-card .status-dot { width: 8px; height: 8px; border-radius: 50%; flex-shrink: 0; }
+.gpu-card .row-metric { display: flex; justify-content: space-between; font-size: 12px; padding: 2px 0; }
+.gpu-card .row-metric .lbl { color: #64748b; }
+.gpu-card .row-metric .val { color: #e2e8f0; font-variant-numeric: tabular-nums; }
+.gpu-card .slot-bar { display: flex; gap: 3px; margin-top: 8px; }
+.gpu-card .slot-bar .s { flex: 1; height: 5px; border-radius: 2px; background: #1e293b; }
+.gpu-card .slot-bar .s.active { background: #38bdf8; }
+.chart-card { background: #111827; border: 1px solid #1e293b; border-radius: 10px; padding: 16px 18px; height: 100%; display: flex; flex-direction: column; }
+.chart-card .title { font-size: 13px; font-weight: 600; color: #e2e8f0; margin-bottom: 12px; }
+.bar-row { margin-bottom: 8px; }
+.bar-label { display: flex; justify-content: space-between; font-size: 11px; margin-bottom: 3px; color: #64748b; }
+.bar-label .name { color: #cbd5e1; }
+.bar-track { height: 5px; background: #1e293b; border-radius: 3px; overflow: hidden; }
+.bar-fill { height: 100%; border-radius: 3px; transition: width 0.6s ease; }
+.table-custom { font-size: 11px; margin: 0; }
+.table-custom th { color: #64748b; font-weight: 500; font-size: 10px; text-transform: uppercase; border-color: #1e293b; padding: 8px 10px; }
+.table-custom td { color: #94a3b8; border-color: rgba(30,41,59,0.5); padding: 6px 10px; }
+.agent-badge { font-size: 10px; padding: 2px 7px; border-radius: 8px; font-weight: 600; }
+.btn-sm-period { font-size: 10px; padding: 3px 10px; border-radius: 6px; border: 1px solid #1e293b; color: #64748b; background: transparent; cursor: pointer; }
+.btn-sm-period.active { background: #1d4ed8; color: #fff; border-color: #1d4ed8; }
+.ring-label { font-size: 22px; font-weight: 700; }
+.ring-sublabel { font-size: 10px; color: #64748b; }
+</style>
+</head>
+<body>
+
+<!-- HEADER -->
+<div class="d-flex justify-content-between align-items-center mb-4">
+  <div>
+    <h5 class="mb-0 text-white fw-bold">&#x26A1; SyslogAI Harness</h5>
+    <div class="small text-secondary" id="live-indicator">
+      <span class="status-dot" id="live-dot" style="width:6px;height:6px;border-radius:50%;display:inline-block;background:#22c55e;animation:pulse 2s infinite"></span>
+      <span id="connection-status">live</span> &middot; <span id="update-time"></span>
+    </div>
+  </div>
+  <div class="d-flex gap-2">
+    <div class="stat-card" style="min-width:100px"><div class="stat-value text-info" id="kpi-total">0</div><div class="stat-label">Requests</div></div>
+    <div class="stat-card" style="min-width:100px"><div class="stat-value text-warning" id="kpi-active">0</div><div class="stat-label">Active</div></div>
+    <div class="stat-card" style="min-width:100px"><div class="stat-value" style="color:#a78bfa" id="kpi-agents">0</div><div class="stat-label">Agents</div></div>
+  </div>
+</div>
+
+<div class="row g-3 align-items-stretch">
+  <!-- ROW 1: Usage Chart (8) + GPU Metrics (4) -->
+  <div class="col-md-8"><div class="chart-card"><div class="title d-flex justify-content-between align-items-center">
+    <span>Usage Over Time</span>
+    <div class="d-flex gap-1">
+      <button class="btn-sm-period active" onclick="switchPeriod('day')">24h</button>
+      <button class="btn-sm-period" onclick="switchPeriod('week')">7d</button>
+      <button class="btn-sm-period" onclick="switchPeriod('month')">30d</button>
+    </div>
+  </div><div id="timeseries-chart" style="height:150px"></div><div id="timeseries-legend" class="d-flex justify-content-center gap-3 mt-2 flex-wrap small"></div></div></div>
+  <div class="col-md-4"><div class="chart-card"><div class="title">GPU Metrics</div><div id="gpu-metrics-card"></div></div></div>
+
+  <!-- ROW 2: 3 GPU Cards -->
+  <div class="col-md-4"><div class="gpu-card" id="gpu-moe"><div class="text-secondary small">Loading...</div></div></div>
+  <div class="col-md-4"><div class="gpu-card" id="gpu-dense"><div class="text-secondary small">Loading...</div></div></div>
+  <div class="col-md-4"><div class="gpu-card" id="gpu-light"><div class="text-secondary small">Loading...</div></div></div>
+
+  <!-- ROW 3: Queue + Model + Agent -->
+  <div class="col-md-4"><div class="chart-card"><div class="title">Queue Status</div><div class="text-center" id="queue-viz"></div></div></div>
+  <div class="col-md-4"><div class="chart-card"><div class="title">Model Distribution</div><div id="route-bars"></div></div></div>
+  <div class="col-md-4"><div class="chart-card"><div class="title">Agent Activity</div><div id="agent-bars"></div></div></div>
+
+  <!-- ROW 4: Performance Analytics -->
+  <div class="col-12 mb-2"><div class="d-flex align-items-center gap-2"><span class="fw-bold text-white" style="font-size:14px">&#x1F4CA; Performance Analytics</span>
+    <div class="d-flex gap-1 ms-auto">
+      <button class="btn-sm-period active" onclick="switchPerfWindow('1')">1h</button>
+      <button class="btn-sm-period" onclick="switchPerfWindow('24')">24h</button>
+    </div>
+  </div></div>
+  <div class="col-md-6"><div class="chart-card"><div class="title">Latency — P50 / P95 / P99 (ms)</div><div id="perf-latency"></div></div></div>
+  <div class="col-md-6"><div class="chart-card"><div class="title">Throughput — Tokens / sec</div><div id="perf-throughput"></div></div></div>
+  <div class="col-md-6"><div class="chart-card"><div class="title">Routing Effectiveness — by Reason</div><div id="perf-reasons"></div></div></div>
+  <div class="col-md-6"><div class="chart-card"><div class="title">Agent Performance</div><div id="perf-agents"></div></div></div>
+
+  <!-- ROW 5: Latency vs Context Scatter -->
+  <div class="col-12"><div class="chart-card"><div class="title d-flex justify-content-between align-items-center">
+    <span>Latency vs Prompt Size — by Model</span>
+    <div class="d-flex gap-2">
+      <select id="scatter-model" onchange="loadScatter()" style="font-size:10px;background:#1e293b;color:#94a3b8;border:1px solid #334155;border-radius:4px;padding:2px 6px">
+        <option value="all">All Models</option>
+        <option value="qwen3.5-9b-vlm">9B VLM</option>
+        <option value="qwen3.6-27B-code">27B Dense</option>
+        <option value="qwen3.6-35B-A3B">35B MoE</option>
+      </select>
+    </div>
+  </div><div id="scatter-plot" style="height:200px;position:relative"></div><div id="scatter-legend" class="d-flex justify-content-center gap-3 mt-2 flex-wrap small"></div></div></div>
+
+  <!-- ROW 6: Live Stream -->
+  <div class="col-12"><div class="chart-card"><div class="title">Live Stream</div>
+    <div class="table-responsive"><table class="table table-custom mb-0">
+      <thead><tr><th>Time</th><th>Agent</th><th>Model</th><th>Reason</th><th>Tier</th></tr></thead>
+      <tbody id="route-tbody"></tbody>
+    </table></div>
+  </div></div>
+</div>
+
+<script>
+var MC={'qwen3.5-9b-vlm':'#22c55e','qwen3.6-27B-code':'#f59e0b','qwen3.6-35B-A3B':'#a78bfa'};
+var ML={'qwen3.5-9b-vlm':'Qwen3.5 9B VLM','qwen3.6-27B-code':'Qwen Code','qwen3.6-35B-A3B':'Qwen MoE'};
+var GL={'qwen3.6-35B-A3B':'MoE - Strix Halo','qwen3.6-27B-code':'Dense - RTX 3090','qwen3.5-9b-vlm':'VLM - RTX 5070'};
+function $(id){return document.getElementById(id);}
+
+function render(data){
+if(!data||!data.gpus)return;
+var t=Object.values(data.route_counts||{}).reduce((a,b)=>a+b,0);
+var ta=0,tm=0;data.gpus.forEach(function(g){ta+=(g.active_requests||0);tm+=(g.max_concurrent||1)});
+$('kpi-total').textContent=t;$('kpi-active').textContent=ta+'/'+tm;$('kpi-agents').textContent=Object.keys(data.agent_counts||{}).length;
+$('update-time').textContent=new Date().toLocaleTimeString();
+var ids={'qwen3.6-35B-A3B':'gpu-moe','qwen3.6-27B-code':'gpu-dense','qwen3.5-9b-vlm':'gpu-light'};
+data.gpus.forEach(function(g){
+var el=$(ids[g.id]);if(!el)return;
+var a=g.active_requests||0,mx=g.max_concurrent||1;
+var sc=g.status==='healthy'?'#22c55e':g.status==='saturated'?'#f59e0b':'#ef4444';
+var ss=g.status==='healthy'?'Online':g.status==='saturated'?'Busy':'Offline';
+var slots='';for(var i=0;i<mx;i++)slots+='<span class=\"s'+(i<a?' active':'')+'\"></span>';
+var h='<div class=\"title\"><span class=\"status-dot\" style=\"background:'+sc+'\"></span>'+GL[g.id]+'<span class=\"ms-auto small\" style=\"color:'+sc+'\">'+ss+'</span></div>';
+h+='<div class=\"row-metric\"><span class=\"lbl\">VRAM</span><span class=\"val\">'+g.vram_used_mb+' / '+g.vram_total_mb+' MB</span></div>';
+h+='<div class=\"row-metric\"><span class=\"lbl\">Utilization</span><span class=\"val\">'+g.gpu_util_pct+'%</span></div>';
+h+='<div class=\"row-metric\"><span class=\"lbl\">Temperature</span><span class=\"val\" style=\"color:'+(g.temp_c>85?'#ef4444':g.temp_c>70?'#f59e0b':'#22c55e')+'\">'+g.temp_c+'C</span></div>';
+if(g.power_w)h+='<div class=\"row-metric\"><span class=\"lbl\">Power</span><span class=\"val\">'+g.power_w+'W'+(g.power_limit_w?'/'+g.power_limit_w+'W':'')+'</span></div>';
+h+='<div class=\"row-metric\"><span class=\"lbl\">Slots</span><span class=\"val\" style=\"color:'+(a>=mx?'#ef4444':'#e2e8f0')+'\">'+a+' / '+mx+'</span></div>';
+h+='<div class=\"slot-bar\">'+slots+'</div>';el.innerHTML=h;
+});
+renderQueue(data);renderGPUMetrics(data);
+var rc=data.route_counts||{},mr=Math.max(1,...Object.values(rc));
+$('route-bars').innerHTML=Object.entries(rc).length?Object.entries(rc).sort((a,b)=>b[1]-a[1]).map(function(e){var m=e[0],c=e[1];return'<div class=\"bar-row\"><div class=\"bar-label\"><span class=\"name\">'+(ML[m]||m)+'</span><span>'+c+' ('+(t?Math.round(c/t*100):0)+'%)</span></div><div class=\"bar-track\"><div class=\"bar-fill\" style=\"width:'+(c/mr*100)+'%;background:'+(MC[m]||'#38bdf8')+'\"></div></div></div>';}).join(''):'<div class=\"text-secondary small\">-</div>';
+var ac=data.agent_counts||{},ma=Math.max(1,...Object.values(ac));
+$('agent-bars').innerHTML=Object.entries(ac).length?Object.entries(ac).sort((a,b)=>b[1]-a[1]).map(function(e){return'<div class=\"bar-row\"><div class=\"bar-label\"><span class=\"name\">'+e[0]+'</span><span>'+e[1]+'</span></div><div class=\"bar-track\"><div class=\"bar-fill\" style=\"width:'+(e[1]/ma*100)+'%;background:#38bdf8\"></div></div></div>';}).join(''):'<div class=\"text-secondary small\">-</div>';
+var recent=data.recent||[];
+$('route-tbody').innerHTML=recent.length?recent.slice(0,20).map(function(r){var d=new Date(r.ts*1000),ag=r.agent||'?';return'<tr><td class=\"text-secondary\">'+d.toLocaleTimeString()+'</td><td><span class=\"agent-badge\" style=\"background:rgba(56,189,248,0.12);color:#38bdf8\">'+ag+'</span></td><td>'+(ML[r.model]||r.model)+'</td><td class=\"text-secondary\">'+(r.reason||'')+'</td><td class=\"text-uppercase\" style=\"font-size:10px;color:'+(r.tier==='enterprise'?'#a78bfa':'#64748b')+'\">'+(r.tier||'')+'</td></tr>';}).join(''):'<tr><td colspan=\"5\" class=\"text-secondary\">Waiting...</td></tr>';
+}
+
+function renderQueue(data){
+var el=$('queue-viz');if(!el)return;
+var ta=0,tm=0;data.gpus.forEach(function(g){ta+=(g.active_requests||0);tm+=(g.max_concurrent||1)});
+var pct=tm>0?Math.round(ta/tm*100):0,st=pct>=100?'SATURATED':pct>=50?'BUSY':'IDLE';
+var sc=pct>=100?'#ef4444':pct>=50?'#f59e0b':'#22c55e';
+var circ=188.5,dash=(pct/100)*circ;
+var h='<div class=\"d-inline-block position-relative mb-2\"><svg width=\"72\" height=\"72\"><circle cx=\"36\" cy=\"36\" r=\"30\" fill=\"none\" stroke=\"#1e293b\" stroke-width=\"6\"/><circle cx=\"36\" cy=\"36\" r=\"30\" fill=\"none\" stroke=\"'+sc+'\" stroke-width=\"6\" stroke-dasharray=\"'+dash+' '+(circ-dash)+'\" stroke-linecap=\"round\" transform=\"rotate(-90 36 36)\"/></svg><div style=\"position:absolute;top:50%;left:50%;transform:translate(-50%,-50%);text-align:center\"><div class=\"ring-label\" style=\"color:'+sc+'\">'+ta+'</div><div class=\"ring-sublabel\">/ '+tm+' slots</div></div></div>';
+h+='<div class=\"fw-bold mb-2 small\" style=\"color:'+sc+'\">'+st+'</div>';
+var lb={'qwen3.6-35B-A3B':'MoE','qwen3.6-27B-code':'Dense','qwen3.5-9b-vlm':'VLM'};
+data.gpus.forEach(function(g){var a=g.active_requests||0,mx=g.max_concurrent||1,gp=mx>0?Math.round(a/mx*100):0;h+='<div class=\"d-flex align-items-center gap-2 mb-1 justify-content-center\"><span class=\"small\" style=\"min-width:32px;text-align:right;font-size:10px\">'+(lb[g.id]||g.id)+'</span><div style=\"flex:1;max-width:70px;height:3px;background:#1e293b;border-radius:2px;overflow:hidden\"><div style=\"height:100%;width:'+gp+'%;background:'+sc+';border-radius:2px\"></div></div><span class=\"small\" style=\"min-width:22px;font-size:10px\">'+a+'/'+mx+'</span></div>'});
+el.innerHTML=h;
+}
+
+function renderGPUMetrics(data){
+var el=$('gpu-metrics-card');if(!el)return;
+var lb={'qwen3.6-35B-A3B':'MoE','qwen3.6-27B-code':'Dense','qwen3.5-9b-vlm':'VLM'};
+var h='';data.gpus.forEach(function(g){
+var nm=lb[g.id]||g.id,tp=g.temp_c||0,ut=g.gpu_util_pct||0,pw=g.power_w||0,pl=g.power_limit_w||0;
+var tc=tp>85?'#ef4444':tp>70?'#f59e0b':'#22c55e',uc=ut>90?'#ef4444':ut>70?'#f59e0b':'#22c55e';
+h+='<div class=\"mb-3\"><div class=\"fw-bold small text-white-50 mb-1\">'+nm+'</div>';
+h+='<div class=\"d-flex align-items-center gap-2 mb-1\"><span class=\"small text-secondary\" style=\"min-width:30px\">T</span><div class=\"flex-grow-1\" style=\"height:3px;background:#1e293b;border-radius:2px;overflow:hidden\"><div style=\"height:100%;width:'+Math.min(tp,100)+'%;background:'+tc+';border-radius:2px\"></div></div><span class=\"small\" style=\"color:'+tc+';min-width:30px;text-align:right\">'+tp+'C</span></div>';
+h+='<div class=\"d-flex align-items-center gap-2 mb-1\"><span class=\"small text-secondary\" style=\"min-width:30px\">U</span><div class=\"flex-grow-1\" style=\"height:3px;background:#1e293b;border-radius:2px;overflow:hidden\"><div style=\"height:100%;width:'+ut+'%;background:'+uc+';border-radius:2px\"></div></div><span class=\"small\" style=\"color:'+uc+';min-width:30px;text-align:right\">'+ut+'%</span></div>';
+if(pw>0){var pp=pl>0?Math.round(pw/pl*100):0,pc=pp>90?'#ef4444':pp>70?'#f59e0b':'#22c55e';h+='<div class=\"d-flex align-items-center gap-2\"><span class=\"small text-secondary\" style=\"min-width:30px\">P</span><div class=\"flex-grow-1\" style=\"height:3px;background:#1e293b;border-radius:2px;overflow:hidden\"><div style=\"height:100%;width:'+pp+'%;background:'+pc+';border-radius:2px\"></div></div><span class=\"small\" style=\"color:'+pc+';min-width:30px;text-align:right\">'+pw+'W</span></div>';}
+h+='</div>';});
+el.innerHTML=h;
+}
+
+var cp='day';
+function switchPeriod(p){cp=p;document.querySelectorAll('.btn-sm-period').forEach(function(b){b.classList.remove('active')});event.target.classList.add('active');loadTS();}
+function loadTS(){fetch('/api/timeseries?period='+cp).then(function(r){return r.json()}).then(renderTS).catch(function(){})}
+function renderTS(d){
+var models=d.models||{},labels=d.labels||[];
+if(!labels.length)return;
+var cn=$('timeseries-chart'),lg=$('timeseries-legend'),mn=Object.keys(models);
+if(!mn.length){cn.innerHTML='<div class=\"text-secondary small text-center py-4\">-</div>';return;}
+var mv=1;for(var m in models)for(var i=0;i<models[m].length;i++)if(models[m][i]>mv)mv=models[m][i];mv=Math.ceil(mv*1.15)||1;
+var W=labels.length>1?100/(labels.length-1):100,H=130;
+var paths='';for(var mi=0;mi<mn.length;mi++){var m=mn[mi],vals=models[m]||[],d='';for(var i=0;i<vals.length;i++){var x=i*W,y=H-(vals[i]/mv)*H;d+=(i===0?'M':'L')+x.toFixed(1)+','+y.toFixed(1)+' ';}paths+='<path d=\"'+d+'\" fill=\"none\" stroke=\"'+(MC[m]||'#38bdf8')+'\" stroke-width=\"2\" stroke-linecap=\"round\" opacity=\"0.8\"/>';}
+var grid='';for(var g=0;g<=4;g++){var y=(g/4)*H;grid+='<line x1=\"0\" y1=\"'+y.toFixed(1)+'\" x2=\"100\" y2=\"'+y.toFixed(1)+'\" stroke=\"#1e293b\" stroke-width=\"1\"/>';}
+cn.innerHTML='<svg viewBox=\"0 0 100 '+(H+16)+'\" style=\"width:100%;height:'+(H+20)+'px;display:block\" preserveAspectRatio=\"none\">'+grid+paths+'</svg>';
+lg.innerHTML=mn.map(function(m){return'<span class=\"d-flex align-items-center gap-1\"><svg width=\"14\" height=\"8\"><line x1=\"0\" y1=\"4\" x2=\"14\" y2=\"4\" stroke=\"'+(MC[m]||'#38bdf8')+'\" stroke-width=\"2\"/></svg>'+(ML[m]||m)+'</span>';}).join('');
+}
+var perfWindow='24';
+function switchPerfWindow(w){perfWindow=w;document.querySelectorAll('.btn-sm-period').forEach(function(b,i){if(i>=4)b.classList.toggle('active',b.textContent.trim().replace('h','')===w)});loadPerf();}
+function loadPerf(){fetch('/api/performance?window='+perfWindow).then(function(r){return r.json()}).then(renderPerf).catch(function(){})}
+function renderPerf(d){
+var models=d.models||[],reasons=d.reasons||[],agents=d.agents||[],sum=d.summary||{};
+// Latency bars: p50/p95/p99 per model
+var mlab={'qwen3.6-35B-A3B':'35B MoE','qwen3.6-27B-code':'27B Dense','qwen3.5-9b-vlm':'9B VLM'};
+var mcol={'qwen3.6-35B-A3B':'#a78bfa','qwen3.6-27B-code':'#f59e0b','qwen3.5-9b-vlm':'#22c55e'};
+if(!models.length){$('perf-latency').innerHTML='<div class="text-secondary small text-center py-4">Accumulating data...</div>';return;}
+var maxLat=Math.max(...models.map(function(m){return m.latency.p99||0}),1);
+var latHTML=models.map(function(m){
+var l=m.latency||{},p50=l.p50||0,p95=l.p95||0,p99=l.p99||0,c=mcol[m.model]||'#38bdf8';
+return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span class="text-secondary">'+m.count+' reqs</span></div>'+
+'<div class="d-flex align-items-center gap-2 mb-1"><span class="text-secondary" style="min-width:28px">p50</span><div class="flex-grow-1" style="height:14px;background:#1e293b;border-radius:4px;overflow:hidden;position:relative"><div style="position:absolute;left:0;top:0;height:100%;width:'+(p50/maxLat*100)+'%;background:'+c+';opacity:0.3;border-radius:4px"></div><div style="position:absolute;left:0;top:0;height:100%;width:'+(p95/maxLat*100)+'%;background:'+c+';opacity:0.5;border-radius:4px"></div><div style="position:absolute;left:0;top:0;height:100%;width:'+(p99/maxLat*100)+'%;background:'+c+';border-radius:4px"></div></div><span style="color:'+c+';min-width:48px;text-align:right;font-variant-numeric:tabular-nums">'+p99+'ms</span></div>'+
+'<div class="d-flex gap-3" style="font-size:10px;color:#64748b;padding-left:32px"><span>p50: '+p50+'ms</span><span>p95: '+p95+'ms</span><span>p99: '+p99+'ms</span></div></div>';
+}).join('');
+$('perf-latency').innerHTML=latHTML;
+// Throughput comparison
+var maxTps=Math.max(...models.map(function(m){return m.throughput.avg_tokens_per_sec||0}),1);
+var tpsHTML=models.map(function(m){
+var t=m.throughput||{},avg=t.avg_tokens_per_sec||0,p50=t.p50||0,c=mcol[m.model]||'#38bdf8';
+var isAllStreaming = avg===0 && p50===0;
+if(isAllStreaming){
+return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span style="color:#64748b;font-style:italic">streaming only</span></div><div class="text-secondary" style="font-size:10px">t/s available for non-streaming requests only</div></div>';
+}
+return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span style="color:'+c+'" class="fw-bold">'+avg+' tok/s</span></div>'+
+'<div class="d-flex align-items-center gap-2"><span class="text-secondary" style="min-width:28px">avg</span><div class="flex-grow-1" style="height:6px;background:#1e293b;border-radius:3px;overflow:hidden"><div style="height:100%;width:'+(Math.max(avg/maxTps*100,6))+'%;background:'+c+';border-radius:3px"></div></div><span class="small" style="color:'+c+';min-width:54px;text-align:right">'+avg+' tok/s</span></div>'+
+'<div class="d-flex align-items-center gap-2 mt-1"><span class="text-secondary" style="min-width:28px;font-size:10px">p50</span><div class="flex-grow-1" style="height:4px;background:#1e293b;border-radius:2px;overflow:hidden"><div style="height:100%;width:'+(Math.max(p50/maxTps*100,4))+'%;background:'+c+';opacity:0.5;border-radius:2px"></div></div><span style="font-size:10px;color:#64748b">'+p50+' tok/s</span></div></div>';
+}).join('');
+$('perf-throughput').innerHTML=tpsHTML;
+// Routing reasons table
+if(reasons.length){
+var rHTML='<table class="table table-custom mb-0"><thead><tr><th>Reason</th><th>Count</th><th>Avg Lat</th><th>P95 Lat</th></tr></thead><tbody>';
+reasons.forEach(function(r){rHTML+='<tr><td>'+r.reason+'</td><td>'+r.count+'</td><td>'+r.avg_total_ms+'ms</td><td>'+r.p95_total_ms+'ms</td></tr>';});
+rHTML+='</tbody></table>';$('perf-reasons').innerHTML=rHTML;
+}else{$('perf-reasons').innerHTML='<div class="text-secondary small text-center py-3">-</div>';}
+// Agent performance
+if(agents.length){
+var maxAc=Math.max(...agents.map(function(a){return a.count||0}),1);
+var aHTML=agents.map(function(a){return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+a.agent+'</span><span class="text-secondary">'+a.count+' reqs</span></div><div class="d-flex align-items-center gap-2"><div class="flex-grow-1" style="height:4px;background:#1e293b;border-radius:2px;overflow:hidden"><div style="height:100%;width:'+(a.count/maxAc*100)+'%;background:#38bdf8;border-radius:2px"></div></div><span class="small" style="color:#38bdf8;min-width:60px;text-align:right">'+a.avg_total_ms+'ms avg</span></div></div>';}).join('');
+$('perf-agents').innerHTML=aHTML;
+}else{$('perf-agents').innerHTML='<div class="text-secondary small text-center py-3">-</div>';}
+}
+function poll(){fetch('/api/state').then(function(r){return r.json()}).then(function(data){render(data);$('connection-status').textContent='live';}).catch(function(){$('connection-status').textContent='reconnecting';});}
+function loadScatter(){
+var m=$('scatter-model').value;
+fetch('/api/scatter?window=24&model='+m).then(function(r){return r.json()}).then(renderScatter).catch(function(){});
+}
+function renderScatter(d){
+var pts=d.points||[],el=$('scatter-plot'),lg=$('scatter-legend');
+if(!pts.length){el.innerHTML='<div class="text-secondary small text-center py-5">No data yet</div>';return;}
+var mcol={'qwen3.6-35B-A3B':'#a78bfa','qwen3.6-27B-code':'#f59e0b','qwen3.5-9b-vlm':'#22c55e','unknown':'#38bdf8'};
+var mlab={'qwen3.6-35B-A3B':'35B MoE','qwen3.6-27B-code':'27B Dense','qwen3.5-9b-vlm':'9B VLM'};
+var maxX=Math.max.apply(null,pts.map(function(p){return p.prompt_tokens||0}))||1000;
+var maxY=Math.max.apply(null,pts.map(function(p){return p.inference_ms||0}))||5000;
+// Log scale for X axis (prompt tokens vary widely)
+var toX=function(t){return Math.log10(Math.max(t,1))/Math.log10(Math.max(maxX,10))*100;};
+var toY=function(t){return (t/maxY)*100;};
+var dots='';
+pts.forEach(function(p){
+var x=toX(p.prompt_tokens),y=toY(p.inference_ms),c=mcol[p.model]||'#38bdf8';
+var r=p.stream?1.5:2.5,o=p.stream?0.4:0.8;
+dots+='<circle cx="'+x+'" cy="'+(100-y)+'" r="'+r+'" fill="'+c+'" opacity="'+o+'"><title>'+mlab[p.model]+' | '+p.prompt_tokens+' tok | '+p.inference_ms+'ms | '+p.agent+'</title></circle>';
+});
+// Grid lines
+var grid='';
+for(var i=1;i<=4;i++){grid+='<line x1="0" y1="'+(i*20)+'" x2="100" y2="'+(i*20)+'" stroke="#1e293b" stroke-width="0.5"/>';}
+for(var i=1;i<=4;i++){grid+='<line x1="'+(i*20)+'" y1="0" x2="'+(i*20)+'" y2="100" stroke="#1e293b" stroke-width="0.5"/>';}
+// Axis labels
+var xTicks='';
+var xVals=[10,100,1000,10000,100000];
+xVals.forEach(function(v){if(v<=maxX)xTicks+='<text x="'+toX(v)+'" y="103" text-anchor="middle" font-size="8" fill="#64748b">'+(v>=1000?(v/1000)+'k':v)+'</text>';});
+var yTicks='';
+var yVals=[500,1000,5000,10000,50000,100000];
+yVals.forEach(function(v){if(v<=maxY)yTicks+='<text x="-2" y="'+(97-toY(v))+'" text-anchor="end" font-size="8" fill="#64748b">'+(v>=1000?(v/1000)+'s':v+'ms')+'</text>';});
+el.innerHTML='<svg viewBox="-35 0 140 115" style="width:100%;height:200px">'+grid+dots+xTicks+yTicks+'<text x="50" y="112" text-anchor="middle" font-size="9" fill="#475569">Prompt Tokens (log scale)</text><text x="-38" y="50" text-anchor="middle" font-size="9" fill="#475569" transform="rotate(-90,-38,50)">Inference Time</text></svg>';
+// Legend
+var models=[];pts.forEach(function(p){if(models.indexOf(p.model)===-1)models.push(p.model);});
+lg.innerHTML=models.map(function(m){return'<span class="d-flex align-items-center gap-1 small"><svg width="10" height="10"><circle cx="5" cy="5" r="3.5" fill="'+(mcol[m]||'#38bdf8')+'"/></svg>'+mlab[m]+'</span>';}).join('');
+}
+poll();setInterval(poll,3000);loadTS();loadPerf();setInterval(loadPerf,15000);loadScatter();setInterval(loadScatter,30000);
+</script>
+</body>
+</html>"""
+
+@app.route("/") 
+def dashboard(): return render_template_string(DASHBOARD_HTML)
+
+@app.route("/api/state")
+def api_state(): return fetch_state()
+
+@app.route("/api/scatter")
+def api_scatter():
+    window = request.args.get("window", "24")
+    model = request.args.get("model", "all")
+    try:
+        r = requests.get(f"http://router:9000/metrics/scatter?window={window}&model={model}", timeout=10)
+        if r.status_code == 200: return r.json()
+    except Exception: pass
+    return {"points": [], "count": 0}
+
+@app.route("/api/performance")
+def api_performance():
+    window = request.args.get("window", "24")
+    model = request.args.get("model", "all")
+    try:
+        r = requests.get(f"http://router:9000/metrics/performance?window={window}&model={model}", timeout=10)
+        if r.status_code == 200: return r.json()
+    except Exception: pass
+    return {"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}}
+
+@app.route("/api/timeseries")
+def api_timeseries():
+    period = request.args.get("period", "day")
+    try:
+        r = requests.get("http://router:9000/metrics/timeseries?period=" + period, timeout=5)
+        if r.status_code == 200: return r.json()
+    except Exception: pass
+    return {"models": {}, "labels": []}
+
+@app.route("/api/stream")
+def api_stream():
+    def ev():
+        q = queue.Queue()
+        with sse_lock: sse_subscribers.append(q)
+        try:
+            yield "data: "+json.dumps(fetch_state())+"\n\n"
+            while True:
+                try: msg = q.get(timeout=3); yield "data: "+msg+"\n\n"
+                except queue.Empty: yield "data: "+json.dumps(fetch_state())+"\n\n"
+        except GeneratorExit: pass
+        finally:
+            with sse_lock:
+                if q in sse_subscribers: sse_subscribers.remove(q)
+    return Response(stream_with_context(ev()), mimetype="text/event-stream", headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"})
+
+@app.route("/health")
+def health(): return {"status":"healthy","service":"harness-dashboard"}
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=3000, debug=False)
@@ -0,0 +1,659 @@
+import os, json, time, logging, traceback, threading, queue, statistics, math
+import requests, redis
+from flask import Flask, request, jsonify, Response, stream_with_context
+
+REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379")
+GPU_MOE_URL = os.environ.get("GPU_MOE_URL", "http://192.168.68.15:8080/v1")
+GPU_DENSE_URL = os.environ.get("GPU_DENSE_URL", "http://192.168.68.8:8080/v1")
+GPU_LIGHT_URL = os.environ.get("GPU_LIGHT_URL", "http://192.168.68.110:8080/v1")
+
+GPU_SIDECARS = {
+    "qwen3.6-35B-A3B": "http://192.168.68.15:8090",
+    "qwen3.6-27B-code": "http://192.168.68.8:8090",
+    "qwen3.5-9b-vlm": "http://192.168.68.110:8090",
+}
+GPU_URLS = {
+    "qwen3.6-35B-A3B": GPU_MOE_URL,
+    "qwen3.6-27B-code": GPU_DENSE_URL,
+    "qwen3.5-9b-vlm": GPU_LIGHT_URL,
+}
+# Max concurrent requests per GPU (based on llama.cpp --parallel)
+GPU_MAX_CONCURRENT = {
+    "qwen3.6-35B-A3B": 2,   # 2 slots (cross-agent spread prevents overheating)
+    "qwen3.6-27B-code": 2,  # 2 slots (128K context frees VRAM)
+    "qwen3.5-9b-vlm": 2,       # 2 slots (12GB VRAM, 4GB headroom)
+}
+
+# Context window sizes (tokens) — used for compaction signals
+GPU_CONTEXT = {
+    "qwen3.6-35B-A3B": 262144,
+    "qwen3.6-27B-code": 131072,
+    "qwen3.5-9b-vlm": 262144,
+}
+
+TIER_MODELS = {
+    "starter": ["qwen3.5-9b-vlm"],
+    "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "qwen3.5-9b-vlm"],
+    "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "qwen3.5-9b-vlm"],
+}
+API_KEYS = {
+    "sk-syslog-local-master-key": {"tier": "enterprise", "agent": "admin"},
+    "sk-syslog-abiba": {"tier": "enterprise", "agent": "Abiba"},
+    "sk-syslog-mumuni": {"tier": "enterprise", "agent": "Mumuni"},
+    "sk-syslog-tanko": {"tier": "enterprise", "agent": "Tanko"},
+    "sk-syslog-koby": {"tier": "enterprise", "agent": "Koby"},
+    "sk-syslog-kagenz0": {"tier": "enterprise", "agent": "Kagenz0"},
+    "sk-syslog-koonimo": {"tier": "enterprise", "agent": "Koonimo"},
+    "sk-starter-abc123": {"tier": "starter", "agent": "test-starter"},
+    "sk-professional-xyz789": {"tier": "professional", "agent": "test-pro"},
+}
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s")
+log = logging.getLogger("router")
+try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping()
+except Exception: r = None
+
+
+def counter_audit_loop():
+    """Every 30s, check GPU slots and reset counters if all slots idle."""
+    while True:
+        time.sleep(30)
+        if not r: continue
+        for model, url in GPU_URLS.items():
+            try:
+                resp = requests.get(url.replace("/v1","") + "/slots",
+                    headers={"Authorization": "Bearer not-needed"}, timeout=5)
+                if resp.status_code == 200:
+                    slots = resp.json()
+                    all_idle = all(not s.get("is_processing", False) for s in slots)
+                    if all_idle:
+                        current = int(r.get("active:" + model) or 0)
+                        if current > 0:
+                            r.set("active:" + model, 0)
+                            log.info("AUDIT: Reset stuck counter for %s (was %d)", model, current)
+            except Exception:
+                pass
+
+threading.Thread(target=counter_audit_loop, daemon=True).start()
+
+app = Flask(__name__)
+sse_subscribers = []; sse_lock = threading.Lock()
+
+def gpu_active_count(model):
+    """Get number of in-flight requests for a GPU."""
+    if r:
+        return int(r.get("active:" + model) or 0)
+    return 0
+
+def gpu_incr(model):
+    if r: r.incr("active:" + model)
+
+def gpu_decr(model):
+    if r:
+        v = r.decr("active:" + model)
+        if v and int(v) < 0:
+            r.set("active:" + model, 0)  # never go negative
+
+def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3):
+    url = GPU_SIDECARS.get(model)
+    if not url: return {"status": "unknown"}
+    try:
+        resp = requests.get(url, timeout=sidecar_timeout)
+        if resp.status_code == 200:
+            d = resp.json()
+            pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100
+            status = "healthy"  # VRAM usage != saturation; busy slots handled by is_gpu_busy()
+            vram_warning = pct >= 95
+            # Also check if llama.cpp endpoint is actually responding
+            gpu_url = GPU_URLS.get(model, "")
+            try:
+                hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout)
+                if hr.status_code != 200:
+                    status = "down"
+            except Exception:
+                status = "down"
+            return {"status": status, "vram_warning": vram_warning, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")}
+    except Exception: pass
+    return {"status": "down"}
+
+def available_models(): return [m for m in GPU_URLS if check_gpu_health(m)["status"] in ("healthy","saturated")]
+
+def estimate_tokens(msgs):
+    """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text)."""
+    return len(json.dumps(msgs, default=str)) // 3.5
+
+def store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, stream):
+    """Store detailed performance record in Redis for analytics."""
+    if not r: return
+    try:
+        total_ms = queue_ms + inference_ms
+        tps = completion_tokens / (inference_ms / 1000) if inference_ms > 0 and completion_tokens > 0 else 0
+        rec = json.dumps({
+            "ts": time.time(),
+            "model": model, "agent": agent, "tier": tier, "reason": reason,
+            "queue_ms": round(queue_ms, 1),
+            "inference_ms": round(inference_ms, 1),
+            "total_ms": round(total_ms, 1),
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "tokens_per_sec": round(tps, 1),
+            "stream": stream
+        })
+        # Global recent list (last 500)
+        r.lpush("perf:recent", rec)
+        r.ltrim("perf:recent", 0, 499)
+        # Per-model list (last 200)
+        r.lpush("perf:model:" + model, rec)
+        r.ltrim("perf:model:" + model, 0, 199)
+        # Per-reason list (last 200)
+        r.lpush("perf:reason:" + reason, rec)
+        r.ltrim("perf:reason:" + reason, 0, 199)
+        # Per-agent list (last 200)
+        r.lpush("perf:agent:" + agent, rec)
+        r.ltrim("perf:agent:" + agent, 0, 199)
+    except Exception:
+        pass
+
+def is_gpu_busy(model):
+    """Check if GPU is at or near max concurrent capacity."""
+    active = gpu_active_count(model)
+    max_c = GPU_MAX_CONCURRENT.get(model, 1)
+    return active >= max_c
+
+def select_best_gpu(candidates, reason, agent=""):
+    """Pick best GPU, spreading agents across GPUs to prevent hotspots."""
+    # Count how many distinct agents are on each GPU
+    gpu_agent_counts = {}
+    if r:
+        for m in GPU_URLS:
+            count = 0
+            for ak in API_KEYS.values():
+                if r.get("agent_gpu:" + ak["agent"] + ":" + m):
+                    count += 1
+            gpu_agent_counts[m] = count
+    # First pass: prefer GPUs with 0 other agents (fresh GPU for this agent)
+    for m in candidates:
+        if not is_gpu_busy(m) and gpu_agent_counts.get(m, 0) == 0:
+            return {"model": m, "reason": reason}
+    # Second pass: prefer GPU this agent is NOT already on (skip own GPU)
+    if agent:
+        for m in candidates:
+            if not is_gpu_busy(m) and not r.get("agent_gpu:" + agent + ":" + m):
+                return {"model": m, "reason": reason}
+    # Third pass: any non-busy GPU
+    for m in candidates:
+        if not is_gpu_busy(m):
+            return {"model": m, "reason": reason}
+    # All busy — pick least loaded
+    best = None
+    best_load = 999
+    for m in candidates:
+        load = gpu_active_count(m)
+        if load < best_load:
+            best_load = load
+            best = m
+    if best:
+        return {"model": best, "reason": "load_balanced_" + reason}
+    return None
+
+def route(rd, tier, agent=""):
+    msgs = rd.get("messages",[]); t = estimate_tokens(msgs)
+    sys = any(m.get("role")=="system" for m in msgs)
+    turns = len([m for m in msgs if m.get("role") in ("user","assistant")])
+    hints = rd.get("routing_hints",{})
+    allowed = TIER_MODELS.get(tier, ["qwen3.5-9b-vlm"])
+    avail = [m for m in available_models() if m in allowed]
+    if not avail: return {"model": allowed[0], "reason": "all_saturated", "saturated": True}
+    # Check if all available GPUs are at max capacity
+    if all(is_gpu_busy(m) for m in avail):
+        return {"model": avail[0], "reason": "all_saturated", "saturated": True}
+    
+    req = rd.get("model","auto")
+    if req != "auto":
+        target = req if req in avail else avail[0]
+        # If explicit model is busy, check if another can take it
+        if is_gpu_busy(target) and req in allowed:
+            alts = [m for m in avail if m != target and m in allowed]
+            if alts:
+                alt = select_best_gpu(alts, "explicit", agent)
+                if alt: return alt
+        return {"model": target, "reason": "explicit"}
+    
+    if hints:
+        if hints.get("priority")=="speed" and "qwen3.5-9b-vlm" in avail:
+            return select_best_gpu(["qwen3.5-9b-vlm"], "hint_speed", agent) or {"model":"qwen3.5-9b-vlm","reason":"hint_speed"}
+        if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail:
+            return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"}
+    
+    first_msg = msgs[0].get("content","") if msgs else ""
+    words = len(first_msg.split()) if isinstance(first_msg, str) else 99
+    
+    # TIER 1: Lightweight — single-turn short queries → VLM (fastest)
+    if not sys and turns <= 1 and t <= 500 and words <= 100 and "qwen3.5-9b-vlm" in avail:
+        if not is_gpu_busy("qwen3.5-9b-vlm"):
+            return {"model":"qwen3.5-9b-vlm","reason":"lightweight"}
+        # VLM busy — Dense is faster for short queries than MoE
+        fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail]
+        result = select_best_gpu(fallback, "lightweight_fallback", agent)
+        if result: return result
+    
+    # TIER 2: Simple conversations — VLM primary (up to 15K tok), fastest for moderate chat
+    if t <= 15000 and turns <= 12 and "qwen3.5-9b-vlm" in avail:
+        if not is_gpu_busy("qwen3.5-9b-vlm"):
+            return {"model":"qwen3.5-9b-vlm","reason":"simple_conv"}
+        # VLM busy — fall back to Dense, then MoE
+        fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail]
+        result = select_best_gpu(fallback, "simple_conv_fallback", agent)
+        if result: return result
+    
+    # TIER 3: Medium complexity — Dense primary, VLM fallback (quality + speed balance)
+    if t <= 25000:
+        candidates = [m for m in ["qwen3.6-27B-code","qwen3.5-9b-vlm","qwen3.6-35B-A3B"] if m in avail]
+        result = select_best_gpu(candidates, "medium", agent)
+        if result: return result
+    
+    # TIER 4: Heavy reasoning — MoE primary (workhorse), Dense fallback
+    if t > 25000:
+        candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","qwen3.5-9b-vlm"] if m in avail]
+        result = select_best_gpu(candidates, "heavy_reasoning", agent)
+        if result: return result
+    
+    # TIER 5: Default — Dense primary, MoE fallback
+    candidates = [m for m in ["qwen3.6-27B-code","qwen3.5-9b-vlm","qwen3.6-35B-A3B"] if m in avail]
+    result = select_best_gpu(candidates, "default", agent)
+    if result: return result
+    return {"model":avail[0],"reason":"last_resort"}
+
+def clean_unicode(text):
+    if not isinstance(text, str): return text
+    text = text.replace(chr(0x2014), "-"); text = text.replace(chr(0x2013), "-")
+    text = text.replace(chr(0x2018), "'"); text = text.replace(chr(0x2019), "'")
+    text = text.replace(chr(0x201C), '"'); text = text.replace(chr(0x201D), '"')
+    text = text.replace(chr(0x2026), "..."); text = text.replace(chr(0x00A0), " ")
+    return text.encode("ascii", "ignore").decode("ascii")
+
+def clean_response(d):
+    if isinstance(d, dict): return {k: clean_response(v) for k,v in d.items()}
+    if isinstance(d, list): return [clean_response(v) for v in d]
+    if isinstance(d, str): return clean_unicode(d)
+    return d
+
+def get_metrics():
+    d = {"gpus":[],"route_counts":{},"agent_counts":{},"tier_counts":{},"recent":[],"timestamp":time.time(),"active_requests":{}}
+    for m in GPU_URLS:
+        h = check_gpu_health(m)
+        d["gpus"].append({"id":m,"gpu_name":h.get("gpu_name",m),"status":h.get("status"),"vram_used_mb":h.get("vram_used_mb"),"vram_total_mb":h.get("vram_total_mb"),"vram_pct":h.get("vram_pct"),"temp_c":h.get("temp_c"),"gpu_util_pct":h.get("gpu_util_pct"),"power_w":h.get("power_w"),"power_limit_w":h.get("power_limit_w"),"active_requests":gpu_active_count(m), "max_concurrent": GPU_MAX_CONCURRENT.get(m, 1)})
+        d["active_requests"][m] = gpu_active_count(m)
+    if r:
+        try:
+            for m in GPU_URLS: d["route_counts"][m] = int(r.get("routes:"+m) or 0)
+            for k,v in API_KEYS.items():
+                c = int(r.get("routes:agent:"+v["agent"]) or 0)
+                if c>0: d["agent_counts"][v["agent"]] = c
+            for t in TIER_MODELS: d["tier_counts"][t] = int(r.get("routes:tier:"+t) or 0)
+            raw = r.lrange("routes:recent",0,49)
+            d["recent"] = [json.loads(x) for x in raw] if raw else []
+        except Exception: pass
+    return d
+
+def bcast():
+    data = get_metrics(); payload = json.dumps(data)
+    with sse_lock:
+        dead = []
+        for q in sse_subscribers:
+            try: q.put(payload)
+            except Exception: dead.append(q)
+        for q in dead: sse_subscribers.remove(q)
+
+QUEUE_TIMEOUT = int(os.environ.get("QUEUE_TIMEOUT", "30"))  # max seconds to queue before 503
+
+@app.route("/v1/chat/completions", methods=["POST"])
+def chat():
+    try:
+        rd = request.get_json(force=True)
+        ak = request.headers.get("Authorization","").replace("Bearer ","")
+        if not ak or ak not in API_KEYS:
+            log.warning("AUTH_REJECTED: no/invalid API key from %s", request.remote_addr)
+            return jsonify({"error": "Unauthorized — valid API key required"}), 401
+        ki = API_KEYS[ak]
+        tier, agent = ki["tier"], ki["agent"]
+        
+        # Allow agent to override queue timeout via header
+        q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT)))
+        
+        # Cross-turn context tracking: accumulate tokens per session
+        session_id = request.headers.get("X-Session-Id", "")
+        session_tokens = 0
+        if session_id and r:
+            try:
+                prev = int(r.get("session:" + session_id) or 0)
+                current = estimate_tokens(rd.get("messages",[]))
+                session_tokens = max(prev, current)  # context only grows
+                r.set("session:" + session_id, session_tokens, ex=86400)  # TTL 24h
+            except Exception: pass
+        
+        d = route(rd, tier, agent)
+        queue_start = time.time()
+        
+        # Queue loop: wait for a GPU slot instead of immediate 503
+        while d.get("saturated"):
+            elapsed = time.time() - queue_start
+            if elapsed > q_timeout:
+                resp = jsonify({"error": "All GPUs saturated", "queued_s": round(elapsed,1), "retry_after_s": 5})
+                resp.headers["Retry-After"] = "5"
+                log.warning("QUEUE_TIMEOUT: %s waited %.1fs, all GPUs saturated", agent, elapsed)
+                return resp, 503
+            time.sleep(0.5)  # poll every 500ms
+            d = route(rd, tier, agent)
+        
+        queue_ms = (time.time() - queue_start) * 1000
+        if queue_ms > 500:
+            log.info("QUEUED: %s waited %.0fms before slot opened", agent, queue_ms)
+        model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]]
+        is_stream = rd.get("stream", False)
+        
+        gpu_incr(model)
+        
+        log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1))
+        # Track which GPU this agent is using (TTL 120s covers typical request)
+        if r and agent:
+            try: r.setex("agent_gpu:" + agent + ":" + model, 120, "1")
+            except: pass
+        if r:
+            try:
+                r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent)
+                r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H"))
+                r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent,"queue_ms": round(queue_ms,1)}))
+                r.ltrim("routes:recent",0,999)
+            except Exception: pass
+        start = time.time()
+        resp = requests.post(url+"/chat/completions", json=rd,
+            headers={"Content-Type":"application/json","Authorization":"Bearer not-needed"}, timeout=300, stream=is_stream)
+        lat = int((time.time()-start)*1000)
+        gpu_decr(model)
+        
+        if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502
+        if is_stream:
+            # Buffer SSE chunks, handle split lines for large responses
+            chunks = []
+            stream_timings = {}
+            buf = ""  # accumulate partial lines
+            for raw in resp.iter_content(chunk_size=None, decode_unicode=True):
+                if raw:
+                    cleaned = clean_unicode(raw)
+                    chunks.append(cleaned)
+                    buf += cleaned
+                    # Process complete lines from buffer
+                    while "\n" in buf:
+                        line, buf = buf.split("\n", 1)
+                        line = line.strip()
+                        if line.startswith("data: ") and not stream_timings:
+                            js = line[6:].strip()
+                            if js.startswith("{") and "timings" in js and "predicted_n" in js:
+                                try:
+                                    tj = json.loads(js).get("timings", {})
+                                    if tj:
+                                        stream_timings = tj
+                                except: pass
+            # Store perf record with real token counts from stream
+            if stream_timings:
+                pt = stream_timings.get("prompt_n", 0)
+                ct = stream_timings.get("predicted_n", 0)
+                tps = stream_timings.get("predicted_per_second", 0)
+                gen_ms = stream_timings.get("predicted_ms", lat)
+                store_perf_record(model, agent, tier, reason, queue_ms, gen_ms, pt, ct, True)
+            else:
+                store_perf_record(model, agent, tier, reason, queue_ms, lat, estimate_tokens(rd.get("messages",[])), 0, True)
+            # Yield all chunks to client
+            def gen():
+                for c in chunks: yield c
+            bcast()
+            ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[])))
+            ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100
+            ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok"))
+            sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream")
+            sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
+            sse_resp.headers["X-Context-Warning"] = ctx_warning
+            sse_resp.headers["X-Context-Model"] = model
+            return sse_resp
+        data = clean_response(resp.json())
+        for c in data.get("choices",[]):
+            msg = c.get("message",{})
+            if not msg.get("content") and msg.get("reasoning_content"):
+                msg["content"] = msg["reasoning_content"]
+        # Extract performance data from llama.cpp response
+        usage = data.get("usage", {})
+        timings = data.get("timings", {})
+        prompt_tokens = usage.get("prompt_tokens", 0)
+        completion_tokens = usage.get("completion_tokens", 0)
+        inference_ms = lat  # total GPU round-trip
+        store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, False)
+        ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[])))
+        ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100
+        ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok"))
+        data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"queue_ms": round(queue_ms,1),"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning}
+        resp = jsonify(data)
+        resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
+        resp.headers["X-Context-Warning"] = ctx_warning
+        resp.headers["X-Context-Model"] = model
+        bcast()
+        return resp
+    except requests.Timeout:
+        gpu_decr(model)
+        log.error("TIMEOUT: %s -> %s", agent, model)
+        return jsonify({"error":"timeout"}), 504
+    except Exception as e:
+        gpu_decr(model)
+        log.error("Error: %s\n%s", e, traceback.format_exc())
+        return jsonify({"error":str(e)}), 500
+
+@app.route("/metrics/performance")
+def performance():
+    """Per-request performance analytics with percentiles per model/reason/agent."""
+    if not r: return jsonify({"error": "Redis unavailable"}), 503
+    try:
+        window_hours = int(request.args.get("window", "24"))
+        model_filter = request.args.get("model", "all")
+
+        # Load recent records
+        cutoff = time.time() - (window_hours * 3600)
+        raw = r.lrange("perf:recent", 0, -1)
+        records = []
+        for x in raw:
+            try:
+                rec = json.loads(x)
+                if rec["ts"] >= cutoff:
+                    records.append(rec)
+            except: pass
+
+        # Filter by model if specified
+        if model_filter != "all":
+            records = [r for r in records if r["model"] == model_filter]
+
+        if not records:
+            return jsonify({"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}})
+
+        def pct(values, p):
+            if len(values) < 2: return round(values[0], 1) if values else 0
+            return round(statistics.quantiles(sorted(values), n=100, method='inclusive')[min(p-1, 98)], 1)
+
+        # Per-model stats
+        model_groups = {}
+        for rec in records:
+            m = rec["model"]
+            if m not in model_groups: model_groups[m] = []
+            model_groups[m].append(rec)
+
+        models = []
+        for m, recs in sorted(model_groups.items()):
+            latencies = [r["total_ms"] for r in recs]
+            tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0]
+            non_stream = [r for r in recs if not r["stream"]]
+            queue_times = [r["queue_ms"] for r in non_stream]
+            models.append({
+                "model": m,
+                "count": len(recs),
+                "stream_pct": round(len([r for r in recs if r["stream"]]) / len(recs) * 100, 1),
+                "latency": {
+                    "avg": round(statistics.mean(latencies), 1),
+                    "p50": pct(latencies, 50),
+                    "p95": pct(latencies, 95),
+                    "p99": pct(latencies, 99)
+                },
+                "throughput": {
+                    "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0,
+                    "p50": pct(tps_vals, 50) if tps_vals else 0,
+                    "p95": pct(tps_vals, 95) if tps_vals else 0,
+                },
+                "queue": {
+                    "avg_ms": round(statistics.mean(queue_times), 1) if queue_times else 0,
+                    "p95_ms": pct(queue_times, 95) if queue_times else 0,
+                } if queue_times else None
+            })
+
+        # Per-reason stats
+        reason_groups = {}
+        for rec in records:
+            rsn = rec["reason"]
+            if rsn not in reason_groups: reason_groups[rsn] = []
+            reason_groups[rsn].append(rec)
+
+        reasons = []
+        for rsn, recs in sorted(reason_groups.items(), key=lambda x: -len(x[1])):
+            latencies = [r["total_ms"] for r in recs]
+            reasons.append({
+                "reason": rsn,
+                "count": len(recs),
+                "avg_total_ms": round(statistics.mean(latencies), 1),
+                "p95_total_ms": pct(latencies, 95)
+            })
+
+        # Per-agent stats
+        agent_groups = {}
+        for rec in records:
+            ag = rec["agent"]
+            if ag not in agent_groups: agent_groups[ag] = []
+            agent_groups[ag].append(rec)
+
+        agents = []
+        for ag, recs in sorted(agent_groups.items(), key=lambda x: -len(x[1])):
+            latencies = [r["total_ms"] for r in recs]
+            tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0]
+            agents.append({
+                "agent": ag,
+                "count": len(recs),
+                "avg_total_ms": round(statistics.mean(latencies), 1),
+                "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0
+            })
+
+        all_lat = [r["total_ms"] for r in records]
+        all_tps = [r["tokens_per_sec"] for r in records if r["tokens_per_sec"] > 0]
+        summary = {
+            "total_requests": len(records),
+            "window_hours": window_hours,
+            "latency": {
+                "avg_ms": round(statistics.mean(all_lat), 1),
+                "p50_ms": pct(all_lat, 50),
+                "p95_ms": pct(all_lat, 95),
+                "p99_ms": pct(all_lat, 99)
+            },
+            "throughput_avg_tps": round(statistics.mean(all_tps), 1) if all_tps else 0
+        }
+
+        return jsonify({"models": models, "reasons": reasons, "agents": agents, "summary": summary})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+@app.route("/metrics/scatter")
+def scatter():
+    """Return individual data points for scatter plots (prompt_tokens vs latency)."""
+    if not r: return jsonify({"error": "Redis unavailable"}), 503
+    try:
+        window_hours = int(request.args.get("window", "24"))
+        model_filter = request.args.get("model", "all")
+        cutoff = time.time() - (window_hours * 3600)
+        raw = r.lrange("perf:recent", 0, -1)
+        points = []
+        for x in raw:
+            try:
+                rec = json.loads(x)
+                if rec["ts"] >= cutoff:
+                    if model_filter == "all" or rec["model"] == model_filter:
+                        points.append({
+                            "model": rec["model"],
+                            "agent": rec["agent"],
+                            "reason": rec["reason"],
+                            "prompt_tokens": int(rec.get("prompt_tokens", 0)),
+                            "completion_tokens": rec.get("completion_tokens", 0),
+                            "inference_ms": round(rec["inference_ms"], 1),
+                            "tokens_per_sec": rec.get("tokens_per_sec", 0),
+                            "stream": rec.get("stream", False)
+                        })
+            except: pass
+        return jsonify({"points": points, "count": len(points)})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+@app.route("/v1/models")
+def models():
+    def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1)
+    return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]})
+
+@app.route("/health")
+def health():
+    gpus = {}
+    for m in GPU_URLS:
+        h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1)
+        h["active_requests"] = gpu_active_count(m)
+        h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1)
+        gpus[m] = h
+    return jsonify({"status":"healthy","redis":"connected" if r else "down","gpus":gpus,"available_models":available_models()})
+
+@app.route("/metrics")
+def metrics(): return jsonify(get_metrics())
+
+@app.route("/metrics/timeseries")
+def metrics_timeseries():
+    period = request.args.get("period", "day"); models_list = list(GPU_URLS.keys())
+    data = {"models": {}, "labels": []}
+    if period == "day":
+        buckets = [time.strftime("%Y%m%d%H", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)]
+        data["labels"] = [time.strftime("%H:00", time.gmtime(time.time()-h*3600)) for h in range(23,-1,-1)]
+    elif period == "week":
+        buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)]
+        data["labels"] = [time.strftime("%a", time.gmtime(time.time()-d*86400)) for d in range(6,-1,-1)]
+    else:
+        buckets = [time.strftime("%Y%m%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)]
+        data["labels"] = [time.strftime("%m/%d", time.gmtime(time.time()-d*86400)) for d in range(29,-1,-1)]
+    if r:
+        for model in models_list:
+            counts = []
+            for bucket in buckets:
+                total = 0
+                if period in ("week","month"):
+                    for hh in range(24): total += int(r.get("ts:"+model+":"+bucket+"{:02d}".format(hh)) or 0)
+                else: total = int(r.get("ts:"+model+":"+bucket) or 0)
+                counts.append(total)
+            data["models"][model] = counts
+    return jsonify(data)
+
+@app.route("/stream")
+def stream():
+    def ev():
+        q = queue.Queue()
+        with sse_lock: sse_subscribers.append(q)
+        try:
+            yield "data: "+json.dumps(get_metrics())+"\n\n"
+            while True:
+                try: yield "data: "+q.get(timeout=3)+"\n\n"
+                except queue.Empty: yield "data: "+json.dumps(get_metrics())+"\n\n"
+        except GeneratorExit: pass
+        finally:
+            with sse_lock:
+                if q in sse_subscribers: sse_subscribers.remove(q)
+    return Response(stream_with_context(ev()), mimetype="text/event-stream",
+                    headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"})
+
+if __name__ == "__main__":
+    log.info("Router on :9000 (load-aware)")
+    app.run(host="0.0.0.0", port=9000, debug=False)
@@ -0,0 +1,80 @@
+# Add time-series tracking and endpoint to router
+with open('/opt/inference-harness/router/router.py') as f:
+    code = f.read()
+
+# Add time-series tracking in the chat handler (after Redis incr)
+old_track = '''r.incr('routes:'+model); r.incr('routes:tier:'+tier); r.incr('routes:agent:'+agent)
+                r.lpush('routes:recent', json.dumps'''
+new_track = '''r.incr('routes:'+model); r.incr('routes:tier:'+tier); r.incr('routes:agent:'+agent)
+                # Time-series: hourly bucket
+                hour_key = 'ts:'+model+':'+time.strftime('%Y%m%d%H')
+                r.incr(hour_key)
+                r.expire(hour_key, 86400*31)  # keep 31 days
+                r.lpush('routes:recent', json.dumps'''
+
+code = code.replace(old_track, new_track)
+
+# Add /metrics/timeseries endpoint before if __name__
+ts_endpoint = '''
+@app.route('/metrics/timeseries')
+def metrics_timeseries():
+    period = request.args.get('period', 'day')
+    models = list(GPU_URLS.keys())
+    data = {'models': {}, 'labels': []}
+    
+    if period == 'day':
+        # Last 24 hours, hourly buckets
+        buckets = []
+        for h in range(23, -1, -1):
+            t = time.time() - h * 3600
+            buckets.append(time.strftime('%Y%m%d%H', time.gmtime(t)))
+        data['labels'] = [time.strftime('%H:00', time.gmtime(time.time() - h*3600)) for h in range(23, -1, -1)]
+    elif period == 'week':
+        # Last 7 days, daily buckets
+        buckets = []
+        for d in range(6, -1, -1):
+            t = time.time() - d * 86400
+            buckets.append(time.strftime('%Y%m%d', time.gmtime(t)))
+        data['labels'] = [time.strftime('%a', time.gmtime(time.time() - d*86400)) for d in range(6, -1, -1)]
+    else:
+        # Month — last 30 days, 3-day buckets
+        buckets = []
+        for d in range(29, -1, -3):
+            t = time.time() - d * 86400
+            buckets.append(time.strftime('%Y%m%d', time.gmtime(t)))
+        data['labels'] = [time.strftime('%m/%d', time.gmtime(time.time() - d*86400)) for d in range(29, -1, -3)]
+    
+    if r:
+        for model in models:
+            counts = []
+            for bucket in buckets:
+                if period == 'month':
+                    # Sum 3 consecutive days per bucket
+                    total = 0
+                    base = time.strptime(bucket, '%Y%m%d')
+                    for offset in range(3):
+                        d = time.strftime('%Y%m%d', time.gmtime(time.mktime(base) + offset*86400))
+                        total += int(r.get('ts:'+model+':'+d) or 0)
+                        # Also check hourly keys for today
+                        for hh in range(24):
+                            total += int(r.get('ts:'+model+':'+d+'{:02d}'.format(hh)) or 0)
+                    counts.append(total)
+                else:
+                    key = 'ts:'+model+':'+bucket
+                    if period == 'week':
+                        # Sum all hours in the day
+                        total = sum(int(r.get(key+'{:02d}'.format(h)) or 0) for h in range(24))
+                    else:
+                        total = int(r.get(key) or 0)
+                    counts.append(total)
+            data['models'][model] = counts
+    
+    return jsonify(data)
+'''
+
+# Insert before if __name__
+code = code.replace(if __name__ == __main__:, ts_endpoint + nif __name__ == __main__:)
+
+with open('/opt/inference-harness/router/router.py', 'w') as f:
+    f.write(code)
+print('Time-series tracking and endpoint added')