Phase 0: Dual-key router — no hardcoded keys, gemma sync, 262K context fix

- Removed all hardcoded API_KEYS from source (env var now REQUIRED)
- Added dual-key transition support (old keys deprecated, logged)
- Model sync: qwen3.5-9b-vlm → gemma-4-12b everywhere
- Dense context: 131K → 262K (all 3 models at 262K)
- Added /admin/keys endpoints for key lifecycle management
- Added ADMIN_KEY env var for admin auth
- Dashboard already synced to gemma-4-12b labels

Deployment: set API_KEYS from phase0-dual-keys.json in docker-compose
Transition: 7 days dual-key, then POST /admin/keys/revoke

dashboard/dashboard.py

@@ -16,7 +16,7 @@ def fetch_state():
 
 def broadcast_loop():
     while True:
-        time.sleep(3)
+        time.sleep(5)
         data = fetch_state(); payload = json.dumps(data)
         with sse_lock:
             dead = [q for q in sse_subscribers if not q.put(payload)]
@@ -101,7 +101,32 @@ body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMac
   <div class="col-md-4"><div class="chart-card"><div class="title">Model Distribution</div><div id="route-bars"></div></div></div>
   <div class="col-md-4"><div class="chart-card"><div class="title">Agent Activity</div><div id="agent-bars"></div></div></div>
 
-  <!-- ROW 4: Live Stream -->
+  <!-- ROW 4: Performance Analytics -->
+  <div class="col-12 mb-2"><div class="d-flex align-items-center gap-2"><span class="fw-bold text-white" style="font-size:14px">&#x1F4CA; Performance Analytics</span>
+    <div class="d-flex gap-1 ms-auto">
+      <button class="btn-sm-period active" onclick="switchPerfWindow('1')">1h</button>
+      <button class="btn-sm-period" onclick="switchPerfWindow('24')">24h</button>
+    </div>
+  </div></div>
+  <div class="col-md-6"><div class="chart-card"><div class="title">Latency — P50 / P95 / P99 (ms)</div><div id="perf-latency"></div></div></div>
+  <div class="col-md-6"><div class="chart-card"><div class="title">Throughput — Tokens / sec</div><div id="perf-throughput"></div></div></div>
+  <div class="col-md-6"><div class="chart-card"><div class="title">Routing Effectiveness — by Reason</div><div id="perf-reasons"></div></div></div>
+  <div class="col-md-6"><div class="chart-card"><div class="title">Agent Performance</div><div id="perf-agents"></div></div></div>
+
+  <!-- ROW 5: Latency vs Context Scatter -->
+  <div class="col-12"><div class="chart-card"><div class="title d-flex justify-content-between align-items-center">
+    <span>Latency vs Prompt Size — by Model</span>
+    <div class="d-flex gap-2">
+      <select id="scatter-model" onchange="loadScatter()" style="font-size:10px;background:#1e293b;color:#94a3b8;border:1px solid #334155;border-radius:4px;padding:2px 6px">
+        <option value="all">All Models</option>
+        <option value="gemma-4-12b">12B VLM</option>
+        <option value="qwen3.6-27B-code">27B Dense</option>
+        <option value="qwen3.6-35B-A3B">35B MoE</option>
+      </select>
+    </div>
+  </div><div id="scatter-plot" style="height:200px;position:relative"></div><div id="scatter-legend" class="d-flex justify-content-center gap-3 mt-2 flex-wrap small"></div></div></div>
+
+  <!-- ROW 6: Live Stream -->
   <div class="col-12"><div class="chart-card"><div class="title">Live Stream</div>
     <div class="table-responsive"><table class="table table-custom mb-0">
       <thead><tr><th>Time</th><th>Agent</th><th>Model</th><th>Reason</th><th>Tier</th></tr></thead>
@@ -113,13 +138,13 @@ body { background: #0b0f17; color: #bcc3cd; font-family: -apple-system, BlinkMac
 <script>
 // ═══ MODEL CONFIG — Update ONLY this array for model migrations ═══
 var MODELS=[
-  {id:'qwen3.6-35B-A3B',label:'Qwen MoE',gpu:'MoE - Strix Halo',color:'#a78bfa',short:'MoE',domId:'gpu-moe'},
+  {id:'qwen3.6-35B-A3B',label:'Qwen MoE',gpu:'MoE - Strix Halo',color:'#a78bfa',short:'MoE',domId:'gpu-moe',perf:'35B MoE'},
-  {id:'qwen3.6-27B-code',label:'Qwen Code',gpu:'Dense - RTX 3090',color:'#f59e0b',short:'Dense',domId:'gpu-dense'},
+  {id:'qwen3.6-27B-code',label:'Qwen Code',gpu:'Dense - RTX 3090',color:'#f59e0b',short:'Dense',domId:'gpu-dense',perf:'27B Dense'},
-  {id:'gemma-4-12b',label:'Gemma 4 12B',gpu:'VLM - RTX 5070',color:'#22c55e',short:'VLM',domId:'gpu-light'}
+  {id:'gemma-4-12b',label:'Gemma 4 12B',gpu:'VLM - RTX 5070',color:'#22c55e',short:'VLM',domId:'gpu-light',perf:'12B VLM'}
 ];
 // Auto-derived lookups — DO NOT EDIT below
-var MC={},ML={},GL={},ids={},lb={};
+var MC={},ML={},GL={},ids={},lb={},mlab={},mcol={};
-MODELS.forEach(function(m){MC[m.id]=m.color;ML[m.id]=m.label;GL[m.id]=m.gpu;ids[m.id]=m.domId;lb[m.id]=m.short;});
+MODELS.forEach(function(m){MC[m.id]=m.color;ML[m.id]=m.label;GL[m.id]=m.gpu;ids[m.id]=m.domId;lb[m.id]=m.short;mlab[m.id]=m.perf;mcol[m.id]=m.color;});
 function $(id){return document.getElementById(id);}
 
 function render(data){
@@ -166,7 +191,7 @@ el.innerHTML=h;
 
 function renderGPUMetrics(data){
 var el=$('gpu-metrics-card');if(!el)return;
-
+var lb={'qwen3.6-35B-A3B':'MoE','qwen3.6-27B-code':'Dense','gemma-4-12b':'VLM'};
 var h='';data.gpus.forEach(function(g){
 var nm=lb[g.id]||g.id,tp=g.temp_c||0,ut=g.gpu_util_pct||0,pw=g.power_w||0,pl=g.power_limit_w||0;
 var tc=tp>85?'#ef4444':tp>70?'#f59e0b':'#22c55e',uc=ut>90?'#ef4444':ut>70?'#f59e0b':'#22c55e';
@@ -193,8 +218,92 @@ var grid='';for(var g=0;g<=4;g++){var y=(g/4)*H;grid+='<line x1=\"0\" y1=\"'+y.t
 cn.innerHTML='<svg viewBox=\"0 0 100 '+(H+16)+'\" style=\"width:100%;height:'+(H+20)+'px;display:block\" preserveAspectRatio=\"none\">'+grid+paths+'</svg>';
 lg.innerHTML=mn.map(function(m){return'<span class=\"d-flex align-items-center gap-1\"><svg width=\"14\" height=\"8\"><line x1=\"0\" y1=\"4\" x2=\"14\" y2=\"4\" stroke=\"'+(MC[m]||'#38bdf8')+'\" stroke-width=\"2\"/></svg>'+(ML[m]||m)+'</span>';}).join('');
 }
-function poll(){fetch('/api/state').then(function(r){return r.json()}).then(function(data){render(data);$('connection-status').textContent='live';}).catch(function(){$('connection-status').textContent='reconnecting';});}
+var perfWindow='24';
-poll();setInterval(poll,3000);loadTS();
+function switchPerfWindow(w){perfWindow=w;document.querySelectorAll('.btn-sm-period').forEach(function(b,i){if(i>=4)b.classList.toggle('active',b.textContent.trim().replace('h','')===w)});loadPerf();}
+function loadPerf(){fetch('/api/performance?window='+perfWindow).then(function(r){return r.json()}).then(renderPerf).catch(function(){})}
+function renderPerf(d){
+var models=d.models||[],reasons=d.reasons||[],agents=d.agents||[],sum=d.summary||{};
+// Latency bars: p50/p95/p99 per model
+// mlab/mcol auto-derived from MODELS above
+if(!models.length){$('perf-latency').innerHTML='<div class="text-secondary small text-center py-4">Accumulating data...</div>';return;}
+var maxLat=Math.max(...models.map(function(m){return m.latency.p99||0}),1);
+var latHTML=models.map(function(m){
+var l=m.latency||{},p50=l.p50||0,p95=l.p95||0,p99=l.p99||0,c=mcol[m.model]||'#38bdf8';
+return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span class="text-secondary">'+m.count+' reqs</span></div>'+
+'<div class="d-flex align-items-center gap-2 mb-1"><span class="text-secondary" style="min-width:28px">p50</span><div class="flex-grow-1" style="height:14px;background:#1e293b;border-radius:4px;overflow:hidden;position:relative"><div style="position:absolute;left:0;top:0;height:100%;width:'+(p50/maxLat*100)+'%;background:'+c+';opacity:0.3;border-radius:4px"></div><div style="position:absolute;left:0;top:0;height:100%;width:'+(p95/maxLat*100)+'%;background:'+c+';opacity:0.5;border-radius:4px"></div><div style="position:absolute;left:0;top:0;height:100%;width:'+(p99/maxLat*100)+'%;background:'+c+';border-radius:4px"></div></div><span style="color:'+c+';min-width:48px;text-align:right;font-variant-numeric:tabular-nums">'+p99+'ms</span></div>'+
+'<div class="d-flex gap-3" style="font-size:10px;color:#64748b;padding-left:32px"><span>p50: '+p50+'ms</span><span>p95: '+p95+'ms</span><span>p99: '+p99+'ms</span></div></div>';
+}).join('');
+$('perf-latency').innerHTML=latHTML;
+// Throughput comparison
+var maxTps=Math.max(...models.map(function(m){return m.throughput.avg_tokens_per_sec||0}),1);
+var tpsHTML=models.map(function(m){
+var t=m.throughput||{},avg=t.avg_tokens_per_sec||0,p50=t.p50||0,c=mcol[m.model]||'#38bdf8';
+var isAllStreaming = avg===0 && p50===0;
+if(isAllStreaming){
+return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span style="color:#64748b;font-style:italic">streaming only</span></div><div class="text-secondary" style="font-size:10px">t/s available for non-streaming requests only</div></div>';
+}
+return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+mlab[m.model]+'</span><span style="color:'+c+'" class="fw-bold">'+avg+' tok/s</span></div>'+
+'<div class="d-flex align-items-center gap-2"><span class="text-secondary" style="min-width:28px">avg</span><div class="flex-grow-1" style="height:6px;background:#1e293b;border-radius:3px;overflow:hidden"><div style="height:100%;width:'+(Math.max(avg/maxTps*100,6))+'%;background:'+c+';border-radius:3px"></div></div><span class="small" style="color:'+c+';min-width:54px;text-align:right">'+avg+' tok/s</span></div>'+
+'<div class="d-flex align-items-center gap-2 mt-1"><span class="text-secondary" style="min-width:28px;font-size:10px">p50</span><div class="flex-grow-1" style="height:4px;background:#1e293b;border-radius:2px;overflow:hidden"><div style="height:100%;width:'+(Math.max(p50/maxTps*100,4))+'%;background:'+c+';opacity:0.5;border-radius:2px"></div></div><span style="font-size:10px;color:#64748b">'+p50+' tok/s</span></div></div>';
+}).join('');
+$('perf-throughput').innerHTML=tpsHTML;
+// Routing reasons table
+if(reasons.length){
+var rHTML='<table class="table table-custom mb-0"><thead><tr><th>Reason</th><th>Count</th><th>Avg Lat</th><th>P95 Lat</th></tr></thead><tbody>';
+reasons.forEach(function(r){rHTML+='<tr><td>'+r.reason+'</td><td>'+r.count+'</td><td>'+r.avg_total_ms+'ms</td><td>'+r.p95_total_ms+'ms</td></tr>';});
+rHTML+='</tbody></table>';$('perf-reasons').innerHTML=rHTML;
+}else{$('perf-reasons').innerHTML='<div class="text-secondary small text-center py-3">-</div>';}
+// Agent performance
+if(agents.length){
+var maxAc=Math.max(...agents.map(function(a){return a.count||0}),1);
+var aHTML=agents.map(function(a){return'<div class="mb-2" style="font-size:11px"><div class="d-flex justify-content-between mb-1"><span style="color:#e2e8f0">'+a.agent+'</span><span class="text-secondary">'+a.count+' reqs</span></div><div class="d-flex align-items-center gap-2"><div class="flex-grow-1" style="height:4px;background:#1e293b;border-radius:2px;overflow:hidden"><div style="height:100%;width:'+(a.count/maxAc*100)+'%;background:#38bdf8;border-radius:2px"></div></div><span class="small" style="color:#38bdf8;min-width:60px;text-align:right">'+a.avg_total_ms+'ms avg</span></div></div>';}).join('');
+$('perf-agents').innerHTML=aHTML;
+}else{$('perf-agents').innerHTML='<div class="text-secondary small text-center py-3">-</div>';}
+}
+var sseConnected = false;
+function poll(){if(sseConnected)return;fetch('/api/state').then(function(r){return r.json()}).then(function(data){render(data);$('connection-status').textContent='live';}).catch(function(){$('connection-status').textContent='reconnecting';});}
+// SSE connection tracking
+var es = new EventSource('/api/stream');
+es.onopen = function(){sseConnected = true; $('connection-status').textContent = 'live (SSE)';};
+es.onmessage = function(e){try{render(JSON.parse(e.data));}catch(ex){}};
+es.onerror = function(){sseConnected = false; $('connection-status').textContent = 'polling';};
+function loadScatter(){
+var m=$('scatter-model').value;
+fetch('/api/scatter?window=24&model='+m).then(function(r){return r.json()}).then(renderScatter).catch(function(){});
+}
+function renderScatter(d){
+var pts=d.points||[],el=$('scatter-plot'),lg=$('scatter-legend');
+if(!pts.length){el.innerHTML='<div class="text-secondary small text-center py-5">No data yet</div>';return;}
+var mcol=Object.assign({unknown:'#38bdf8'},mcol);
+// mlab auto-derived from MODELS above
+var maxX=Math.max.apply(null,pts.map(function(p){return p.prompt_tokens||0}))||1000;
+var maxY=Math.max.apply(null,pts.map(function(p){return p.inference_ms||0}))||5000;
+// Log scale for X axis (prompt tokens vary widely)
+var toX=function(t){return Math.log10(Math.max(t,1))/Math.log10(Math.max(maxX,10))*100;};
+var toY=function(t){return (t/maxY)*100;};
+var dots='';
+pts.forEach(function(p){
+var x=toX(p.prompt_tokens),y=toY(p.inference_ms),c=mcol[p.model]||'#38bdf8';
+var r=p.stream?1.5:2.5,o=p.stream?0.4:0.8;
+dots+='<circle cx="'+x+'" cy="'+(100-y)+'" r="'+r+'" fill="'+c+'" opacity="'+o+'"><title>'+mlab[p.model]+' | '+p.prompt_tokens+' tok | '+p.inference_ms+'ms | '+p.agent+'</title></circle>';
+});
+// Grid lines
+var grid='';
+for(var i=1;i<=4;i++){grid+='<line x1="0" y1="'+(i*20)+'" x2="100" y2="'+(i*20)+'" stroke="#1e293b" stroke-width="0.5"/>';}
+for(var i=1;i<=4;i++){grid+='<line x1="'+(i*20)+'" y1="0" x2="'+(i*20)+'" y2="100" stroke="#1e293b" stroke-width="0.5"/>';}
+// Axis labels
+var xTicks='';
+var xVals=[10,100,1000,10000,100000];
+xVals.forEach(function(v){if(v<=maxX)xTicks+='<text x="'+toX(v)+'" y="103" text-anchor="middle" font-size="8" fill="#64748b">'+(v>=1000?(v/1000)+'k':v)+'</text>';});
+var yTicks='';
+var yVals=[500,1000,5000,10000,50000,100000];
+yVals.forEach(function(v){if(v<=maxY)yTicks+='<text x="-2" y="'+(97-toY(v))+'" text-anchor="end" font-size="8" fill="#64748b">'+(v>=1000?(v/1000)+'s':v+'ms')+'</text>';});
+el.innerHTML='<svg viewBox="-35 0 140 115" style="width:100%;height:200px">'+grid+dots+xTicks+yTicks+'<text x="50" y="112" text-anchor="middle" font-size="9" fill="#475569">Prompt Tokens (log scale)</text><text x="-38" y="50" text-anchor="middle" font-size="9" fill="#475569" transform="rotate(-90,-38,50)">Inference Time</text></svg>';
+// Legend
+var models=[];pts.forEach(function(p){if(models.indexOf(p.model)===-1)models.push(p.model);});
+lg.innerHTML=models.map(function(m){return'<span class="d-flex align-items-center gap-1 small"><svg width="10" height="10"><circle cx="5" cy="5" r="3.5" fill="'+(mcol[m]||'#38bdf8')+'"/></svg>'+mlab[m]+'</span>';}).join('');
+}
+poll();setInterval(poll,10000);loadTS();loadPerf();setInterval(loadPerf,15000);loadScatter();setInterval(loadScatter,30000);
 </script>
 </body>
 </html>"""
@@ -205,6 +314,26 @@ def dashboard(): return render_template_string(DASHBOARD_HTML)
 @app.route("/api/state")
 def api_state(): return fetch_state()
 
+@app.route("/api/scatter")
+def api_scatter():
+    window = request.args.get("window", "24")
+    model = request.args.get("model", "all")
+    try:
+        r = requests.get(f"http://router:9000/metrics/scatter?window={window}&model={model}", timeout=10)
+        if r.status_code == 200: return r.json()
+    except Exception: pass
+    return {"points": [], "count": 0}
+
+@app.route("/api/performance")
+def api_performance():
+    window = request.args.get("window", "24")
+    model = request.args.get("model", "all")
+    try:
+        r = requests.get(f"http://router:9000/metrics/performance?window={window}&model={model}", timeout=10)
+        if r.status_code == 200: return r.json()
+    except Exception: pass
+    return {"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}}
+
 @app.route("/api/timeseries")
 def api_timeseries():
     period = request.args.get("period", "day")

phase0-dual-keys.json

@@ -0,0 +1,20 @@
+{
+  "sk-syslog-local-master-key": {"tier": "enterprise", "agent": "admin", "deprecated": true},
+  "sk-33a0d6e6-a6da12fb483770d6f63f543b7e16a742": {"tier": "enterprise", "agent": "admin"},
+  "sk-syslog-abiba": {"tier": "enterprise", "agent": "Abiba", "deprecated": true},
+  "sk-6a6e49d0-e35e27c524ce0ba2de8f862a0e966b0c": {"tier": "enterprise", "agent": "Abiba"},
+  "sk-syslog-mumuni": {"tier": "enterprise", "agent": "Mumuni", "deprecated": true},
+  "sk-ba249c99-29e3b163a8d8ab7abb8663cae0dceb37": {"tier": "enterprise", "agent": "Mumuni"},
+  "sk-syslog-tanko": {"tier": "enterprise", "agent": "Tanko", "deprecated": true},
+  "sk-4a67112a-d9b62caf5a5881a15837df506f782708": {"tier": "enterprise", "agent": "Tanko"},
+  "sk-syslog-koby": {"tier": "enterprise", "agent": "Koby", "deprecated": true},
+  "sk-9dd66bba-316f616ec40cb51f3b489a0146bb986f": {"tier": "enterprise", "agent": "Koby"},
+  "sk-syslog-kagenz0": {"tier": "enterprise", "agent": "Kagenz0", "deprecated": true},
+  "sk-5a05020f-a984a8a20f731c5a65dfc07d51495bb3": {"tier": "enterprise", "agent": "Kagenz0"},
+  "sk-syslog-koonimo": {"tier": "enterprise", "agent": "Koonimo", "deprecated": true},
+  "sk-b2f0e1b2-70b7cf1c5a5e402a49b5469e88d9e335": {"tier": "enterprise", "agent": "Koonimo"},
+  "sk-starter-abc123": {"tier": "starter", "agent": "test-starter", "deprecated": true},
+  "sk-a8307a00-cd76daa07dd61c41aeb4545656f8b8c4": {"tier": "starter", "agent": "test-starter"},
+  "sk-professional-xyz789": {"tier": "professional", "agent": "test-pro", "deprecated": true},
+  "sk-b519a91e-4d2774c6c7da3fc31326e69ccc426781": {"tier": "professional", "agent": "test-pro"}
+}

router/router.py

@@ -1,4 +1,4 @@
-import os, json, time, logging, traceback, threading, queue
+import os, json, time, logging, traceback, threading, queue, statistics, math
 import requests, redis
 from flask import Flask, request, jsonify, Response, stream_with_context
 
@@ -19,16 +19,16 @@ GPU_URLS = {
 }
 # Max concurrent requests per GPU (based on llama.cpp --parallel)
 GPU_MAX_CONCURRENT = {
-    "qwen3.6-35B-A3B": 2,   # 2 slots
+    "qwen3.6-35B-A3B": 2,   # 2 slots (cross-agent spread prevents overheating)
     "qwen3.6-27B-code": 2,  # 2 slots
-    "gemma-4-12b": 2,       # 2 slots (7.1GB VRAM)
+    "gemma-4-12b": 2,       # 2 slots (12GB VRAM, 4GB headroom)
 }
 
 # Context window sizes (tokens) — used for compaction signals
 GPU_CONTEXT = {
-    "qwen3.6-35B-A3B": 131072,
+    "qwen3.6-35B-A3B": 262144,
     "qwen3.6-27B-code": 262144,
-    "gemma-4-12b": 262144,  # 262K context (gemma-4-12b)
+    "gemma-4-12b": 262144,
 }
 
 TIER_MODELS = {
@@ -36,18 +36,45 @@ TIER_MODELS = {
     "professional": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"],
     "enterprise": ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"],
 }
-API_KEYS = {
+# ── PHASE 0: Dual-Key API Key System ──
-    "sk-syslog-local-master-key": {"tier": "enterprise", "agent": "admin"},
+# API_KEYS env var is REQUIRED (JSON string). No hardcoded fallback.
-    "sk-syslog-abiba": {"tier": "enterprise", "agent": "Abiba"},
+# Format: {"sk-xxx": {"tier": "enterprise", "agent": "Name"}}
-    "sk-syslog-mumuni": {"tier": "enterprise", "agent": "Mumuni"},
+# Deprecated keys get {"deprecated": true} — still accepted, logged with warning.
-    "sk-syslog-tanko": {"tier": "enterprise", "agent": "Tanko"},
+_raw_keys = os.environ.get("API_KEYS")
-    "sk-syslog-koby": {"tier": "enterprise", "agent": "Koby"},
+if not _raw_keys:
-    "sk-syslog-kagenz0": {"tier": "enterprise", "agent": "Kagenz0"},
+    raise RuntimeError("FATAL: API_KEYS environment variable is required. "
-    "sk-syslog-koonimo": {"tier": "enterprise", "agent": "Koonimo"},
+                       "Set it in docker-compose.yml or .env file. "
-    "sk-starter-abc123": {"tier": "starter", "agent": "test-starter"},
+                       "No hardcoded keys fallback — this is a security feature.")
-    "sk-professional-xyz789": {"tier": "professional", "agent": "test-pro"},
+API_KEYS = json.loads(_raw_keys)
+log.info("Loaded %d API keys from env var (%d deprecated)",
+         len(API_KEYS), sum(1 for v in API_KEYS.values() if v.get("deprecated")))
+# Rate limits: requests per minute per API key tier
+RATE_LIMIT_RPM = {
+    "enterprise": 120,
+    "professional": 60,
+    "starter": 20,
 }
 
+def check_rate_limit(api_key, tier):
+    """Token bucket rate limiter using Redis. Returns (allowed, retry_after_or_remaining, reset_seconds)."""
+    if not r:
+        return True, 999, 60
+    limit = RATE_LIMIT_RPM.get(tier, 30)
+    key = f"ratelimit:{api_key}"
+    current = int(r.get(key) or 0)
+    if current >= limit:
+        ttl = r.ttl(key)
+        retry = max(ttl, 1) if ttl and ttl > 0 else 60
+        return False, retry, 0
+    pipe = r.pipeline()
+    pipe.incr(key)
+    pipe.expire(key, 60)  # 1-minute sliding window
+    pipe.execute()
+    remaining = limit - (current + 1)
+    reset_seconds = r.ttl(key) or 60
+    return True, remaining, reset_seconds
+
+
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [ROUTER] %(levelname)s %(message)s")
 log = logging.getLogger("router")
 try: r = redis.from_url(REDIS_URL, decode_responses=True); r.ping()
@@ -94,24 +121,25 @@ def gpu_decr(model):
         if v and int(v) < 0:
             r.set("active:" + model, 0)  # never go negative
 
-def check_gpu_health(model):
+def check_gpu_health(model, sidecar_timeout=5, gpu_timeout=3):
     url = GPU_SIDECARS.get(model)
     if not url: return {"status": "unknown"}
     try:
-        resp = requests.get(url, timeout=5)
+        resp = requests.get(url, timeout=sidecar_timeout)
         if resp.status_code == 200:
             d = resp.json()
             pct = (d.get("vram_used_mb",0) / max(d.get("vram_total_mb",1), 1)) * 100
-            status = "healthy" if pct < 90 else "saturated"
+            status = "healthy"  # VRAM usage != saturation; busy slots handled by is_gpu_busy()
+            vram_warning = pct >= 95
             # Also check if llama.cpp endpoint is actually responding
             gpu_url = GPU_URLS.get(model, "")
             try:
-                hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=3)
+                hr = requests.get(gpu_url.replace("/v1","") + "/health", headers={"Authorization": "Bearer not-needed"}, timeout=gpu_timeout)
                 if hr.status_code != 200:
                     status = "down"
             except Exception:
                 status = "down"
-            return {"status": status, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")}
+            return {"status": status, "vram_warning": vram_warning, "vram_used_mb": d.get("vram_used_mb"), "vram_total_mb": d.get("vram_total_mb"), "vram_pct": round(pct,1), "temp_c": d.get("temp_c"), "gpu_util_pct": d.get("gpu_util_pct"), "gpu_name": d.get("gpu_name"), "power_w": d.get("power_w"), "power_limit_w": d.get("power_limit_w")}
     except Exception: pass
     return {"status": "down"}
 
@@ -121,14 +149,65 @@ def estimate_tokens(msgs):
     """Estimate token count from messages. Uses JSON length / 3.5 (closer to real tokenizer ratios for dense text)."""
     return len(json.dumps(msgs, default=str)) // 3.5
 
+def store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, stream):
+    """Store detailed performance record in Redis for analytics."""
+    if not r: return
+    try:
+        total_ms = queue_ms + inference_ms
+        tps = completion_tokens / (inference_ms / 1000) if inference_ms > 0 and completion_tokens > 0 else 0
+        rec = json.dumps({
+            "ts": time.time(),
+            "model": model, "agent": agent, "tier": tier, "reason": reason,
+            "queue_ms": round(queue_ms, 1),
+            "inference_ms": round(inference_ms, 1),
+            "total_ms": round(total_ms, 1),
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "tokens_per_sec": round(tps, 1),
+            "stream": stream
+        })
+        # Global recent list (last 500)
+        r.lpush("perf:recent", rec)
+        r.ltrim("perf:recent", 0, 499)
+        # Per-model list (last 200)
+        r.lpush("perf:model:" + model, rec)
+        r.ltrim("perf:model:" + model, 0, 199)
+        # Per-reason list (last 200)
+        r.lpush("perf:reason:" + reason, rec)
+        r.ltrim("perf:reason:" + reason, 0, 199)
+        # Per-agent list (last 200)
+        r.lpush("perf:agent:" + agent, rec)
+        r.ltrim("perf:agent:" + agent, 0, 199)
+    except Exception:
+        pass
+
 def is_gpu_busy(model):
     """Check if GPU is at or near max concurrent capacity."""
     active = gpu_active_count(model)
     max_c = GPU_MAX_CONCURRENT.get(model, 1)
     return active >= max_c
 
-def select_best_gpu(candidates, reason):
+def select_best_gpu(candidates, reason, agent=""):
-    """Pick the best GPU from candidates IN ORDER — first non-busy one wins."""
+    """Pick best GPU, spreading agents across GPUs to prevent hotspots."""
+    # Count how many distinct agents are on each GPU
+    gpu_agent_counts = {}
+    if r:
+        for m in GPU_URLS:
+            count = 0
+            for ak in API_KEYS.values():
+                if r.get("agent_gpu:" + ak["agent"] + ":" + m):
+                    count += 1
+            gpu_agent_counts[m] = count
+    # First pass: prefer GPUs with 0 other agents (fresh GPU for this agent)
+    for m in candidates:
+        if not is_gpu_busy(m) and gpu_agent_counts.get(m, 0) == 0:
+            return {"model": m, "reason": reason}
+    # Second pass: prefer GPU this agent is NOT already on (skip own GPU)
+    if agent:
+        for m in candidates:
+            if not is_gpu_busy(m) and not r.get("agent_gpu:" + agent + ":" + m):
+                return {"model": m, "reason": reason}
+    # Third pass: any non-busy GPU
     for m in candidates:
         if not is_gpu_busy(m):
             return {"model": m, "reason": reason}
@@ -144,7 +223,7 @@ def select_best_gpu(candidates, reason):
         return {"model": best, "reason": "load_balanced_" + reason}
     return None
 
-def route(rd, tier):
+def route(rd, tier, agent=""):
     msgs = rd.get("messages",[]); t = estimate_tokens(msgs)
     sys = any(m.get("role")=="system" for m in msgs)
     turns = len([m for m in msgs if m.get("role") in ("user","assistant")])
@@ -163,53 +242,52 @@ def route(rd, tier):
         if is_gpu_busy(target) and req in allowed:
             alts = [m for m in avail if m != target and m in allowed]
             if alts:
-                alt = select_best_gpu(alts, "explicit")
+                alt = select_best_gpu(alts, "explicit", agent)
                 if alt: return alt
         return {"model": target, "reason": "explicit"}
     
     if hints:
         if hints.get("priority")=="speed" and "gemma-4-12b" in avail:
-            return select_best_gpu(["gemma-4-12b"], "hint_speed") or {"model":"gemma-4-12b","reason":"hint_speed"}
+            return select_best_gpu(["gemma-4-12b"], "hint_speed", agent) or {"model":"gemma-4-12b","reason":"hint_speed"}
-        if hints.get("priority")=="quality" and "qwen3.6-27B-code" in avail:
+        if hints.get("priority")=="quality" and "qwen3.6-35B-A3B" in avail:
-            return select_best_gpu(["qwen3.6-27B-code"], "hint_quality") or {"model":"qwen3.6-27B-code","reason":"hint_quality"}
+            return select_best_gpu(["qwen3.6-35B-A3B"], "hint_quality", agent) or {"model":"qwen3.6-35B-A3B","reason":"hint_quality"}
     
     first_msg = msgs[0].get("content","") if msgs else ""
     words = len(first_msg.split()) if isinstance(first_msg, str) else 99
     
-    # TIER 1: Lightweight — single-turn short queries → VLM first
+    # TIER 1: Lightweight — single-turn short queries → VLM (fastest)
-    if not sys and turns <= 1 and words <= 100 and "gemma-4-12b" in avail:
+    if not sys and turns <= 1 and t <= 500 and words <= 100 and "gemma-4-12b" in avail:
         if not is_gpu_busy("gemma-4-12b"):
             return {"model":"gemma-4-12b","reason":"lightweight"}
-        # VLM busy — fall back to Dense, then MoE
+        # VLM busy — Dense is faster for short queries than MoE
-        fallback = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code"] if m in avail]
+        fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail]
-        result = select_best_gpu(fallback, "lightweight_fallback")
+        result = select_best_gpu(fallback, "lightweight_fallback", agent)
         if result: return result
     
-    # TIER 2: Simple conversations — short context, any prompt → VLM preferred
+    # TIER 2: Simple conversations — VLM primary (up to 15K tok), fastest for moderate chat
-    if t <= 1000 and turns <= 4 and "gemma-4-12b" in avail:
+    if t <= 15000 and turns <= 12 and "gemma-4-12b" in avail:
         if not is_gpu_busy("gemma-4-12b"):
             return {"model":"gemma-4-12b","reason":"simple_conv"}
-        # VLM busy — try Dense
+        # VLM busy — fall back to Dense, then MoE
-        if "qwen3.6-27B-code" in avail and not is_gpu_busy("qwen3.6-27B-code"):
+        fallback = [m for m in ["qwen3.6-27B-code","qwen3.6-35B-A3B"] if m in avail]
-            return {"model":"qwen3.6-27B-code","reason":"simple_conv_fallback"}
+        result = select_best_gpu(fallback, "simple_conv_fallback", agent)
+        if result: return result
     
-    # TIER 3: Heavy reasoning — extremely large context or very long conversations
+    # TIER 3: Medium complexity — Dense primary, VLM fallback (quality + speed balance)
-    if t > 50000 or turns > 25:
+    if t <= 25000:
-        # MoE first (131K context handles heavy sessions), then Dense (98K reasoning), then Light (131K fallback)
+        candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail]
+        result = select_best_gpu(candidates, "medium", agent)
+        if result: return result
+    
+    # TIER 4: Heavy reasoning — MoE primary (workhorse), Dense fallback
+    if t > 25000:
         candidates = [m for m in ["qwen3.6-35B-A3B","qwen3.6-27B-code","gemma-4-12b"] if m in avail]
-        result = select_best_gpu(candidates, "heavy_reasoning")
+        result = select_best_gpu(candidates, "heavy_reasoning", agent)
         if result: return result
     
-    # TIER 4: Default — MoE first, VLM helps, Dense last (slow)
+    # TIER 5: Default — Dense primary, MoE fallback
-    if t <= 50000:
+    candidates = [m for m in ["qwen3.6-27B-code","gemma-4-12b","qwen3.6-35B-A3B"] if m in avail]
-        candidates = [m for m in ["qwen3.6-35B-A3B","gemma-4-12b","qwen3.6-27B-code"] if m in avail]
+    result = select_best_gpu(candidates, "default", agent)
-        result = select_best_gpu(candidates, "default")
-        if result: return result
-    
-    # Fallback — best available
-    if "qwen3.6-35B-A3B" in avail and not is_gpu_busy("qwen3.6-35B-A3B"):
-        return {"model":"qwen3.6-35B-A3B","reason":"default_moe"}
-    result = select_best_gpu([m for m in avail], "fallback")
     if result: return result
     return {"model":avail[0],"reason":"last_resort"}
 
@@ -266,6 +344,27 @@ def chat():
             return jsonify({"error": "Unauthorized — valid API key required"}), 401
         ki = API_KEYS[ak]
         tier, agent = ki["tier"], ki["agent"]
+        # Phase 0: dual-key transition — log deprecated key usage
+        if ki.get("deprecated"):
+            new_key = next((k for k, v in API_KEYS.items()
+                           if v.get("agent") == agent and not v.get("deprecated")), None)
+            log.warning("DEPRECATED_KEY: agent=%s using old key %s...%s — switch to %s...%s",
+                       agent, ak[:12], ak[-8:],
+                       new_key[:12] if new_key else "N/A",
+                       new_key[-8:] if new_key else "N/A")
+            if r:
+                r.incr("deprecated_usage:" + agent)
+        
+        # Rate limit check
+        allowed, rl_val, reset_sec = check_rate_limit(ak, tier)
+        if not allowed:
+            resp = jsonify({"error": "Rate limit exceeded", "retry_after_s": rl_val})
+            resp.headers["Retry-After"] = str(rl_val)
+            resp.headers["X-RateLimit-Limit"] = str(RATE_LIMIT_RPM.get(tier, 30))
+            resp.headers["X-RateLimit-Remaining"] = "0"
+            resp.headers["X-RateLimit-Reset"] = str(int(time.time() + rl_val))
+            log.warning("RATE_LIMIT: %s (%s) exceeded limit", agent, ak[-8:])
+            return resp, 429
         
         # Allow agent to override queue timeout via header
         q_timeout = int(request.headers.get("X-Queue-Timeout", str(QUEUE_TIMEOUT)))
@@ -281,7 +380,7 @@ def chat():
                 r.set("session:" + session_id, session_tokens, ex=86400)  # TTL 24h
             except Exception: pass
         
-        d = route(rd, tier)
+        d = route(rd, tier, agent)
         queue_start = time.time()
         
         # Queue loop: wait for a GPU slot instead of immediate 503
@@ -293,22 +392,31 @@ def chat():
                 log.warning("QUEUE_TIMEOUT: %s waited %.1fs, all GPUs saturated", agent, elapsed)
                 return resp, 503
             time.sleep(0.5)  # poll every 500ms
-            d = route(rd, tier)
+            d = route(rd, tier, agent)
         
-        waited = time.time() - queue_start
+        queue_ms = (time.time() - queue_start) * 1000
-        if waited > 0.5:
+        if queue_ms > 500:
-            log.info("QUEUED: %s waited %.1fs before slot opened", agent, waited)
+            log.info("QUEUED: %s waited %.0fms before slot opened", agent, queue_ms)
         model, reason, url = d["model"], d["reason"], GPU_URLS[d["model"]]
+        
+        # Stash rate limit values for response headers
+        _rl_remaining = rl_val
+        _rl_limit = RATE_LIMIT_RPM.get(tier, 30)
+        _rl_reset = reset_sec
         is_stream = rd.get("stream", False)
         
         gpu_incr(model)
         
         log.info("ROUTE: %s -> %s (%s) stream=%s active=%d/%d", agent, model, reason, is_stream, gpu_active_count(model), GPU_MAX_CONCURRENT.get(model,1))
+        # Track which GPU this agent is using (TTL 120s covers typical request)
+        if r and agent:
+            try: r.setex("agent_gpu:" + agent + ":" + model, 120, "1")
+            except: pass
         if r:
             try:
                 r.incr("routes:"+model); r.incr("routes:tier:"+tier); r.incr("routes:agent:"+agent)
                 r.incr("ts:"+model+":"+time.strftime("%Y%m%d%H"))
-                r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent}))
+                r.lpush("routes:recent", json.dumps({"ts":time.time(),"model":model,"reason":reason,"tier":tier,"agent":agent,"queue_ms": round(queue_ms,1)}))
                 r.ltrim("routes:recent",0,999)
             except Exception: pass
         start = time.time()
@@ -319,14 +427,47 @@ def chat():
         
         if resp.status_code != 200: return jsonify({"error":"GPU error "+str(resp.status_code)}), 502
         if is_stream:
+            # Buffer SSE chunks, handle split lines for large responses
+            chunks = []
+            stream_timings = {}
+            buf = ""  # accumulate partial lines
+            for raw in resp.iter_content(chunk_size=None, decode_unicode=True):
+                if raw:
+                    cleaned = clean_unicode(raw)
+                    chunks.append(cleaned)
+                    buf += cleaned
+                    # Process complete lines from buffer
+                    while "\n" in buf:
+                        line, buf = buf.split("\n", 1)
+                        line = line.strip()
+                        if line.startswith("data: ") and not stream_timings:
+                            js = line[6:].strip()
+                            if js.startswith("{") and "timings" in js and "predicted_n" in js:
+                                try:
+                                    tj = json.loads(js).get("timings", {})
+                                    if tj:
+                                        stream_timings = tj
+                                except: pass
+            # Store perf record with real token counts from stream
+            if stream_timings:
+                pt = stream_timings.get("prompt_n", 0)
+                ct = stream_timings.get("predicted_n", 0)
+                tps = stream_timings.get("predicted_per_second", 0)
+                gen_ms = stream_timings.get("predicted_ms", lat)
+                store_perf_record(model, agent, tier, reason, queue_ms, gen_ms, pt, ct, True)
+            else:
+                store_perf_record(model, agent, tier, reason, queue_ms, lat, estimate_tokens(rd.get("messages",[])), 0, True)
+            # Yield all chunks to client
             def gen():
-                for raw in resp.iter_content(chunk_size=None, decode_unicode=True):
+                for c in chunks: yield c
-                    if raw: yield clean_unicode(raw)
             bcast()
             ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[])))
             ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100
             ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok"))
             sse_resp = Response(stream_with_context(gen()), mimetype="text/event-stream")
+            sse_resp.headers["X-RateLimit-Limit"] = str(_rl_limit)
+            sse_resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining))
+            sse_resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset))
             sse_resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
             sse_resp.headers["X-Context-Warning"] = ctx_warning
             sse_resp.headers["X-Context-Model"] = model
@@ -336,11 +477,21 @@ def chat():
             msg = c.get("message",{})
             if not msg.get("content") and msg.get("reasoning_content"):
                 msg["content"] = msg["reasoning_content"]
+        # Extract performance data from llama.cpp response
+        usage = data.get("usage", {})
+        timings = data.get("timings", {})
+        prompt_tokens = usage.get("prompt_tokens", 0)
+        completion_tokens = usage.get("completion_tokens", 0)
+        inference_ms = lat  # total GPU round-trip
+        store_perf_record(model, agent, tier, reason, queue_ms, inference_ms, prompt_tokens, completion_tokens, False)
         ctx_remaining = GPU_CONTEXT.get(model, 65536) - max(session_tokens, estimate_tokens(rd.get("messages",[])))
         ctx_pct = ctx_remaining / GPU_CONTEXT.get(model, 65536) * 100
         ctx_warning = "compact_urgent" if ctx_pct < 5 else ("compact_recommended" if ctx_pct < 15 else ("compact_soon" if ctx_pct < 30 else "ok"))
-        data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning}
+        data["routing"] = {"model":model,"reason":reason,"gpu":url,"tier":tier,"agent":agent,"latency_ms":lat,"queue_ms": round(queue_ms,1),"active_gpu":gpu_active_count(model),"context_remaining": max(0, ctx_remaining),"context_pct": round(ctx_pct,1),"context_warning": ctx_warning}
         resp = jsonify(data)
+        resp.headers["X-RateLimit-Limit"] = str(_rl_limit)
+        resp.headers["X-RateLimit-Remaining"] = str(max(0, _rl_remaining))
+        resp.headers["X-RateLimit-Reset"] = str(int(time.time() + _rl_reset))
         resp.headers["X-Context-Remaining"] = str(max(0, ctx_remaining))
         resp.headers["X-Context-Warning"] = ctx_warning
         resp.headers["X-Context-Model"] = model
@@ -355,14 +506,163 @@ def chat():
         log.error("Error: %s\n%s", e, traceback.format_exc())
         return jsonify({"error":str(e)}), 500
 
+@app.route("/metrics/performance")
+def performance():
+    """Per-request performance analytics with percentiles per model/reason/agent."""
+    if not r: return jsonify({"error": "Redis unavailable"}), 503
+    try:
+        window_hours = int(request.args.get("window", "24"))
+        model_filter = request.args.get("model", "all")
+
+        # Load recent records
+        cutoff = time.time() - (window_hours * 3600)
+        raw = r.lrange("perf:recent", 0, -1)
+        records = []
+        for x in raw:
+            try:
+                rec = json.loads(x)
+                if rec["ts"] >= cutoff:
+                    records.append(rec)
+            except: pass
+
+        # Filter by model if specified
+        if model_filter != "all":
+            records = [r for r in records if r["model"] == model_filter]
+
+        if not records:
+            return jsonify({"models": [], "reasons": [], "agents": [], "summary": {"total_requests": 0}})
+
+        def pct(values, p):
+            if len(values) < 2: return round(values[0], 1) if values else 0
+            return round(statistics.quantiles(sorted(values), n=100, method='inclusive')[min(p-1, 98)], 1)
+
+        # Per-model stats
+        model_groups = {}
+        for rec in records:
+            m = rec["model"]
+            if m not in model_groups: model_groups[m] = []
+            model_groups[m].append(rec)
+
+        models = []
+        for m, recs in sorted(model_groups.items()):
+            latencies = [r["total_ms"] for r in recs]
+            tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0]
+            non_stream = [r for r in recs if not r["stream"]]
+            queue_times = [r["queue_ms"] for r in non_stream]
+            models.append({
+                "model": m,
+                "count": len(recs),
+                "stream_pct": round(len([r for r in recs if r["stream"]]) / len(recs) * 100, 1),
+                "latency": {
+                    "avg": round(statistics.mean(latencies), 1),
+                    "p50": pct(latencies, 50),
+                    "p95": pct(latencies, 95),
+                    "p99": pct(latencies, 99)
+                },
+                "throughput": {
+                    "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0,
+                    "p50": pct(tps_vals, 50) if tps_vals else 0,
+                    "p95": pct(tps_vals, 95) if tps_vals else 0,
+                },
+                "queue": {
+                    "avg_ms": round(statistics.mean(queue_times), 1) if queue_times else 0,
+                    "p95_ms": pct(queue_times, 95) if queue_times else 0,
+                } if queue_times else None
+            })
+
+        # Per-reason stats
+        reason_groups = {}
+        for rec in records:
+            rsn = rec["reason"]
+            if rsn not in reason_groups: reason_groups[rsn] = []
+            reason_groups[rsn].append(rec)
+
+        reasons = []
+        for rsn, recs in sorted(reason_groups.items(), key=lambda x: -len(x[1])):
+            latencies = [r["total_ms"] for r in recs]
+            reasons.append({
+                "reason": rsn,
+                "count": len(recs),
+                "avg_total_ms": round(statistics.mean(latencies), 1),
+                "p95_total_ms": pct(latencies, 95)
+            })
+
+        # Per-agent stats
+        agent_groups = {}
+        for rec in records:
+            ag = rec["agent"]
+            if ag not in agent_groups: agent_groups[ag] = []
+            agent_groups[ag].append(rec)
+
+        agents = []
+        for ag, recs in sorted(agent_groups.items(), key=lambda x: -len(x[1])):
+            latencies = [r["total_ms"] for r in recs]
+            tps_vals = [r["tokens_per_sec"] for r in recs if r["tokens_per_sec"] > 0]
+            agents.append({
+                "agent": ag,
+                "count": len(recs),
+                "avg_total_ms": round(statistics.mean(latencies), 1),
+                "avg_tokens_per_sec": round(statistics.mean(tps_vals), 1) if tps_vals else 0
+            })
+
+        all_lat = [r["total_ms"] for r in records]
+        all_tps = [r["tokens_per_sec"] for r in records if r["tokens_per_sec"] > 0]
+        summary = {
+            "total_requests": len(records),
+            "window_hours": window_hours,
+            "latency": {
+                "avg_ms": round(statistics.mean(all_lat), 1),
+                "p50_ms": pct(all_lat, 50),
+                "p95_ms": pct(all_lat, 95),
+                "p99_ms": pct(all_lat, 99)
+            },
+            "throughput_avg_tps": round(statistics.mean(all_tps), 1) if all_tps else 0
+        }
+
+        return jsonify({"models": models, "reasons": reasons, "agents": agents, "summary": summary})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+@app.route("/metrics/scatter")
+def scatter():
+    """Return individual data points for scatter plots (prompt_tokens vs latency)."""
+    if not r: return jsonify({"error": "Redis unavailable"}), 503
+    try:
+        window_hours = int(request.args.get("window", "24"))
+        model_filter = request.args.get("model", "all")
+        cutoff = time.time() - (window_hours * 3600)
+        raw = r.lrange("perf:recent", 0, -1)
+        points = []
+        for x in raw:
+            try:
+                rec = json.loads(x)
+                if rec["ts"] >= cutoff:
+                    if model_filter == "all" or rec["model"] == model_filter:
+                        points.append({
+                            "model": rec["model"],
+                            "agent": rec["agent"],
+                            "reason": rec["reason"],
+                            "prompt_tokens": int(rec.get("prompt_tokens", 0)),
+                            "completion_tokens": rec.get("completion_tokens", 0),
+                            "inference_ms": round(rec["inference_ms"], 1),
+                            "tokens_per_sec": rec.get("tokens_per_sec", 0),
+                            "stream": rec.get("stream", False)
+                        })
+            except: pass
+        return jsonify({"points": points, "count": len(points)})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
 @app.route("/v1/models")
-def models(): return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":check_gpu_health(m).get("status"),"gpu":check_gpu_health(m).get("gpu_name")} for m in GPU_URLS]})
+def models():
+    def _h(m): return check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1)
+    return jsonify({"object":"list","data":[{"id":m,"object":"model","owned_by":"syslog","status":_h(m).get("status"),"gpu":_h(m).get("gpu_name")} for m in GPU_URLS]})
 
 @app.route("/health")
 def health():
     gpus = {}
     for m in GPU_URLS:
-        h = check_gpu_health(m)
+        h = check_gpu_health(m, sidecar_timeout=1.5, gpu_timeout=1)
         h["active_requests"] = gpu_active_count(m)
         h["max_concurrent"] = GPU_MAX_CONCURRENT.get(m, 1)
         gpus[m] = h
@@ -413,6 +713,119 @@ def stream():
     return Response(stream_with_context(ev()), mimetype="text/event-stream",
                     headers={"Cache-Control":"no-cache","X-Accel-Buffering":"no","Access-Control-Allow-Origin":"*"})
 
+# ── Phase 0: Admin Key Management ──
+ADMIN_KEY = os.environ.get("ADMIN_KEY", "")
+
+def _admin_auth():
+    """Require admin key for management endpoints."""
+    if not ADMIN_KEY:
+        return False, "ADMIN_KEY not configured on server"
+    ak = request.headers.get("Authorization","").replace("Bearer ","")
+    if ak != ADMIN_KEY:
+        return False, "Admin key required"
+    return True, None
+
+@app.route("/admin/keys")
+def admin_keys():
+    """List all API keys (masked) with agent, tier, and deprecation status."""
+    ok, err = _admin_auth()
+    if not ok: return jsonify({"error": err}), 401
+    keys = []
+    for key, info in API_KEYS.items():
+        masked = key[:8] + "..." + key[-8:]
+        keys.append({
+            "masked": masked,
+            "prefix": key[:8],
+            "agent": info["agent"],
+            "tier": info["tier"],
+            "deprecated": info.get("deprecated", False),
+            "length": len(key)
+        })
+    return jsonify({
+        "total": len(keys),
+        "active": sum(1 for k in keys if not k["deprecated"]),
+        "deprecated": sum(1 for k in keys if k["deprecated"]),
+        "keys": sorted(keys, key=lambda k: (k["deprecated"], k["agent"]))
+    })
+
+@app.route("/admin/keys/deprecation-summary")
+def admin_deprecation_summary():
+    """Summary of deprecated key usage (from Redis logs, if available)."""
+    ok, err = _admin_auth()
+    if not ok: return jsonify({"error": err}), 401
+    deprecated_agents = []
+    for key, info in API_KEYS.items():
+        if info.get("deprecated"):
+            # Check Redis for usage count
+            count = 0
+            if r:
+                count = int(r.get("deprecated_usage:" + info["agent"]) or 0)
+            deprecated_agents.append({
+                "agent": info["agent"],
+                "deprecated_uses": count,
+                "needs_migration": count > 0
+            })
+    return jsonify({
+        "deprecated_agents": sorted(deprecated_agents, key=lambda d: -d["deprecated_uses"]),
+        "recommendation": "Run POST /admin/keys/revoke to remove keys with 0 usage"
+    })
+
+@app.route("/admin/keys/generate", methods=["POST"])
+def admin_generate_key():
+    """Generate a new API key for an agent. Body: {"agent": "Name", "tier": "enterprise"}"""
+    ok, err = _admin_auth()
+    if not ok: return jsonify({"error": err}), 401
+    body = request.get_json(force=True)
+    agent = body.get("agent", "").strip()
+    tier = body.get("tier", "enterprise")
+    if not agent:
+        return jsonify({"error": "agent field required"}), 400
+    if tier not in ("starter", "professional", "enterprise"):
+        return jsonify({"error": "tier must be starter/professional/enterprise"}), 400
+    # Generate secure key
+    import secrets, hashlib
+    prefix = hashlib.sha256(secrets.token_bytes(12)).hexdigest()[:8]
+    suffix = secrets.token_hex(20)
+    new_key = f"sk-{prefix}-{suffix}"
+    # Update in-memory dict (note: not persisted across restarts without env var update)
+    API_KEYS[new_key] = {"tier": tier, "agent": agent}
+    log.info("KEY_GENERATED: agent=%s tier=%s key=%s...%s", agent, tier, new_key[:8], new_key[-8:])
+    return jsonify({
+        "agent": agent,
+        "tier": tier,
+        "key": new_key,
+        "masked": new_key[:8] + "..." + new_key[-8:],
+        "warning": "This key exists in memory only. Update API_KEYS env var and redeploy to persist."
+    }), 201
+
+@app.route("/admin/keys/revoke", methods=["POST"])
+def admin_revoke_key():
+    """Revoke a deprecated key. Body: {"agent": "Name"} or {"key_prefix": "sk-xxxx"}"""
+    ok, err = _admin_auth()
+    if not ok: return jsonify({"error": err}), 401
+    body = request.get_json(force=True)
+    agent = body.get("agent", "")
+    key_prefix = body.get("key_prefix", "")
+    revoked = []
+    keys_to_remove = []
+    for key, info in API_KEYS.items():
+        if not info.get("deprecated"):
+            continue
+        if agent and info["agent"] == agent:
+            keys_to_remove.append(key)
+        elif key_prefix and key.startswith(key_prefix):
+            keys_to_remove.append(key)
+    for key in keys_to_remove:
+        info = API_KEYS.pop(key)
+        revoked.append({"agent": info["agent"], "masked": key[:8] + "..." + key[-8:]})
+        log.warning("KEY_REVOKED: agent=%s key=%s...%s", info["agent"], key[:8], key[-8:])
+    return jsonify({
+        "revoked": len(revoked),
+        "keys": revoked,
+        "remaining_total": len(API_KEYS),
+        "warning": "Memory-only revoke. Update API_KEYS env var and redeploy to persist."
+    })
+
 if __name__ == "__main__":
     log.info("Router on :9000 (load-aware)")
     app.run(host="0.0.0.0", port=9000, debug=False)