sync: bug fixes + eval system from private repo

- Fix chunk_status: pass chunk_status/chunk/metadata to context builder - Fix vector search: scope by node_id BEFORE similarity search - Add eval logging system (RAH_EVALS_LOG=1) - Add eval dashboard at /evals - Add vitest for testing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-28 16:09:25 +11:00
parent e15f223ed8
commit 2f2ef10ec9
29 changed files with 1489 additions and 50 deletions
@@ -0,0 +1,24 @@
+import { NextRequest } from 'next/server';
+import { upsertEvalComment } from '@/services/evals/evalsStore';
+
+export async function POST(request: NextRequest) {
+  const evalsEnabled = process.env.RAH_EVALS_LOG === '1' || process.env.RAH_EVALS_LOG === 'true';
+  if (process.env.NODE_ENV === 'production' || !evalsEnabled) {
+    return new Response('Not Found', { status: 404 });
+  }
+
+  const body = await request.json();
+  const traceId = typeof body?.traceId === 'string' ? body.traceId : null;
+  const scenarioId = typeof body?.scenarioId === 'string' ? body.scenarioId : null;
+  const comment = typeof body?.comment === 'string' ? body.comment : null;
+
+  if (!traceId || comment === null) {
+    return new Response(JSON.stringify({ error: 'Invalid payload' }), { status: 400 });
+  }
+
+  upsertEvalComment(traceId, scenarioId, comment);
+  return new Response(JSON.stringify({ ok: true }), {
+    status: 200,
+    headers: { 'Content-Type': 'application/json' },
+  });
+}
@@ -0,0 +1,376 @@
+'use client';
+
+import { useMemo, useState } from 'react';
+import type { EvalTrace } from '@/services/evals/evalsStore';
+
+type Props = {
+  traces: EvalTrace[];
+  scenarioList: Array<{
+    id: string;
+    name: string;
+    description?: string;
+    tools?: string[];
+    enabled?: boolean;
+    notes?: string;
+  }>;
+};
+
+function formatPreview(text: string | null, max = 80) {
+  if (!text) return '';
+  const compact = text.replace(/\s+/g, ' ').trim();
+  return compact.length > max ? `${compact.slice(0, max)}...` : compact;
+}
+
+function statusLabel(success: number | null) {
+  if (success === null) return 'n/a';
+  return success ? 'success' : 'fail';
+}
+
+function badgeStyle(kind: 'success' | 'fail' | 'neutral') {
+  const base = { display: 'inline-block', padding: '2px 8px', borderRadius: 999, fontSize: 12 };
+  if (kind === 'success') return { ...base, background: '#e6f4ea', color: '#146c2e' };
+  if (kind === 'fail') return { ...base, background: '#fdecea', color: '#b42318' };
+  return { ...base, background: '#f2f2f2', color: '#333' };
+}
+
+function prettyJson(value: string | null) {
+  if (!value) return '';
+  try {
+    return JSON.stringify(JSON.parse(value), null, 2);
+  } catch {
+    return value;
+  }
+}
+
+export default function EvalsClient({ traces, scenarioList }: Props) {
+  const [openTraceId, setOpenTraceId] = useState<string | null>(traces[0]?.chat.trace_id || null);
+  const [comments, setComments] = useState<Record<string, string>>(() => {
+    const initial: Record<string, string> = {};
+    traces.forEach((trace) => {
+      if (trace.comment) {
+        initial[trace.chat.trace_id] = trace.comment;
+      }
+    });
+    return initial;
+  });
+  const [sourceFilter, setSourceFilter] = useState<string>('all'); // 'all' | 'live' | 'scenario'
+  const [scenarioFilter, setScenarioFilter] = useState<string>('all');
+  const [statusFilter, setStatusFilter] = useState<string>('all');
+  const [searchFilter, setSearchFilter] = useState<string>('');
+
+  const rows = useMemo(() => {
+    return traces.map((trace) => {
+      const { chat, toolCalls } = trace;
+      const status = statusLabel(chat.success);
+      const isLive = !chat.scenario_id;
+      return {
+        trace,
+        id: chat.trace_id,
+        source: isLive ? 'live' : 'scenario',
+        scenario: chat.scenario_id || '—',
+        model: chat.model || 'n/a',
+        latency: chat.latency_ms ?? 'n/a',
+        tokens: `${chat.input_tokens ?? 0}/${chat.output_tokens ?? 0} (${chat.total_tokens ?? 0})`,
+        cost: chat.estimated_cost_usd ?? null,
+        cache: chat.cache_hit == null ? 'n/a' : chat.cache_hit ? 'hit' : 'miss',
+        cacheTokens: `${chat.cache_read_tokens ?? 0}/${chat.cache_write_tokens ?? 0}`,
+        toolCount: toolCalls.length,
+        status,
+        userPreview: formatPreview(chat.user_message),
+        timestamp: chat.ts,
+        mode: chat.mode || 'n/a',
+        workflow: chat.workflow_key || '—',
+      };
+    });
+  }, [traces]);
+
+  const filteredRows = useMemo(() => {
+    return rows.filter((row) => {
+      if (sourceFilter !== 'all' && row.source !== sourceFilter) return false;
+      if (scenarioFilter !== 'all' && row.scenario !== scenarioFilter) return false;
+      if (statusFilter !== 'all' && row.status !== statusFilter) return false;
+      if (searchFilter.trim()) {
+        const needle = searchFilter.toLowerCase();
+        if (!row.userPreview.toLowerCase().includes(needle)) return false;
+      }
+      return true;
+    });
+  }, [rows, sourceFilter, scenarioFilter, statusFilter, searchFilter]);
+
+  const openTrace = traces.find((trace) => trace.chat.trace_id === openTraceId) || traces[0];
+
+  return (
+    <div>
+      <div style={{ marginBottom: 24, border: '1px solid #ddd', borderRadius: 8, background: '#fff', padding: 16 }}>
+        <div style={{ fontWeight: 600, marginBottom: 8 }}>Scenario Set</div>
+        <div style={{ overflowX: 'auto' }}>
+          <table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}>
+            <thead>
+              <tr style={{ textAlign: 'left', borderBottom: '1px solid #ddd' }}>
+                <th style={{ padding: '6px 6px' }}>ID</th>
+                <th style={{ padding: '6px 6px' }}>Name</th>
+                <th style={{ padding: '6px 6px' }}>Description</th>
+                <th style={{ padding: '6px 6px' }}>Tools</th>
+                <th style={{ padding: '6px 6px' }}>Enabled</th>
+                <th style={{ padding: '6px 6px' }}>Notes</th>
+              </tr>
+            </thead>
+            <tbody>
+              {scenarioList.map((scenario) => (
+                <tr key={scenario.id} style={{ borderBottom: '1px solid #eee' }}>
+                  <td style={{ padding: '6px 6px' }}>{scenario.id}</td>
+                  <td style={{ padding: '6px 6px' }}>{scenario.name}</td>
+                  <td style={{ padding: '6px 6px', color: '#555' }}>{scenario.description || '—'}</td>
+                  <td style={{ padding: '6px 6px', color: '#555' }}>{scenario.tools?.join(', ') || '—'}</td>
+                  <td style={{ padding: '6px 6px' }}>{scenario.enabled === false ? 'no' : 'yes'}</td>
+                  <td style={{ padding: '6px 6px', color: '#555' }}>{scenario.notes || '—'}</td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </div>
+      </div>
+
+      <div style={{ overflowX: 'auto' }}>
+        <div style={{ display: 'flex', gap: 12, alignItems: 'center', flexWrap: 'wrap', marginBottom: 12 }}>
+          <label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
+            <span>Source</span>
+            <select
+              value={sourceFilter}
+              onChange={(event) => setSourceFilter(event.target.value)}
+              style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd', fontWeight: 500 }}
+            >
+              <option value="all">All</option>
+              <option value="live">🟢 Live Runs</option>
+              <option value="scenario">🔬 Scenarios</option>
+            </select>
+          </label>
+          <label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
+            <span>Scenario</span>
+            <select
+              value={scenarioFilter}
+              onChange={(event) => setScenarioFilter(event.target.value)}
+              style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd' }}
+            >
+              <option value="all">All</option>
+              {Array.from(new Set(rows.filter(row => row.scenario !== '—').map(row => row.scenario))).map((scenario) => (
+                <option key={scenario} value={scenario}>{scenario}</option>
+              ))}
+            </select>
+          </label>
+          <label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
+            <span>Status</span>
+            <select
+              value={statusFilter}
+              onChange={(event) => setStatusFilter(event.target.value)}
+              style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd' }}
+            >
+              <option value="all">All</option>
+              <option value="success">Success</option>
+              <option value="fail">Fail</option>
+              <option value="n/a">N/A</option>
+            </select>
+          </label>
+          <label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
+            <span>Search</span>
+            <input
+              value={searchFilter}
+              onChange={(event) => setSearchFilter(event.target.value)}
+              placeholder="User input contains..."
+              style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd', minWidth: 240 }}
+            />
+          </label>
+        </div>
+        <table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}>
+          <thead>
+            <tr style={{ textAlign: 'left', borderBottom: '1px solid #ddd' }}>
+              <th style={{ padding: '8px 6px' }}>Source</th>
+              <th style={{ padding: '8px 6px' }}>Scenario</th>
+              <th style={{ padding: '8px 6px' }}>Status</th>
+              <th style={{ padding: '8px 6px' }}>Latency</th>
+              <th style={{ padding: '8px 6px' }}>Tokens</th>
+              <th style={{ padding: '8px 6px' }}>Cost</th>
+              <th style={{ padding: '8px 6px' }}>Cache</th>
+              <th style={{ padding: '8px 6px' }}>Tools</th>
+              <th style={{ padding: '8px 6px' }}>Model</th>
+              <th style={{ padding: '8px 6px' }}>Mode</th>
+              <th style={{ padding: '8px 6px' }}>Workflow</th>
+              <th style={{ padding: '8px 6px' }}>User Input</th>
+              <th style={{ padding: '8px 6px' }}>Time</th>
+              <th style={{ padding: '8px 6px' }}>Comment</th>
+            </tr>
+          </thead>
+          <tbody>
+            {filteredRows.map((row) => {
+              const isOpen = row.id === openTraceId;
+              const statusKind = row.status === 'success' ? 'success' : row.status === 'fail' ? 'fail' : 'neutral';
+              const commentValue = comments[row.id] || '';
+              return (
+                <tr
+                  key={row.id}
+                  onClick={() => setOpenTraceId(row.id)}
+                  style={{
+                    cursor: 'pointer',
+                    background: isOpen ? '#eef4ff' : '#fff',
+                    borderBottom: '1px solid #eee',
+                  }}
+                >
+                  <td style={{ padding: '8px 6px' }}>
+                    <span style={{
+                      ...badgeStyle(row.source === 'live' ? 'success' : 'neutral'),
+                      fontSize: 11,
+                    }}>
+                      {row.source === 'live' ? '🟢 Live' : '🔬 Scenario'}
+                    </span>
+                  </td>
+                  <td style={{ padding: '8px 6px' }}>{row.scenario}</td>
+                  <td style={{ padding: '8px 6px' }}><span style={badgeStyle(statusKind)}>{row.status}</span></td>
+                  <td style={{ padding: '8px 6px' }}>{row.latency} ms</td>
+                  <td style={{ padding: '8px 6px' }}>{row.tokens}</td>
+                  <td style={{ padding: '8px 6px' }}>{row.cost == null ? 'n/a' : `$${row.cost.toFixed(4)}`}</td>
+                  <td style={{ padding: '8px 6px' }}>{row.cache} ({row.cacheTokens})</td>
+                  <td style={{ padding: '8px 6px' }}>{row.toolCount}</td>
+                  <td style={{ padding: '8px 6px' }}>{row.model}</td>
+                  <td style={{ padding: '8px 6px' }}>{row.mode}</td>
+                  <td style={{ padding: '8px 6px' }}>{row.workflow}</td>
+                  <td style={{ padding: '8px 6px', color: '#555' }}>{row.userPreview}</td>
+                  <td style={{ padding: '8px 6px', color: '#666' }}>{row.timestamp}</td>
+                  <td style={{ padding: '8px 6px' }}>
+                    <textarea
+                      value={commentValue}
+                      onChange={(event) => {
+                        const nextValue = event.target.value;
+                        setComments((prev) => ({ ...prev, [row.id]: nextValue }));
+                      }}
+                      onBlur={async (event) => {
+                        const nextValue = event.target.value;
+                        try {
+                          await fetch('/api/evals/comment', {
+                            method: 'POST',
+                            headers: { 'Content-Type': 'application/json' },
+                            body: JSON.stringify({
+                              traceId: row.id,
+                              scenarioId: row.scenario,
+                              comment: nextValue,
+                            }),
+                          });
+                        } catch {
+                          // Ignore failures in the UI, keep local state.
+                        }
+                      }}
+                      onClick={(event) => event.stopPropagation()}
+                      rows={2}
+                      style={{
+                        width: 220,
+                        resize: 'vertical',
+                        fontFamily: 'inherit',
+                        fontSize: 12,
+                        padding: 6,
+                        borderRadius: 6,
+                        border: '1px solid #ddd',
+                      }}
+                      placeholder="Add notes..."
+                    />
+                  </td>
+                </tr>
+              );
+            })}
+          </tbody>
+        </table>
+      </div>
+
+      {openTrace ? (
+        <div style={{ marginTop: 24, border: '1px solid #ddd', borderRadius: 8, background: '#fff', padding: 16 }}>
+          <div style={{ display: 'flex', gap: 16, flexWrap: 'wrap', marginBottom: 12 }}>
+            <div><strong>Source:</strong> {openTrace.chat.scenario_id ? '🔬 Scenario' : '🟢 Live Run'}</div>
+            <div><strong>Scenario:</strong> {openTrace.chat.scenario_id || '—'}</div>
+            <div><strong>Trace:</strong> {openTrace.chat.trace_id}</div>
+            <div><strong>Helper:</strong> {openTrace.chat.helper_name || 'n/a'}</div>
+            <div><strong>Model:</strong> {openTrace.chat.model || 'n/a'}</div>
+            <div><strong>Mode:</strong> {openTrace.chat.mode || 'n/a'}</div>
+            <div><strong>Workflow:</strong> {openTrace.chat.workflow_key || '—'}</div>
+            <div><strong>Latency:</strong> {openTrace.chat.latency_ms ?? 'n/a'} ms</div>
+            <div><strong>Tokens:</strong> {openTrace.chat.input_tokens ?? 0}/{openTrace.chat.output_tokens ?? 0} (total {openTrace.chat.total_tokens ?? 0})</div>
+            <div><strong>Cost:</strong> {openTrace.chat.estimated_cost_usd == null ? 'n/a' : `$${openTrace.chat.estimated_cost_usd.toFixed(4)}`}</div>
+            <div><strong>Cache:</strong> {openTrace.chat.cache_hit == null ? 'n/a' : openTrace.chat.cache_hit ? 'hit' : 'miss'} (read {openTrace.chat.cache_read_tokens ?? 0} / write {openTrace.chat.cache_write_tokens ?? 0})</div>
+            <div><strong>Tools:</strong> {openTrace.toolCalls.length}</div>
+          </div>
+
+          <div style={{ marginBottom: 12 }}>
+            <div><strong>Comment:</strong></div>
+            <div style={{ color: '#555' }}>{comments[openTrace.chat.trace_id] || '—'}</div>
+          </div>
+
+          <details open>
+            <summary style={{ cursor: 'pointer', marginBottom: 8 }}><strong>System Message</strong></summary>
+            <pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.system_message || 'n/a'}</pre>
+          </details>
+
+          <div style={{ marginTop: 16 }}>
+            <div style={{ fontWeight: 600, marginBottom: 8 }}>Trace Steps</div>
+            {[
+              {
+                type: 'chat',
+                ts: openTrace.chat.ts,
+                title: 'LLM Chat',
+                spanId: openTrace.chat.span_id,
+                parentSpanId: null,
+                latency: openTrace.chat.latency_ms,
+                success: openTrace.chat.success,
+                content: (
+                  <>
+                    <div><strong>User</strong></div>
+                    <pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.user_message}</pre>
+                    <div><strong>Assistant</strong></div>
+                    <pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.assistant_message}</pre>
+                  </>
+                ),
+              },
+              ...openTrace.toolCalls.map((tool) => ({
+                type: 'tool',
+                ts: tool.ts,
+                title: `Tool: ${tool.tool_name}`,
+                spanId: tool.span_id,
+                parentSpanId: tool.parent_span_id,
+                latency: tool.latency_ms,
+                success: tool.success,
+                content: (
+                  <>
+                    <div><strong>Args</strong></div>
+                    <pre style={{ whiteSpace: 'pre-wrap' }}>{prettyJson(tool.args_json)}</pre>
+                    <div><strong>Result</strong></div>
+                    <pre style={{ whiteSpace: 'pre-wrap' }}>{prettyJson(tool.result_json)}</pre>
+                    {tool.error ? (
+                      <>
+                        <div><strong>Error</strong></div>
+                        <pre style={{ whiteSpace: 'pre-wrap' }}>{tool.error}</pre>
+                      </>
+                    ) : null}
+                  </>
+                ),
+              })),
+            ].sort((a, b) => a.ts.localeCompare(b.ts)).map((step, index) => {
+              const stepStatus = statusLabel(step.success);
+              const stepKind = stepStatus === 'success' ? 'success' : stepStatus === 'fail' ? 'fail' : 'neutral';
+              return (
+                <div key={`${openTrace.chat.trace_id}-${step.type}-${index}`} style={{ borderTop: '1px solid #eee', paddingTop: 12, marginTop: 12 }}>
+                  <div style={{ display: 'flex', gap: 12, alignItems: 'center', flexWrap: 'wrap' }}>
+                    <div><strong>Step {index + 1}:</strong> {step.title}</div>
+                    <span style={badgeStyle(stepKind)}>{stepStatus}</span>
+                    <div style={{ color: '#666' }}>{step.ts}</div>
+                  </div>
+                  <div style={{ marginTop: 6, color: '#555' }}>
+                    <div><strong>Span:</strong> {step.spanId || 'n/a'}</div>
+                    {step.parentSpanId ? <div><strong>Parent Span:</strong> {step.parentSpanId}</div> : null}
+                    <div><strong>Latency:</strong> {step.latency ?? 'n/a'} ms</div>
+                  </div>
+                  <div style={{ marginTop: 8 }}>{step.content}</div>
+                </div>
+              );
+            })}
+          </div>
+        </div>
+      ) : null}
+    </div>
+  );
+}
@@ -0,0 +1,39 @@
+import { notFound } from 'next/navigation';
+import { loadEvalTraces } from '@/services/evals/evalsStore';
+import EvalsClient from './EvalsClient';
+import { scenarios } from '../../tests/evals/scenarios';
+
+export const dynamic = 'force-dynamic';
+
+export default function EvalsPage() {
+  const evalsEnabled = process.env.RAH_EVALS_LOG === '1' || process.env.RAH_EVALS_LOG === 'true';
+  if (process.env.NODE_ENV === 'production' || !evalsEnabled) {
+    notFound();
+  }
+
+  const traces = loadEvalTraces(25);
+  const scenarioList = scenarios.map((scenario) => ({
+    id: scenario.id,
+    name: scenario.name,
+    description: scenario.description,
+    tools: scenario.tools,
+    enabled: scenario.enabled,
+    notes: scenario.notes,
+  }));
+
+  return (
+    <div style={{ height: '100vh', overflowY: 'auto', background: '#f5f5f5', color: '#111' }}>
+      <div style={{ padding: 24, maxWidth: 1200, margin: '0 auto', fontFamily: 'ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace' }}>
+        <div style={{ position: 'sticky', top: 0, background: '#f5f5f5', paddingBottom: 12, marginBottom: 12, zIndex: 1 }}>
+          <h1 style={{ fontSize: 24, marginBottom: 6 }}>Eval Traces</h1>
+          <div style={{ color: '#444' }}>Trace table on top, full span detail below.</div>
+        </div>
+        {traces.length === 0 ? (
+          <p>No eval traces found. Run evals with RAH_EVALS_LOG=1.</p>
+        ) : (
+          <EvalsClient traces={traces} scenarioList={scenarioList} />
+        )}
+      </div>
+    </div>
+  );
+}
@@ -15,7 +15,10 @@
    "type-check": "tsc --noEmit",
    "setup": "npm install",
    "sqlite:backup": "bash scripts/database/sqlite-backup.sh",
-    "sqlite:restore": "bash scripts/database/sqlite-restore.sh"
+    "sqlite:restore": "bash scripts/database/sqlite-restore.sh",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "evals": "RAH_EVALS_LOG=1 tsx tests/evals/runner.ts"
  },
  "dependencies": {
    "@ai-sdk/anthropic": "^2.0.27",
@@ -62,6 +65,8 @@
    "prettier": "^3.3.3",
    "tailwindcss": "^3.4.17",
    "tailwindcss-animate": "^1.0.7",
-    "typescript": "^5.7.2"
+    "tsx": "^4.19.4",
+    "typescript": "^5.7.2",
+    "vitest": "^3.2.4"
  }
 }
@@ -75,7 +75,10 @@ export default function ThreePanelLayout() {
        content: node.content,
        dimensions: node.dimensions,
        created_at: node.created_at,
-        updated_at: node.updated_at
+        updated_at: node.updated_at,
+        chunk_status: node.chunk_status,
+        chunk: node.chunk,
+        metadata: node.metadata,
      }));
      setOpenTabsData(validNodes);
    } catch (error) {
@@ -12,6 +12,10 @@ interface RequestContext {
    openai?: string;
    anthropic?: string;
  };
+  helperName?: string;
+  evalDatasetId?: string;
+  evalScenarioId?: string;
+  evalChatSpanId?: string;
 }

 let currentContext: RequestContext = {};
@@ -176,29 +176,51 @@ export class ChunkService {
    nodeIds?: number[]
  ): Promise<Array<Chunk & { similarity: number }>> {
    const sqlite = getSQLiteClient();
+    const startTime = Date.now();
+    const vectorString = `[${queryEmbedding.join(',')}]`;

-    // Step 1: Determine vector search limit - more conservative approach
-    let vectorLimit = 50; // Start smaller for efficiency
-    
+    // When searching specific nodes, first get their chunk_ids to scope the vector search
    if (nodeIds && nodeIds.length > 0) {
-      // When searching specific nodes, get exact count and use reasonable multiplier
-      const candidateCountQuery = `SELECT COUNT(*) as count FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
-      const candidateResult = sqlite.query<{count: number}>(candidateCountQuery, nodeIds);
-      const candidateCount = Number(candidateResult.rows[0].count);
+      // Get chunk IDs for the target nodes
+      const chunkIdsQuery = `SELECT id FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
+      const chunkIdsResult = sqlite.query<{id: number}>(chunkIdsQuery, nodeIds);
+      const chunkIds = chunkIdsResult.rows.map(r => r.id);

-      // Use 2x the candidate count but cap at reasonable limits
-      vectorLimit = Math.min(Math.max(candidateCount * 2, 50), 1000);
-      console.log(`🔍 Node-scoped search: ${candidateCount} candidates, using vector limit ${vectorLimit}`);
-    } else {
-      // For global search, use adaptive limit based on expected results
-      vectorLimit = Math.max(matchCount * 10, 50);
+      if (chunkIds.length === 0) {
+        console.log(`🔍 Node-scoped search: no chunks found for nodes ${nodeIds.join(', ')}`);
+        return [];
+      }
+
+      console.log(`🔍 Node-scoped search: ${chunkIds.length} chunks in nodes ${nodeIds.join(', ')}`);
+
+      // Search ONLY within those chunks using rowid filter
+      const query = `
+        SELECT c.*, (1.0 / (1.0 + v.distance)) AS similarity
+        FROM vec_chunks v
+        JOIN chunks c ON c.id = v.chunk_id
+        WHERE v.embedding MATCH ?
+          AND v.chunk_id IN (${chunkIds.map(() => '?').join(',')})
+          AND (1.0 / (1.0 + v.distance)) >= ?
+        ORDER BY v.distance
+        LIMIT ?
+      `;
+
+      const params = [vectorString, ...chunkIds, similarityThreshold, matchCount];
+      const result = sqlite.query<Chunk & { similarity: number }>(query, params);
+      const searchTime = Date.now() - startTime;
+
+      console.log(`📊 Vector search (node-scoped): ${result.rows.length} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
+      if (result.rows.length > 0) {
+        console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
+      }
+
+      return result.rows;
    }

-    const startTime = Date.now();
+    // Global search (no node filter)
+    const vectorLimit = Math.max(matchCount * 10, 50);

-    // Step 2: Use CTE-based query to avoid UNION ALL explosion
-    const vectorString = `[${queryEmbedding.join(',')}]`;
-    let query = `
+    const query = `
      WITH vector_results AS (
        SELECT chunk_id, distance
        FROM vec_chunks
@@ -209,32 +231,16 @@ export class ChunkService {
      SELECT c.*, (1.0 / (1.0 + vr.distance)) AS similarity
      FROM vector_results vr
      JOIN chunks c ON c.id = vr.chunk_id
+      WHERE (1.0 / (1.0 + vr.distance)) >= ?
+      ORDER BY similarity DESC
+      LIMIT ?
    `;

-    const params: any[] = [vectorString, vectorLimit];
-    const conditions = [];
-    
-    // Node ID filter if provided
-    if (nodeIds && nodeIds.length > 0) {
-      conditions.push(`c.node_id IN (${nodeIds.map(() => '?').join(',')})`);
-      params.push(...nodeIds);
-    }
-    
-    // Similarity threshold filter
-    conditions.push(`(1.0 / (1.0 + vr.distance)) >= ?`);
-    params.push(similarityThreshold);
-    
-    if (conditions.length > 0) {
-      query += ` WHERE ${conditions.join(' AND ')}`;
-    }
-
-    query += ` ORDER BY similarity DESC LIMIT ?`;
-    params.push(matchCount);
-
+    const params = [vectorString, vectorLimit, similarityThreshold, matchCount];
    const result = sqlite.query<Chunk & { similarity: number }>(query, params);
    const searchTime = Date.now() - startTime;

-    console.log(`📊 Vector search: ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
+    console.log(`📊 Vector search (global): ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
    if (result.rows.length > 0) {
      console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
    }
@@ -0,0 +1,269 @@
+import Database from 'better-sqlite3';
+import fs from 'fs';
+import path from 'path';
+import { randomUUID } from 'crypto';
+import { RequestContext } from '@/services/context/requestContext';
+
+type EvalToolCallLog = {
+  toolName: string;
+  args?: unknown;
+  result?: unknown;
+  error?: unknown;
+  latencyMs?: number;
+};
+
+type EvalChatLog = {
+  traceId?: string;
+  spanId?: string;
+  helperName?: string;
+  model?: string;
+  promptVersion?: string;
+  systemMessage?: string | null;
+  userMessage?: string | null;
+  assistantMessage?: string | null;
+  inputTokens?: number;
+  outputTokens?: number;
+  totalTokens?: number;
+  cacheWriteTokens?: number;
+  cacheReadTokens?: number;
+  cacheHit?: boolean;
+  cacheSavingsPct?: number;
+  estimatedCostUsd?: number;
+  provider?: string | null;
+  mode?: string | null;
+  workflowKey?: string | null;
+  workflowNodeId?: number | null;
+  latencyMs?: number;
+  success?: boolean;
+  error?: string | null;
+};
+
+const EVALS_LOG_FLAG = process.env.RAH_EVALS_LOG;
+const EVALS_LOG_ENABLED = EVALS_LOG_FLAG === '1' || EVALS_LOG_FLAG === 'true';
+const LOG_DIR = path.join(process.cwd(), 'logs');
+const DB_PATH = path.join(LOG_DIR, 'evals.sqlite');
+
+let evalsDb: Database.Database | null = null;
+
+function shouldLogEvals() {
+  // Log ALL interactions when RAH_EVALS_LOG=1
+  // - Real app interactions: scenario_id will be NULL
+  // - Synthetic scenarios: scenario_id will be set
+  return EVALS_LOG_ENABLED;
+}
+
+function ensureSchema(db: Database.Database) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS tool_calls (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      ts TEXT NOT NULL,
+      trace_id TEXT,
+      span_id TEXT,
+      parent_span_id TEXT,
+      helper_name TEXT,
+      tool_name TEXT NOT NULL,
+      args_json TEXT,
+      result_json TEXT,
+      success INTEGER,
+      latency_ms INTEGER,
+      error TEXT,
+      dataset_id TEXT,
+      scenario_id TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_tool_calls_trace ON tool_calls(trace_id);
+    CREATE INDEX IF NOT EXISTS idx_tool_calls_scenario ON tool_calls(scenario_id);
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS llm_chats (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      ts TEXT NOT NULL,
+      trace_id TEXT,
+      span_id TEXT,
+      helper_name TEXT,
+      model TEXT,
+      prompt_version TEXT,
+      system_message TEXT,
+      user_message TEXT,
+      assistant_message TEXT,
+      input_tokens INTEGER,
+      output_tokens INTEGER,
+      total_tokens INTEGER,
+      cache_write_tokens INTEGER,
+      cache_read_tokens INTEGER,
+      cache_hit INTEGER,
+      cache_savings_pct INTEGER,
+      estimated_cost_usd REAL,
+      provider TEXT,
+      mode TEXT,
+      workflow_key TEXT,
+      workflow_node_id INTEGER,
+      latency_ms INTEGER,
+      success INTEGER,
+      error TEXT,
+      dataset_id TEXT,
+      scenario_id TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_llm_chats_trace ON llm_chats(trace_id);
+    CREATE INDEX IF NOT EXISTS idx_llm_chats_scenario ON llm_chats(scenario_id);
+  `);
+
+  const columns = db.prepare(`PRAGMA table_info(llm_chats);`).all() as { name: string }[];
+  const columnNames = new Set(columns.map(column => column.name));
+  if (!columnNames.has('system_message')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN system_message TEXT;`);
+  }
+  if (!columnNames.has('cache_write_tokens')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_write_tokens INTEGER;`);
+  }
+  if (!columnNames.has('cache_read_tokens')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_read_tokens INTEGER;`);
+  }
+  if (!columnNames.has('cache_hit')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_hit INTEGER;`);
+  }
+  if (!columnNames.has('cache_savings_pct')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_savings_pct INTEGER;`);
+  }
+  if (!columnNames.has('estimated_cost_usd')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN estimated_cost_usd REAL;`);
+  }
+  if (!columnNames.has('provider')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN provider TEXT;`);
+  }
+  if (!columnNames.has('mode')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN mode TEXT;`);
+  }
+  if (!columnNames.has('workflow_key')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN workflow_key TEXT;`);
+  }
+  if (!columnNames.has('workflow_node_id')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN workflow_node_id INTEGER;`);
+  }
+}
+
+function getDb() {
+  if (!EVALS_LOG_ENABLED) return null;
+  if (evalsDb) return evalsDb;
+
+  fs.mkdirSync(LOG_DIR, { recursive: true });
+  evalsDb = new Database(DB_PATH);
+  evalsDb.pragma('journal_mode = WAL');
+  evalsDb.pragma('synchronous = NORMAL');
+  evalsDb.pragma('busy_timeout = 3000');
+  ensureSchema(evalsDb);
+  return evalsDb;
+}
+
+function stringifySafe(value: unknown) {
+  if (value === undefined) return null;
+  try {
+    return JSON.stringify(value);
+  } catch (error) {
+    return JSON.stringify({
+      error: 'Failed to serialize payload',
+      message: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+export function logEvalToolCall(entry: EvalToolCallLog) {
+  if (!shouldLogEvals()) return;
+  const context = RequestContext.get();
+  const traceId = context.traceId;
+  if (!traceId) return;
+
+  const db = getDb();
+  if (!db) return;
+
+  const now = new Date().toISOString();
+  const spanId = randomUUID();
+  const parentSpanId = context.evalChatSpanId || null;
+  const helperName = context.helperName || null;
+  const datasetId = context.evalDatasetId || null;
+  const scenarioId = context.evalScenarioId || null;
+  const errorMessage = entry.error instanceof Error
+    ? entry.error.message
+    : entry.error
+      ? String(entry.error)
+      : null;
+  const success = typeof entry.error === 'undefined'
+    ? entry.result && typeof (entry.result as any).success === 'boolean'
+      ? ((entry.result as any).success ? 1 : 0)
+      : 1
+    : 0;
+
+  db.prepare(`
+    INSERT INTO tool_calls (
+      ts, trace_id, span_id, parent_span_id, helper_name,
+      tool_name, args_json, result_json, success, latency_ms, error,
+      dataset_id, scenario_id
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    now,
+    traceId,
+    spanId,
+    parentSpanId,
+    helperName,
+    entry.toolName,
+    stringifySafe(entry.args),
+    stringifySafe(entry.result),
+    success,
+    entry.latencyMs ?? null,
+    errorMessage,
+    datasetId,
+    scenarioId
+  );
+}
+
+export function logEvalChat(entry: EvalChatLog) {
+  if (!shouldLogEvals()) return;
+  const context = RequestContext.get();
+  const traceId = entry.traceId || context.traceId;
+  if (!traceId) return;
+
+  const db = getDb();
+  if (!db) return;
+
+  const now = new Date().toISOString();
+  const spanId = entry.spanId || context.evalChatSpanId || randomUUID();
+  const datasetId = context.evalDatasetId || null;
+  const scenarioId = context.evalScenarioId || null;
+
+  db.prepare(`
+    INSERT INTO llm_chats (
+      ts, trace_id, span_id, helper_name, model, prompt_version, system_message,
+      user_message, assistant_message, input_tokens, output_tokens, total_tokens,
+      cache_write_tokens, cache_read_tokens, cache_hit, cache_savings_pct,
+      estimated_cost_usd, provider, mode, workflow_key, workflow_node_id,
+      latency_ms, success, error, dataset_id, scenario_id
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    now,
+    traceId,
+    spanId,
+    entry.helperName ?? null,
+    entry.model ?? null,
+    entry.promptVersion ?? null,
+    entry.systemMessage ?? null,
+    entry.userMessage ?? null,
+    entry.assistantMessage ?? null,
+    entry.inputTokens ?? null,
+    entry.outputTokens ?? null,
+    entry.totalTokens ?? null,
+    entry.cacheWriteTokens ?? null,
+    entry.cacheReadTokens ?? null,
+    typeof entry.cacheHit === 'boolean' ? (entry.cacheHit ? 1 : 0) : null,
+    entry.cacheSavingsPct ?? null,
+    entry.estimatedCostUsd ?? null,
+    entry.provider ?? null,
+    entry.mode ?? null,
+    entry.workflowKey ?? null,
+    entry.workflowNodeId ?? null,
+    entry.latencyMs ?? null,
+    typeof entry.success === 'boolean' ? (entry.success ? 1 : 0) : null,
+    entry.error ?? null,
+    datasetId,
+    scenarioId
+  );
+}
@@ -0,0 +1,141 @@
+import Database from 'better-sqlite3';
+import fs from 'fs';
+import path from 'path';
+
+export type EvalChatRow = {
+  id: number;
+  ts: string;
+  trace_id: string;
+  span_id: string | null;
+  helper_name: string | null;
+  model: string | null;
+  prompt_version: string | null;
+  system_message: string | null;
+  user_message: string | null;
+  assistant_message: string | null;
+  input_tokens: number | null;
+  output_tokens: number | null;
+  total_tokens: number | null;
+  cache_write_tokens: number | null;
+  cache_read_tokens: number | null;
+  cache_hit: number | null;
+  cache_savings_pct: number | null;
+  estimated_cost_usd: number | null;
+  provider: string | null;
+  mode: string | null;
+  workflow_key: string | null;
+  workflow_node_id: number | null;
+  latency_ms: number | null;
+  success: number | null;
+  error: string | null;
+  dataset_id: string | null;
+  scenario_id: string | null;
+};
+
+export type EvalToolCallRow = {
+  id: number;
+  ts: string;
+  trace_id: string;
+  span_id: string | null;
+  parent_span_id: string | null;
+  helper_name: string | null;
+  tool_name: string;
+  args_json: string | null;
+  result_json: string | null;
+  success: number | null;
+  latency_ms: number | null;
+  error: string | null;
+  dataset_id: string | null;
+  scenario_id: string | null;
+};
+
+export type EvalTrace = {
+  chat: EvalChatRow;
+  toolCalls: EvalToolCallRow[];
+  comment: string | null;
+};
+
+const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
+
+function ensureCommentSchema(db: Database.Database) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS eval_comments (
+      trace_id TEXT PRIMARY KEY,
+      scenario_id TEXT,
+      comment TEXT,
+      updated_at TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_eval_comments_scenario ON eval_comments(scenario_id);
+  `);
+}
+
+export function upsertEvalComment(traceId: string, scenarioId: string | null, comment: string) {
+  if (!fs.existsSync(LOG_DB_PATH)) {
+    return;
+  }
+  const db = new Database(LOG_DB_PATH);
+  ensureCommentSchema(db);
+  const now = new Date().toISOString();
+  db.prepare(`
+    INSERT INTO eval_comments (trace_id, scenario_id, comment, updated_at)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(trace_id) DO UPDATE SET
+      comment = excluded.comment,
+      updated_at = excluded.updated_at,
+      scenario_id = excluded.scenario_id
+  `).run(traceId, scenarioId, comment, now);
+}
+
+export function loadEvalTraces(limit = 25): EvalTrace[] {
+  if (!fs.existsSync(LOG_DB_PATH)) {
+    return [];
+  }
+
+  const db = new Database(LOG_DB_PATH, { readonly: true, fileMustExist: true });
+  const chats = db.prepare(`
+    SELECT *
+    FROM llm_chats
+    ORDER BY ts DESC
+    LIMIT ?
+  `).all(limit) as EvalChatRow[];
+
+  if (chats.length === 0) return [];
+
+  const traceIds = Array.from(new Set(chats.map(chat => chat.trace_id)));
+  const placeholders = traceIds.map(() => '?').join(', ');
+  const toolCalls = db.prepare(`
+    SELECT *
+    FROM tool_calls
+    WHERE trace_id IN (${placeholders})
+    ORDER BY ts ASC
+  `).all(...traceIds) as EvalToolCallRow[];
+
+  const toolCallsByTrace = new Map<string, EvalToolCallRow[]>();
+  toolCalls.forEach(call => {
+    const list = toolCallsByTrace.get(call.trace_id) || [];
+    list.push(call);
+    toolCallsByTrace.set(call.trace_id, list);
+  });
+
+  const commentsByTrace = new Map<string, string | null>();
+  if (placeholders.length > 0) {
+    try {
+      const commentRows = db.prepare(`
+        SELECT trace_id, comment
+        FROM eval_comments
+        WHERE trace_id IN (${placeholders})
+      `).all(...traceIds) as Array<{ trace_id: string; comment: string | null }>;
+      commentRows.forEach(row => {
+        commentsByTrace.set(row.trace_id, row.comment ?? null);
+      });
+    } catch {
+      // Comments table doesn't exist yet in readonly mode.
+    }
+  }
+
+  return chats.map(chat => ({
+    chat,
+    toolCalls: toolCallsByTrace.get(chat.trace_id) || [],
+    comment: commentsByTrace.get(chat.trace_id) ?? null,
+  }));
+}
@@ -0,0 +1,5 @@
+{
+  "id": "golden-v1",
+  "name": "Golden Dataset v1",
+  "description": "Baseline eval scenarios for core RA-H helper behavior."
+}
@@ -0,0 +1,258 @@
+import fs from 'fs';
+import path from 'path';
+import Database from 'better-sqlite3';
+import { randomUUID } from 'crypto';
+import { scenarios } from './scenarios';
+import { Scenario } from './types';
+
+type EvalChatRow = {
+  trace_id: string;
+  scenario_id: string;
+  assistant_message: string | null;
+  latency_ms: number | null;
+  input_tokens: number | null;
+  output_tokens: number | null;
+  total_tokens: number | null;
+};
+
+type EvalToolCallRow = {
+  tool_name: string;
+};
+
+type EvalResult = {
+  scenario: string;
+  passed: boolean;
+  failures: string[];
+  warnings: string[];
+  latencyMs?: number | null;
+};
+
+const BASE_URL = process.env.RAH_EVALS_BASE_URL || 'http://localhost:3000';
+const DATASET_ENV = process.env.RAH_EVALS_DATASET_ID;
+const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
+const RAH_DB_PATH = process.env.SQLITE_DB_PATH || path.join(
+  process.env.HOME || '~',
+  'Library/Application Support/RA-H/db/rah.sqlite'
+);
+
+function loadDatasetId() {
+  if (DATASET_ENV) return DATASET_ENV;
+  const datasetPath = path.join(process.cwd(), 'tests', 'evals', 'dataset.json');
+  const raw = fs.readFileSync(datasetPath, 'utf-8');
+  const parsed = JSON.parse(raw);
+  return parsed.id || 'default';
+}
+
+function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): number | null {
+  if (!query) return null;
+  if (!fs.existsSync(RAH_DB_PATH)) return null;
+  const db = new Database(RAH_DB_PATH, { readonly: true, fileMustExist: true });
+  if (query.titleEquals) {
+    const row = db.prepare(`
+      SELECT id
+      FROM nodes
+      WHERE title = ?
+      ORDER BY datetime(created_at) DESC
+      LIMIT 1
+    `).get(query.titleEquals) as { id?: number } | undefined;
+    return row?.id ?? null;
+  }
+  if (query.titleContains) {
+    const row = db.prepare(`
+      SELECT id
+      FROM nodes
+      WHERE title LIKE ?
+      ORDER BY datetime(created_at) DESC
+      LIMIT 1
+    `).get(`%${query.titleContains}%`) as { id?: number } | undefined;
+    return row?.id ?? null;
+  }
+  return null;
+}
+
+async function drainResponse(response: Response) {
+  if (!response.body) return;
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    decoder.decode(value, { stream: true });
+  }
+}
+
+function openEvalDb() {
+  if (!fs.existsSync(LOG_DB_PATH)) {
+    return null;
+  }
+  return new Database(LOG_DB_PATH, { readonly: true, fileMustExist: true });
+}
+
+function getEvalChatRow(db: Database.Database, traceId: string, scenarioId: string) {
+  return db.prepare(`
+    SELECT trace_id, scenario_id, assistant_message, latency_ms, input_tokens, output_tokens, total_tokens
+    FROM llm_chats
+    WHERE trace_id = ? AND scenario_id = ?
+    ORDER BY id DESC
+    LIMIT 1
+  `).get(traceId, scenarioId) as EvalChatRow | undefined;
+}
+
+function getEvalToolCalls(db: Database.Database, traceId: string, scenarioId: string) {
+  return db.prepare(`
+    SELECT tool_name
+    FROM tool_calls
+    WHERE trace_id = ? AND scenario_id = ?
+  `).all(traceId, scenarioId) as EvalToolCallRow[];
+}
+
+async function waitForEvalRow(traceId: string, scenarioId: string, timeoutMs = 10000) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    const db = openEvalDb();
+    if (db) {
+      const row = getEvalChatRow(db, traceId, scenarioId);
+      if (row) return { row, db };
+    }
+    await new Promise(resolve => setTimeout(resolve, 200));
+  }
+  return { row: undefined, db: null };
+}
+
+function normalizeContains(text: string) {
+  return text.toLowerCase();
+}
+
+function checkScenario(
+  scenario: Scenario,
+  chatRow: EvalChatRow,
+  toolCalls: EvalToolCallRow[]
+): EvalResult {
+  const failures: string[] = [];
+  const warnings: string[] = [];
+  const expect = scenario.expect || {};
+  const toolNames = toolCalls.map(call => call.tool_name);
+  const responseText = chatRow.assistant_message || '';
+
+  (expect.toolsCalled || []).forEach(tool => {
+    if (!toolNames.includes(tool)) {
+      failures.push(`Expected tool "${tool}" not called`);
+    }
+  });
+
+  (expect.toolsCalledSoft || []).forEach(tool => {
+    if (!toolNames.includes(tool)) {
+      warnings.push(`(soft) Expected tool "${tool}" not called`);
+    }
+  });
+
+  (expect.responseContains || []).forEach(text => {
+    if (!normalizeContains(responseText).includes(normalizeContains(text))) {
+      failures.push(`Response missing "${text}"`);
+    }
+  });
+
+  (expect.responseContainsSoft || []).forEach(text => {
+    if (!normalizeContains(responseText).includes(normalizeContains(text))) {
+      warnings.push(`(soft) Response missing "${text}"`);
+    }
+  });
+
+  (expect.responseNotContains || []).forEach(text => {
+    if (normalizeContains(responseText).includes(normalizeContains(text))) {
+      failures.push(`Response should not contain "${text}"`);
+    }
+  });
+
+  if (typeof expect.maxLatencyMs === 'number' && chatRow.latency_ms !== null) {
+    if (chatRow.latency_ms > expect.maxLatencyMs) {
+      failures.push(`Latency ${chatRow.latency_ms}ms exceeded ${expect.maxLatencyMs}ms`);
+    }
+  }
+
+  return {
+    scenario: scenario.name,
+    passed: failures.length === 0,
+    failures,
+    warnings,
+    latencyMs: chatRow.latency_ms,
+  };
+}
+
+async function runScenario(scenario: Scenario, datasetId: string): Promise<EvalResult> {
+  const traceId = `eval_${Date.now()}_${randomUUID().slice(0, 8)}`;
+  const resolvedFocusedNodeId =
+    scenario.input.focusedNodeId ?? resolveFocusedNodeId(scenario.input.focusedNodeQuery);
+  const response = await fetch(`${BASE_URL}/api/rah/chat`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      messages: [{ role: 'user', content: scenario.input.message }],
+      openTabs: [],
+      activeTabId: resolvedFocusedNodeId ?? null,
+      currentView: 'nodes',
+      mode: scenario.input.mode ?? 'easy',
+      traceId,
+      evals: {
+        datasetId,
+        scenarioId: scenario.id,
+      },
+    }),
+  });
+
+  if (!response.ok) {
+    return {
+      scenario: scenario.name,
+      passed: false,
+      failures: [`HTTP ${response.status} from /api/rah/chat`],
+      warnings: [],
+    };
+  }
+
+  await drainResponse(response);
+
+  const { row, db } = await waitForEvalRow(traceId, scenario.id);
+  if (!row) {
+    return {
+      scenario: scenario.name,
+      passed: false,
+      failures: ['Timed out waiting for eval logs'],
+      warnings: [],
+    };
+  }
+
+  const toolCalls = db ? getEvalToolCalls(db, traceId, scenario.id) : [];
+  return checkScenario(scenario, row, toolCalls);
+}
+
+async function runAll() {
+  const datasetId = loadDatasetId();
+  console.log(`Running ${scenarios.length} scenarios (dataset: ${datasetId})...\n`);
+
+  const results: EvalResult[] = [];
+  for (const scenario of scenarios.filter(s => s.enabled !== false)) {
+    const result = await runScenario(scenario, datasetId);
+    results.push(result);
+    const icon = result.passed ? '✓' : '✗';
+    const latency = result.latencyMs ? ` (${result.latencyMs}ms)` : '';
+    console.log(`${icon} ${result.scenario}${latency}`);
+    result.failures.forEach(failure => console.log(`  - ${failure}`));
+    result.warnings.forEach(warning => console.log(`  - ${warning}`));
+  }
+
+  const failed = results.filter(result => !result.passed);
+  const warnings = results.filter(result => result.warnings.length > 0);
+  console.log('\nSummary');
+  console.log(`- Passed: ${results.length - failed.length}`);
+  console.log(`- Failed: ${failed.length}`);
+  console.log(`- With warnings: ${warnings.length}`);
+
+  if (failed.length > 0) {
+    process.exit(1);
+  }
+}
+
+runAll().catch(error => {
+  console.error('Eval runner failed:', error);
+  process.exit(1);
+});
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'create-edge',
+  name: 'Create edge between two new nodes',
+  description: 'Create two nodes and connect them with an edge.',
+  tools: ['createNode', 'createEdge'],
+  input: {
+    message: 'Create nodes titled "Eval Edge A" and "Eval Edge B" (one sentence each), then create an edge from A to B labeled "related".',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'createEdge'],
+    responseContainsSoft: ['edge', 'connected'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'create-node',
+  name: 'Create node request',
+  description: 'Create a new node via chat.',
+  tools: ['createNode'],
+  input: {
+    message: 'Create a node titled "Eval: Test Node" with a short summary about evals.',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode'],
+    responseContainsSoft: ['node', 'created', 'added'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'delegate-mini',
+  name: 'Delegate to Mini RAH',
+  description: 'Ask the system to delegate a short task to mini helper.',
+  tools: ['delegateToMiniRAH'],
+  input: {
+    message: 'Delegate to mini RAH: summarize my notes on plaintext productivity in 3 bullets.',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['delegateToMiniRAH'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'delegate-wise',
+  name: 'Delegate to Wise RAH',
+  description: 'Ask the system to delegate a research comparison task.',
+  tools: ['delegateToWiseRAH'],
+  input: {
+    message: 'Delegate to wise RAH: compare SQLite vs markdown storage for PKM in 5 bullets.',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['delegateToWiseRAH'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'dimension-lifecycle',
+  name: 'Dimension lifecycle (create/update/lock/unlock)',
+  description: 'Create a dimension, update its description, lock it, then unlock it.',
+  tools: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
+  input: {
+    message: 'Create a dimension named "eval-dim" with description "temporary eval dimension", then update the description to "eval dimension updated", then lock it, then unlock it.',
+  },
+  expect: {
+    toolsCalledSoft: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
+    responseContainsSoft: ['eval-dim'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'get-dimension',
+  name: 'Get single dimension',
+  description: 'Fetch detail for a known dimension.',
+  tools: ['getDimension'],
+  input: {
+    message: 'Get details for the dimension "ai".',
+  },
+  expect: {
+    toolsCalledSoft: ['getDimension'],
+    responseContainsSoft: ['ai'],
+  },
+};
@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'hard-mode-query',
+  name: 'Hard mode retrieval query',
+  description: 'Run a baseline retrieval query in hard mode.',
+  tools: ['queryNodes', 'searchContentEmbeddings'],
+  input: {
+    message: 'What have I captured about plaintext productivity and tools?',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    responseContainsSoft: ['plaintext', 'productivity'],
+  },
+};
@@ -0,0 +1,33 @@
+import { scenario as simpleQuery } from './simple-query';
+import { scenario as searchEmbeddings } from './search-embeddings';
+import { scenario as createNode } from './create-node';
+import { scenario as hardModeQuery } from './hard-mode-query';
+import { scenario as workflowIntegrate } from './workflow-integrate';
+import { scenario as updateNode } from './update-node';
+import { scenario as createEdge } from './create-edge';
+import { scenario as queryDimensions } from './query-dimensions';
+import { scenario as getDimension } from './get-dimension';
+import { scenario as dimensionLifecycle } from './dimension-lifecycle';
+import { scenario as delegateMini } from './delegate-mini';
+import { scenario as delegateWise } from './delegate-wise';
+import { scenario as youtubeExtract } from './youtube-extract';
+import { scenario as websiteExtract } from './website-extract';
+import { scenario as paperExtract } from './paper-extract';
+
+export const scenarios = [
+  simpleQuery,
+  searchEmbeddings,
+  createNode,
+  updateNode,
+  createEdge,
+  queryDimensions,
+  getDimension,
+  dimensionLifecycle,
+  hardModeQuery,
+  workflowIntegrate,
+  delegateMini,
+  delegateWise,
+  youtubeExtract,
+  websiteExtract,
+  paperExtract,
+];
@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'paper-extract',
+  name: 'Paper extract (optional)',
+  description: 'Extract and ingest a PDF (requires network).',
+  tools: ['paperExtract'],
+  enabled: false,
+  notes: 'Enable when network is available; can be slow.',
+  input: {
+    message: 'Add this paper: https://arxiv.org/pdf/1706.03762.pdf',
+  },
+  expect: {
+    toolsCalledSoft: ['paperExtract'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'query-dimensions',
+  name: 'Query dimensions',
+  description: 'List available dimensions and basic info.',
+  tools: ['queryDimensions'],
+  input: {
+    message: 'List my top dimensions and briefly describe what they represent.',
+  },
+  expect: {
+    toolsCalledSoft: ['queryDimensions'],
+    responseContainsSoft: ['dimension'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'search-embeddings',
+  name: 'Embedding search triggers retrieval',
+  description: 'Semantic search over stored knowledge.',
+  tools: ['searchContentEmbeddings', 'queryNodes'],
+  input: {
+    message: 'Find my notes about deep learning architectures.',
+  },
+  expect: {
+    toolsCalledSoft: ['searchContentEmbeddings'],
+    responseContainsSoft: ['found', 'node'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'simple-query',
+  name: 'Simple query routes to helper',
+  description: 'Baseline retrieval query against existing notes.',
+  tools: ['queryNodes'],
+  input: {
+    message: 'What do I know about machine learning?',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    responseContainsSoft: ['node', 'found'],
+  },
+};
@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'update-node',
+  name: 'Update node content',
+  description: 'Create and then update a node in one request.',
+  tools: ['createNode', 'updateNode'],
+  input: {
+    message: 'Create a node titled "Eval: Update Node" with one sentence, then append one more sentence to it.',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'updateNode'],
+    responseContainsSoft: ['updated', 'appended'],
+  },
+};
@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'website-extract',
+  name: 'Website extract (optional)',
+  description: 'Extract and ingest a webpage (requires network).',
+  tools: ['websiteExtract'],
+  enabled: false,
+  notes: 'Enable when network is available; can be slow.',
+  input: {
+    message: 'Add this article: https://sive.rs/plaintext',
+  },
+  expect: {
+    toolsCalledSoft: ['websiteExtract'],
+  },
+};
@@ -0,0 +1,17 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'workflow-integrate',
+  name: 'Integrate workflow on focused node',
+  description: 'Execute integrate workflow on a real node.',
+  tools: ['executeWorkflow'],
+  input: {
+    message: 'Integrate this.',
+    focusedNodeQuery: { titleContains: 'Markdown vs database backends for PKM' },
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['executeWorkflow'],
+    responseContainsSoft: ['integrate', 'updated'],
+  },
+};
@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'youtube-extract',
+  name: 'YouTube extract (optional)',
+  description: 'Extract and ingest a YouTube video (requires network + API keys).',
+  tools: ['youtubeExtract'],
+  enabled: false,
+  notes: 'Enable when network + API keys are configured; can be flaky.',
+  input: {
+    message: 'Add this video: https://www.youtube.com/watch?v=dQw4w9WgXcQ',
+  },
+  expect: {
+    toolsCalledSoft: ['youtubeExtract'],
+  },
+};
@@ -0,0 +1,29 @@
+export type ScenarioExpectations = {
+  toolsCalled?: string[];
+  toolsCalledSoft?: string[];
+  responseContains?: string[];
+  responseContainsSoft?: string[];
+  responseNotContains?: string[];
+  maxLatencyMs?: number;
+};
+
+export type ScenarioInput = {
+  message: string;
+  focusedNodeId?: number;
+  focusedNodeQuery?: {
+    titleContains?: string;
+    titleEquals?: string;
+  };
+  mode?: 'easy' | 'hard';
+};
+
+export type Scenario = {
+  id: string;
+  name: string;
+  input: ScenarioInput;
+  expect?: ScenarioExpectations;
+  description?: string;
+  tools?: string[];
+  enabled?: boolean;
+  notes?: string;
+};
@@ -0,0 +1,16 @@
+import { defineConfig } from 'vitest/config';
+import path from 'path';
+
+export default defineConfig({
+  test: {
+    environment: 'node',
+    include: ['tests/**/*.test.ts'],
+    globals: true,
+    setupFiles: ['tests/unit/setup.ts'],
+  },
+  resolve: {
+    alias: {
+      '@': path.resolve(__dirname, './src'),
+    },
+  },
+});