sync: bug fixes + eval system from private repo

- Fix chunk_status: pass chunk_status/chunk/metadata to context builder
- Fix vector search: scope by node_id BEFORE similarity search
- Add eval logging system (RAH_EVALS_LOG=1)
- Add eval dashboard at /evals
- Add vitest for testing

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

tests/evals/dataset.json

@@ -0,0 +1,5 @@
+{
+  "id": "golden-v1",
+  "name": "Golden Dataset v1",
+  "description": "Baseline eval scenarios for core RA-H helper behavior."
+}

tests/evals/runner.ts

@@ -0,0 +1,258 @@
+import fs from 'fs';
+import path from 'path';
+import Database from 'better-sqlite3';
+import { randomUUID } from 'crypto';
+import { scenarios } from './scenarios';
+import { Scenario } from './types';
+
+type EvalChatRow = {
+  trace_id: string;
+  scenario_id: string;
+  assistant_message: string | null;
+  latency_ms: number | null;
+  input_tokens: number | null;
+  output_tokens: number | null;
+  total_tokens: number | null;
+};
+
+type EvalToolCallRow = {
+  tool_name: string;
+};
+
+type EvalResult = {
+  scenario: string;
+  passed: boolean;
+  failures: string[];
+  warnings: string[];
+  latencyMs?: number | null;
+};
+
+const BASE_URL = process.env.RAH_EVALS_BASE_URL || 'http://localhost:3000';
+const DATASET_ENV = process.env.RAH_EVALS_DATASET_ID;
+const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
+const RAH_DB_PATH = process.env.SQLITE_DB_PATH || path.join(
+  process.env.HOME || '~',
+  'Library/Application Support/RA-H/db/rah.sqlite'
+);
+
+function loadDatasetId() {
+  if (DATASET_ENV) return DATASET_ENV;
+  const datasetPath = path.join(process.cwd(), 'tests', 'evals', 'dataset.json');
+  const raw = fs.readFileSync(datasetPath, 'utf-8');
+  const parsed = JSON.parse(raw);
+  return parsed.id || 'default';
+}
+
+function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): number | null {
+  if (!query) return null;
+  if (!fs.existsSync(RAH_DB_PATH)) return null;
+  const db = new Database(RAH_DB_PATH, { readonly: true, fileMustExist: true });
+  if (query.titleEquals) {
+    const row = db.prepare(`
+      SELECT id
+      FROM nodes
+      WHERE title = ?
+      ORDER BY datetime(created_at) DESC
+      LIMIT 1
+    `).get(query.titleEquals) as { id?: number } | undefined;
+    return row?.id ?? null;
+  }
+  if (query.titleContains) {
+    const row = db.prepare(`
+      SELECT id
+      FROM nodes
+      WHERE title LIKE ?
+      ORDER BY datetime(created_at) DESC
+      LIMIT 1
+    `).get(`%${query.titleContains}%`) as { id?: number } | undefined;
+    return row?.id ?? null;
+  }
+  return null;
+}
+
+async function drainResponse(response: Response) {
+  if (!response.body) return;
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    decoder.decode(value, { stream: true });
+  }
+}
+
+function openEvalDb() {
+  if (!fs.existsSync(LOG_DB_PATH)) {
+    return null;
+  }
+  return new Database(LOG_DB_PATH, { readonly: true, fileMustExist: true });
+}
+
+function getEvalChatRow(db: Database.Database, traceId: string, scenarioId: string) {
+  return db.prepare(`
+    SELECT trace_id, scenario_id, assistant_message, latency_ms, input_tokens, output_tokens, total_tokens
+    FROM llm_chats
+    WHERE trace_id = ? AND scenario_id = ?
+    ORDER BY id DESC
+    LIMIT 1
+  `).get(traceId, scenarioId) as EvalChatRow | undefined;
+}
+
+function getEvalToolCalls(db: Database.Database, traceId: string, scenarioId: string) {
+  return db.prepare(`
+    SELECT tool_name
+    FROM tool_calls
+    WHERE trace_id = ? AND scenario_id = ?
+  `).all(traceId, scenarioId) as EvalToolCallRow[];
+}
+
+async function waitForEvalRow(traceId: string, scenarioId: string, timeoutMs = 10000) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    const db = openEvalDb();
+    if (db) {
+      const row = getEvalChatRow(db, traceId, scenarioId);
+      if (row) return { row, db };
+    }
+    await new Promise(resolve => setTimeout(resolve, 200));
+  }
+  return { row: undefined, db: null };
+}
+
+function normalizeContains(text: string) {
+  return text.toLowerCase();
+}
+
+function checkScenario(
+  scenario: Scenario,
+  chatRow: EvalChatRow,
+  toolCalls: EvalToolCallRow[]
+): EvalResult {
+  const failures: string[] = [];
+  const warnings: string[] = [];
+  const expect = scenario.expect || {};
+  const toolNames = toolCalls.map(call => call.tool_name);
+  const responseText = chatRow.assistant_message || '';
+
+  (expect.toolsCalled || []).forEach(tool => {
+    if (!toolNames.includes(tool)) {
+      failures.push(`Expected tool "${tool}" not called`);
+    }
+  });
+
+  (expect.toolsCalledSoft || []).forEach(tool => {
+    if (!toolNames.includes(tool)) {
+      warnings.push(`(soft) Expected tool "${tool}" not called`);
+    }
+  });
+
+  (expect.responseContains || []).forEach(text => {
+    if (!normalizeContains(responseText).includes(normalizeContains(text))) {
+      failures.push(`Response missing "${text}"`);
+    }
+  });
+
+  (expect.responseContainsSoft || []).forEach(text => {
+    if (!normalizeContains(responseText).includes(normalizeContains(text))) {
+      warnings.push(`(soft) Response missing "${text}"`);
+    }
+  });
+
+  (expect.responseNotContains || []).forEach(text => {
+    if (normalizeContains(responseText).includes(normalizeContains(text))) {
+      failures.push(`Response should not contain "${text}"`);
+    }
+  });
+
+  if (typeof expect.maxLatencyMs === 'number' && chatRow.latency_ms !== null) {
+    if (chatRow.latency_ms > expect.maxLatencyMs) {
+      failures.push(`Latency ${chatRow.latency_ms}ms exceeded ${expect.maxLatencyMs}ms`);
+    }
+  }
+
+  return {
+    scenario: scenario.name,
+    passed: failures.length === 0,
+    failures,
+    warnings,
+    latencyMs: chatRow.latency_ms,
+  };
+}
+
+async function runScenario(scenario: Scenario, datasetId: string): Promise<EvalResult> {
+  const traceId = `eval_${Date.now()}_${randomUUID().slice(0, 8)}`;
+  const resolvedFocusedNodeId =
+    scenario.input.focusedNodeId ?? resolveFocusedNodeId(scenario.input.focusedNodeQuery);
+  const response = await fetch(`${BASE_URL}/api/rah/chat`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      messages: [{ role: 'user', content: scenario.input.message }],
+      openTabs: [],
+      activeTabId: resolvedFocusedNodeId ?? null,
+      currentView: 'nodes',
+      mode: scenario.input.mode ?? 'easy',
+      traceId,
+      evals: {
+        datasetId,
+        scenarioId: scenario.id,
+      },
+    }),
+  });
+
+  if (!response.ok) {
+    return {
+      scenario: scenario.name,
+      passed: false,
+      failures: [`HTTP ${response.status} from /api/rah/chat`],
+      warnings: [],
+    };
+  }
+
+  await drainResponse(response);
+
+  const { row, db } = await waitForEvalRow(traceId, scenario.id);
+  if (!row) {
+    return {
+      scenario: scenario.name,
+      passed: false,
+      failures: ['Timed out waiting for eval logs'],
+      warnings: [],
+    };
+  }
+
+  const toolCalls = db ? getEvalToolCalls(db, traceId, scenario.id) : [];
+  return checkScenario(scenario, row, toolCalls);
+}
+
+async function runAll() {
+  const datasetId = loadDatasetId();
+  console.log(`Running ${scenarios.length} scenarios (dataset: ${datasetId})...\n`);
+
+  const results: EvalResult[] = [];
+  for (const scenario of scenarios.filter(s => s.enabled !== false)) {
+    const result = await runScenario(scenario, datasetId);
+    results.push(result);
+    const icon = result.passed ? '✓' : '✗';
+    const latency = result.latencyMs ? ` (${result.latencyMs}ms)` : '';
+    console.log(`${icon} ${result.scenario}${latency}`);
+    result.failures.forEach(failure => console.log(`  - ${failure}`));
+    result.warnings.forEach(warning => console.log(`  - ${warning}`));
+  }
+
+  const failed = results.filter(result => !result.passed);
+  const warnings = results.filter(result => result.warnings.length > 0);
+  console.log('\nSummary');
+  console.log(`- Passed: ${results.length - failed.length}`);
+  console.log(`- Failed: ${failed.length}`);
+  console.log(`- With warnings: ${warnings.length}`);
+
+  if (failed.length > 0) {
+    process.exit(1);
+  }
+}
+
+runAll().catch(error => {
+  console.error('Eval runner failed:', error);
+  process.exit(1);
+});

tests/evals/scenarios/create-edge.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'create-edge',
+  name: 'Create edge between two new nodes',
+  description: 'Create two nodes and connect them with an edge.',
+  tools: ['createNode', 'createEdge'],
+  input: {
+    message: 'Create nodes titled "Eval Edge A" and "Eval Edge B" (one sentence each), then create an edge from A to B labeled "related".',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'createEdge'],
+    responseContainsSoft: ['edge', 'connected'],
+  },
+};

tests/evals/scenarios/create-node.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'create-node',
+  name: 'Create node request',
+  description: 'Create a new node via chat.',
+  tools: ['createNode'],
+  input: {
+    message: 'Create a node titled "Eval: Test Node" with a short summary about evals.',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode'],
+    responseContainsSoft: ['node', 'created', 'added'],
+  },
+};

tests/evals/scenarios/delegate-mini.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'delegate-mini',
+  name: 'Delegate to Mini RAH',
+  description: 'Ask the system to delegate a short task to mini helper.',
+  tools: ['delegateToMiniRAH'],
+  input: {
+    message: 'Delegate to mini RAH: summarize my notes on plaintext productivity in 3 bullets.',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['delegateToMiniRAH'],
+  },
+};

tests/evals/scenarios/delegate-wise.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'delegate-wise',
+  name: 'Delegate to Wise RAH',
+  description: 'Ask the system to delegate a research comparison task.',
+  tools: ['delegateToWiseRAH'],
+  input: {
+    message: 'Delegate to wise RAH: compare SQLite vs markdown storage for PKM in 5 bullets.',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['delegateToWiseRAH'],
+  },
+};

tests/evals/scenarios/dimension-lifecycle.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'dimension-lifecycle',
+  name: 'Dimension lifecycle (create/update/lock/unlock)',
+  description: 'Create a dimension, update its description, lock it, then unlock it.',
+  tools: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
+  input: {
+    message: 'Create a dimension named "eval-dim" with description "temporary eval dimension", then update the description to "eval dimension updated", then lock it, then unlock it.',
+  },
+  expect: {
+    toolsCalledSoft: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
+    responseContainsSoft: ['eval-dim'],
+  },
+};

tests/evals/scenarios/get-dimension.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'get-dimension',
+  name: 'Get single dimension',
+  description: 'Fetch detail for a known dimension.',
+  tools: ['getDimension'],
+  input: {
+    message: 'Get details for the dimension "ai".',
+  },
+  expect: {
+    toolsCalledSoft: ['getDimension'],
+    responseContainsSoft: ['ai'],
+  },
+};

tests/evals/scenarios/hard-mode-query.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'hard-mode-query',
+  name: 'Hard mode retrieval query',
+  description: 'Run a baseline retrieval query in hard mode.',
+  tools: ['queryNodes', 'searchContentEmbeddings'],
+  input: {
+    message: 'What have I captured about plaintext productivity and tools?',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    responseContainsSoft: ['plaintext', 'productivity'],
+  },
+};

tests/evals/scenarios/index.ts

@@ -0,0 +1,33 @@
+import { scenario as simpleQuery } from './simple-query';
+import { scenario as searchEmbeddings } from './search-embeddings';
+import { scenario as createNode } from './create-node';
+import { scenario as hardModeQuery } from './hard-mode-query';
+import { scenario as workflowIntegrate } from './workflow-integrate';
+import { scenario as updateNode } from './update-node';
+import { scenario as createEdge } from './create-edge';
+import { scenario as queryDimensions } from './query-dimensions';
+import { scenario as getDimension } from './get-dimension';
+import { scenario as dimensionLifecycle } from './dimension-lifecycle';
+import { scenario as delegateMini } from './delegate-mini';
+import { scenario as delegateWise } from './delegate-wise';
+import { scenario as youtubeExtract } from './youtube-extract';
+import { scenario as websiteExtract } from './website-extract';
+import { scenario as paperExtract } from './paper-extract';
+
+export const scenarios = [
+  simpleQuery,
+  searchEmbeddings,
+  createNode,
+  updateNode,
+  createEdge,
+  queryDimensions,
+  getDimension,
+  dimensionLifecycle,
+  hardModeQuery,
+  workflowIntegrate,
+  delegateMini,
+  delegateWise,
+  youtubeExtract,
+  websiteExtract,
+  paperExtract,
+];

tests/evals/scenarios/paper-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'paper-extract',
+  name: 'Paper extract (optional)',
+  description: 'Extract and ingest a PDF (requires network).',
+  tools: ['paperExtract'],
+  enabled: false,
+  notes: 'Enable when network is available; can be slow.',
+  input: {
+    message: 'Add this paper: https://arxiv.org/pdf/1706.03762.pdf',
+  },
+  expect: {
+    toolsCalledSoft: ['paperExtract'],
+  },
+};

tests/evals/scenarios/query-dimensions.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'query-dimensions',
+  name: 'Query dimensions',
+  description: 'List available dimensions and basic info.',
+  tools: ['queryDimensions'],
+  input: {
+    message: 'List my top dimensions and briefly describe what they represent.',
+  },
+  expect: {
+    toolsCalledSoft: ['queryDimensions'],
+    responseContainsSoft: ['dimension'],
+  },
+};

tests/evals/scenarios/search-embeddings.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'search-embeddings',
+  name: 'Embedding search triggers retrieval',
+  description: 'Semantic search over stored knowledge.',
+  tools: ['searchContentEmbeddings', 'queryNodes'],
+  input: {
+    message: 'Find my notes about deep learning architectures.',
+  },
+  expect: {
+    toolsCalledSoft: ['searchContentEmbeddings'],
+    responseContainsSoft: ['found', 'node'],
+  },
+};

tests/evals/scenarios/simple-query.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'simple-query',
+  name: 'Simple query routes to helper',
+  description: 'Baseline retrieval query against existing notes.',
+  tools: ['queryNodes'],
+  input: {
+    message: 'What do I know about machine learning?',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    responseContainsSoft: ['node', 'found'],
+  },
+};

tests/evals/scenarios/update-node.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'update-node',
+  name: 'Update node content',
+  description: 'Create and then update a node in one request.',
+  tools: ['createNode', 'updateNode'],
+  input: {
+    message: 'Create a node titled "Eval: Update Node" with one sentence, then append one more sentence to it.',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'updateNode'],
+    responseContainsSoft: ['updated', 'appended'],
+  },
+};

tests/evals/scenarios/website-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'website-extract',
+  name: 'Website extract (optional)',
+  description: 'Extract and ingest a webpage (requires network).',
+  tools: ['websiteExtract'],
+  enabled: false,
+  notes: 'Enable when network is available; can be slow.',
+  input: {
+    message: 'Add this article: https://sive.rs/plaintext',
+  },
+  expect: {
+    toolsCalledSoft: ['websiteExtract'],
+  },
+};

tests/evals/scenarios/workflow-integrate.ts

@@ -0,0 +1,17 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'workflow-integrate',
+  name: 'Integrate workflow on focused node',
+  description: 'Execute integrate workflow on a real node.',
+  tools: ['executeWorkflow'],
+  input: {
+    message: 'Integrate this.',
+    focusedNodeQuery: { titleContains: 'Markdown vs database backends for PKM' },
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['executeWorkflow'],
+    responseContainsSoft: ['integrate', 'updated'],
+  },
+};

tests/evals/scenarios/youtube-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'youtube-extract',
+  name: 'YouTube extract (optional)',
+  description: 'Extract and ingest a YouTube video (requires network + API keys).',
+  tools: ['youtubeExtract'],
+  enabled: false,
+  notes: 'Enable when network + API keys are configured; can be flaky.',
+  input: {
+    message: 'Add this video: https://www.youtube.com/watch?v=dQw4w9WgXcQ',
+  },
+  expect: {
+    toolsCalledSoft: ['youtubeExtract'],
+  },
+};

tests/evals/types.ts

@@ -0,0 +1,29 @@
+export type ScenarioExpectations = {
+  toolsCalled?: string[];
+  toolsCalledSoft?: string[];
+  responseContains?: string[];
+  responseContainsSoft?: string[];
+  responseNotContains?: string[];
+  maxLatencyMs?: number;
+};
+
+export type ScenarioInput = {
+  message: string;
+  focusedNodeId?: number;
+  focusedNodeQuery?: {
+    titleContains?: string;
+    titleEquals?: string;
+  };
+  mode?: 'easy' | 'hard';
+};
+
+export type Scenario = {
+  id: string;
+  name: string;
+  input: ScenarioInput;
+  expect?: ScenarioExpectations;
+  description?: string;
+  tools?: string[];
+  enabled?: boolean;
+  notes?: string;
+};