feat: sync runtime search and schema quality updates from app repo

- port retrieval, validation, and eval improvements relevant to os
- align prompts and dimensions with the flat single-agent model
- replace the old eval suite with the focused core scenarios

Generated with Codex

tests/evals/dataset.json

@@ -1,5 +1,13 @@
 {
-  "id": "golden-v1",
-  "name": "Golden Dataset v1",
-  "description": "Baseline eval scenarios for core RA-H helper behavior."
+  "id": "golden-ra-h-core-v2",
+  "name": "Golden RA-H Core v2",
+  "description": "Slim 5-scenario eval set for core RA-H usage: focused graph writes, skill-guided writes, indexed node search, chunk-grounded insight creation, and hub traversal.",
+  "inherits": "golden-v1",
+  "version": 2,
+  "focus": [
+    "tool-call correctness",
+    "skill usage only when appropriate",
+    "core retrieval/write behavior",
+    "latency/tokens/cost guards"
+  ]
 }

tests/evals/runner.ts

@@ -13,10 +13,12 @@ type EvalChatRow = {
   input_tokens: number | null;
   output_tokens: number | null;
   total_tokens: number | null;
+  estimated_cost_usd: number | null;
 };
 
 type EvalToolCallRow = {
   tool_name: string;
+  args_json: string | null;
 };
 
 type EvalResult = {
@@ -25,10 +27,19 @@ type EvalResult = {
   failures: string[];
   warnings: string[];
   latencyMs?: number | null;
+  totalTokens?: number | null;
+  estimatedCostUsd?: number | null;
+  triggerStats?: {
+    tp: number;
+    fp: number;
+    fn: number;
+  };
 };
 
 const BASE_URL = process.env.RAH_EVALS_BASE_URL || 'http://localhost:3000';
 const DATASET_ENV = process.env.RAH_EVALS_DATASET_ID;
+const SUITE_ENV = (process.env.RAH_EVALS_SUITE || 'all').toLowerCase();
+const WAIT_TIMEOUT_MS = Number(process.env.RAH_EVALS_TIMEOUT_MS || 60000);
 const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
 const RAH_DB_PATH = process.env.SQLITE_DB_PATH || path.join(
   process.env.HOME || '~',
@@ -43,6 +54,39 @@ function loadDatasetId() {
   return parsed.id || 'default';
 }
 
+type EvalSuite = 'all' | 'tools' | 'skills' | 'traversal' | 'internal' | 'external';
+
+type FocusedNodeContext = {
+  id: number;
+  title: string;
+  description: string | null;
+  link: string | null;
+  chunk_status: string | null;
+  chunk: string | null;
+  metadata: string | null;
+};
+
+function getDefaultScenarioSuites(scenario: Scenario): string[] {
+  const suites = ['internal'];
+  if (scenario.id.startsWith('skill-trigger-')) {
+    suites.push('skills');
+  } else {
+    suites.push('tools');
+  }
+  if (scenario.id.includes('traverse')) {
+    suites.push('traversal');
+  }
+  return suites;
+}
+
+function shouldRunScenario(scenario: Scenario, suite: EvalSuite) {
+  if (suite === 'all') return true;
+  const suites = scenario.suites && scenario.suites.length > 0
+    ? scenario.suites
+    : getDefaultScenarioSuites(scenario);
+  return suites.includes(suite);
+}
+
 function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): number | null {
   if (!query) return null;
   if (!fs.existsSync(RAH_DB_PATH)) return null;
@@ -70,6 +114,18 @@ function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): num
   return null;
 }
 
+function loadFocusedNodeContext(nodeId: number | null): FocusedNodeContext[] {
+  if (!nodeId || !fs.existsSync(RAH_DB_PATH)) return [];
+  const db = new Database(RAH_DB_PATH, { readonly: true, fileMustExist: true });
+  const row = db.prepare(`
+    SELECT id, title, description, link, chunk_status, chunk, metadata
+    FROM nodes
+    WHERE id = ?
+    LIMIT 1
+  `).get(nodeId) as FocusedNodeContext | undefined;
+  return row ? [row] : [];
+}
+
 async function drainResponse(response: Response) {
   if (!response.body) return;
   const reader = response.body.getReader();
@@ -90,7 +146,7 @@ function openEvalDb() {
 
 function getEvalChatRow(db: Database.Database, traceId: string, scenarioId: string) {
   return db.prepare(`
-    SELECT trace_id, scenario_id, assistant_message, latency_ms, input_tokens, output_tokens, total_tokens
+    SELECT trace_id, scenario_id, assistant_message, latency_ms, input_tokens, output_tokens, total_tokens, estimated_cost_usd
     FROM llm_chats
     WHERE trace_id = ? AND scenario_id = ?
     ORDER BY id DESC
@@ -100,7 +156,7 @@ function getEvalChatRow(db: Database.Database, traceId: string, scenarioId: stri
 
 function getEvalToolCalls(db: Database.Database, traceId: string, scenarioId: string) {
   return db.prepare(`
-    SELECT tool_name
+    SELECT tool_name, args_json
     FROM tool_calls
     WHERE trace_id = ? AND scenario_id = ?
   `).all(traceId, scenarioId) as EvalToolCallRow[];
@@ -119,10 +175,83 @@ async function waitForEvalRow(traceId: string, scenarioId: string, timeoutMs = 1
   return { row: undefined, db: null };
 }
 
+async function ensureServerReady() {
+  try {
+    const response = await fetch(`${BASE_URL}/api/health/ping`);
+    if (!response.ok) {
+      throw new Error(`Health check returned HTTP ${response.status}`);
+    }
+  } catch (error) {
+    throw new Error(
+      `RA-H dev server is not reachable at ${BASE_URL}. Start it with "npm run dev:evals".` +
+      ` ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+}
+
+async function ensureEvalsEnabled() {
+  try {
+    const response = await fetch(`${BASE_URL}/evals`, {
+      redirect: 'manual',
+    });
+
+    if (response.status === 404) {
+      throw new Error('The /evals route returned 404');
+    }
+
+    if (!response.ok && response.status !== 307 && response.status !== 308) {
+      throw new Error(`The /evals route returned HTTP ${response.status}`);
+    }
+  } catch (error) {
+    throw new Error(
+      `Eval logging does not appear to be enabled on the dev server. Start the server with "npm run dev:evals" before running "npm run evals".` +
+      ` ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+}
+
 function normalizeContains(text: string) {
   return text.toLowerCase();
 }
 
+function normalizeSkillId(value: string) {
+  return value
+    .trim()
+    .toLowerCase()
+    .replace(/[\s_]+/g, '-')
+    .replace(/[^a-z0-9-]/g, '')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '');
+}
+
+function safeJsonParse(value: string | null): Record<string, unknown> | null {
+  if (!value) return null;
+  try {
+    const parsed = JSON.parse(value);
+    if (parsed && typeof parsed === 'object') {
+      return parsed as Record<string, unknown>;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+function getSkillsRead(toolCalls: EvalToolCallRow[]) {
+  const names = new Set<string>();
+  for (const call of toolCalls) {
+    if (call.tool_name !== 'readSkill' && call.tool_name !== 'rah_read_skill') {
+      continue;
+    }
+    const parsed = safeJsonParse(call.args_json);
+    const rawName = parsed?.name;
+    if (typeof rawName === 'string' && rawName.trim().length > 0) {
+      names.add(normalizeSkillId(rawName));
+    }
+  }
+  return names;
+}
+
 function checkScenario(
   scenario: Scenario,
   chatRow: EvalChatRow,
@@ -133,6 +262,12 @@ function checkScenario(
   const expect = scenario.expect || {};
   const toolNames = toolCalls.map(call => call.tool_name);
   const responseText = chatRow.assistant_message || '';
+  const skillsRead = getSkillsRead(toolCalls);
+  const requiredSkills = new Set((expect.skillsRead || []).map(normalizeSkillId));
+  const requiredSkillsSoft = new Set((expect.skillsReadSoft || []).map(normalizeSkillId));
+  const forbiddenSkills = new Set((expect.skillsNotRead || []).map(normalizeSkillId));
+  const forbiddenSkillsSoft = new Set((expect.skillsNotReadSoft || []).map(normalizeSkillId));
+  const triggerStats = { tp: 0, fp: 0, fn: 0 };
 
   (expect.toolsCalled || []).forEach(tool => {
     if (!toolNames.includes(tool)) {
@@ -146,6 +281,54 @@ function checkScenario(
     }
   });
 
+  (expect.toolsNotCalled || []).forEach(tool => {
+    if (toolNames.includes(tool)) {
+      failures.push(`Tool "${tool}" should not be called`);
+    }
+  });
+
+  (expect.toolsNotCalledSoft || []).forEach(tool => {
+    if (toolNames.includes(tool)) {
+      warnings.push(`(soft) Tool "${tool}" should not be called`);
+    }
+  });
+
+  requiredSkills.forEach(skill => {
+    if (!skillsRead.has(skill)) {
+      failures.push(`Expected skill "${skill}" not read`);
+      triggerStats.fn += 1;
+    } else {
+      triggerStats.tp += 1;
+    }
+  });
+
+  requiredSkillsSoft.forEach(skill => {
+    if (!skillsRead.has(skill)) {
+      warnings.push(`(soft) Expected skill "${skill}" not read`);
+    }
+  });
+
+  forbiddenSkills.forEach(skill => {
+    if (skillsRead.has(skill)) {
+      failures.push(`Skill "${skill}" should not be read`);
+      triggerStats.fp += 1;
+    }
+  });
+
+  forbiddenSkillsSoft.forEach(skill => {
+    if (skillsRead.has(skill)) {
+      warnings.push(`(soft) Skill "${skill}" should not be read`);
+    }
+  });
+
+  if (requiredSkills.size > 0) {
+    skillsRead.forEach(skill => {
+      if (!requiredSkills.has(skill)) {
+        triggerStats.fp += 1;
+      }
+    });
+  }
+
   (expect.responseContains || []).forEach(text => {
     if (!normalizeContains(responseText).includes(normalizeContains(text))) {
       failures.push(`Response missing "${text}"`);
@@ -170,12 +353,27 @@ function checkScenario(
     }
   }
 
+  if (typeof expect.maxTotalTokens === 'number' && chatRow.total_tokens !== null) {
+    if (chatRow.total_tokens > expect.maxTotalTokens) {
+      failures.push(`Total tokens ${chatRow.total_tokens} exceeded ${expect.maxTotalTokens}`);
+    }
+  }
+
+  if (typeof expect.maxEstimatedCostUsd === 'number' && chatRow.estimated_cost_usd !== null) {
+    if (chatRow.estimated_cost_usd > expect.maxEstimatedCostUsd) {
+      failures.push(`Estimated cost ${chatRow.estimated_cost_usd} exceeded ${expect.maxEstimatedCostUsd}`);
+    }
+  }
+
   return {
     scenario: scenario.name,
     passed: failures.length === 0,
     failures,
     warnings,
     latencyMs: chatRow.latency_ms,
+    totalTokens: chatRow.total_tokens,
+    estimatedCostUsd: chatRow.estimated_cost_usd,
+    triggerStats: triggerStats.tp || triggerStats.fp || triggerStats.fn ? triggerStats : undefined,
   };
 }
 
@@ -183,15 +381,15 @@ async function runScenario(scenario: Scenario, datasetId: string): Promise<EvalR
   const traceId = `eval_${Date.now()}_${randomUUID().slice(0, 8)}`;
   const resolvedFocusedNodeId =
     scenario.input.focusedNodeId ?? resolveFocusedNodeId(scenario.input.focusedNodeQuery);
+  const openTabs = loadFocusedNodeContext(resolvedFocusedNodeId ?? null);
   const response = await fetch(`${BASE_URL}/api/rah/chat`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({
       messages: [{ role: 'user', content: scenario.input.message }],
-      openTabs: [],
+      openTabs,
       activeTabId: resolvedFocusedNodeId ?? null,
       currentView: 'nodes',
-      mode: scenario.input.mode ?? 'easy',
       traceId,
       evals: {
         datasetId,
@@ -211,7 +409,7 @@ async function runScenario(scenario: Scenario, datasetId: string): Promise<EvalR
 
   await drainResponse(response);
 
-  const { row, db } = await waitForEvalRow(traceId, scenario.id);
+  const { row, db } = await waitForEvalRow(traceId, scenario.id, WAIT_TIMEOUT_MS);
   if (!row) {
     return {
       scenario: scenario.name,
@@ -226,26 +424,58 @@ async function runScenario(scenario: Scenario, datasetId: string): Promise<EvalR
 }
 
 async function runAll() {
+  const suite: EvalSuite = ['skills', 'tools', 'traversal', 'internal', 'external'].includes(SUITE_ENV)
+    ? (SUITE_ENV as EvalSuite)
+    : 'all';
   const datasetId = loadDatasetId();
-  console.log(`Running ${scenarios.length} scenarios (dataset: ${datasetId})...\n`);
+  await ensureServerReady();
+  await ensureEvalsEnabled();
+  const runnable = scenarios.filter(s => s.enabled !== false && shouldRunScenario(s, suite));
+  console.log(`Running ${runnable.length} scenarios (dataset: ${datasetId}, suite: ${suite})...\n`);
 
   const results: EvalResult[] = [];
-  for (const scenario of scenarios.filter(s => s.enabled !== false)) {
+  for (const scenario of runnable) {
     const result = await runScenario(scenario, datasetId);
     results.push(result);
     const icon = result.passed ? '✓' : '✗';
-    const latency = result.latencyMs ? ` (${result.latencyMs}ms)` : '';
-    console.log(`${icon} ${result.scenario}${latency}`);
+    const parts: string[] = [];
+    if (typeof result.latencyMs === 'number') parts.push(`${result.latencyMs}ms`);
+    if (typeof result.totalTokens === 'number') parts.push(`${result.totalTokens} tok`);
+    if (typeof result.estimatedCostUsd === 'number') parts.push(`$${result.estimatedCostUsd.toFixed(4)}`);
+    const details = parts.length > 0 ? ` (${parts.join(', ')})` : '';
+    console.log(`${icon} ${result.scenario}${details}`);
     result.failures.forEach(failure => console.log(`  - ${failure}`));
     result.warnings.forEach(warning => console.log(`  - ${warning}`));
   }
 
   const failed = results.filter(result => !result.passed);
   const warnings = results.filter(result => result.warnings.length > 0);
+  const triggerTotals = results.reduce(
+    (acc, result) => {
+      if (result.triggerStats) {
+        acc.tp += result.triggerStats.tp;
+        acc.fp += result.triggerStats.fp;
+        acc.fn += result.triggerStats.fn;
+      }
+      return acc;
+    },
+    { tp: 0, fp: 0, fn: 0 }
+  );
+  const triggerPrecision = triggerTotals.tp + triggerTotals.fp > 0
+    ? triggerTotals.tp / (triggerTotals.tp + triggerTotals.fp)
+    : null;
+  const triggerRecall = triggerTotals.tp + triggerTotals.fn > 0
+    ? triggerTotals.tp / (triggerTotals.tp + triggerTotals.fn)
+    : null;
   console.log('\nSummary');
   console.log(`- Passed: ${results.length - failed.length}`);
   console.log(`- Failed: ${failed.length}`);
   console.log(`- With warnings: ${warnings.length}`);
+  if (triggerPrecision !== null || triggerRecall !== null) {
+    console.log(`- Trigger TP/FP/FN: ${triggerTotals.tp}/${triggerTotals.fp}/${triggerTotals.fn}`);
+    console.log(`- Trigger precision: ${triggerPrecision === null ? 'n/a' : triggerPrecision.toFixed(3)}`);
+    console.log(`- Trigger recall: ${triggerRecall === null ? 'n/a' : triggerRecall.toFixed(3)}`);
+  }
 
   if (failed.length > 0) {
     process.exit(1);

tests/evals/scenarios/archive/types.ts

@@ -0,0 +1 @@
+export * from '../../types';

tests/evals/scenarios/chunk-quote-insight.ts

@@ -0,0 +1,19 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'chunk-quote-insight',
+  name: 'Chunk quote to insight',
+  description: 'Search a focused transcript for a specific quote, then create a grounded insight node from it.',
+  tools: ['searchContentEmbeddings', 'createNode', 'createEdge'],
+  input: {
+    message: 'Search inside this focused transcript for a quote about verification being harder than generating solutions. Quote it briefly, then create a new insight node titled "Lange on verification difficulty" and connect it back to this transcript with explanation "Insight extracted from quoted passage."',
+    focusedNodeQuery: { titleContains: 'When AI Discovers the Next Transformer' },
+  },
+  expect: {
+    toolsCalledSoft: ['searchContentEmbeddings', 'createNode', 'createEdge'],
+    responseContainsSoft: ['Lange on verification difficulty'],
+    maxLatencyMs: 35000,
+    maxTotalTokens: 14000,
+    maxEstimatedCostUsd: 0.14,
+  },
+};

tests/evals/scenarios/focused-graph-write.ts

@@ -0,0 +1,20 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'focused-graph-write',
+  name: 'Focused graph write',
+  description: 'Create one new node from the focused transcript and connect it back without unnecessary retrieval.',
+  tools: ['createNode', 'createEdge'],
+  input: {
+    message: 'Create a new node titled "Lange: Verification bottleneck" for the claim that generating many solutions is easier than verifying them, then connect it to this focused transcript with explanation "Claim extracted from this transcript source."',
+    focusedNodeQuery: { titleContains: 'When AI Discovers the Next Transformer' },
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'createEdge'],
+    toolsNotCalledSoft: ['readSkill', 'queryNodes', 'searchContentEmbeddings'],
+    responseContainsSoft: ['Lange: Verification bottleneck'],
+    maxLatencyMs: 25000,
+    maxTotalTokens: 9000,
+    maxEstimatedCostUsd: 0.08,
+  },
+};

tests/evals/scenarios/hub-traversal.ts

@@ -0,0 +1,19 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'hub-traversal',
+  name: 'Hub traversal',
+  description: 'Traverse from core hubs and connected nodes to synthesize what the user should focus on next.',
+  tools: ['queryEdge'],
+  suites: ['traversal', 'internal'],
+  input: {
+    message: 'Traverse from my hub nodes "Building RA-H — Personal Knowledge Graph" and "Nature of Intelligence & Consciousness" and tell me what I should focus on next and why.',
+  },
+  expect: {
+    toolsCalledSoft: ['queryEdge'],
+    responseContainsSoft: ['focus', 'why'],
+    maxLatencyMs: 45000,
+    maxTotalTokens: 16000,
+    maxEstimatedCostUsd: 0.18,
+  },
+};

tests/evals/scenarios/index.ts

@@ -1,27 +1,13 @@
-import { scenario as simpleQuery } from './simple-query';
-import { scenario as searchEmbeddings } from './search-embeddings';
-import { scenario as createNode } from './create-node';
-import { scenario as hardModeQuery } from './hard-mode-query';
-import { scenario as updateNode } from './update-node';
-import { scenario as createEdge } from './create-edge';
-import { scenario as queryDimensions } from './query-dimensions';
-import { scenario as getDimension } from './get-dimension';
-import { scenario as dimensionLifecycle } from './dimension-lifecycle';
-import { scenario as youtubeExtract } from './youtube-extract';
-import { scenario as websiteExtract } from './website-extract';
-import { scenario as paperExtract } from './paper-extract';
+import { scenario as focusedGraphWrite } from './focused-graph-write';
+import { scenario as skillGuidedWrite } from './skill-guided-write';
+import { scenario as nodeIndexSearch } from './node-index-search';
+import { scenario as chunkQuoteInsight } from './chunk-quote-insight';
+import { scenario as hubTraversal } from './hub-traversal';
 
 export const scenarios = [
-  simpleQuery,
-  searchEmbeddings,
-  createNode,
-  updateNode,
-  createEdge,
-  queryDimensions,
-  getDimension,
-  dimensionLifecycle,
-  hardModeQuery,
-  youtubeExtract,
-  websiteExtract,
-  paperExtract,
+  focusedGraphWrite,
+  skillGuidedWrite,
+  nodeIndexSearch,
+  chunkQuoteInsight,
+  hubTraversal,
 ];

tests/evals/scenarios/node-index-search.ts

@@ -0,0 +1,19 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'node-index-search',
+  name: 'Node index search',
+  description: 'Simple lookup should stay in node search and avoid chunk retrieval.',
+  tools: ['queryNodes'],
+  input: {
+    message: 'Find me the node about Plaintext Productivity. Just return the matching node.',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    toolsNotCalledSoft: ['searchContentEmbeddings', 'readSkill'],
+    responseContainsSoft: ['Plaintext Productivity'],
+    maxLatencyMs: 15000,
+    maxTotalTokens: 7000,
+    maxEstimatedCostUsd: 0.06,
+  },
+};

tests/evals/scenarios/skill-guided-write.ts

@@ -0,0 +1,20 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'skill-guided-write',
+  name: 'Skill-guided graph write',
+  description: 'Explicit policy-guided graph work should read the DB policy skill, then either create the requested node+edge or correctly reuse the existing node+edge without duplicating them.',
+  tools: ['readSkill', 'queryNodes', 'createNode', 'createEdge'],
+  suites: ['skills', 'internal'],
+  input: {
+    message: 'Using your DB operations policy, create a node titled "Eval: SQLite-first retrieval audit" with an explicit description and connect it to "Building RA-H — Personal Knowledge Graph" with explanation "Improves RA-H retrieval architecture."',
+  },
+  expect: {
+    skillsReadSoft: ['db-operations'],
+    toolsCalledSoft: ['readSkill'],
+    responseContainsSoft: ['SQLite-first retrieval audit'],
+    maxLatencyMs: 35000,
+    maxTotalTokens: 12000,
+    maxEstimatedCostUsd: 0.12,
+  },
+};

tests/evals/scenarios/types.ts

@@ -0,0 +1 @@
+export type { Scenario, ScenarioExpectations, ScenarioInput } from '../types';

tests/evals/types.ts

@@ -1,10 +1,18 @@
 export type ScenarioExpectations = {
+  skillsRead?: string[];
+  skillsReadSoft?: string[];
+  skillsNotRead?: string[];
+  skillsNotReadSoft?: string[];
   toolsCalled?: string[];
   toolsCalledSoft?: string[];
+  toolsNotCalled?: string[];
+  toolsNotCalledSoft?: string[];
   responseContains?: string[];
   responseContainsSoft?: string[];
   responseNotContains?: string[];
   maxLatencyMs?: number;
+  maxTotalTokens?: number;
+  maxEstimatedCostUsd?: number;
 };
 
 export type ScenarioInput = {
@@ -24,6 +32,7 @@ export type Scenario = {
   expect?: ScenarioExpectations;
   description?: string;
   tools?: string[];
+  suites?: string[];
   enabled?: boolean;
   notes?: string;
 };