feat: sync runtime search and schema quality updates from app repo

- port retrieval, validation, and eval improvements relevant to os
- align prompts and dimensions with the flat single-agent model
- replace the old eval suite with the focused core scenarios

Generated with Codex

tests/evals/scenarios/archive/types.ts

@@ -0,0 +1 @@
+export * from '../../types';

tests/evals/scenarios/chunk-quote-insight.ts

@@ -0,0 +1,19 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'chunk-quote-insight',
+  name: 'Chunk quote to insight',
+  description: 'Search a focused transcript for a specific quote, then create a grounded insight node from it.',
+  tools: ['searchContentEmbeddings', 'createNode', 'createEdge'],
+  input: {
+    message: 'Search inside this focused transcript for a quote about verification being harder than generating solutions. Quote it briefly, then create a new insight node titled "Lange on verification difficulty" and connect it back to this transcript with explanation "Insight extracted from quoted passage."',
+    focusedNodeQuery: { titleContains: 'When AI Discovers the Next Transformer' },
+  },
+  expect: {
+    toolsCalledSoft: ['searchContentEmbeddings', 'createNode', 'createEdge'],
+    responseContainsSoft: ['Lange on verification difficulty'],
+    maxLatencyMs: 35000,
+    maxTotalTokens: 14000,
+    maxEstimatedCostUsd: 0.14,
+  },
+};

tests/evals/scenarios/focused-graph-write.ts

@@ -0,0 +1,20 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'focused-graph-write',
+  name: 'Focused graph write',
+  description: 'Create one new node from the focused transcript and connect it back without unnecessary retrieval.',
+  tools: ['createNode', 'createEdge'],
+  input: {
+    message: 'Create a new node titled "Lange: Verification bottleneck" for the claim that generating many solutions is easier than verifying them, then connect it to this focused transcript with explanation "Claim extracted from this transcript source."',
+    focusedNodeQuery: { titleContains: 'When AI Discovers the Next Transformer' },
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'createEdge'],
+    toolsNotCalledSoft: ['readSkill', 'queryNodes', 'searchContentEmbeddings'],
+    responseContainsSoft: ['Lange: Verification bottleneck'],
+    maxLatencyMs: 25000,
+    maxTotalTokens: 9000,
+    maxEstimatedCostUsd: 0.08,
+  },
+};

tests/evals/scenarios/hub-traversal.ts

@@ -0,0 +1,19 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'hub-traversal',
+  name: 'Hub traversal',
+  description: 'Traverse from core hubs and connected nodes to synthesize what the user should focus on next.',
+  tools: ['queryEdge'],
+  suites: ['traversal', 'internal'],
+  input: {
+    message: 'Traverse from my hub nodes "Building RA-H — Personal Knowledge Graph" and "Nature of Intelligence & Consciousness" and tell me what I should focus on next and why.',
+  },
+  expect: {
+    toolsCalledSoft: ['queryEdge'],
+    responseContainsSoft: ['focus', 'why'],
+    maxLatencyMs: 45000,
+    maxTotalTokens: 16000,
+    maxEstimatedCostUsd: 0.18,
+  },
+};

tests/evals/scenarios/index.ts

@@ -1,27 +1,13 @@
-import { scenario as simpleQuery } from './simple-query';
-import { scenario as searchEmbeddings } from './search-embeddings';
-import { scenario as createNode } from './create-node';
-import { scenario as hardModeQuery } from './hard-mode-query';
-import { scenario as updateNode } from './update-node';
-import { scenario as createEdge } from './create-edge';
-import { scenario as queryDimensions } from './query-dimensions';
-import { scenario as getDimension } from './get-dimension';
-import { scenario as dimensionLifecycle } from './dimension-lifecycle';
-import { scenario as youtubeExtract } from './youtube-extract';
-import { scenario as websiteExtract } from './website-extract';
-import { scenario as paperExtract } from './paper-extract';
+import { scenario as focusedGraphWrite } from './focused-graph-write';
+import { scenario as skillGuidedWrite } from './skill-guided-write';
+import { scenario as nodeIndexSearch } from './node-index-search';
+import { scenario as chunkQuoteInsight } from './chunk-quote-insight';
+import { scenario as hubTraversal } from './hub-traversal';
 
 export const scenarios = [
-  simpleQuery,
-  searchEmbeddings,
-  createNode,
-  updateNode,
-  createEdge,
-  queryDimensions,
-  getDimension,
-  dimensionLifecycle,
-  hardModeQuery,
-  youtubeExtract,
-  websiteExtract,
-  paperExtract,
+  focusedGraphWrite,
+  skillGuidedWrite,
+  nodeIndexSearch,
+  chunkQuoteInsight,
+  hubTraversal,
 ];

tests/evals/scenarios/node-index-search.ts

@@ -0,0 +1,19 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'node-index-search',
+  name: 'Node index search',
+  description: 'Simple lookup should stay in node search and avoid chunk retrieval.',
+  tools: ['queryNodes'],
+  input: {
+    message: 'Find me the node about Plaintext Productivity. Just return the matching node.',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    toolsNotCalledSoft: ['searchContentEmbeddings', 'readSkill'],
+    responseContainsSoft: ['Plaintext Productivity'],
+    maxLatencyMs: 15000,
+    maxTotalTokens: 7000,
+    maxEstimatedCostUsd: 0.06,
+  },
+};

tests/evals/scenarios/skill-guided-write.ts

@@ -0,0 +1,20 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'skill-guided-write',
+  name: 'Skill-guided graph write',
+  description: 'Explicit policy-guided graph work should read the DB policy skill, then either create the requested node+edge or correctly reuse the existing node+edge without duplicating them.',
+  tools: ['readSkill', 'queryNodes', 'createNode', 'createEdge'],
+  suites: ['skills', 'internal'],
+  input: {
+    message: 'Using your DB operations policy, create a node titled "Eval: SQLite-first retrieval audit" with an explicit description and connect it to "Building RA-H — Personal Knowledge Graph" with explanation "Improves RA-H retrieval architecture."',
+  },
+  expect: {
+    skillsReadSoft: ['db-operations'],
+    toolsCalledSoft: ['readSkill'],
+    responseContainsSoft: ['SQLite-first retrieval audit'],
+    maxLatencyMs: 35000,
+    maxTotalTokens: 12000,
+    maxEstimatedCostUsd: 0.12,
+  },
+};

tests/evals/scenarios/types.ts

@@ -0,0 +1 @@
+export type { Scenario, ScenarioExpectations, ScenarioInput } from '../types';