feat: sync runtime search and schema quality updates from app repo

- port retrieval, validation, and eval improvements relevant to os
- align prompts and dimensions with the flat single-agent model
- replace the old eval suite with the focused core scenarios

Generated with Codex

tests/evals/scenarios/archive/create-edge.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'create-edge',
+  name: 'Create edge between two new nodes',
+  description: 'Create two nodes and connect them with an edge.',
+  tools: ['createNode', 'createEdge'],
+  input: {
+    message: 'Create nodes titled "Eval Edge A" and "Eval Edge B" (one sentence each), then create an edge from A to B labeled "related".',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'createEdge'],
+    responseContainsSoft: ['edge', 'connected'],
+  },
+};

tests/evals/scenarios/archive/create-node.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'create-node',
+  name: 'Create node request',
+  description: 'Create a new node via chat.',
+  tools: ['createNode'],
+  input: {
+    message: 'Create a node titled "Eval: Test Node" with a short summary about evals.',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode'],
+    responseContainsSoft: ['node', 'created', 'added'],
+  },
+};

tests/evals/scenarios/archive/dimension-lifecycle.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'dimension-lifecycle',
+  name: 'Dimension lifecycle (create/update/lock/unlock)',
+  description: 'Create a dimension, update its description, lock it, then unlock it.',
+  tools: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
+  input: {
+    message: 'Create a dimension named "eval-dim" with description "temporary eval dimension", then update the description to "eval dimension updated", then lock it, then unlock it.',
+  },
+  expect: {
+    toolsCalledSoft: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
+    responseContainsSoft: ['eval-dim'],
+  },
+};

tests/evals/scenarios/archive/get-dimension.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'get-dimension',
+  name: 'Get single dimension',
+  description: 'Fetch detail for a known dimension.',
+  tools: ['getDimension'],
+  input: {
+    message: 'Get details for the dimension "ai".',
+  },
+  expect: {
+    toolsCalledSoft: ['getDimension'],
+    responseContainsSoft: ['ai'],
+  },
+};

tests/evals/scenarios/archive/hard-mode-query.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'hard-mode-query',
+  name: 'Hard mode retrieval query',
+  description: 'Run a baseline retrieval query in hard mode.',
+  tools: ['queryNodes', 'searchContentEmbeddings'],
+  input: {
+    message: 'What have I captured about plaintext productivity and tools?',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    responseContainsSoft: ['plaintext', 'productivity'],
+  },
+};

tests/evals/scenarios/archive/paper-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'paper-extract',
+  name: 'Paper extract (optional)',
+  description: 'Extract and ingest a PDF (requires network).',
+  tools: ['paperExtract'],
+  enabled: false,
+  notes: 'Enable when network is available; can be slow.',
+  input: {
+    message: 'Add this paper: https://arxiv.org/pdf/1706.03762.pdf',
+  },
+  expect: {
+    toolsCalledSoft: ['paperExtract'],
+  },
+};

tests/evals/scenarios/archive/query-dimensions.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'query-dimensions',
+  name: 'Query dimensions',
+  description: 'List available dimensions and basic info.',
+  tools: ['queryDimensions'],
+  input: {
+    message: 'List my top dimensions and briefly describe what they represent.',
+  },
+  expect: {
+    toolsCalledSoft: ['queryDimensions'],
+    responseContainsSoft: ['dimension'],
+  },
+};

tests/evals/scenarios/archive/search-embeddings.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'search-embeddings',
+  name: 'Embedding search triggers retrieval',
+  description: 'Semantic search over stored knowledge.',
+  tools: ['searchContentEmbeddings', 'queryNodes'],
+  input: {
+    message: 'Find my notes about deep learning architectures.',
+  },
+  expect: {
+    toolsCalledSoft: ['searchContentEmbeddings'],
+    responseContainsSoft: ['found', 'node'],
+  },
+};

tests/evals/scenarios/archive/simple-query.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'simple-query',
+  name: 'Simple query routes to helper',
+  description: 'Baseline retrieval query against existing notes.',
+  tools: ['queryNodes'],
+  input: {
+    message: 'What do I know about machine learning?',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    responseContainsSoft: ['node', 'found'],
+  },
+};

tests/evals/scenarios/archive/types.ts

@@ -0,0 +1 @@
+export * from '../../types';

tests/evals/scenarios/archive/update-node.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'update-node',
+  name: 'Update node content',
+  description: 'Create and then update a node in one request.',
+  tools: ['createNode', 'updateNode'],
+  input: {
+    message: 'Create a node titled "Eval: Update Node" with one sentence, then append one more sentence to it.',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'updateNode'],
+    responseContainsSoft: ['updated', 'appended'],
+  },
+};

tests/evals/scenarios/archive/website-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'website-extract',
+  name: 'Website extract (optional)',
+  description: 'Extract and ingest a webpage (requires network).',
+  tools: ['websiteExtract'],
+  enabled: false,
+  notes: 'Enable when network is available; can be slow.',
+  input: {
+    message: 'Add this article: https://sive.rs/plaintext',
+  },
+  expect: {
+    toolsCalledSoft: ['websiteExtract'],
+  },
+};

tests/evals/scenarios/archive/youtube-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'youtube-extract',
+  name: 'YouTube extract (optional)',
+  description: 'Extract and ingest a YouTube video (requires network + API keys).',
+  tools: ['youtubeExtract'],
+  enabled: false,
+  notes: 'Enable when network + API keys are configured; can be flaky.',
+  input: {
+    message: 'Add this video: https://www.youtube.com/watch?v=dQw4w9WgXcQ',
+  },
+  expect: {
+    toolsCalledSoft: ['youtubeExtract'],
+  },
+};