sync: bug fixes + eval system from private repo

- Fix chunk_status: pass chunk_status/chunk/metadata to context builder
- Fix vector search: scope by node_id BEFORE similarity search
- Add eval logging system (RAH_EVALS_LOG=1)
- Add eval dashboard at /evals
- Add vitest for testing

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

tests/evals/scenarios/create-edge.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'create-edge',
+  name: 'Create edge between two new nodes',
+  description: 'Create two nodes and connect them with an edge.',
+  tools: ['createNode', 'createEdge'],
+  input: {
+    message: 'Create nodes titled "Eval Edge A" and "Eval Edge B" (one sentence each), then create an edge from A to B labeled "related".',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'createEdge'],
+    responseContainsSoft: ['edge', 'connected'],
+  },
+};

tests/evals/scenarios/create-node.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'create-node',
+  name: 'Create node request',
+  description: 'Create a new node via chat.',
+  tools: ['createNode'],
+  input: {
+    message: 'Create a node titled "Eval: Test Node" with a short summary about evals.',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode'],
+    responseContainsSoft: ['node', 'created', 'added'],
+  },
+};

tests/evals/scenarios/delegate-mini.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'delegate-mini',
+  name: 'Delegate to Mini RAH',
+  description: 'Ask the system to delegate a short task to mini helper.',
+  tools: ['delegateToMiniRAH'],
+  input: {
+    message: 'Delegate to mini RAH: summarize my notes on plaintext productivity in 3 bullets.',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['delegateToMiniRAH'],
+  },
+};

tests/evals/scenarios/delegate-wise.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'delegate-wise',
+  name: 'Delegate to Wise RAH',
+  description: 'Ask the system to delegate a research comparison task.',
+  tools: ['delegateToWiseRAH'],
+  input: {
+    message: 'Delegate to wise RAH: compare SQLite vs markdown storage for PKM in 5 bullets.',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['delegateToWiseRAH'],
+  },
+};

tests/evals/scenarios/dimension-lifecycle.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'dimension-lifecycle',
+  name: 'Dimension lifecycle (create/update/lock/unlock)',
+  description: 'Create a dimension, update its description, lock it, then unlock it.',
+  tools: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
+  input: {
+    message: 'Create a dimension named "eval-dim" with description "temporary eval dimension", then update the description to "eval dimension updated", then lock it, then unlock it.',
+  },
+  expect: {
+    toolsCalledSoft: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
+    responseContainsSoft: ['eval-dim'],
+  },
+};

tests/evals/scenarios/get-dimension.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'get-dimension',
+  name: 'Get single dimension',
+  description: 'Fetch detail for a known dimension.',
+  tools: ['getDimension'],
+  input: {
+    message: 'Get details for the dimension "ai".',
+  },
+  expect: {
+    toolsCalledSoft: ['getDimension'],
+    responseContainsSoft: ['ai'],
+  },
+};

tests/evals/scenarios/hard-mode-query.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'hard-mode-query',
+  name: 'Hard mode retrieval query',
+  description: 'Run a baseline retrieval query in hard mode.',
+  tools: ['queryNodes', 'searchContentEmbeddings'],
+  input: {
+    message: 'What have I captured about plaintext productivity and tools?',
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    responseContainsSoft: ['plaintext', 'productivity'],
+  },
+};

tests/evals/scenarios/index.ts

@@ -0,0 +1,33 @@
+import { scenario as simpleQuery } from './simple-query';
+import { scenario as searchEmbeddings } from './search-embeddings';
+import { scenario as createNode } from './create-node';
+import { scenario as hardModeQuery } from './hard-mode-query';
+import { scenario as workflowIntegrate } from './workflow-integrate';
+import { scenario as updateNode } from './update-node';
+import { scenario as createEdge } from './create-edge';
+import { scenario as queryDimensions } from './query-dimensions';
+import { scenario as getDimension } from './get-dimension';
+import { scenario as dimensionLifecycle } from './dimension-lifecycle';
+import { scenario as delegateMini } from './delegate-mini';
+import { scenario as delegateWise } from './delegate-wise';
+import { scenario as youtubeExtract } from './youtube-extract';
+import { scenario as websiteExtract } from './website-extract';
+import { scenario as paperExtract } from './paper-extract';
+
+export const scenarios = [
+  simpleQuery,
+  searchEmbeddings,
+  createNode,
+  updateNode,
+  createEdge,
+  queryDimensions,
+  getDimension,
+  dimensionLifecycle,
+  hardModeQuery,
+  workflowIntegrate,
+  delegateMini,
+  delegateWise,
+  youtubeExtract,
+  websiteExtract,
+  paperExtract,
+];

tests/evals/scenarios/paper-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'paper-extract',
+  name: 'Paper extract (optional)',
+  description: 'Extract and ingest a PDF (requires network).',
+  tools: ['paperExtract'],
+  enabled: false,
+  notes: 'Enable when network is available; can be slow.',
+  input: {
+    message: 'Add this paper: https://arxiv.org/pdf/1706.03762.pdf',
+  },
+  expect: {
+    toolsCalledSoft: ['paperExtract'],
+  },
+};

tests/evals/scenarios/query-dimensions.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'query-dimensions',
+  name: 'Query dimensions',
+  description: 'List available dimensions and basic info.',
+  tools: ['queryDimensions'],
+  input: {
+    message: 'List my top dimensions and briefly describe what they represent.',
+  },
+  expect: {
+    toolsCalledSoft: ['queryDimensions'],
+    responseContainsSoft: ['dimension'],
+  },
+};

tests/evals/scenarios/search-embeddings.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'search-embeddings',
+  name: 'Embedding search triggers retrieval',
+  description: 'Semantic search over stored knowledge.',
+  tools: ['searchContentEmbeddings', 'queryNodes'],
+  input: {
+    message: 'Find my notes about deep learning architectures.',
+  },
+  expect: {
+    toolsCalledSoft: ['searchContentEmbeddings'],
+    responseContainsSoft: ['found', 'node'],
+  },
+};

tests/evals/scenarios/simple-query.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'simple-query',
+  name: 'Simple query routes to helper',
+  description: 'Baseline retrieval query against existing notes.',
+  tools: ['queryNodes'],
+  input: {
+    message: 'What do I know about machine learning?',
+  },
+  expect: {
+    toolsCalledSoft: ['queryNodes'],
+    responseContainsSoft: ['node', 'found'],
+  },
+};

tests/evals/scenarios/update-node.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'update-node',
+  name: 'Update node content',
+  description: 'Create and then update a node in one request.',
+  tools: ['createNode', 'updateNode'],
+  input: {
+    message: 'Create a node titled "Eval: Update Node" with one sentence, then append one more sentence to it.',
+  },
+  expect: {
+    toolsCalledSoft: ['createNode', 'updateNode'],
+    responseContainsSoft: ['updated', 'appended'],
+  },
+};

tests/evals/scenarios/website-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'website-extract',
+  name: 'Website extract (optional)',
+  description: 'Extract and ingest a webpage (requires network).',
+  tools: ['websiteExtract'],
+  enabled: false,
+  notes: 'Enable when network is available; can be slow.',
+  input: {
+    message: 'Add this article: https://sive.rs/plaintext',
+  },
+  expect: {
+    toolsCalledSoft: ['websiteExtract'],
+  },
+};

tests/evals/scenarios/workflow-integrate.ts

@@ -0,0 +1,17 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'workflow-integrate',
+  name: 'Integrate workflow on focused node',
+  description: 'Execute integrate workflow on a real node.',
+  tools: ['executeWorkflow'],
+  input: {
+    message: 'Integrate this.',
+    focusedNodeQuery: { titleContains: 'Markdown vs database backends for PKM' },
+    mode: 'hard',
+  },
+  expect: {
+    toolsCalledSoft: ['executeWorkflow'],
+    responseContainsSoft: ['integrate', 'updated'],
+  },
+};

tests/evals/scenarios/youtube-extract.ts

@@ -0,0 +1,16 @@
+import { Scenario } from '../types';
+
+export const scenario: Scenario = {
+  id: 'youtube-extract',
+  name: 'YouTube extract (optional)',
+  description: 'Extract and ingest a YouTube video (requires network + API keys).',
+  tools: ['youtubeExtract'],
+  enabled: false,
+  notes: 'Enable when network + API keys are configured; can be flaky.',
+  input: {
+    message: 'Add this video: https://www.youtube.com/watch?v=dQw4w9WgXcQ',
+  },
+  expect: {
+    toolsCalledSoft: ['youtubeExtract'],
+  },
+};