sync: bug fixes + eval system from private repo

- Fix chunk_status: pass chunk_status/chunk/metadata to context builder
- Fix vector search: scope by node_id BEFORE similarity search
- Add eval logging system (RAH_EVALS_LOG=1)
- Add eval dashboard at /evals
- Add vitest for testing

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/components/layout/ThreePanelLayout.tsx

@@ -75,7 +75,10 @@ export default function ThreePanelLayout() {
         content: node.content,
         dimensions: node.dimensions,
         created_at: node.created_at,
-        updated_at: node.updated_at
+        updated_at: node.updated_at,
+        chunk_status: node.chunk_status,
+        chunk: node.chunk,
+        metadata: node.metadata,
       }));
       setOpenTabsData(validNodes);
     } catch (error) {

src/services/context/requestContext.ts

@@ -12,6 +12,10 @@ interface RequestContext {
     openai?: string;
     anthropic?: string;
   };
+  helperName?: string;
+  evalDatasetId?: string;
+  evalScenarioId?: string;
+  evalChatSpanId?: string;
 }
 
 let currentContext: RequestContext = {};
@@ -26,11 +30,11 @@ export const RequestContext = {
       }
     });
   },
-  
+
   get(): RequestContext {
     return currentContext;
   },
-  
+
   clear() {
     currentContext = {};
   },

src/services/database/chunks.ts

@@ -170,35 +170,57 @@ export class ChunkService {
   // PostgreSQL path removed in SQLite-only consolidation
 
   private async searchChunksSQLite(
-    queryEmbedding: number[], 
+    queryEmbedding: number[],
     similarityThreshold = 0.5,
     matchCount = 5,
     nodeIds?: number[]
   ): Promise<Array<Chunk & { similarity: number }>> {
     const sqlite = getSQLiteClient();
-    
-    // Step 1: Determine vector search limit - more conservative approach
-    let vectorLimit = 50; // Start smaller for efficiency
-    
-    if (nodeIds && nodeIds.length > 0) {
-      // When searching specific nodes, get exact count and use reasonable multiplier
-      const candidateCountQuery = `SELECT COUNT(*) as count FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
-      const candidateResult = sqlite.query<{count: number}>(candidateCountQuery, nodeIds);
-      const candidateCount = Number(candidateResult.rows[0].count);
-      
-      // Use 2x the candidate count but cap at reasonable limits
-      vectorLimit = Math.min(Math.max(candidateCount * 2, 50), 1000);
-      console.log(`🔍 Node-scoped search: ${candidateCount} candidates, using vector limit ${vectorLimit}`);
-    } else {
-      // For global search, use adaptive limit based on expected results
-      vectorLimit = Math.max(matchCount * 10, 50);
-    }
-    
     const startTime = Date.now();
-    
-    // Step 2: Use CTE-based query to avoid UNION ALL explosion
     const vectorString = `[${queryEmbedding.join(',')}]`;
-    let query = `
+
+    // When searching specific nodes, first get their chunk_ids to scope the vector search
+    if (nodeIds && nodeIds.length > 0) {
+      // Get chunk IDs for the target nodes
+      const chunkIdsQuery = `SELECT id FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
+      const chunkIdsResult = sqlite.query<{id: number}>(chunkIdsQuery, nodeIds);
+      const chunkIds = chunkIdsResult.rows.map(r => r.id);
+
+      if (chunkIds.length === 0) {
+        console.log(`🔍 Node-scoped search: no chunks found for nodes ${nodeIds.join(', ')}`);
+        return [];
+      }
+
+      console.log(`🔍 Node-scoped search: ${chunkIds.length} chunks in nodes ${nodeIds.join(', ')}`);
+
+      // Search ONLY within those chunks using rowid filter
+      const query = `
+        SELECT c.*, (1.0 / (1.0 + v.distance)) AS similarity
+        FROM vec_chunks v
+        JOIN chunks c ON c.id = v.chunk_id
+        WHERE v.embedding MATCH ?
+          AND v.chunk_id IN (${chunkIds.map(() => '?').join(',')})
+          AND (1.0 / (1.0 + v.distance)) >= ?
+        ORDER BY v.distance
+        LIMIT ?
+      `;
+
+      const params = [vectorString, ...chunkIds, similarityThreshold, matchCount];
+      const result = sqlite.query<Chunk & { similarity: number }>(query, params);
+      const searchTime = Date.now() - startTime;
+
+      console.log(`📊 Vector search (node-scoped): ${result.rows.length} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
+      if (result.rows.length > 0) {
+        console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
+      }
+
+      return result.rows;
+    }
+
+    // Global search (no node filter)
+    const vectorLimit = Math.max(matchCount * 10, 50);
+
+    const query = `
       WITH vector_results AS (
         SELECT chunk_id, distance
         FROM vec_chunks
@@ -209,36 +231,20 @@ export class ChunkService {
       SELECT c.*, (1.0 / (1.0 + vr.distance)) AS similarity
       FROM vector_results vr
       JOIN chunks c ON c.id = vr.chunk_id
+      WHERE (1.0 / (1.0 + vr.distance)) >= ?
+      ORDER BY similarity DESC
+      LIMIT ?
     `;
-    
-    const params: any[] = [vectorString, vectorLimit];
-    const conditions = [];
-    
-    // Node ID filter if provided
-    if (nodeIds && nodeIds.length > 0) {
-      conditions.push(`c.node_id IN (${nodeIds.map(() => '?').join(',')})`);
-      params.push(...nodeIds);
-    }
-    
-    // Similarity threshold filter
-    conditions.push(`(1.0 / (1.0 + vr.distance)) >= ?`);
-    params.push(similarityThreshold);
-    
-    if (conditions.length > 0) {
-      query += ` WHERE ${conditions.join(' AND ')}`;
-    }
-
-    query += ` ORDER BY similarity DESC LIMIT ?`;
-    params.push(matchCount);
 
+    const params = [vectorString, vectorLimit, similarityThreshold, matchCount];
     const result = sqlite.query<Chunk & { similarity: number }>(query, params);
     const searchTime = Date.now() - startTime;
-    
-    console.log(`📊 Vector search: ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
+
+    console.log(`📊 Vector search (global): ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
     if (result.rows.length > 0) {
       console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
     }
-    
+
     return result.rows;
   }
 

src/services/evals/evalsLogger.ts

@@ -0,0 +1,269 @@
+import Database from 'better-sqlite3';
+import fs from 'fs';
+import path from 'path';
+import { randomUUID } from 'crypto';
+import { RequestContext } from '@/services/context/requestContext';
+
+type EvalToolCallLog = {
+  toolName: string;
+  args?: unknown;
+  result?: unknown;
+  error?: unknown;
+  latencyMs?: number;
+};
+
+type EvalChatLog = {
+  traceId?: string;
+  spanId?: string;
+  helperName?: string;
+  model?: string;
+  promptVersion?: string;
+  systemMessage?: string | null;
+  userMessage?: string | null;
+  assistantMessage?: string | null;
+  inputTokens?: number;
+  outputTokens?: number;
+  totalTokens?: number;
+  cacheWriteTokens?: number;
+  cacheReadTokens?: number;
+  cacheHit?: boolean;
+  cacheSavingsPct?: number;
+  estimatedCostUsd?: number;
+  provider?: string | null;
+  mode?: string | null;
+  workflowKey?: string | null;
+  workflowNodeId?: number | null;
+  latencyMs?: number;
+  success?: boolean;
+  error?: string | null;
+};
+
+const EVALS_LOG_FLAG = process.env.RAH_EVALS_LOG;
+const EVALS_LOG_ENABLED = EVALS_LOG_FLAG === '1' || EVALS_LOG_FLAG === 'true';
+const LOG_DIR = path.join(process.cwd(), 'logs');
+const DB_PATH = path.join(LOG_DIR, 'evals.sqlite');
+
+let evalsDb: Database.Database | null = null;
+
+function shouldLogEvals() {
+  // Log ALL interactions when RAH_EVALS_LOG=1
+  // - Real app interactions: scenario_id will be NULL
+  // - Synthetic scenarios: scenario_id will be set
+  return EVALS_LOG_ENABLED;
+}
+
+function ensureSchema(db: Database.Database) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS tool_calls (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      ts TEXT NOT NULL,
+      trace_id TEXT,
+      span_id TEXT,
+      parent_span_id TEXT,
+      helper_name TEXT,
+      tool_name TEXT NOT NULL,
+      args_json TEXT,
+      result_json TEXT,
+      success INTEGER,
+      latency_ms INTEGER,
+      error TEXT,
+      dataset_id TEXT,
+      scenario_id TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_tool_calls_trace ON tool_calls(trace_id);
+    CREATE INDEX IF NOT EXISTS idx_tool_calls_scenario ON tool_calls(scenario_id);
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS llm_chats (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      ts TEXT NOT NULL,
+      trace_id TEXT,
+      span_id TEXT,
+      helper_name TEXT,
+      model TEXT,
+      prompt_version TEXT,
+      system_message TEXT,
+      user_message TEXT,
+      assistant_message TEXT,
+      input_tokens INTEGER,
+      output_tokens INTEGER,
+      total_tokens INTEGER,
+      cache_write_tokens INTEGER,
+      cache_read_tokens INTEGER,
+      cache_hit INTEGER,
+      cache_savings_pct INTEGER,
+      estimated_cost_usd REAL,
+      provider TEXT,
+      mode TEXT,
+      workflow_key TEXT,
+      workflow_node_id INTEGER,
+      latency_ms INTEGER,
+      success INTEGER,
+      error TEXT,
+      dataset_id TEXT,
+      scenario_id TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_llm_chats_trace ON llm_chats(trace_id);
+    CREATE INDEX IF NOT EXISTS idx_llm_chats_scenario ON llm_chats(scenario_id);
+  `);
+
+  const columns = db.prepare(`PRAGMA table_info(llm_chats);`).all() as { name: string }[];
+  const columnNames = new Set(columns.map(column => column.name));
+  if (!columnNames.has('system_message')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN system_message TEXT;`);
+  }
+  if (!columnNames.has('cache_write_tokens')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_write_tokens INTEGER;`);
+  }
+  if (!columnNames.has('cache_read_tokens')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_read_tokens INTEGER;`);
+  }
+  if (!columnNames.has('cache_hit')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_hit INTEGER;`);
+  }
+  if (!columnNames.has('cache_savings_pct')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_savings_pct INTEGER;`);
+  }
+  if (!columnNames.has('estimated_cost_usd')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN estimated_cost_usd REAL;`);
+  }
+  if (!columnNames.has('provider')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN provider TEXT;`);
+  }
+  if (!columnNames.has('mode')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN mode TEXT;`);
+  }
+  if (!columnNames.has('workflow_key')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN workflow_key TEXT;`);
+  }
+  if (!columnNames.has('workflow_node_id')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN workflow_node_id INTEGER;`);
+  }
+}
+
+function getDb() {
+  if (!EVALS_LOG_ENABLED) return null;
+  if (evalsDb) return evalsDb;
+
+  fs.mkdirSync(LOG_DIR, { recursive: true });
+  evalsDb = new Database(DB_PATH);
+  evalsDb.pragma('journal_mode = WAL');
+  evalsDb.pragma('synchronous = NORMAL');
+  evalsDb.pragma('busy_timeout = 3000');
+  ensureSchema(evalsDb);
+  return evalsDb;
+}
+
+function stringifySafe(value: unknown) {
+  if (value === undefined) return null;
+  try {
+    return JSON.stringify(value);
+  } catch (error) {
+    return JSON.stringify({
+      error: 'Failed to serialize payload',
+      message: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+export function logEvalToolCall(entry: EvalToolCallLog) {
+  if (!shouldLogEvals()) return;
+  const context = RequestContext.get();
+  const traceId = context.traceId;
+  if (!traceId) return;
+
+  const db = getDb();
+  if (!db) return;
+
+  const now = new Date().toISOString();
+  const spanId = randomUUID();
+  const parentSpanId = context.evalChatSpanId || null;
+  const helperName = context.helperName || null;
+  const datasetId = context.evalDatasetId || null;
+  const scenarioId = context.evalScenarioId || null;
+  const errorMessage = entry.error instanceof Error
+    ? entry.error.message
+    : entry.error
+      ? String(entry.error)
+      : null;
+  const success = typeof entry.error === 'undefined'
+    ? entry.result && typeof (entry.result as any).success === 'boolean'
+      ? ((entry.result as any).success ? 1 : 0)
+      : 1
+    : 0;
+
+  db.prepare(`
+    INSERT INTO tool_calls (
+      ts, trace_id, span_id, parent_span_id, helper_name,
+      tool_name, args_json, result_json, success, latency_ms, error,
+      dataset_id, scenario_id
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    now,
+    traceId,
+    spanId,
+    parentSpanId,
+    helperName,
+    entry.toolName,
+    stringifySafe(entry.args),
+    stringifySafe(entry.result),
+    success,
+    entry.latencyMs ?? null,
+    errorMessage,
+    datasetId,
+    scenarioId
+  );
+}
+
+export function logEvalChat(entry: EvalChatLog) {
+  if (!shouldLogEvals()) return;
+  const context = RequestContext.get();
+  const traceId = entry.traceId || context.traceId;
+  if (!traceId) return;
+
+  const db = getDb();
+  if (!db) return;
+
+  const now = new Date().toISOString();
+  const spanId = entry.spanId || context.evalChatSpanId || randomUUID();
+  const datasetId = context.evalDatasetId || null;
+  const scenarioId = context.evalScenarioId || null;
+
+  db.prepare(`
+    INSERT INTO llm_chats (
+      ts, trace_id, span_id, helper_name, model, prompt_version, system_message,
+      user_message, assistant_message, input_tokens, output_tokens, total_tokens,
+      cache_write_tokens, cache_read_tokens, cache_hit, cache_savings_pct,
+      estimated_cost_usd, provider, mode, workflow_key, workflow_node_id,
+      latency_ms, success, error, dataset_id, scenario_id
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    now,
+    traceId,
+    spanId,
+    entry.helperName ?? null,
+    entry.model ?? null,
+    entry.promptVersion ?? null,
+    entry.systemMessage ?? null,
+    entry.userMessage ?? null,
+    entry.assistantMessage ?? null,
+    entry.inputTokens ?? null,
+    entry.outputTokens ?? null,
+    entry.totalTokens ?? null,
+    entry.cacheWriteTokens ?? null,
+    entry.cacheReadTokens ?? null,
+    typeof entry.cacheHit === 'boolean' ? (entry.cacheHit ? 1 : 0) : null,
+    entry.cacheSavingsPct ?? null,
+    entry.estimatedCostUsd ?? null,
+    entry.provider ?? null,
+    entry.mode ?? null,
+    entry.workflowKey ?? null,
+    entry.workflowNodeId ?? null,
+    entry.latencyMs ?? null,
+    typeof entry.success === 'boolean' ? (entry.success ? 1 : 0) : null,
+    entry.error ?? null,
+    datasetId,
+    scenarioId
+  );
+}

src/services/evals/evalsStore.ts

@@ -0,0 +1,141 @@
+import Database from 'better-sqlite3';
+import fs from 'fs';
+import path from 'path';
+
+export type EvalChatRow = {
+  id: number;
+  ts: string;
+  trace_id: string;
+  span_id: string | null;
+  helper_name: string | null;
+  model: string | null;
+  prompt_version: string | null;
+  system_message: string | null;
+  user_message: string | null;
+  assistant_message: string | null;
+  input_tokens: number | null;
+  output_tokens: number | null;
+  total_tokens: number | null;
+  cache_write_tokens: number | null;
+  cache_read_tokens: number | null;
+  cache_hit: number | null;
+  cache_savings_pct: number | null;
+  estimated_cost_usd: number | null;
+  provider: string | null;
+  mode: string | null;
+  workflow_key: string | null;
+  workflow_node_id: number | null;
+  latency_ms: number | null;
+  success: number | null;
+  error: string | null;
+  dataset_id: string | null;
+  scenario_id: string | null;
+};
+
+export type EvalToolCallRow = {
+  id: number;
+  ts: string;
+  trace_id: string;
+  span_id: string | null;
+  parent_span_id: string | null;
+  helper_name: string | null;
+  tool_name: string;
+  args_json: string | null;
+  result_json: string | null;
+  success: number | null;
+  latency_ms: number | null;
+  error: string | null;
+  dataset_id: string | null;
+  scenario_id: string | null;
+};
+
+export type EvalTrace = {
+  chat: EvalChatRow;
+  toolCalls: EvalToolCallRow[];
+  comment: string | null;
+};
+
+const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
+
+function ensureCommentSchema(db: Database.Database) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS eval_comments (
+      trace_id TEXT PRIMARY KEY,
+      scenario_id TEXT,
+      comment TEXT,
+      updated_at TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_eval_comments_scenario ON eval_comments(scenario_id);
+  `);
+}
+
+export function upsertEvalComment(traceId: string, scenarioId: string | null, comment: string) {
+  if (!fs.existsSync(LOG_DB_PATH)) {
+    return;
+  }
+  const db = new Database(LOG_DB_PATH);
+  ensureCommentSchema(db);
+  const now = new Date().toISOString();
+  db.prepare(`
+    INSERT INTO eval_comments (trace_id, scenario_id, comment, updated_at)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(trace_id) DO UPDATE SET
+      comment = excluded.comment,
+      updated_at = excluded.updated_at,
+      scenario_id = excluded.scenario_id
+  `).run(traceId, scenarioId, comment, now);
+}
+
+export function loadEvalTraces(limit = 25): EvalTrace[] {
+  if (!fs.existsSync(LOG_DB_PATH)) {
+    return [];
+  }
+
+  const db = new Database(LOG_DB_PATH, { readonly: true, fileMustExist: true });
+  const chats = db.prepare(`
+    SELECT *
+    FROM llm_chats
+    ORDER BY ts DESC
+    LIMIT ?
+  `).all(limit) as EvalChatRow[];
+
+  if (chats.length === 0) return [];
+
+  const traceIds = Array.from(new Set(chats.map(chat => chat.trace_id)));
+  const placeholders = traceIds.map(() => '?').join(', ');
+  const toolCalls = db.prepare(`
+    SELECT *
+    FROM tool_calls
+    WHERE trace_id IN (${placeholders})
+    ORDER BY ts ASC
+  `).all(...traceIds) as EvalToolCallRow[];
+
+  const toolCallsByTrace = new Map<string, EvalToolCallRow[]>();
+  toolCalls.forEach(call => {
+    const list = toolCallsByTrace.get(call.trace_id) || [];
+    list.push(call);
+    toolCallsByTrace.set(call.trace_id, list);
+  });
+
+  const commentsByTrace = new Map<string, string | null>();
+  if (placeholders.length > 0) {
+    try {
+      const commentRows = db.prepare(`
+        SELECT trace_id, comment
+        FROM eval_comments
+        WHERE trace_id IN (${placeholders})
+      `).all(...traceIds) as Array<{ trace_id: string; comment: string | null }>;
+      commentRows.forEach(row => {
+        commentsByTrace.set(row.trace_id, row.comment ?? null);
+      });
+    } catch {
+      // Comments table doesn't exist yet in readonly mode.
+    }
+  }
+
+  return chats.map(chat => ({
+    chat,
+    toolCalls: toolCallsByTrace.get(chat.trace_id) || [],
+    comment: commentsByTrace.get(chat.trace_id) ?? null,
+  }));
+}