feat: sync runtime search and schema quality updates from app repo

- port retrieval, validation, and eval improvements relevant to os - align prompts and dimensions with the flat single-agent model - replace the old eval suite with the focused core scenarios Generated with Codex
2026-03-15 14:55:45 +11:00
parent 053c163e31
commit 4c75df101f
57 changed files with 1809 additions and 534 deletions
@@ -1,6 +1,7 @@
 import { getSQLiteClient } from '@/services/database/sqlite-client';
 import { UsageData } from '@/types/analytics';
 import { RequestContext } from '@/services/context/requestContext';
+import { logEvalChat } from '@/services/evals/evalsLogger';

 interface ChatLogEntry {
  chat_type: string;
@@ -26,14 +27,64 @@ interface StreamMetadata {
  traceId?: string;
  parentChatId?: number;
  systemMessage?: string;
+  promptVersion?: string;
+  modelUsed?: string;
  workflowKey?: string;
  workflowNodeId?: number;
-  mode?: 'easy' | 'hard';
  toolCallsData?: any[];
  backendUsage?: Array<{
    provider: string;
    headers: Record<string, string>;
  }>;
+  requestStartedAt?: number;
+  timingBreakdown?: {
+    promptBuildMs?: number;
+    toolsBuildMs?: number;
+    modelResolveMs?: number;
+    messageAssemblyMs?: number;
+    streamSetupMs?: number;
+    toolLoopMs?: number;
+  };
+  toolTimingData?: Array<{
+    toolName: string;
+    durationMs: number;
+  }>;
+  latencyMs?: number;
+  firstTokenLatencyMs?: number | null;
+  firstChunkLatencyMs?: number | null;
+}
+
+function normalizeToolResult(result: unknown): unknown {
+  if (result == null) return null;
+  if (typeof result === 'object') return result;
+  return { value: result };
+}
+
+function collectToolCalls(result: any): any[] | undefined {
+  const collected: any[] = [];
+
+  const pushCall = (call: any) => {
+    if (!call?.toolName) return;
+    collected.push({
+      toolName: call.toolName,
+      args: call.args ?? null,
+      result: normalizeToolResult(call.result),
+    });
+  };
+
+  if (Array.isArray(result?.toolCalls)) {
+    result.toolCalls.forEach(pushCall);
+  }
+
+  if (Array.isArray(result?.steps)) {
+    result.steps.forEach((step: any) => {
+      if (Array.isArray(step?.toolCalls)) {
+        step.toolCalls.forEach(pushCall);
+      }
+    });
+  }
+
+  return collected.length > 0 ? collected : undefined;
 }

 export class ChatLoggingMiddleware {
@@ -94,7 +145,7 @@ export class ChatLoggingMiddleware {
        focused_node_id: metadata.activeTabId ?? null,
        helper_name: metadata.helperName,
        agent_type: metadata.agentType || 'orchestrator',
-        delegation_id: metadata.delegationId ?? null,
+        delegation_id: null,
        metadata: {
          timestamp: new Date().toISOString(),
          session_id: metadata.sessionId,
@@ -102,7 +153,6 @@ export class ChatLoggingMiddleware {
          open_tab_count: metadata.openTabs?.length || 0,
          has_focused_node: !!metadata.activeTabId,
          message_count: messages.length,
-          ...(metadata.mode && { mode: metadata.mode }),
          // System message
          ...(metadata.systemMessage && { system_message: metadata.systemMessage }),
          // Enhanced usage data
@@ -125,6 +175,14 @@ export class ChatLoggingMiddleware {
            validation_message: metadata.usageData.validationMessage,
            fallback_action: metadata.usageData.fallbackAction,
          }),
+          ...((metadata.toolCallsData && metadata.toolCallsData.length > 0) ? {
+            tools_used: metadata.usageData?.toolsUsed ?? Array.from(new Set(
+              metadata.toolCallsData
+                .map((call: any) => call?.toolName)
+                .filter((toolName: unknown): toolName is string => typeof toolName === 'string' && toolName.length > 0)
+            )),
+            tool_calls_count: metadata.usageData?.toolCallsCount ?? metadata.toolCallsData.length,
+          } : {}),
          // Tool calls data
          ...(metadata.toolCallsData && metadata.toolCallsData.length > 0 && {
            tool_calls: metadata.toolCallsData
@@ -132,16 +190,17 @@ export class ChatLoggingMiddleware {
          // Trace grouping
          ...(metadata.traceId && { trace_id: metadata.traceId }),
          ...(metadata.parentChatId && { parent_chat_id: metadata.parentChatId }),
-          // Workflow metadata
-          ...(metadata.workflowKey && {
-            workflow_key: metadata.workflowKey,
-            workflow_node_id: metadata.workflowNodeId,
-            is_workflow: true,
-          }),
          // Backend usage (for Supabase sync correlation)
          ...(metadata.backendUsage && metadata.backendUsage.length > 0 && {
            backend_usage: metadata.backendUsage,
          }),
+          ...(metadata.timingBreakdown && { timing_breakdown: metadata.timingBreakdown }),
+          ...(metadata.toolTimingData && metadata.toolTimingData.length > 0 && {
+            tool_timings: metadata.toolTimingData,
+          }),
+          ...(metadata.latencyMs !== undefined && { latency_ms: metadata.latencyMs }),
+          ...(metadata.firstTokenLatencyMs !== undefined && { first_token_latency_ms: metadata.firstTokenLatencyMs }),
+          ...(metadata.firstChunkLatencyMs !== undefined && { first_chunk_latency_ms: metadata.firstChunkLatencyMs }),
        }
      };

@@ -165,7 +224,7 @@ export class ChatLoggingMiddleware {

      const lastInsertedChatId = Number(result.lastInsertRowid);

-      if (metadata.agentType === 'orchestrator' && (metadata.helperName === 'ra-h' || metadata.helperName === 'ra-h-easy')) {
+      if (metadata.agentType === 'orchestrator' && metadata.helperName === 'ra-h') {
        RequestContext.set({ 
          traceId: metadata.traceId, 
          parentChatId: lastInsertedChatId 
@@ -180,20 +239,21 @@ export class ChatLoggingMiddleware {
  static createLoggingHandlers(metadata: StreamMetadata, messages: any[]) {
    let assistantResponse = '';
    const userMessage = this.extractUserMessage(messages);
+    const startedAt = metadata.requestStartedAt ?? Date.now();
+    const streamStartedAt = Date.now();
+    const streamSetupMs = Math.max(0, streamStartedAt - startedAt);
+    let firstTextDeltaAt: number | null = null;
+    let firstChunkAt: number | null = null;

    return {
      onFinish: async (result: any) => {
        const { text, toolCalls, steps } = result;
        // Log if we have a user message and either text OR tool activity
-        const hasActivity = text || toolCalls?.length > 0 || steps?.length > 0;
+        const hasActivity = Boolean(text || toolCalls?.length > 0 || steps?.length > 0);
        
        if (userMessage && hasActivity) {
          // Capture tool calls if present
-          const toolCallsData = toolCalls && toolCalls.length > 0 ? toolCalls.map((tc: any) => ({
-            toolName: tc.toolName,
-            args: tc.args,
-            result: typeof tc.result === 'object' ? tc.result : { value: tc.result }
-          })) : undefined;
+          const toolCallsData = collectToolCalls(result);
          
          if (toolCallsData) {
            console.log(`🔧 Captured ${toolCallsData.length} tool calls for logging`);
@@ -201,7 +261,16 @@ export class ChatLoggingMiddleware {
          
          const enhancedMetadata = { 
            ...metadata,
-            toolCallsData
+            toolCallsData,
+            timingBreakdown: {
+              ...metadata.timingBreakdown,
+              streamSetupMs,
+            },
+            latencyMs: Date.now() - startedAt,
+            firstChunkLatencyMs: firstChunkAt ? firstChunkAt - startedAt : null,
+            firstTokenLatencyMs: firstTextDeltaAt
+              ? firstTextDeltaAt - startedAt
+              : (firstChunkAt ? firstChunkAt - startedAt : null),
          };
          
          await this.logChatInteraction(
@@ -213,9 +282,73 @@ export class ChatLoggingMiddleware {
        } else if (userMessage && !hasActivity) {
          console.warn(`⚠️ Skipping chat log - no text or tool activity for user message: ${userMessage.substring(0, 50)}...`);
        }
+
+        const evalContext = RequestContext.get();
+        if (userMessage) {
+          const toolCallsData = collectToolCalls(result);
+          const timingBreakdown = {
+            ...metadata.timingBreakdown,
+            streamSetupMs,
+          };
+          const evalMetadata = {
+            ...metadata,
+            toolCallsData,
+            timingBreakdown,
+          };
+          logEvalChat({
+            traceId: evalMetadata.traceId,
+            spanId: evalContext.evalChatSpanId,
+            helperName: evalMetadata.helperName,
+            model: evalMetadata.modelUsed || evalMetadata.usageData?.modelUsed,
+            promptVersion: evalMetadata.promptVersion,
+            systemMessage: evalMetadata.systemMessage || null,
+            userMessage,
+            assistantMessage: text || assistantResponse || null,
+            inputTokens: evalMetadata.usageData?.inputTokens,
+            outputTokens: evalMetadata.usageData?.outputTokens,
+            totalTokens: evalMetadata.usageData?.totalTokens,
+            cacheWriteTokens: evalMetadata.usageData?.cacheWriteTokens,
+            cacheReadTokens: evalMetadata.usageData?.cacheReadTokens,
+            cacheHit: evalMetadata.usageData?.cacheHit,
+            cacheSavingsPct: evalMetadata.usageData?.cacheSavingsPct,
+            estimatedCostUsd: evalMetadata.usageData?.estimatedCostUsd,
+            provider: evalMetadata.usageData?.provider ?? null,
+            workflowKey: evalMetadata.workflowKey ?? null,
+            workflowNodeId: evalMetadata.workflowNodeId ?? null,
+            latencyMs: Date.now() - startedAt,
+            firstChunkLatencyMs: firstChunkAt ? firstChunkAt - startedAt : null,
+            firstTokenLatencyMs: firstTextDeltaAt
+              ? firstTextDeltaAt - startedAt
+              : (firstChunkAt ? firstChunkAt - startedAt : null),
+            promptBuildMs: timingBreakdown?.promptBuildMs ?? null,
+            toolsBuildMs: timingBreakdown?.toolsBuildMs ?? null,
+            modelResolveMs: timingBreakdown?.modelResolveMs ?? null,
+            messageAssemblyMs: timingBreakdown?.messageAssemblyMs ?? null,
+            streamSetupMs,
+            toolLoopMs: timingBreakdown?.toolLoopMs ?? null,
+            toolsUsed: evalMetadata.usageData?.toolsUsed ?? (
+              evalMetadata.toolCallsData
+                ? Array.from(new Set(
+                    evalMetadata.toolCallsData
+                      .map((call: any) => call?.toolName)
+                      .filter((toolName: unknown): toolName is string => typeof toolName === 'string' && toolName.length > 0)
+                  ))
+                : null
+            ),
+            toolCallsCount: evalMetadata.usageData?.toolCallsCount ?? evalMetadata.toolCallsData?.length ?? null,
+            success: hasActivity,
+            error: null,
+          });
+        }
      },
      onChunk: ({ chunk }: { chunk: any }) => {
+        if (firstChunkAt === null) {
+          firstChunkAt = Date.now();
+        }
        if (chunk.type === 'text-delta' && chunk.textDelta) {
+          if (firstTextDeltaAt === null) {
+            firstTextDeltaAt = Date.now();
+          }
          assistantResponse += chunk.textDelta;
        }
      }
@@ -1,6 +1,75 @@
 import { getSQLiteClient } from './sqlite-client';
 import { Chunk, ChunkData } from '@/types/database';

+type RankedChunk = Chunk & { similarity: number };
+
+function sanitizeFtsQuery(input: string): string {
+  return input
+    .replace(/['"()*:^~{}[\]]/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(word => word.length > 0 && !/^(AND|OR|NOT|NEAR)$/i.test(word))
+    .join(' ');
+}
+
+function extractRelaxedSearchTerms(query: string): string[] {
+  const stopWords = new Set([
+    'a', 'an', 'and', 'are', 'as', 'at', 'be', 'being', 'briefly', 'by', 'find',
+    'focused', 'for', 'from', 'in', 'inside', 'is', 'it', 'me', 'my', 'of', 'on',
+    'or', 'quote', 'search', 'solutions', 'specific', 'that', 'the', 'then', 'this',
+    'to', 'transcript', 'with'
+  ]);
+
+  const rawTerms = query
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, ' ')
+    .split(/\s+/)
+    .map(term => term.trim())
+    .filter(term => term.length >= 4 && !stopWords.has(term));
+
+  const expanded = new Set<string>();
+  for (const term of rawTerms) {
+    expanded.add(term);
+    if (term.length >= 6) {
+      expanded.add(term.slice(0, 5));
+      expanded.add(term.slice(0, 6));
+    }
+    if (term.endsWith('ing') && term.length > 6) {
+      expanded.add(term.slice(0, -3));
+    }
+    if (term.endsWith('tion') && term.length > 7) {
+      expanded.add(term.slice(0, -4));
+    }
+    if (term.endsWith('ions') && term.length > 7) {
+      expanded.add(term.slice(0, -4));
+    }
+  }
+
+  return Array.from(expanded).slice(0, 12);
+}
+
+function reciprocalRankFuse<T extends { id: number }>(rankedLists: T[][], limit: number): T[] {
+  const scores = new Map<number, { score: number; item: T }>();
+  const k = 60;
+
+  rankedLists.forEach((list) => {
+    list.forEach((item, index) => {
+      const entry = scores.get(item.id);
+      const score = 1 / (k + index + 1);
+      if (entry) {
+        entry.score += score;
+      } else {
+        scores.set(item.id, { score, item });
+      }
+    });
+  });
+
+  return Array.from(scores.values())
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit)
+    .map(entry => entry.item);
+}
+
 export class ChunkService {
  async getChunksByNodeId(nodeId: number): Promise<Chunk[]> {
    const sqlite = getSQLiteClient();
@@ -157,7 +226,18 @@ export class ChunkService {
    fallbackQuery?: string
  ): Promise<Array<Chunk & { similarity: number }>> {
    try {
-      return await this.searchChunksSQLite(queryEmbedding, similarityThreshold, matchCount, nodeIds);
+      const vectorResults = await this.searchChunksSQLite(queryEmbedding, similarityThreshold, matchCount, nodeIds);
+
+      if (!fallbackQuery) {
+        return vectorResults;
+      }
+
+      const textResults = await this.textSearchFallback(fallbackQuery, matchCount, nodeIds);
+      if (textResults.length === 0) {
+        return vectorResults;
+      }
+
+      return reciprocalRankFuse<RankedChunk>([vectorResults, textResults], matchCount);
    } catch (error) {
      console.warn('Vector search failed, attempting text fallback:', error);
      if (fallbackQuery) {
@@ -255,6 +335,12 @@ export class ChunkService {
  ): Promise<Array<Chunk & { similarity: number }>> {
    const sqlite = getSQLiteClient();
    const startTime = Date.now();
+    const ftsResults = this.ftsSearchChunks(sqlite, query, matchCount, nodeIds);
+    if (ftsResults.length > 0) {
+      const searchTime = Date.now() - startTime;
+      console.log(`📝 Text fallback (FTS): ${ftsResults.length} chunks found, time=${searchTime}ms`);
+      return ftsResults;
+    }
    
    // Clean query for LIKE search
    const cleanQuery = query.trim().toLowerCase();
@@ -285,12 +371,98 @@ export class ChunkService {
    textQuery += ` ORDER BY LENGTH(text) ASC LIMIT ?`;
    params.push(String(matchCount));
    
-    const result = sqlite.query<Chunk & { similarity: number }>(textQuery, params);
+    const result = sqlite.query<RankedChunk>(textQuery, params);
    const searchTime = Date.now() - startTime;
    
    console.log(`📝 Text fallback: ${result.rows.length} chunks found, time=${searchTime}ms`);
-    
-    return result.rows;
+    if (result.rows.length > 0) {
+      return result.rows;
+    }
+
+    const relaxedTerms = extractRelaxedSearchTerms(query);
+    if (relaxedTerms.length === 0) {
+      return [];
+    }
+
+    const scoreClauses = relaxedTerms.map(() => 'CASE WHEN LOWER(text) LIKE ? THEN 1 ELSE 0 END');
+    const scoreParams = relaxedTerms.map(term => `%${term}%`);
+    const relaxedParams = [...scoreParams];
+
+    let relaxedQuery = `
+      SELECT *,
+        (${scoreClauses.join(' + ')}) * 0.7 AS similarity
+      FROM chunks
+      WHERE (${scoreClauses.join(' + ')}) > 0
+    `;
+
+    if (nodeIds && nodeIds.length > 0) {
+      relaxedQuery += ` AND node_id IN (${nodeIds.map(() => '?').join(',')})`;
+      relaxedParams.push(...nodeIds.map(String));
+    }
+
+    relaxedQuery += ' ORDER BY similarity DESC, chunk_idx ASC LIMIT ?';
+    relaxedParams.push(String(matchCount));
+
+    const relaxedResult = sqlite.query<RankedChunk>(
+      relaxedQuery,
+      [...scoreParams, ...relaxedParams]
+    );
+    const relaxedSearchTime = Date.now() - startTime;
+    console.log(`📝 Text fallback (relaxed): ${relaxedResult.rows.length} chunks found, time=${relaxedSearchTime}ms`);
+
+    return relaxedResult.rows;
+  }
+
+  private ftsSearchChunks(
+    sqlite: ReturnType<typeof getSQLiteClient>,
+    query: string,
+    matchCount: number,
+    nodeIds?: number[]
+  ): RankedChunk[] {
+    const ftsExists = sqlite.prepare(
+      "SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks_fts'"
+    ).get();
+    if (!ftsExists) return [];
+
+    const ftsQuery = sanitizeFtsQuery(query);
+    if (!ftsQuery) return [];
+
+    try {
+      if (nodeIds && nodeIds.length > 0) {
+        const result = sqlite.query<RankedChunk>(`
+          SELECT c.*, 0.85 as similarity
+          FROM chunks c
+          WHERE c.node_id IN (${nodeIds.map(() => '?').join(',')})
+          AND c.id IN (
+            SELECT rowid
+            FROM chunks_fts
+            WHERE chunks_fts MATCH ?
+          )
+          ORDER BY c.chunk_idx ASC
+          LIMIT ?
+        `, [...nodeIds, ftsQuery, matchCount]);
+
+        return result.rows;
+      }
+
+      const result = sqlite.query<RankedChunk>(`
+        WITH fts_matches AS (
+          SELECT rowid, rank
+          FROM chunks_fts
+          WHERE chunks_fts MATCH ?
+          LIMIT ?
+        )
+        SELECT c.*, 0.85 as similarity
+        FROM fts_matches fm
+        JOIN chunks c ON c.id = fm.rowid
+        ORDER BY fm.rank
+      `, [ftsQuery, matchCount]);
+
+      return result.rows;
+    } catch (error) {
+      console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error);
+      return [];
+    }
  }

  async getChunkCount(): Promise<number> {
@@ -1,7 +1,4 @@
 import { getSQLiteClient } from './sqlite-client';
-import { openai as openaiProvider } from '@ai-sdk/openai';
-import { generateText } from 'ai';
-import { hasValidOpenAiKey } from '../storage/apiKeys';

 export interface Dimension {
  name: string;
@@ -18,85 +15,23 @@ export interface LockedDimension {

 export class DimensionService {
  /**
-   * Get all locked (priority) dimensions with their descriptions
+   * Legacy compatibility shim. Dimensions are now flat, so there is no locked subset.
   */
  static async getLockedDimensions(): Promise<LockedDimension[]> {
-    const sqlite = getSQLiteClient();
-    
-    const result = sqlite.query(`
-      WITH dimension_counts AS (
-        SELECT nd.dimension, COUNT(*) AS count 
-        FROM node_dimensions nd 
-        GROUP BY nd.dimension
-      )
-      SELECT 
-        d.name,
-        d.description,
-        COALESCE(dc.count, 0) AS count
-      FROM dimensions d
-      LEFT JOIN dimension_counts dc ON dc.dimension = d.name
-      WHERE d.is_priority = 1
-      ORDER BY d.name ASC
-    `);
-
-    return result.rows.map((row: any) => ({
-      name: row.name,
-      description: row.description,
-      count: Number(row.count)
-    }));
+    return [];
  }

  /**
-   * Automatically assign locked dimensions + suggest keyword dimensions
-   * Returns { locked: string[], keywords: string[] }
-   *
-   * IMPORTANT: Returns empty result immediately if no valid API key is configured.
-   * This prevents slow node creation when OpenAI is unavailable.
+   * Automatic special-dimension assignment has been removed. Callers must provide dimensions explicitly.
   */
  static async assignDimensions(nodeData: {
    title: string;
-    notes?: string;
+    content?: string;
    link?: string;
    description?: string;
  }): Promise<{ locked: string[]; keywords: string[] }> {
-    // Fast path: skip AI if no valid API key
-    if (!hasValidOpenAiKey()) {
-      console.log(`[DimensionAssignment] No valid OpenAI key, skipping for: "${nodeData.title}"`);
-      return { locked: [], keywords: [] };
-    }
-
-    try {
-      const lockedDimensions = await this.getLockedDimensions();
-
-      if (lockedDimensions.length === 0) {
-        console.log('[DimensionAssignment] No locked dimensions available');
-        return { locked: [], keywords: [] };
-      }
-
-      const prompt = this.buildAssignmentPrompt(nodeData, lockedDimensions);
-
-      console.log(`[DimensionAssignment] Processing: "${nodeData.title}"`);
-
-      const response = await generateText({
-        model: openaiProvider('gpt-4o-mini'),
-        prompt,
-        maxOutputTokens: 300, // Increased to accommodate more dimensions
-        temperature: 0.1,
-      });
-
-      console.log(`[DimensionAssignment] AI Response:\n${response.text}`);
-
-      const result = this.parseAssignmentResponse(response.text, lockedDimensions);
-
-      console.log(`[DimensionAssignment] Locked: ${result.locked.join(', ') || 'none'}`);
-      console.log(`[DimensionAssignment] Keywords: ${result.keywords.join(', ') || 'none'}`);
-
-      return result;
-
-    } catch (error) {
-      console.error('[DimensionAssignment] Error:', error);
-      return { locked: [], keywords: [] };
-    }
+    console.log(`[DimensionAssignment] Skipped for "${nodeData.title}" — flat dimensions require explicit assignment.`);
+    return { locked: [], keywords: [] };
  }

  /**
@@ -153,7 +88,7 @@ export class DimensionService {
  }

  /**
-   * Build AI prompt for dimension assignment (locked dimensions only)
+   * Legacy no-op prompt builder retained only for backward compatibility.
   */
  private static buildAssignmentPrompt(
    nodeData: { title: string; notes?: string; link?: string; description?: string },
@@ -162,13 +97,13 @@ export class DimensionService {
    // Use description as primary context, content as fallback
    let nodeContextSection: string;
    if (nodeData.description) {
-      const contentPreview = nodeData.notes?.slice(0, 500) || '';
+      const notesPreview = nodeData.notes?.slice(0, 500) || '';
      nodeContextSection = `DESCRIPTION: ${nodeData.description}

-NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`;
+NOTES PREVIEW: ${notesPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`;
    } else {
-      const contentPreview = nodeData.notes?.slice(0, 2000) || '';
-      nodeContextSection = `NOTES: ${contentPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`;
+      const notesPreview = nodeData.notes?.slice(0, 2000) || '';
+      nodeContextSection = `NOTES: ${notesPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`;
    }

    // Include ALL locked dimensions, using fallback text for those without descriptions
@@ -181,7 +116,7 @@ NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500
      })
      .join('\n---\n');

-    return `You are categorizing a knowledge node into locked dimensions.
+    return `Dimensions are now flat categories with no locked subset.

 === NODE TO CATEGORIZE ===
 Title: ${nodeData.title}
@@ -203,7 +138,7 @@ LOCKED:
  }

  /**
-   * Parse AI response and extract locked dimensions
+   * Legacy no-op parser retained only for backward compatibility.
   */
  private static parseAssignmentResponse(
    response: string,
@@ -251,4 +186,4 @@ LOCKED:
  }
 }

-export const dimensionService = new DimensionService();
+export const dimensionService = new DimensionService();
@@ -2,9 +2,11 @@ import { getSQLiteClient } from './sqlite-client';
 import { Edge, EdgeContext, EdgeData, EdgeCreatedVia, NodeConnection, Node } from '@/types/database';
 import { eventBroadcaster } from '../events';
 import { nodeService } from './nodes';
+import { getOpenAiKey } from '../storage/apiKeys';
 import { generateText } from 'ai';
 import { createOpenAI } from '@ai-sdk/openai';
 import { z } from 'zod';
+import { validateEdgeExplanation } from './quality';

 const inferredEdgeContextSchema = z.object({
  type: z.enum(['created_by', 'part_of', 'source_of', 'related_to']),
@@ -53,7 +55,7 @@ async function inferEdgeContext(params: {

  // If no API key is configured, degrade gracefully.
  // We still enforce explanation, but fall back to "related_to" classification.
-  const apiKey = process.env.OPENAI_API_KEY;
+  const apiKey = getOpenAiKey();
  if (!apiKey) {
    return { type: 'related_to', confidence: 0.0, swap_direction: false };
  }
@@ -118,11 +120,11 @@ async function autoInferEdge(params: {
 }): Promise<{ explanation: string; type: EdgeContext['type']; confidence: number; swap_direction: boolean }> {
  const { fromNode, toNode } = params;

-  const apiKey = process.env.OPENAI_API_KEY;
+  const apiKey = getOpenAiKey();
  if (!apiKey) {
    // Fallback without AI
    return {
-      explanation: `Related to ${toNode.title}`,
+      explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
      type: 'related_to',
      confidence: 0.0,
      swap_direction: false,
@@ -181,8 +183,8 @@ async function autoInferEdge(params: {

    const parsed = schema.safeParse(parsedJson);
    if (!parsed.success) {
-      return {
-        explanation: `Related to ${toNode.title}`,
+        return {
+        explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
        type: 'related_to',
        confidence: 0.2,
        swap_direction: false,
@@ -193,7 +195,7 @@ async function autoInferEdge(params: {
  } catch (error) {
    console.warn('[edges] autoInferEdge failed; falling back', error);
    return {
-      explanation: `Related to ${toNode.title}`,
+      explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
      type: 'related_to',
      confidence: 0.2,
      swap_direction: false,
@@ -205,13 +207,33 @@ export class EdgeService {
  async getEdges(): Promise<Edge[]> {
    const sqlite = getSQLiteClient();
    const result = sqlite.query<Edge>('SELECT * FROM edges ORDER BY created_at DESC');
-    return result.rows;
+    return result.rows.map((row: any) => {
+      let context: any = row.context;
+      if (typeof context === 'string') {
+        try {
+          context = JSON.parse(context);
+        } catch {
+          // Keep raw context string if JSON parsing fails.
+        }
+      }
+      return { ...row, context };
+    });
  }

  async getEdgeById(id: number): Promise<Edge | null> {
    const sqlite = getSQLiteClient();
    const result = sqlite.query<Edge>('SELECT * FROM edges WHERE id = ?', [id]);
-    return result.rows[0] || null;
+    const row: any = result.rows[0];
+    if (!row) return null;
+    let context: any = row.context;
+    if (typeof context === 'string') {
+      try {
+        context = JSON.parse(context);
+      } catch {
+        // Keep raw context string if JSON parsing fails.
+      }
+    }
+    return { ...row, context };
  }

  async createEdge(edgeData: EdgeData): Promise<Edge> {
@@ -249,8 +271,12 @@ export class EdgeService {
      };
    } else if (edgeData.skip_inference) {
      inferred = { type: 'related_to' as const, confidence: 0.0, swap_direction: false };
-      if (!explanation) explanation = `Related to ${toNode.title}`;
+      if (!explanation) explanation = `Connection to ${toNode.title}; exact relationship uncertain.`;
    } else {
+      const explanationError = validateEdgeExplanation(explanation);
+      if (explanationError) {
+        throw new Error(explanationError);
+      }
      inferred = await inferEdgeContext({ explanation, fromNode, toNode });
    }

@@ -267,14 +293,15 @@ export class EdgeService {
    };

    const result = sqlite.prepare(`
-      INSERT INTO edges (from_node_id, to_node_id, context, source, created_at)
-      VALUES (?, ?, ?, ?, ?)
+      INSERT INTO edges (from_node_id, to_node_id, context, source, created_at, explanation)
+      VALUES (?, ?, ?, ?, ?, ?)
    `).run(
      finalFromId,
      finalToId,
      JSON.stringify(context),
      edgeData.source,
-      now
+      now,
+      explanation
    );

    const edgeId = Number(result.lastInsertRowid);
@@ -316,6 +343,10 @@ export class EdgeService {
        if (!explanation) {
          throw new Error('Edge explanation is required');
        }
+        const explanationError = validateEdgeExplanation(explanation);
+        if (explanationError) {
+          throw new Error(explanationError);
+        }

        const existingEdge = await this.getEdgeById(id);
        if (!existingEdge) {
@@ -342,8 +373,13 @@ export class EdgeService {
          (existingContext?.created_via as EdgeCreatedVia) ||
          'ui';

-        // Note: On update, we don't swap direction - the edge already exists with its direction.
-        // We only update the type and confidence based on the new explanation.
+        const nextFromId = inferred.swap_direction ? existingEdge.to_node_id : existingEdge.from_node_id;
+        const nextToId = inferred.swap_direction ? existingEdge.from_node_id : existingEdge.to_node_id;
+        if (inferred.swap_direction) {
+          updates.from_node_id = nextFromId;
+          updates.to_node_id = nextToId;
+        }
+
        updates.context = {
          ...existingContext,
          ...incomingContext,
@@ -368,6 +404,18 @@ export class EdgeService {
      }
    });

+    if (Object.prototype.hasOwnProperty.call(updates, 'explanation')) {
+      const rawExplanation = (updates as any).explanation;
+      if (typeof rawExplanation === 'string') {
+        const explanationError = validateEdgeExplanation(rawExplanation);
+        if (explanationError) {
+          throw new Error(explanationError);
+        }
+        updateFields.push('explanation = ?');
+        params.push(rawExplanation.trim());
+      }
+    }
+
    if (updateFields.length === 0) {
      throw new Error('No valid fields to update');
    }
@@ -429,7 +477,7 @@ export class EdgeService {
          WHEN e.from_node_id = ? THEN n_to.title
          ELSE n_from.title
        END as connected_node_title,
-        CASE 
+        CASE
          WHEN e.from_node_id = ? THEN n_to.notes
          ELSE n_from.notes
        END as connected_node_notes,
@@ -1,6 +1,44 @@
 import { getSQLiteClient } from './sqlite-client';
 import { Node, NodeFilters } from '@/types/database';
 import { eventBroadcaster } from '../events';
+import { EmbeddingService } from '@/services/embeddings';
+
+type NodeRow = Node & { dimensions_json: string };
+type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
+
+function sanitizeFtsQuery(input: string): string {
+  return input
+    .replace(/['"()*:^~{}[\]]/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(word => word.length > 0 && !/^(AND|OR|NOT|NEAR)$/i.test(word))
+    .join(' ');
+}
+
+function reciprocalRankFuse<T extends { id: number }>(
+  rankedLists: T[][],
+  limit: number,
+): T[] {
+  const scores = new Map<number, { score: number; item: T }>();
+  const k = 60;
+
+  rankedLists.forEach((list) => {
+    list.forEach((item, index) => {
+      const existing = scores.get(item.id);
+      const score = 1 / (k + index + 1);
+      if (existing) {
+        existing.score += score;
+      } else {
+        scores.set(item.id, { score, item });
+      }
+    });
+  });
+
+  return Array.from(scores.values())
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit)
+    .map(entry => entry.item);
+}

 export class NodeService {
  async getNodes(filters: NodeFilters = {}): Promise<Node[]> {
@@ -10,6 +48,11 @@ export class NodeService {
  async countNodes(filters: NodeFilters = {}): Promise<number> {
    const { dimensions, search, dimensionsMatch = 'any',
            createdAfter, createdBefore, eventAfter, eventBefore } = filters;
+
+    if (search?.trim()) {
+      return this.countSearchNodesSQLite(filters);
+    }
+
    const sqlite = getSQLiteClient();

    let query = `SELECT COUNT(*) as total FROM nodes n WHERE 1=1`;
@@ -52,6 +95,11 @@ export class NodeService {
  private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
    const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any',
            createdAfter, createdBefore, eventAfter, eventBefore } = filters;
+
+    if (search?.trim()) {
+      return this.searchNodesSQLite(filters);
+    }
+
    const sqlite = getSQLiteClient();
    
    // Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
@@ -135,6 +183,7 @@ export class NodeService {
    } else if (sortBy === 'created') {
      query += ' ORDER BY n.created_at DESC';
    } else if (sortBy === 'event_date') {
+      // Nodes with event_date first (DESC), then by updated_at for nulls
      query += ' ORDER BY n.event_date IS NULL, n.event_date DESC, n.updated_at DESC';
    } else {
      query += ' ORDER BY n.updated_at DESC';
@@ -150,14 +199,10 @@ export class NodeService {
      params.push(offset);
    }

-    const result = sqlite.query<Node & { dimensions_json: string }>(query, params);
+    const result = sqlite.query<NodeRow>(query, params);
    
    // Parse dimensions_json and metadata back for compatibility
-    return result.rows.map(row => ({
-      ...row,
-      dimensions: JSON.parse(row.dimensions_json || '[]'),
-      metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
-    }));
+    return result.rows.map(row => this.mapNodeRow(row));
  }

  async getNodeById(id: number): Promise<Node | null> {
@@ -177,16 +222,12 @@ export class NodeService {
      FROM nodes n
      WHERE n.id = ?
    `;
-    const result = sqlite.query<Node & { dimensions_json: string }>(query, [id]);
+    const result = sqlite.query<NodeRow>(query, [id]);
    
    if (result.rows.length === 0) return null;
    
    const row = result.rows[0];
-    return {
-      ...row,
-      dimensions: JSON.parse(row.dimensions_json || '[]'),
-      metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
-    };
+    return this.mapNodeRow(row);
  }

  async createNode(nodeData: Partial<Node>): Promise<Node> {
@@ -363,6 +404,264 @@ export class NodeService {
    return this.getNodes({ search: searchTerm, limit });
  }

+  private mapNodeRow(row: NodeRow): Node {
+    return {
+      ...row,
+      dimensions: JSON.parse(row.dimensions_json || '[]'),
+      metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
+    };
+  }
+
+  private buildNodeFilterClauses(filters: NodeFilters, alias = 'n'): { clauses: string[]; params: any[] } {
+    const {
+      dimensions,
+      dimensionsMatch = 'any',
+      createdAfter,
+      createdBefore,
+      eventAfter,
+      eventBefore,
+    } = filters;
+
+    const clauses: string[] = [];
+    const params: any[] = [];
+
+    if (dimensions && dimensions.length > 0) {
+      if (dimensionsMatch === 'all' && dimensions.length > 1) {
+        clauses.push(`(
+          SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd
+          WHERE nd.node_id = ${alias}.id
+          AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
+        ) = ?`);
+        params.push(...dimensions, dimensions.length);
+      } else {
+        clauses.push(`EXISTS (
+          SELECT 1 FROM node_dimensions nd
+          WHERE nd.node_id = ${alias}.id
+          AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
+        )`);
+        params.push(...dimensions);
+      }
+    }
+
+    if (createdAfter) { clauses.push(`${alias}.created_at >= ?`); params.push(createdAfter); }
+    if (createdBefore) { clauses.push(`${alias}.created_at < ?`); params.push(createdBefore); }
+    if (eventAfter) { clauses.push(`${alias}.event_date >= ?`); params.push(eventAfter); }
+    if (eventBefore) { clauses.push(`${alias}.event_date < ?`); params.push(eventBefore); }
+
+    return { clauses, params };
+  }
+
+  private async searchNodesSQLite(filters: NodeFilters): Promise<Node[]> {
+    const sqlite = getSQLiteClient();
+    const search = filters.search?.trim();
+    const limit = Math.min(Math.max(filters.limit ?? 100, 1), 100);
+    const offset = Math.max(filters.offset ?? 0, 0);
+
+    if (!search) {
+      return [];
+    }
+
+    const searchLimit = Math.max(limit + offset, Math.min(limit * 5, 100));
+    let rankedRows = this.searchNodesFts(sqlite, search, filters, searchLimit);
+
+    if (rankedRows.length === 0) {
+      rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit);
+    }
+
+    if ((filters.searchMode ?? 'standard') === 'hybrid') {
+      const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
+      if (vectorRows.length > 0) {
+        rankedRows = reciprocalRankFuse<NodeSearchRow>([rankedRows, vectorRows], searchLimit);
+      }
+    }
+
+    return rankedRows
+      .slice(offset, offset + limit)
+      .map(row => this.mapNodeRow(row));
+  }
+
+  private countSearchNodesSQLite(filters: NodeFilters): number {
+    const sqlite = getSQLiteClient();
+    const search = filters.search?.trim();
+    if (!search) return 0;
+
+    const ftsQuery = sanitizeFtsQuery(search);
+    const ftsExists = sqlite.prepare(
+      "SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
+    ).get();
+    const { clauses, params } = this.buildNodeFilterClauses(filters);
+
+    if (ftsExists && ftsQuery) {
+      const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
+      const result = sqlite.query<{ total: number }>(`
+        WITH matched_nodes AS (
+          SELECT rowid
+          FROM nodes_fts
+          WHERE nodes_fts MATCH ?
+        )
+        SELECT COUNT(*) as total
+        FROM matched_nodes mn
+        JOIN nodes n ON n.id = mn.rowid
+        ${whereClauses}
+      `, [ftsQuery, ...params]);
+
+      return Number(result.rows[0]?.total ?? 0);
+    }
+
+    const words = search.split(/\s+/).filter(Boolean);
+    let query = `SELECT COUNT(*) as total FROM nodes n WHERE 1=1`;
+    const queryParams = [...params];
+
+    if (clauses.length > 0) {
+      query += ` AND ${clauses.join(' AND ')}`;
+    }
+
+    for (const word of words) {
+      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
+      queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
+    }
+
+    const result = sqlite.query<{ total: number }>(query, queryParams);
+    return Number(result.rows[0]?.total ?? 0);
+  }
+
+  private searchNodesFts(
+    sqlite: ReturnType<typeof getSQLiteClient>,
+    search: string,
+    filters: NodeFilters,
+    limit: number,
+  ): NodeSearchRow[] {
+    const ftsQuery = sanitizeFtsQuery(search);
+    if (!ftsQuery) return [];
+
+    const ftsExists = sqlite.prepare(
+      "SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
+    ).get();
+    if (!ftsExists) return [];
+
+    const { clauses, params } = this.buildNodeFilterClauses(filters);
+    const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
+
+    try {
+      const result = sqlite.query<NodeSearchRow>(`
+        WITH fts_matches AS (
+          SELECT rowid, rank
+          FROM nodes_fts
+          WHERE nodes_fts MATCH ?
+          LIMIT ?
+        )
+        SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+               n.chunk_status, n.embedding_updated_at, n.embedding_text,
+               n.created_at, n.updated_at,
+               COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
+                         FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
+               fm.rank
+        FROM fts_matches fm
+        JOIN nodes n ON n.id = fm.rowid
+        ${whereClauses}
+        ORDER BY fm.rank
+        LIMIT ?
+      `, [ftsQuery, Math.max(limit * 2, 50), ...params, limit]);
+
+      return result.rows;
+    } catch (error) {
+      console.warn('[NodeSearch] FTS search failed, falling back to LIKE:', error);
+      return [];
+    }
+  }
+
+  private searchNodesLike(
+    sqlite: ReturnType<typeof getSQLiteClient>,
+    search: string,
+    filters: NodeFilters,
+    limit: number,
+  ): NodeSearchRow[] {
+    const words = search.split(/\s+/).filter(Boolean);
+    const { clauses, params } = this.buildNodeFilterClauses(filters);
+    let query = `
+      SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+             n.chunk_status, n.embedding_updated_at, n.embedding_text,
+             n.created_at, n.updated_at,
+             COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
+                       FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
+      FROM nodes n
+      WHERE 1=1
+    `;
+    const queryParams = [...params];
+
+    if (clauses.length > 0) {
+      query += ` AND ${clauses.join(' AND ')}`;
+    }
+
+    for (const word of words) {
+      query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
+      queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
+    }
+
+    query += ` ORDER BY
+      CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 6 END,
+      CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
+      CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
+      CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
+      CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
+      n.updated_at DESC
+      LIMIT ?`;
+
+    queryParams.push(search, `${search}%`, `%${search}%`, `%${search}%`, `%${search}%`, limit);
+
+    const result = sqlite.query<NodeSearchRow>(query, queryParams);
+    return result.rows;
+  }
+
+  private async searchNodesVector(
+    sqlite: ReturnType<typeof getSQLiteClient>,
+    search: string,
+    filters: NodeFilters,
+    limit: number,
+  ): Promise<NodeSearchRow[]> {
+    try {
+      const embedding = await EmbeddingService.generateQueryEmbedding(search);
+      if (!EmbeddingService.validateEmbedding(embedding)) {
+        return [];
+      }
+
+      const vecExists = sqlite.prepare(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='vec_nodes'"
+      ).get();
+      if (!vecExists) return [];
+
+      const vectorString = `[${embedding.join(',')}]`;
+      const { clauses, params } = this.buildNodeFilterClauses(filters);
+      const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
+
+      const result = sqlite.query<NodeSearchRow>(`
+        WITH vector_matches AS (
+          SELECT node_id, distance
+          FROM vec_nodes
+          WHERE embedding MATCH ?
+          ORDER BY distance
+          LIMIT ?
+        )
+        SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
+               n.chunk_status, n.embedding_updated_at, n.embedding_text,
+               n.created_at, n.updated_at,
+               COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
+                         FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
+               (1.0 / (1.0 + vm.distance)) AS similarity
+        FROM vector_matches vm
+        JOIN nodes n ON n.id = vm.node_id
+        ${whereClauses}
+        ORDER BY vm.distance
+        LIMIT ?
+      `, [vectorString, Math.max(limit * 2, 50), ...params, limit]);
+
+      return result.rows;
+    } catch (error) {
+      console.warn('[NodeSearch] Vector search unavailable, continuing without it:', error);
+      return [];
+    }
+  }
+
  async getNodeCount(): Promise<number> {
    const sqlite = getSQLiteClient();
    const result = sqlite.query('SELECT COUNT(*) as count FROM nodes');
@@ -0,0 +1,64 @@
+const WEAK_DESCRIPTION_PATTERNS = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i;
+const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|conversation|dataset|decision|dimension|document|episode|essay|guide|idea|insight|interview|node|note|paper|person|plan|placeholder|podcast|presentation|project|question|record|research|skill|source|summary|talk|target|test node|thread|tool|transcript|tweet|video|website|workflow)\b/i;
+const UNCERTAINTY_PATTERNS = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i;
+const GENERIC_EDGE_PATTERNS = /^(related|related to|connected|connected to|association|associated with)$/i;
+
+export function normalizeDimensionName(value: string): string {
+  return value.trim().replace(/\s+/g, ' ');
+}
+
+export function normalizeDimensions(values: unknown, max = 5): string[] {
+  if (!Array.isArray(values)) return [];
+
+  const seen = new Set<string>();
+  const normalized: string[] = [];
+
+  for (const value of values) {
+    if (typeof value !== 'string') continue;
+    const trimmed = normalizeDimensionName(value);
+    if (!trimmed) continue;
+    const key = trimmed.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    normalized.push(trimmed);
+    if (normalized.length >= max) break;
+  }
+
+  return normalized;
+}
+
+export function validateExplicitDescription(description: string): string | null {
+  const text = description.trim();
+  if (text.length < 24) {
+    return 'Description must be explicit and substantial (at least 24 characters).';
+  }
+  if (WEAK_DESCRIPTION_PATTERNS.test(text)) {
+    return 'Description is too vague. State exactly what this is and why it matters.';
+  }
+  if (!EXPLICIT_ENTITY_PATTERNS.test(text) && !UNCERTAINTY_PATTERNS.test(text)) {
+    return 'Description must explicitly identify what this thing is, or state uncertainty explicitly.';
+  }
+  return null;
+}
+
+export function validateEdgeExplanation(explanation: string): string | null {
+  const text = explanation.trim();
+  if (text.length < 8) {
+    return 'Edge explanation must be explicit enough to describe the relationship.';
+  }
+  if (GENERIC_EDGE_PATTERNS.test(text)) {
+    return 'Edge explanation is too generic. State the actual relationship or explicitly note uncertainty.';
+  }
+  return null;
+}
+
+export function validateDimensionDescription(description: string): string | null {
+  const text = description.trim();
+  if (!text) {
+    return 'Dimension description is required.';
+  }
+  if (text.length > 500) {
+    return 'Description must be 500 characters or less.';
+  }
+  return null;
+}
@@ -19,12 +19,10 @@ class SQLiteClient {
  private db: Database.Database;
  private config: SQLiteConfig;
  private readonly readOnly: boolean;
-  private readonly embeddingsDisabled: boolean;

  private constructor() {
    this.config = this.getSQLiteConfig();
    this.readOnly = process.env.SQLITE_READONLY === 'true';
-    this.embeddingsDisabled = process.env.DISABLE_EMBEDDINGS === 'true';
    
    // Initialize database connection
    const dbDirectory = path.dirname(this.config.dbPath);
@@ -35,15 +33,13 @@ class SQLiteClient {
      ? new Database(this.config.dbPath, { readonly: true, fileMustExist: true })
      : new Database(this.config.dbPath);
    
-    // Load sqlite-vec extension (skip entirely if embeddings are disabled)
-    if (!this.embeddingsDisabled) {
-      try {
-        this.db.loadExtension(this.config.vecExtensionPath);
-        console.log('SQLite vector extension loaded successfully');
-      } catch (error) {
-        // Do not fail hard — allow the app to run without vector features
-        console.error('Warning: Failed to load vector extension:', error);
-      }
+    // Load sqlite-vec extension
+    try {
+      this.db.loadExtension(this.config.vecExtensionPath);
+      console.log('SQLite vector extension loaded successfully');
+    } catch (error) {
+      // Do not fail hard — allow the app to run without vector features
+      console.error('Warning: Failed to load vector extension:', error);
    }

    // Configure SQLite settings
@@ -61,11 +57,9 @@ class SQLiteClient {
      this.db.pragma('temp_store = memory');
      this.db.pragma('busy_timeout = 5000');

-      // Ensure vector virtual tables are present and healthy (skip if disabled)
-      if (!this.embeddingsDisabled) {
-        this.ensureVectorTables();
-        this.healVectorTablesIfCorrupt();
-      }
+      // Ensure vector virtual tables are present and healthy
+      this.ensureVectorTables();
+      this.healVectorTablesIfCorrupt();

      // Ensure logging schema (rename memory->logs if needed, create triggers/views)
      this.ensureLoggingAndMemorySchema();
@@ -140,9 +134,7 @@ class SQLiteClient {
      } as DatabaseError;
    }
    // Proactively validate/repair vec vtables before any write transaction
-    if (!this.embeddingsDisabled) {
-      this.healVectorTablesIfCorrupt();
-    }
+    this.healVectorTablesIfCorrupt();
    const txn = this.db.transaction(callback);
    try {
      return txn();
@@ -162,9 +154,6 @@ class SQLiteClient {
  }

  public async checkVectorExtension(): Promise<boolean> {
-    if (this.embeddingsDisabled) {
-      return false;
-    }
    try {
      const result = this.query('SELECT vec_version() as version');
      return result.rows.length > 0;
@@ -266,7 +255,6 @@ class SQLiteClient {
          }
        };
        ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;");
-        // type column removed in final schema pass
      } catch (nodeErr) {
        console.warn('Failed to ensure nodes columns:', nodeErr);
      }
@@ -363,6 +351,17 @@ class SQLiteClient {
                     'cache_hit', COALESCE(json_extract(NEW.metadata, '$.cache_hit'), 0),
                     'model', COALESCE(json_extract(NEW.metadata, '$.model_used'), ''),
                     'tools_count', COALESCE(json_extract(NEW.metadata, '$.tool_calls_count'), 0),
+                     'tools_used', COALESCE(json_extract(NEW.metadata, '$.tools_used'), json('[]')),
+                     'latency_ms', COALESCE(json_extract(NEW.metadata, '$.latency_ms'), 0),
+                     'prompt_build_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.promptBuildMs'), 0),
+                     'tools_build_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.toolsBuildMs'), 0),
+                     'model_resolve_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.modelResolveMs'), 0),
+                     'message_assembly_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.messageAssemblyMs'), 0),
+                     'stream_setup_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.streamSetupMs'), 0),
+                     'tool_loop_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.toolLoopMs'), 0),
+                     'first_token_latency_ms', COALESCE(json_extract(NEW.metadata, '$.first_token_latency_ms'), 0),
+                     'first_chunk_latency_ms', COALESCE(json_extract(NEW.metadata, '$.first_chunk_latency_ms'), 0),
+                     'tool_timings', COALESCE(json_extract(NEW.metadata, '$.tool_timings'), json('[]')),
                     'trace_id', COALESCE(json_extract(NEW.metadata, '$.trace_id'), ''),
                     'voice_tts_chars', COALESCE(json_extract(NEW.metadata, '$.voice_tts_chars'), 0),
                     'voice_tts_cost_usd', COALESCE(json_extract(NEW.metadata, '$.voice_tts_cost_usd'), 0),
@@ -435,26 +434,18 @@ class SQLiteClient {
      }
      // Do not recreate memory_v; alias has been removed.

-      // 6) Drop orphaned chat_memory_state table (removed in final schema pass)
-      this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`);
-
-      // Agent delegation table for orchestrator/worker coordination
+      // 6) Clean up removed chat_memory_state table
      try {
-        this.db.exec(`
-          CREATE TABLE IF NOT EXISTS agent_delegations (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            session_id TEXT UNIQUE NOT NULL,
-            task TEXT NOT NULL,
-            context TEXT,
-            expected_outcome TEXT,
-            status TEXT NOT NULL DEFAULT 'queued',
-            summary TEXT,
-            created_at TEXT DEFAULT CURRENT_TIMESTAMP,
-            updated_at TEXT DEFAULT CURRENT_TIMESTAMP
-          );
-        `);
+        this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`);
      } catch (e) {
-        console.warn('Failed to ensure agent_delegations table:', e);
+        // Ignore if table doesn't exist
+      }
+
+      // Clean up removed agent_delegations table
+      try {
+        this.db.exec(`DROP TABLE IF EXISTS agent_delegations;`);
+      } catch (e) {
+        console.warn('Failed to drop agent_delegations table:', e);
      }

      // 8) Logs retention trigger (~10k most recent rows)
@@ -623,77 +614,62 @@ class SQLiteClient {
        }
      }

-      // 10) Final schema pass migrations (content→notes, event_date, icon, drop dead columns)
+      // 10) Final schema pass migrations (content→notes, event_date, dimensions.icon, drop dead columns)
      try {
        const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
-        const nodeColNames = nodeCols2.map((c: any) => c.name);
+        const nodeColNames = nodeCols2.map(c => c.name);

-        // Rename content → notes
+        // Rename content → notes (additive first)
        if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) {
-          console.log('Renaming nodes.content → nodes.notes');
+          console.log('Migrating nodes.content → nodes.notes...');
          this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;');
        }

-        // Add event_date with backfill from metadata
+        // Add event_date
        if (!nodeColNames.includes('event_date')) {
-          console.log('Adding nodes.event_date column');
          this.db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;');
-          this.db.exec(`
-            UPDATE nodes SET event_date = json_extract(metadata, '$.published_date')
-            WHERE metadata IS NOT NULL
-              AND json_extract(metadata, '$.published_date') IS NOT NULL
-              AND json_extract(metadata, '$.published_date') != '';
-          `);
+          // Backfill from metadata.published_date where available
+          try {
+            this.db.exec(`
+              UPDATE nodes SET event_date = json_extract(metadata, '$.published_date')
+              WHERE event_date IS NULL AND json_extract(metadata, '$.published_date') IS NOT NULL;
+            `);
+          } catch {}
        }

        // Add dimensions.icon
        const dimCols2 = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
-        if (!dimCols2.some((c: any) => c.name === 'icon')) {
-          console.log('Adding dimensions.icon column');
+        if (!dimCols2.some(c => c.name === 'icon')) {
          this.db.exec('ALTER TABLE dimensions ADD COLUMN icon TEXT;');
        }

-        // Drop dead columns (SQLite 3.35+)
+        // Drop dead columns (requires SQLite 3.35+)
+        // nodes.type
        if (nodeColNames.includes('type')) {
-          console.log('Dropping nodes.type column');
-          try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_type;'); } catch {}
-          try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch (e) {
-            console.warn('Could not drop nodes.type (SQLite < 3.35?):', e);
-          }
+          try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch {}
        }
+        // nodes.is_pinned
        if (nodeColNames.includes('is_pinned')) {
-          console.log('Dropping nodes.is_pinned column');
-          try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_pinned;'); } catch {}
-          try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch (e) {
-            console.warn('Could not drop nodes.is_pinned:', e);
-          }
+          try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch {}
        }
-
-        // Drop edges.user_feedback
+        // edges.user_feedback
        const edgeCols = this.db.prepare('PRAGMA table_info(edges)').all() as Array<{ name: string }>;
-        if (edgeCols.some((c: any) => c.name === 'user_feedback')) {
-          console.log('Dropping edges.user_feedback column');
-          try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch (e) {
-            console.warn('Could not drop edges.user_feedback:', e);
-          }
+        if (edgeCols.some(c => c.name === 'user_feedback')) {
+          try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch {}
        }

-        // Rebuild FTS if it references 'content' instead of 'notes'
+        // Recreate nodes_fts to index title + description + notes
        try {
-          const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as any;
-          if (ftsCheck && ftsCheck.sql && ftsCheck.sql.includes('content')) {
-            console.log('Rebuilding nodes_fts to reference notes instead of content');
+          const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined;
+          if (ftsCheck?.sql && (!ftsCheck.sql.includes('description') || ftsCheck.sql.includes('content'))) {
            this.db.exec('DROP TABLE IF EXISTS nodes_fts;');
-            this.db.exec(`
-              CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content=nodes, content_rowid=id);
-              INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');
-            `);
+            this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content='nodes', content_rowid='id');");
+            // Rebuild FTS index
+            this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
          }
        } catch (ftsErr) {
-          console.warn('FTS rebuild skipped:', ftsErr);
+          console.warn('Failed to rebuild nodes_fts:', ftsErr);
        }
-
-        console.log('Final schema pass migrations complete');
      } catch (schemaErr) {
        console.warn('Final schema pass migration error:', schemaErr);
      }
@@ -34,6 +34,16 @@ type EvalChatLog = {
  workflowKey?: string | null;
  workflowNodeId?: number | null;
  latencyMs?: number;
+  firstChunkLatencyMs?: number | null;
+  firstTokenLatencyMs?: number | null;
+  promptBuildMs?: number | null;
+  toolsBuildMs?: number | null;
+  modelResolveMs?: number | null;
+  messageAssemblyMs?: number | null;
+  streamSetupMs?: number | null;
+  toolLoopMs?: number | null;
+  toolsUsed?: string[] | null;
+  toolCallsCount?: number | null;
  success?: boolean;
  error?: string | null;
 };
@@ -99,6 +109,16 @@ function ensureSchema(db: Database.Database) {
      workflow_key TEXT,
      workflow_node_id INTEGER,
      latency_ms INTEGER,
+      first_chunk_latency_ms INTEGER,
+      first_token_latency_ms INTEGER,
+      prompt_build_ms INTEGER,
+      tools_build_ms INTEGER,
+      model_resolve_ms INTEGER,
+      message_assembly_ms INTEGER,
+      stream_setup_ms INTEGER,
+      tool_loop_ms INTEGER,
+      tools_used_json TEXT,
+      tool_calls_count INTEGER,
      success INTEGER,
      error TEXT,
      dataset_id TEXT,
@@ -140,6 +160,36 @@ function ensureSchema(db: Database.Database) {
  if (!columnNames.has('workflow_node_id')) {
    db.exec(`ALTER TABLE llm_chats ADD COLUMN workflow_node_id INTEGER;`);
  }
+  if (!columnNames.has('first_chunk_latency_ms')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN first_chunk_latency_ms INTEGER;`);
+  }
+  if (!columnNames.has('first_token_latency_ms')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN first_token_latency_ms INTEGER;`);
+  }
+  if (!columnNames.has('prompt_build_ms')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN prompt_build_ms INTEGER;`);
+  }
+  if (!columnNames.has('tools_build_ms')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN tools_build_ms INTEGER;`);
+  }
+  if (!columnNames.has('model_resolve_ms')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN model_resolve_ms INTEGER;`);
+  }
+  if (!columnNames.has('message_assembly_ms')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN message_assembly_ms INTEGER;`);
+  }
+  if (!columnNames.has('stream_setup_ms')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN stream_setup_ms INTEGER;`);
+  }
+  if (!columnNames.has('tool_loop_ms')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN tool_loop_ms INTEGER;`);
+  }
+  if (!columnNames.has('tools_used_json')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN tools_used_json TEXT;`);
+  }
+  if (!columnNames.has('tool_calls_count')) {
+    db.exec(`ALTER TABLE llm_chats ADD COLUMN tool_calls_count INTEGER;`);
+  }
 }

 function getDb() {
@@ -236,8 +286,11 @@ export function logEvalChat(entry: EvalChatLog) {
      user_message, assistant_message, input_tokens, output_tokens, total_tokens,
      cache_write_tokens, cache_read_tokens, cache_hit, cache_savings_pct,
      estimated_cost_usd, provider, mode, workflow_key, workflow_node_id,
-      latency_ms, success, error, dataset_id, scenario_id
-    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      latency_ms, first_chunk_latency_ms, first_token_latency_ms,
+      prompt_build_ms, tools_build_ms, model_resolve_ms, message_assembly_ms,
+      stream_setup_ms, tool_loop_ms, tools_used_json, tool_calls_count,
+      success, error, dataset_id, scenario_id
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `).run(
    now,
    traceId,
@@ -261,6 +314,16 @@ export function logEvalChat(entry: EvalChatLog) {
    entry.workflowKey ?? null,
    entry.workflowNodeId ?? null,
    entry.latencyMs ?? null,
+    entry.firstChunkLatencyMs ?? null,
+    entry.firstTokenLatencyMs ?? null,
+    entry.promptBuildMs ?? null,
+    entry.toolsBuildMs ?? null,
+    entry.modelResolveMs ?? null,
+    entry.messageAssemblyMs ?? null,
+    entry.streamSetupMs ?? null,
+    entry.toolLoopMs ?? null,
+    stringifySafe(entry.toolsUsed ?? null),
+    entry.toolCallsCount ?? null,
    typeof entry.success === 'boolean' ? (entry.success ? 1 : 0) : null,
    entry.error ?? null,
    datasetId,
@@ -26,6 +26,16 @@ export type EvalChatRow = {
  workflow_key: string | null;
  workflow_node_id: number | null;
  latency_ms: number | null;
+  first_chunk_latency_ms: number | null;
+  first_token_latency_ms: number | null;
+  prompt_build_ms: number | null;
+  tools_build_ms: number | null;
+  model_resolve_ms: number | null;
+  message_assembly_ms: number | null;
+  stream_setup_ms: number | null;
+  tool_loop_ms: number | null;
+  tools_used_json: string | null;
+  tool_calls_count: number | null;
  success: number | null;
  error: string | null;
  dataset_id: string | null;
@@ -0,0 +1,12 @@
+import { RequestContext } from '@/services/context/requestContext';
+
+export function getInternalApiBaseUrl(): string {
+  const requestOrigin = (RequestContext.get() as { requestOrigin?: string }).requestOrigin;
+  if (requestOrigin) {
+    return requestOrigin;
+  }
+
+  return process.env.NEXT_PUBLIC_BASE_URL
+    || process.env.NEXT_PUBLIC_APP_URL
+    || 'http://localhost:3000';
+}