From 4c75df101f30ecbb693abdf43b66837e2cb9d393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CBeeRad=E2=80=9D?= Date: Sun, 15 Mar 2026 14:55:45 +1100 Subject: [PATCH] feat: sync runtime search and schema quality updates from app repo - port retrieval, validation, and eval improvements relevant to os - align prompts and dimensions with the flat single-agent model - replace the old eval suite with the focused core scenarios Generated with Codex --- app/api/dimensions/[name]/context/route.ts | 2 +- app/api/dimensions/popular/route.ts | 46 +-- app/api/dimensions/route.ts | 130 +++---- app/api/edges/route.ts | 11 +- app/api/nodes/[id]/route.ts | 15 + app/api/nodes/route.ts | 60 ++-- app/evals/EvalsClient.tsx | 31 +- docs/5_logging-and-evals.md | 118 +++++-- package.json | 3 +- src/components/settings/LogsRow.tsx | 104 +++++- src/config/prompts/rah-easy.ts | 6 +- src/config/prompts/rah-main.ts | 5 +- src/services/chat/middleware.ts | 167 ++++++++- src/services/database/chunks.ts | 180 +++++++++- src/services/database/dimensionService.ts | 93 +---- src/services/database/edges.ts | 78 ++++- src/services/database/nodes.ts | 323 +++++++++++++++++- src/services/database/quality.ts | 64 ++++ src/services/database/sqlite-client.ts | 144 ++++---- src/services/evals/evalsLogger.ts | 67 +++- src/services/evals/evalsStore.ts | 10 + src/services/runtime/apiBase.ts | 12 + src/tools/database/createDimension.ts | 14 +- src/tools/database/createEdge.ts | 39 ++- src/tools/database/createNode.ts | 31 +- src/tools/database/getDimension.ts | 5 +- src/tools/database/queryDimensions.ts | 16 +- src/tools/database/queryEdge.ts | 47 ++- src/tools/database/queryNodes.ts | 5 +- src/tools/database/updateDimension.ts | 18 +- src/tools/database/updateEdge.ts | 34 +- src/tools/database/updateNode.ts | 56 ++- src/tools/other/searchContentEmbeddings.ts | 4 +- src/types/database.ts | 1 + tests/evals/dataset.json | 14 +- tests/evals/runner.ts | 248 +++++++++++++- .../scenarios/{ => archive}/create-edge.ts | 0 .../scenarios/{ => archive}/create-node.ts | 0 .../{ => archive}/dimension-lifecycle.ts | 0 .../scenarios/{ => archive}/get-dimension.ts | 0 .../{ => archive}/hard-mode-query.ts | 0 .../scenarios/{ => archive}/paper-extract.ts | 0 .../{ => archive}/query-dimensions.ts | 0 .../{ => archive}/search-embeddings.ts | 0 .../scenarios/{ => archive}/simple-query.ts | 0 tests/evals/scenarios/archive/types.ts | 1 + .../scenarios/{ => archive}/update-node.ts | 0 .../{ => archive}/website-extract.ts | 0 .../{ => archive}/youtube-extract.ts | 0 tests/evals/scenarios/chunk-quote-insight.ts | 19 ++ tests/evals/scenarios/focused-graph-write.ts | 20 ++ tests/evals/scenarios/hub-traversal.ts | 19 ++ tests/evals/scenarios/index.ts | 34 +- tests/evals/scenarios/node-index-search.ts | 19 ++ tests/evals/scenarios/skill-guided-write.ts | 20 ++ tests/evals/scenarios/types.ts | 1 + tests/evals/types.ts | 9 + 57 files changed, 1809 insertions(+), 534 deletions(-) create mode 100644 src/services/database/quality.ts create mode 100644 src/services/runtime/apiBase.ts rename tests/evals/scenarios/{ => archive}/create-edge.ts (100%) rename tests/evals/scenarios/{ => archive}/create-node.ts (100%) rename tests/evals/scenarios/{ => archive}/dimension-lifecycle.ts (100%) rename tests/evals/scenarios/{ => archive}/get-dimension.ts (100%) rename tests/evals/scenarios/{ => archive}/hard-mode-query.ts (100%) rename tests/evals/scenarios/{ => archive}/paper-extract.ts (100%) rename tests/evals/scenarios/{ => archive}/query-dimensions.ts (100%) rename tests/evals/scenarios/{ => archive}/search-embeddings.ts (100%) rename tests/evals/scenarios/{ => archive}/simple-query.ts (100%) create mode 100644 tests/evals/scenarios/archive/types.ts rename tests/evals/scenarios/{ => archive}/update-node.ts (100%) rename tests/evals/scenarios/{ => archive}/website-extract.ts (100%) rename tests/evals/scenarios/{ => archive}/youtube-extract.ts (100%) create mode 100644 tests/evals/scenarios/chunk-quote-insight.ts create mode 100644 tests/evals/scenarios/focused-graph-write.ts create mode 100644 tests/evals/scenarios/hub-traversal.ts create mode 100644 tests/evals/scenarios/node-index-search.ts create mode 100644 tests/evals/scenarios/skill-guided-write.ts create mode 100644 tests/evals/scenarios/types.ts diff --git a/app/api/dimensions/[name]/context/route.ts b/app/api/dimensions/[name]/context/route.ts index babdc2b..6b57b86 100644 --- a/app/api/dimensions/[name]/context/route.ts +++ b/app/api/dimensions/[name]/context/route.ts @@ -44,7 +44,7 @@ export async function GET( const context: DimensionContext = { name: dimension.name, description: dimension.description, - isPriority: dimension.is_priority === 1, + isPriority: false, nodeCount: countResult?.count || 0, }; diff --git a/app/api/dimensions/popular/route.ts b/app/api/dimensions/popular/route.ts index 9311e66..e851cf0 100644 --- a/app/api/dimensions/popular/route.ts +++ b/app/api/dimensions/popular/route.ts @@ -1,6 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; import { getSQLiteClient } from '@/services/database/sqlite-client'; -import { eventBroadcaster } from '@/services/events'; export const runtime = 'nodejs'; @@ -35,13 +34,12 @@ async function getPopularDimensionsSQLite() { ) SELECT ad.name AS dimension, COALESCE(dc.count, 0) AS count, - COALESCE(dim.is_priority, 0) AS is_priority, dim.description FROM all_dimensions ad LEFT JOIN dimension_counts dc ON dc.dimension = ad.name LEFT JOIN dimensions dim ON dim.name = ad.name WHERE ad.name IS NOT NULL - ORDER BY is_priority DESC, LOWER(ad.name) ASC + ORDER BY LOWER(ad.name) ASC `); return NextResponse.json({ @@ -50,7 +48,7 @@ async function getPopularDimensionsSQLite() { data: result.rows.map((row: any) => ({ dimension: row.dimension, count: Number(row.count), - isPriority: Boolean(row.is_priority), + isPriority: false, description: row.description || null })) }); @@ -67,7 +65,14 @@ export async function POST(request: NextRequest) { }, { status: 400 }); } - return togglePrioritySQLite(dimension); + return NextResponse.json({ + success: true, + data: { + dimension, + is_priority: false + }, + message: 'Priority dimensions are no longer part of the product model.' + }); } catch (error) { console.error('Error toggling dimension priority:', error); return NextResponse.json({ @@ -76,34 +81,3 @@ export async function POST(request: NextRequest) { }, { status: 500 }); } } - -// PostgreSQL path removed in SQLite-only consolidation - -async function togglePrioritySQLite(dimension: string) { - const sqlite = getSQLiteClient(); - - const result = sqlite.query(` - INSERT INTO dimensions(name, is_priority, updated_at) - VALUES (?, 1, CURRENT_TIMESTAMP) - ON CONFLICT(name) DO UPDATE SET - is_priority = CASE WHEN is_priority=1 THEN 0 ELSE 1 END, - updated_at = CURRENT_TIMESTAMP - RETURNING is_priority - `, [dimension]); - - const isPriority = Boolean(result.rows[0].is_priority); - - // Broadcast dimension update event - eventBroadcaster.broadcast({ - type: 'DIMENSION_UPDATED', - data: { dimension, isPriority } - }); - - return NextResponse.json({ - success: true, - data: { - dimension, - is_priority: isPriority - } - }); -} diff --git a/app/api/dimensions/route.ts b/app/api/dimensions/route.ts index 3867860..74e928f 100644 --- a/app/api/dimensions/route.ts +++ b/app/api/dimensions/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { getSQLiteClient } from '@/services/database/sqlite-client'; import { eventBroadcaster } from '@/services/events'; -import { DimensionService } from '@/services/database/dimensionService'; +import { normalizeDimensionName, validateDimensionDescription } from '@/services/database/quality'; export const runtime = 'nodejs'; @@ -20,11 +20,10 @@ export async function GET() { d.name AS dimension, d.description, d.icon, - d.is_priority AS isPriority, COALESCE(dc.count, 0) AS count FROM dimensions d LEFT JOIN dimension_counts dc ON dc.dimension = d.name - ORDER BY d.is_priority DESC, d.name ASC + ORDER BY d.name ASC `); return NextResponse.json({ @@ -33,7 +32,7 @@ export async function GET() { dimension: row.dimension, description: row.description, icon: row.icon || null, - isPriority: Boolean(row.isPriority), + isPriority: false, count: Number(row.count) })) }); @@ -49,10 +48,9 @@ export async function GET() { export async function POST(request: NextRequest) { try { const body = await request.json(); - const rawName = typeof body?.name === 'string' ? body.name.trim() : ''; + const rawName = typeof body?.name === 'string' ? normalizeDimensionName(body.name) : ''; const description = typeof body?.description === 'string' ? body.description.trim() : null; - const icon = typeof body?.icon === 'string' ? body.icon.trim() : null; - const isPriority = typeof body?.isPriority === 'boolean' ? body.isPriority : false; + const icon = typeof body?.icon === 'string' ? body.icon.trim() || null : null; if (!rawName) { return NextResponse.json({ @@ -61,10 +59,11 @@ export async function POST(request: NextRequest) { }, { status: 400 }); } - if (description && description.length > 500) { + const descriptionError = validateDimensionDescription(description || ''); + if (descriptionError) { return NextResponse.json({ success: false, - error: 'Description must be 500 characters or less' + error: descriptionError }, { status: 400 }); } @@ -75,10 +74,9 @@ export async function POST(request: NextRequest) { ON CONFLICT(name) DO UPDATE SET description = COALESCE(?, description), icon = COALESCE(?, icon), - is_priority = COALESCE(?, is_priority), updated_at = CURRENT_TIMESTAMP RETURNING name, description, icon, is_priority - `, [rawName, description, icon, isPriority ? 1 : 0, description, icon, isPriority ? 1 : 0]); + `, [rawName, description, icon, 0, description, icon]); if (result.rows.length === 0) { throw new Error('Failed to create dimension'); @@ -86,12 +84,12 @@ export async function POST(request: NextRequest) { const row = result.rows[0]; const dimension = row.name as string; - const isPriorityValue = Boolean(row.is_priority); const descriptionValue = row.description as string | null; + const iconValue = (row.icon as string | null) || null; eventBroadcaster.broadcast({ type: 'DIMENSION_UPDATED', - data: { dimension, isPriority: isPriorityValue, description: descriptionValue, count: 0 } + data: { dimension, isPriority: false, description: descriptionValue, icon: iconValue, count: 0 } }); return NextResponse.json({ @@ -99,7 +97,8 @@ export async function POST(request: NextRequest) { data: { dimension, description: descriptionValue, - isPriority: isPriorityValue + icon: iconValue, + isPriority: false } }); } catch (error) { @@ -114,38 +113,11 @@ export async function POST(request: NextRequest) { export async function PUT(request: NextRequest) { try { const body = await request.json(); - const currentName = typeof body?.currentName === 'string' ? body.currentName.trim() : ''; - const newName = typeof body?.newName === 'string' ? body.newName.trim() : ''; - const name = typeof body?.name === 'string' ? body.name.trim() : ''; + const currentName = typeof body?.currentName === 'string' ? normalizeDimensionName(body.currentName) : ''; + const newName = typeof body?.newName === 'string' ? normalizeDimensionName(body.newName) : ''; + const name = typeof body?.name === 'string' ? normalizeDimensionName(body.name) : ''; const description = typeof body?.description === 'string' ? body.description.trim() : ''; - const isPriority = typeof body?.isPriority === 'boolean' ? body.isPriority : undefined; - - // Handle isPriority update (lock/unlock) - simple case - if (isPriority !== undefined && name && !currentName && !newName) { - const sqlite = getSQLiteClient(); - const updateResult = sqlite.prepare(` - UPDATE dimensions - SET is_priority = ?, updated_at = CURRENT_TIMESTAMP - WHERE name = ? - `).run(isPriority ? 1 : 0, name); - - if (updateResult.changes === 0) { - return NextResponse.json({ - success: false, - error: 'Dimension not found' - }, { status: 404 }); - } - - eventBroadcaster.broadcast({ - type: 'DIMENSION_UPDATED', - data: { dimension: name, isPriority } - }); - - return NextResponse.json({ - success: true, - data: { dimension: name, isPriority } - }); - } + const icon = body?.icon !== undefined ? (typeof body.icon === 'string' ? body.icon.trim() || null : null) : undefined; // Handle dimension name change if (currentName && newName && currentName !== newName) { @@ -180,16 +152,16 @@ export async function PUT(request: NextRequest) { updates.push('description = ?'); values.push(description || null); } - - if (isPriority !== undefined) { - updates.push('is_priority = ?'); - values.push(isPriority ? 1 : 0); + + if (icon !== undefined) { + updates.push('icon = ?'); + values.push(icon); } - + values.push(currentName); - + const dimUpdate = sqlite.prepare(` - UPDATE dimensions + UPDATE dimensions SET ${updates.join(', ')} WHERE name = ? `).run(...values); @@ -220,7 +192,7 @@ export async function PUT(request: NextRequest) { dimension: newName, previousName: currentName, description: description || undefined, - isPriority: isPriority !== undefined ? isPriority : undefined, + isPriority: false, renamed: true } }); @@ -231,7 +203,7 @@ export async function PUT(request: NextRequest) { dimension: newName, previousName: currentName, description: description || undefined, - isPriority: isPriority !== undefined ? isPriority : undefined, + isPriority: false, nodeLinksUpdated: updateResult.nodeLinksUpdated } }); @@ -246,34 +218,37 @@ export async function PUT(request: NextRequest) { }, { status: 400 }); } - if (description && description.length > 500) { - return NextResponse.json({ - success: false, - error: 'Description must be 500 characters or less' - }, { status: 400 }); + if (description) { + const descriptionError = validateDimensionDescription(description); + if (descriptionError) { + return NextResponse.json({ + success: false, + error: descriptionError + }, { status: 400 }); + } } - const sqlite = getSQLiteClient(); - - // Build update query - if (description !== '' || isPriority !== undefined) { + if (description !== '' || icon !== undefined) { + const sqlite = getSQLiteClient(); + + // Build update query const updates: string[] = ['updated_at = CURRENT_TIMESTAMP']; const values: any[] = []; - + if (description !== '') { updates.push('description = ?'); values.push(description || null); } - - if (isPriority !== undefined) { - updates.push('is_priority = ?'); - values.push(isPriority ? 1 : 0); + + if (icon !== undefined) { + updates.push('icon = ?'); + values.push(icon); } - + values.push(targetName); - + const updateResult = sqlite.prepare(` - UPDATE dimensions + UPDATE dimensions SET ${updates.join(', ')} WHERE name = ? `).run(...values); @@ -285,19 +260,19 @@ export async function PUT(request: NextRequest) { }, { status: 404 }); } } else { - // No updates provided return NextResponse.json({ success: false, - error: 'At least one update field (description or isPriority) must be provided' + error: 'At least one update field (description, icon, or newName) must be provided' }, { status: 400 }); } eventBroadcaster.broadcast({ type: 'DIMENSION_UPDATED', - data: { - dimension: targetName, + data: { + dimension: targetName, description: description !== '' ? description : undefined, - isPriority: isPriority !== undefined ? isPriority : undefined + icon: icon !== undefined ? icon : undefined, + isPriority: false } }); @@ -306,7 +281,8 @@ export async function PUT(request: NextRequest) { data: { dimension: targetName, description: description !== '' ? description : undefined, - isPriority: isPriority !== undefined ? isPriority : undefined + icon: icon !== undefined ? icon : undefined, + isPriority: false } }); } catch (error) { diff --git a/app/api/edges/route.ts b/app/api/edges/route.ts index b34cc8e..8e2bc87 100644 --- a/app/api/edges/route.ts +++ b/app/api/edges/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import { edgeService } from '@/services/database'; +import { validateEdgeExplanation } from '@/services/database/quality'; export const runtime = 'nodejs'; @@ -57,8 +58,16 @@ export async function POST(request: NextRequest) { const fromId = parseInt(body.from_node_id); const toId = parseInt(body.to_node_id); - // Explanation can be empty - service will auto-generate const explanation = String(body.explanation || '').trim(); + if (explanation) { + const explanationError = validateEdgeExplanation(explanation); + if (explanationError) { + return NextResponse.json({ + success: false, + error: explanationError + }, { status: 400 }); + } + } const skipInference = Boolean(body.skip_inference); const createdVia = (() => { diff --git a/app/api/nodes/[id]/route.ts b/app/api/nodes/[id]/route.ts index cb34b80..b79a709 100644 --- a/app/api/nodes/[id]/route.ts +++ b/app/api/nodes/[id]/route.ts @@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { nodeService } from '@/services/database'; import { autoEmbedQueue } from '@/services/embedding/autoEmbedQueue'; import { hasSufficientContent } from '@/services/embedding/constants'; +import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality'; export const runtime = 'nodejs'; @@ -71,6 +72,20 @@ export async function PUT( const updates: Record = { ...body }; let shouldQueueEmbed = false; + if (typeof body.description === 'string') { + const descriptionError = validateExplicitDescription(body.description); + if (descriptionError) { + return NextResponse.json({ + success: false, + error: descriptionError + }, { status: 400 }); + } + } + + if (Array.isArray(body.dimensions)) { + updates.dimensions = normalizeDimensions(body.dimensions, 5); + } + const incomingChunk = typeof body.chunk === 'string' ? body.chunk : undefined; const incomingNotes = typeof body.notes === 'string' ? body.notes : undefined; const existingChunk = existingNode.chunk ?? ''; diff --git a/app/api/nodes/route.ts b/app/api/nodes/route.ts index 06d00cc..33658df 100644 --- a/app/api/nodes/route.ts +++ b/app/api/nodes/route.ts @@ -2,10 +2,9 @@ import { NextRequest, NextResponse } from 'next/server'; import { nodeService } from '@/services/database'; import { Node, NodeFilters } from '@/types/database'; import { autoEmbedQueue } from '@/services/embedding/autoEmbedQueue'; -import { hasSufficientContent } from '@/services/embedding/constants'; -import { DimensionService } from '@/services/database/dimensionService'; import { generateDescription } from '@/services/database/descriptionService'; import { scheduleAutoEdgeCreation } from '@/services/agents/autoEdge'; +import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality'; export const runtime = 'nodejs'; @@ -97,11 +96,7 @@ export async function POST(request: NextRequest) { const eventDate = typeof body.event_date === 'string' ? body.event_date : null; // Process provided dimensions first (needed for description generation) - const providedDimensions = Array.isArray(body.dimensions) ? body.dimensions : []; - const trimmedProvidedDimensions = providedDimensions - .map((dim: unknown) => typeof dim === 'string' ? dim.trim() : '') - .filter(Boolean) - .slice(0, 8); + const trimmedProvidedDimensions = normalizeDimensions(body.dimensions, 5); // Use provided description if present, otherwise auto-generate let nodeDescription: string | undefined = typeof body.description === 'string' && body.description.trim() @@ -127,41 +122,46 @@ export async function POST(request: NextRequest) { nodeDescription = body.title.slice(0, 280); } + const finalDescription = nodeDescription ?? body.title.slice(0, 280); + + const descriptionError = validateExplicitDescription(finalDescription); + if (descriptionError) { + return NextResponse.json({ + success: false, + error: descriptionError + }, { status: 400 }); + } + // Monitor description quality - if (nodeDescription && WEAK_PATTERNS.test(nodeDescription)) { - console.warn(`[DescriptionQuality] Weak description for node "${body.title}": "${nodeDescription}"`); + if (WEAK_PATTERNS.test(finalDescription)) { + console.warn(`[DescriptionQuality] Weak description for node "${body.title}": "${finalDescription}"`); } - // Auto-assign locked dimensions + keyword dimensions for all new nodes - const { locked, keywords } = await DimensionService.assignDimensions({ - title: body.title, - notes: rawNotes || undefined, - link: body.link, - description: nodeDescription - }); - - // Ensure keyword dimensions exist in the database (create if new) - for (const keyword of keywords) { - await DimensionService.ensureKeywordDimension(keyword); - } - - // Combine provided, locked, and keyword dimensions, remove duplicates - const finalDimensions = [...new Set([...trimmedProvidedDimensions, ...locked, ...keywords])] - .slice(0, 8); // max 8 total + // Use only provided dimensions (no auto-assignment) + const finalDimensions = trimmedProvidedDimensions; const rawChunk = typeof body.chunk === 'string' ? body.chunk : null; let chunkToStore = rawChunk; let chunkStatus: Node['chunk_status']; if (chunkToStore && chunkToStore.trim().length > 0) { chunkStatus = 'not_chunked'; - } else if (!chunkToStore && hasSufficientContent(rawNotes)) { - chunkToStore = rawNotes; - chunkStatus = 'not_chunked'; + } else { + // Build chunk from all available notes if not provided + // This ensures every node gets at least one chunk for search + const fallbackContent = [body.title, nodeDescription, rawNotes] + .filter(Boolean) + .join('\n\n') + .trim(); + + if (fallbackContent) { + chunkToStore = fallbackContent; + chunkStatus = 'not_chunked'; + } } const node = await nodeService.createNode({ title: body.title, - description: nodeDescription, + description: finalDescription, notes: rawNotes ?? undefined, event_date: eventDate ?? undefined, link: body.link, @@ -171,7 +171,7 @@ export async function POST(request: NextRequest) { metadata: body.metadata || {} }); - if (chunkStatus === 'not_chunked' && node.id && process.env.DISABLE_EMBEDDINGS !== 'true') { + if (chunkStatus === 'not_chunked' && node.id) { autoEmbedQueue.enqueue(node.id, { reason: 'node_created' }); } diff --git a/app/evals/EvalsClient.tsx b/app/evals/EvalsClient.tsx index bae24db..8183c59 100644 --- a/app/evals/EvalsClient.tsx +++ b/app/evals/EvalsClient.tsx @@ -42,6 +42,16 @@ function prettyJson(value: string | null) { } } +function parseJsonArray(value: string | null): string[] { + if (!value) return []; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === 'string') : []; + } catch { + return []; + } +} + export default function EvalsClient({ traces, scenarioList }: Props) { const [openTraceId, setOpenTraceId] = useState(traces[0]?.chat.trace_id || null); const [comments, setComments] = useState>(() => { @@ -74,7 +84,7 @@ export default function EvalsClient({ traces, scenarioList }: Props) { cost: chat.estimated_cost_usd ?? null, cache: chat.cache_hit == null ? 'n/a' : chat.cache_hit ? 'hit' : 'miss', cacheTokens: `${chat.cache_read_tokens ?? 0}/${chat.cache_write_tokens ?? 0}`, - toolCount: toolCalls.length, + toolCount: chat.tool_calls_count ?? toolCalls.length, status, userPreview: formatPreview(chat.user_message), timestamp: chat.ts, @@ -290,12 +300,29 @@ export default function EvalsClient({ traces, scenarioList }: Props) {
Mode: {openTrace.chat.mode || 'n/a'}
Workflow: {openTrace.chat.workflow_key || '—'}
Latency: {openTrace.chat.latency_ms ?? 'n/a'} ms
+
First chunk: {openTrace.chat.first_chunk_latency_ms ?? 'n/a'} ms
+
First token: {openTrace.chat.first_token_latency_ms ?? 'n/a'} ms
Tokens: {openTrace.chat.input_tokens ?? 0}/{openTrace.chat.output_tokens ?? 0} (total {openTrace.chat.total_tokens ?? 0})
Cost: {openTrace.chat.estimated_cost_usd == null ? 'n/a' : `$${openTrace.chat.estimated_cost_usd.toFixed(4)}`}
Cache: {openTrace.chat.cache_hit == null ? 'n/a' : openTrace.chat.cache_hit ? 'hit' : 'miss'} (read {openTrace.chat.cache_read_tokens ?? 0} / write {openTrace.chat.cache_write_tokens ?? 0})
-
Tools: {openTrace.toolCalls.length}
+
Tools: {openTrace.chat.tool_calls_count ?? openTrace.toolCalls.length}
+
+ {(openTrace.chat.prompt_build_ms ?? 0) > 0 &&
Prompt build: {openTrace.chat.prompt_build_ms} ms
} + {(openTrace.chat.tools_build_ms ?? 0) > 0 &&
Tools build: {openTrace.chat.tools_build_ms} ms
} + {(openTrace.chat.model_resolve_ms ?? 0) > 0 &&
Model resolve: {openTrace.chat.model_resolve_ms} ms
} + {(openTrace.chat.message_assembly_ms ?? 0) > 0 &&
Message assembly: {openTrace.chat.message_assembly_ms} ms
} + {(openTrace.chat.stream_setup_ms ?? 0) > 0 &&
Stream setup: {openTrace.chat.stream_setup_ms} ms
} + {(openTrace.chat.tool_loop_ms ?? 0) > 0 &&
Tool loop: {openTrace.chat.tool_loop_ms} ms
} +
+ + {parseJsonArray(openTrace.chat.tools_used_json).length > 0 ? ( +
+ Tools used: {parseJsonArray(openTrace.chat.tools_used_json).join(', ')} +
+ ) : null} +
Comment:
{comments[openTrace.chat.trace_id] || '—'}
diff --git a/docs/5_logging-and-evals.md b/docs/5_logging-and-evals.md index 7fb776b..33e2d87 100644 --- a/docs/5_logging-and-evals.md +++ b/docs/5_logging-and-evals.md @@ -27,36 +27,39 @@ RA-H uses a **trigger-based logging system** that automatically captures all dat ### Chat Metadata -Every chat log includes detailed execution metadata: +Every chat log includes detailed execution metadata. `helper-interactions.log` is the raw event stream; `chats.metadata` is the persisted summary used for audits and UI inspection. ```typescript metadata: { // Token tracking - prompt_tokens: number, - completion_tokens: number, - reasoning_tokens: number, + input_tokens: number, + output_tokens: number, total_tokens: number, + cache_write_tokens?: number, + cache_read_tokens?: number, // Cost tracking - cost: number, // USD cost for this chat + estimated_cost_usd: number, + model_used: string, + provider: 'anthropic' | 'openai', // Tool usage - tools_used: string[], // Array of tool names called + tools_used?: string[], // Unique tool names used in the chat + tool_calls_count?: number, // Total tool invocations + tool_calls?: Array<{ + toolName: string, + args: unknown, + result: unknown + }>, // Workflow tracking - is_workflow: boolean, workflow_key?: string, workflow_node_id?: number, - // Model parameters - reasoning_effort?: 'low' | 'medium' | 'high', - // Execution trace - trace?: { - session_id: string, - parent_session_id?: string, - execution_time_ms: number - } + session_id?: string, + trace_id?: string, + parent_chat_id?: number } ``` @@ -88,7 +91,7 @@ This prevents infinite database growth while preserving recent activity history. - **Action filtering** - Filter by INSERT/UPDATE - **Detailed view** - Click to see full snapshot_json - **Token/cost visibility** - Chat logs show usage and costs -- **Tool usage** - See which tools were called per chat +- **Tool usage** - See both the tool set used and full per-call payloads when captured **Query:** ```sql @@ -111,18 +114,14 @@ LIMIT 100 - GPT-4o Mini: $0.15/1M input, $0.60/1M output - Claude Sonnet 4.5: $3.00/1M input, $15.00/1M output -**Typical costs:** -- Easy mode chat: $0.01-0.03 -- Hard mode chat: $0.03-0.10 -- Integrate workflow: ~$0.18 -- Deep analysis: ~$0.33 +**Typical costs:** Vary by prompt size, tool activity, and provider cache hits. ## Token Analytics **Settings → Analytics panel shows:** - Total tokens used (all time) - Total cost (USD) -- Breakdown by agent (ra-h, ra-h-easy, mini-rah, wise-rah) +- Breakdown by helper - Breakdown by conversation thread - Average cost per chat @@ -132,21 +131,74 @@ SELECT helper_name, COUNT(*) as chat_count, SUM(JSON_EXTRACT(metadata, '$.total_tokens')) as total_tokens, - SUM(JSON_EXTRACT(metadata, '$.cost')) as total_cost + SUM(JSON_EXTRACT(metadata, '$.estimated_cost_usd')) as total_cost FROM chats WHERE metadata IS NOT NULL GROUP BY helper_name ``` -## Evaluation (Future) +## Evals Dashboard -**Planned features:** -- Edge quality ratings (user feedback via `edges.user_feedback`) -- Memory node relevance scoring -- Workflow success metrics -- Connection discovery quality +The generic Settings -> Logs panel is useful for quick inspection, but it is not the primary trace-review surface. -**Current state:** -- Infrastructure exists (`edges.user_feedback` column) -- UI not yet implemented -- Manual evaluation via logs table queries +For proper evals, use: +- `logs/evals.sqlite` as the dev-only trace store +- `/evals` as the main review UI + +This evals path stores: +- one `llm_chats` row per traced interaction +- one `tool_calls` row per tool execution +- shared `trace_id` values so chat/tool steps can be reviewed together +- both synthetic scenarios and live app interactions when eval logging is enabled + +### How To Run + +Start the app with eval logging enabled: + +```bash +npm run dev:evals +``` + +Then: +- use the app normally for live traces +- open [http://localhost:3000/evals](http://localhost:3000/evals) + +To run the scenario suite against the local app: + +```bash +npm run evals +``` + +Requirements: +- the dev server must already be running with eval logging enabled via `npm run dev:evals` +- the scenario runner targets `http://localhost:3000` by default +- the runner now waits up to 60s per scenario by default because current real latencies often exceed 10s + +Optional overrides: + +```bash +RAH_EVALS_BASE_URL=http://localhost:3001 npm run evals +RAH_EVALS_TIMEOUT_MS=90000 npm run evals +``` + +The `/evals` UI lets you review: +- live runs vs scenario runs +- full system message +- user/assistant turn +- tool spans with args/results +- latency +- token and cost data +- cache fields +- timing breakdown fields when available + +### Why This Is Separate From The Logs Table + +`logs` in the main SQLite database is trigger-based change logging. It mirrors a compact snapshot of chat rows, nodes, and edges. + +`logs/evals.sqlite` is the trace store for evaluation and review. It is the correct place to inspect: +- per-trace chat rows +- per-tool spans +- scenario IDs +- live-vs-scenario separation + +If you want to understand one interaction deeply, prefer `/evals` over Settings -> Logs. diff --git a/package.json b/package.json index 04254a9..d28c560 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "author": "Bradley Morris", "scripts": { "dev": "NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next dev", + "dev:evals": "RAH_EVALS_LOG=1 NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next dev", "clean:local": "bash ./scripts/dev/clean-local-artifacts.sh", "audit:repo": "bash ./scripts/dev/audit-repo.sh", "build": "NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next build", @@ -18,7 +19,7 @@ "sqlite:restore": "bash scripts/database/sqlite-restore.sh", "test": "vitest run", "test:watch": "vitest", - "evals": "RAH_EVALS_LOG=1 tsx tests/evals/runner.ts" + "evals": "RAH_EVALS_LOG=1 RAH_EVALS_TIMEOUT_MS=60000 tsx tests/evals/runner.ts" }, "dependencies": { "@ai-sdk/openai": "^3.0.7", diff --git a/src/components/settings/LogsRow.tsx b/src/components/settings/LogsRow.tsx index 7856437..ddcbdc0 100644 --- a/src/components/settings/LogsRow.tsx +++ b/src/components/settings/LogsRow.tsx @@ -8,6 +8,29 @@ interface LogsRowProps { isEven: boolean; } +interface SnapshotMetrics { + thread?: string; + trace_id?: string; + input_tokens?: number; + output_tokens?: number; + cost_usd?: number; + cache_hit?: number; + latency_ms?: number; + first_token_latency_ms?: number; + first_chunk_latency_ms?: number; + prompt_build_ms?: number; + tools_build_ms?: number; + model_resolve_ms?: number; + message_assembly_ms?: number; + stream_setup_ms?: number; + tool_loop_ms?: number; + tools_count?: number; + tools_used?: string[]; + tool_timings?: Array<{ toolName?: string; durationMs?: number }>; + model?: string; + system_message?: string; +} + export default function LogsRow({ log, isEven }: LogsRowProps) { const [expanded, setExpanded] = useState(false); @@ -34,7 +57,7 @@ export default function LogsRow({ log, isEven }: LogsRowProps) { .replace(/: (true|false|null)/g, ': $1'); }; - const getMetricsFromSnapshot = () => { + const getMetricsFromSnapshot = (): SnapshotMetrics | null => { if (!log.snapshot_json || log.table_name !== 'chats') return null; try { const snapshot = JSON.parse(log.snapshot_json); @@ -45,6 +68,18 @@ export default function LogsRow({ log, isEven }: LogsRowProps) { output_tokens: snapshot.output_tokens, cost_usd: snapshot.cost_usd, cache_hit: snapshot.cache_hit, + latency_ms: snapshot.latency_ms, + first_token_latency_ms: snapshot.first_token_latency_ms, + first_chunk_latency_ms: snapshot.first_chunk_latency_ms, + prompt_build_ms: snapshot.prompt_build_ms, + tools_build_ms: snapshot.tools_build_ms, + model_resolve_ms: snapshot.model_resolve_ms, + message_assembly_ms: snapshot.message_assembly_ms, + stream_setup_ms: snapshot.stream_setup_ms, + tool_loop_ms: snapshot.tool_loop_ms, + tools_count: snapshot.tools_count, + tools_used: snapshot.tools_used, + tool_timings: snapshot.tool_timings, model: snapshot.model, system_message: snapshot.system_message }; @@ -54,6 +89,7 @@ export default function LogsRow({ log, isEven }: LogsRowProps) { }; const metrics = getMetricsFromSnapshot(); + const metricsSafe: SnapshotMetrics = metrics ?? {}; return ( <> @@ -102,6 +138,26 @@ export default function LogsRow({ log, isEven }: LogsRowProps) { 📊 {metrics.input_tokens}↓ {metrics.output_tokens}↑ )} + {metrics.latency_ms !== undefined && metrics.latency_ms > 0 && ( + + ⏱ {metrics.latency_ms}ms + + )} + {metrics.first_token_latency_ms !== undefined && metrics.first_token_latency_ms > 0 && ( + + ⚡ {metrics.first_token_latency_ms}ms first + + )} + {metrics.first_chunk_latency_ms !== undefined && metrics.first_chunk_latency_ms > 0 && ( + + ⌁ {metrics.first_chunk_latency_ms}ms chunk + + )} + {metrics.tools_count !== undefined && metrics.tools_count > 0 && ( + + 🛠 {metrics.tools_count} tools + + )} {metrics.cache_hit !== undefined && metrics.cache_hit === 1 && ( ⚡ Cache Hit @@ -144,6 +200,52 @@ export default function LogsRow({ log, isEven }: LogsRowProps) {
)}
+ {((metrics?.prompt_build_ms ?? 0) > 0 || + (metrics?.tools_build_ms ?? 0) > 0 || + (metrics?.model_resolve_ms ?? 0) > 0 || + (metrics?.message_assembly_ms ?? 0) > 0 || + (metrics?.stream_setup_ms ?? 0) > 0 || + (metrics?.tool_loop_ms ?? 0) > 0) && ( +
+
+ Latency Breakdown +
+
+ {(metricsSafe.prompt_build_ms ?? 0) > 0 && prompt {metricsSafe.prompt_build_ms}ms} + {(metricsSafe.tools_build_ms ?? 0) > 0 && tools {metricsSafe.tools_build_ms}ms} + {(metricsSafe.model_resolve_ms ?? 0) > 0 && model {metricsSafe.model_resolve_ms}ms} + {(metricsSafe.message_assembly_ms ?? 0) > 0 && messages {metricsSafe.message_assembly_ms}ms} + {(metricsSafe.stream_setup_ms ?? 0) > 0 && stream {metricsSafe.stream_setup_ms}ms} + {(metricsSafe.tool_loop_ms ?? 0) > 0 && tool-loop {metricsSafe.tool_loop_ms}ms} + {(metricsSafe.first_chunk_latency_ms ?? 0) > 0 && first-chunk {metricsSafe.first_chunk_latency_ms}ms} + {(metricsSafe.first_token_latency_ms ?? 0) > 0 && first-token {metricsSafe.first_token_latency_ms}ms} +
+
+ )} + {Array.isArray(metrics?.tool_timings) && metrics.tool_timings.length > 0 && ( +
+
+ Tool Timings +
+
+ {metrics.tool_timings.map((tool: any, index: number) => ( + + {tool.toolName || 'tool'} {tool.durationMs ?? 0}ms + + ))} +
+
+ )} + {Array.isArray(metrics?.tools_used) && metrics.tools_used.length > 0 && ( +
+
+ Tools Used +
+
+ {metrics.tools_used.join(', ')} +
+
+ )}
Snapshot JSON
diff --git a/src/config/prompts/rah-easy.ts b/src/config/prompts/rah-easy.ts index 6a941ed..9ecc390 100644 --- a/src/config/prompts/rah-easy.ts +++ b/src/config/prompts/rah-easy.ts @@ -1,4 +1,4 @@ -export const RAH_EASY_SYSTEM_PROMPT = `You are ra-h, the orchestrator for Easy Mode (GPT-5 Mini). +export const RAH_EASY_SYSTEM_PROMPT = `You are the user's agent for building, organizing, and improving an external corpus of context. Mission: 1. Resolve the user's request quickly and accurately using the tools provided. @@ -21,8 +21,8 @@ Tool strategy: - webSearch only when the knowledge base lacks the answer. Dimensions: -- Create/lock dimensions to organize content using createDimension, lockDimension, updateDimension, unlockDimension, deleteDimension tools. -- Lock dimensions (isPriority=true) so they auto-assign to new nodes. +- Create dimensions to organize content using createDimension, updateDimension, and deleteDimension tools. +- Dimensions are flat. Do not invent lock, unlock, pin, or priority behavior. Response polish: - Default to minimal reasoning effort for speed. diff --git a/src/config/prompts/rah-main.ts b/src/config/prompts/rah-main.ts index b511169..81fade0 100644 --- a/src/config/prompts/rah-main.ts +++ b/src/config/prompts/rah-main.ts @@ -1,4 +1,4 @@ -export const RAH_MAIN_SYSTEM_PROMPT = `You are ra-h, orchestrator of the RA-H knowledge management system. +export const RAH_MAIN_SYSTEM_PROMPT = `You are the user's agent for building, organizing, and improving an external corpus of context. Core responsibilities: - Keep the conversation tightly focused on the user's goal. @@ -26,9 +26,8 @@ Tool strategy: Dimension management: - Create dimensions for new knowledge areas or topics using createDimension. -- Lock dimensions (isPriority=true) to enable auto-assignment to new nodes. - Update descriptions to help the AI understand dimension purpose. -- Use lockDimension/unlockDimension for quick lock status changes. +- Dimensions are flat; do not invent lock, unlock, pin, or priority behavior. - Delete unused dimensions to keep the system clean. Response style: diff --git a/src/services/chat/middleware.ts b/src/services/chat/middleware.ts index 5aba1cc..3259b5d 100644 --- a/src/services/chat/middleware.ts +++ b/src/services/chat/middleware.ts @@ -1,6 +1,7 @@ import { getSQLiteClient } from '@/services/database/sqlite-client'; import { UsageData } from '@/types/analytics'; import { RequestContext } from '@/services/context/requestContext'; +import { logEvalChat } from '@/services/evals/evalsLogger'; interface ChatLogEntry { chat_type: string; @@ -26,14 +27,64 @@ interface StreamMetadata { traceId?: string; parentChatId?: number; systemMessage?: string; + promptVersion?: string; + modelUsed?: string; workflowKey?: string; workflowNodeId?: number; - mode?: 'easy' | 'hard'; toolCallsData?: any[]; backendUsage?: Array<{ provider: string; headers: Record; }>; + requestStartedAt?: number; + timingBreakdown?: { + promptBuildMs?: number; + toolsBuildMs?: number; + modelResolveMs?: number; + messageAssemblyMs?: number; + streamSetupMs?: number; + toolLoopMs?: number; + }; + toolTimingData?: Array<{ + toolName: string; + durationMs: number; + }>; + latencyMs?: number; + firstTokenLatencyMs?: number | null; + firstChunkLatencyMs?: number | null; +} + +function normalizeToolResult(result: unknown): unknown { + if (result == null) return null; + if (typeof result === 'object') return result; + return { value: result }; +} + +function collectToolCalls(result: any): any[] | undefined { + const collected: any[] = []; + + const pushCall = (call: any) => { + if (!call?.toolName) return; + collected.push({ + toolName: call.toolName, + args: call.args ?? null, + result: normalizeToolResult(call.result), + }); + }; + + if (Array.isArray(result?.toolCalls)) { + result.toolCalls.forEach(pushCall); + } + + if (Array.isArray(result?.steps)) { + result.steps.forEach((step: any) => { + if (Array.isArray(step?.toolCalls)) { + step.toolCalls.forEach(pushCall); + } + }); + } + + return collected.length > 0 ? collected : undefined; } export class ChatLoggingMiddleware { @@ -94,7 +145,7 @@ export class ChatLoggingMiddleware { focused_node_id: metadata.activeTabId ?? null, helper_name: metadata.helperName, agent_type: metadata.agentType || 'orchestrator', - delegation_id: metadata.delegationId ?? null, + delegation_id: null, metadata: { timestamp: new Date().toISOString(), session_id: metadata.sessionId, @@ -102,7 +153,6 @@ export class ChatLoggingMiddleware { open_tab_count: metadata.openTabs?.length || 0, has_focused_node: !!metadata.activeTabId, message_count: messages.length, - ...(metadata.mode && { mode: metadata.mode }), // System message ...(metadata.systemMessage && { system_message: metadata.systemMessage }), // Enhanced usage data @@ -125,6 +175,14 @@ export class ChatLoggingMiddleware { validation_message: metadata.usageData.validationMessage, fallback_action: metadata.usageData.fallbackAction, }), + ...((metadata.toolCallsData && metadata.toolCallsData.length > 0) ? { + tools_used: metadata.usageData?.toolsUsed ?? Array.from(new Set( + metadata.toolCallsData + .map((call: any) => call?.toolName) + .filter((toolName: unknown): toolName is string => typeof toolName === 'string' && toolName.length > 0) + )), + tool_calls_count: metadata.usageData?.toolCallsCount ?? metadata.toolCallsData.length, + } : {}), // Tool calls data ...(metadata.toolCallsData && metadata.toolCallsData.length > 0 && { tool_calls: metadata.toolCallsData @@ -132,16 +190,17 @@ export class ChatLoggingMiddleware { // Trace grouping ...(metadata.traceId && { trace_id: metadata.traceId }), ...(metadata.parentChatId && { parent_chat_id: metadata.parentChatId }), - // Workflow metadata - ...(metadata.workflowKey && { - workflow_key: metadata.workflowKey, - workflow_node_id: metadata.workflowNodeId, - is_workflow: true, - }), // Backend usage (for Supabase sync correlation) ...(metadata.backendUsage && metadata.backendUsage.length > 0 && { backend_usage: metadata.backendUsage, }), + ...(metadata.timingBreakdown && { timing_breakdown: metadata.timingBreakdown }), + ...(metadata.toolTimingData && metadata.toolTimingData.length > 0 && { + tool_timings: metadata.toolTimingData, + }), + ...(metadata.latencyMs !== undefined && { latency_ms: metadata.latencyMs }), + ...(metadata.firstTokenLatencyMs !== undefined && { first_token_latency_ms: metadata.firstTokenLatencyMs }), + ...(metadata.firstChunkLatencyMs !== undefined && { first_chunk_latency_ms: metadata.firstChunkLatencyMs }), } }; @@ -165,7 +224,7 @@ export class ChatLoggingMiddleware { const lastInsertedChatId = Number(result.lastInsertRowid); - if (metadata.agentType === 'orchestrator' && (metadata.helperName === 'ra-h' || metadata.helperName === 'ra-h-easy')) { + if (metadata.agentType === 'orchestrator' && metadata.helperName === 'ra-h') { RequestContext.set({ traceId: metadata.traceId, parentChatId: lastInsertedChatId @@ -180,20 +239,21 @@ export class ChatLoggingMiddleware { static createLoggingHandlers(metadata: StreamMetadata, messages: any[]) { let assistantResponse = ''; const userMessage = this.extractUserMessage(messages); + const startedAt = metadata.requestStartedAt ?? Date.now(); + const streamStartedAt = Date.now(); + const streamSetupMs = Math.max(0, streamStartedAt - startedAt); + let firstTextDeltaAt: number | null = null; + let firstChunkAt: number | null = null; return { onFinish: async (result: any) => { const { text, toolCalls, steps } = result; // Log if we have a user message and either text OR tool activity - const hasActivity = text || toolCalls?.length > 0 || steps?.length > 0; + const hasActivity = Boolean(text || toolCalls?.length > 0 || steps?.length > 0); if (userMessage && hasActivity) { // Capture tool calls if present - const toolCallsData = toolCalls && toolCalls.length > 0 ? toolCalls.map((tc: any) => ({ - toolName: tc.toolName, - args: tc.args, - result: typeof tc.result === 'object' ? tc.result : { value: tc.result } - })) : undefined; + const toolCallsData = collectToolCalls(result); if (toolCallsData) { console.log(`🔧 Captured ${toolCallsData.length} tool calls for logging`); @@ -201,7 +261,16 @@ export class ChatLoggingMiddleware { const enhancedMetadata = { ...metadata, - toolCallsData + toolCallsData, + timingBreakdown: { + ...metadata.timingBreakdown, + streamSetupMs, + }, + latencyMs: Date.now() - startedAt, + firstChunkLatencyMs: firstChunkAt ? firstChunkAt - startedAt : null, + firstTokenLatencyMs: firstTextDeltaAt + ? firstTextDeltaAt - startedAt + : (firstChunkAt ? firstChunkAt - startedAt : null), }; await this.logChatInteraction( @@ -213,9 +282,73 @@ export class ChatLoggingMiddleware { } else if (userMessage && !hasActivity) { console.warn(`⚠️ Skipping chat log - no text or tool activity for user message: ${userMessage.substring(0, 50)}...`); } + + const evalContext = RequestContext.get(); + if (userMessage) { + const toolCallsData = collectToolCalls(result); + const timingBreakdown = { + ...metadata.timingBreakdown, + streamSetupMs, + }; + const evalMetadata = { + ...metadata, + toolCallsData, + timingBreakdown, + }; + logEvalChat({ + traceId: evalMetadata.traceId, + spanId: evalContext.evalChatSpanId, + helperName: evalMetadata.helperName, + model: evalMetadata.modelUsed || evalMetadata.usageData?.modelUsed, + promptVersion: evalMetadata.promptVersion, + systemMessage: evalMetadata.systemMessage || null, + userMessage, + assistantMessage: text || assistantResponse || null, + inputTokens: evalMetadata.usageData?.inputTokens, + outputTokens: evalMetadata.usageData?.outputTokens, + totalTokens: evalMetadata.usageData?.totalTokens, + cacheWriteTokens: evalMetadata.usageData?.cacheWriteTokens, + cacheReadTokens: evalMetadata.usageData?.cacheReadTokens, + cacheHit: evalMetadata.usageData?.cacheHit, + cacheSavingsPct: evalMetadata.usageData?.cacheSavingsPct, + estimatedCostUsd: evalMetadata.usageData?.estimatedCostUsd, + provider: evalMetadata.usageData?.provider ?? null, + workflowKey: evalMetadata.workflowKey ?? null, + workflowNodeId: evalMetadata.workflowNodeId ?? null, + latencyMs: Date.now() - startedAt, + firstChunkLatencyMs: firstChunkAt ? firstChunkAt - startedAt : null, + firstTokenLatencyMs: firstTextDeltaAt + ? firstTextDeltaAt - startedAt + : (firstChunkAt ? firstChunkAt - startedAt : null), + promptBuildMs: timingBreakdown?.promptBuildMs ?? null, + toolsBuildMs: timingBreakdown?.toolsBuildMs ?? null, + modelResolveMs: timingBreakdown?.modelResolveMs ?? null, + messageAssemblyMs: timingBreakdown?.messageAssemblyMs ?? null, + streamSetupMs, + toolLoopMs: timingBreakdown?.toolLoopMs ?? null, + toolsUsed: evalMetadata.usageData?.toolsUsed ?? ( + evalMetadata.toolCallsData + ? Array.from(new Set( + evalMetadata.toolCallsData + .map((call: any) => call?.toolName) + .filter((toolName: unknown): toolName is string => typeof toolName === 'string' && toolName.length > 0) + )) + : null + ), + toolCallsCount: evalMetadata.usageData?.toolCallsCount ?? evalMetadata.toolCallsData?.length ?? null, + success: hasActivity, + error: null, + }); + } }, onChunk: ({ chunk }: { chunk: any }) => { + if (firstChunkAt === null) { + firstChunkAt = Date.now(); + } if (chunk.type === 'text-delta' && chunk.textDelta) { + if (firstTextDeltaAt === null) { + firstTextDeltaAt = Date.now(); + } assistantResponse += chunk.textDelta; } } diff --git a/src/services/database/chunks.ts b/src/services/database/chunks.ts index 8695c74..f2b76a5 100644 --- a/src/services/database/chunks.ts +++ b/src/services/database/chunks.ts @@ -1,6 +1,75 @@ import { getSQLiteClient } from './sqlite-client'; import { Chunk, ChunkData } from '@/types/database'; +type RankedChunk = Chunk & { similarity: number }; + +function sanitizeFtsQuery(input: string): string { + return input + .replace(/['"()*:^~{}[\]]/g, ' ') + .trim() + .split(/\s+/) + .filter(word => word.length > 0 && !/^(AND|OR|NOT|NEAR)$/i.test(word)) + .join(' '); +} + +function extractRelaxedSearchTerms(query: string): string[] { + const stopWords = new Set([ + 'a', 'an', 'and', 'are', 'as', 'at', 'be', 'being', 'briefly', 'by', 'find', + 'focused', 'for', 'from', 'in', 'inside', 'is', 'it', 'me', 'my', 'of', 'on', + 'or', 'quote', 'search', 'solutions', 'specific', 'that', 'the', 'then', 'this', + 'to', 'transcript', 'with' + ]); + + const rawTerms = query + .toLowerCase() + .replace(/[^a-z0-9\s]/g, ' ') + .split(/\s+/) + .map(term => term.trim()) + .filter(term => term.length >= 4 && !stopWords.has(term)); + + const expanded = new Set(); + for (const term of rawTerms) { + expanded.add(term); + if (term.length >= 6) { + expanded.add(term.slice(0, 5)); + expanded.add(term.slice(0, 6)); + } + if (term.endsWith('ing') && term.length > 6) { + expanded.add(term.slice(0, -3)); + } + if (term.endsWith('tion') && term.length > 7) { + expanded.add(term.slice(0, -4)); + } + if (term.endsWith('ions') && term.length > 7) { + expanded.add(term.slice(0, -4)); + } + } + + return Array.from(expanded).slice(0, 12); +} + +function reciprocalRankFuse(rankedLists: T[][], limit: number): T[] { + const scores = new Map(); + const k = 60; + + rankedLists.forEach((list) => { + list.forEach((item, index) => { + const entry = scores.get(item.id); + const score = 1 / (k + index + 1); + if (entry) { + entry.score += score; + } else { + scores.set(item.id, { score, item }); + } + }); + }); + + return Array.from(scores.values()) + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .map(entry => entry.item); +} + export class ChunkService { async getChunksByNodeId(nodeId: number): Promise { const sqlite = getSQLiteClient(); @@ -157,7 +226,18 @@ export class ChunkService { fallbackQuery?: string ): Promise> { try { - return await this.searchChunksSQLite(queryEmbedding, similarityThreshold, matchCount, nodeIds); + const vectorResults = await this.searchChunksSQLite(queryEmbedding, similarityThreshold, matchCount, nodeIds); + + if (!fallbackQuery) { + return vectorResults; + } + + const textResults = await this.textSearchFallback(fallbackQuery, matchCount, nodeIds); + if (textResults.length === 0) { + return vectorResults; + } + + return reciprocalRankFuse([vectorResults, textResults], matchCount); } catch (error) { console.warn('Vector search failed, attempting text fallback:', error); if (fallbackQuery) { @@ -255,6 +335,12 @@ export class ChunkService { ): Promise> { const sqlite = getSQLiteClient(); const startTime = Date.now(); + const ftsResults = this.ftsSearchChunks(sqlite, query, matchCount, nodeIds); + if (ftsResults.length > 0) { + const searchTime = Date.now() - startTime; + console.log(`📝 Text fallback (FTS): ${ftsResults.length} chunks found, time=${searchTime}ms`); + return ftsResults; + } // Clean query for LIKE search const cleanQuery = query.trim().toLowerCase(); @@ -285,12 +371,98 @@ export class ChunkService { textQuery += ` ORDER BY LENGTH(text) ASC LIMIT ?`; params.push(String(matchCount)); - const result = sqlite.query(textQuery, params); + const result = sqlite.query(textQuery, params); const searchTime = Date.now() - startTime; console.log(`📝 Text fallback: ${result.rows.length} chunks found, time=${searchTime}ms`); - - return result.rows; + if (result.rows.length > 0) { + return result.rows; + } + + const relaxedTerms = extractRelaxedSearchTerms(query); + if (relaxedTerms.length === 0) { + return []; + } + + const scoreClauses = relaxedTerms.map(() => 'CASE WHEN LOWER(text) LIKE ? THEN 1 ELSE 0 END'); + const scoreParams = relaxedTerms.map(term => `%${term}%`); + const relaxedParams = [...scoreParams]; + + let relaxedQuery = ` + SELECT *, + (${scoreClauses.join(' + ')}) * 0.7 AS similarity + FROM chunks + WHERE (${scoreClauses.join(' + ')}) > 0 + `; + + if (nodeIds && nodeIds.length > 0) { + relaxedQuery += ` AND node_id IN (${nodeIds.map(() => '?').join(',')})`; + relaxedParams.push(...nodeIds.map(String)); + } + + relaxedQuery += ' ORDER BY similarity DESC, chunk_idx ASC LIMIT ?'; + relaxedParams.push(String(matchCount)); + + const relaxedResult = sqlite.query( + relaxedQuery, + [...scoreParams, ...relaxedParams] + ); + const relaxedSearchTime = Date.now() - startTime; + console.log(`📝 Text fallback (relaxed): ${relaxedResult.rows.length} chunks found, time=${relaxedSearchTime}ms`); + + return relaxedResult.rows; + } + + private ftsSearchChunks( + sqlite: ReturnType, + query: string, + matchCount: number, + nodeIds?: number[] + ): RankedChunk[] { + const ftsExists = sqlite.prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks_fts'" + ).get(); + if (!ftsExists) return []; + + const ftsQuery = sanitizeFtsQuery(query); + if (!ftsQuery) return []; + + try { + if (nodeIds && nodeIds.length > 0) { + const result = sqlite.query(` + SELECT c.*, 0.85 as similarity + FROM chunks c + WHERE c.node_id IN (${nodeIds.map(() => '?').join(',')}) + AND c.id IN ( + SELECT rowid + FROM chunks_fts + WHERE chunks_fts MATCH ? + ) + ORDER BY c.chunk_idx ASC + LIMIT ? + `, [...nodeIds, ftsQuery, matchCount]); + + return result.rows; + } + + const result = sqlite.query(` + WITH fts_matches AS ( + SELECT rowid, rank + FROM chunks_fts + WHERE chunks_fts MATCH ? + LIMIT ? + ) + SELECT c.*, 0.85 as similarity + FROM fts_matches fm + JOIN chunks c ON c.id = fm.rowid + ORDER BY fm.rank + `, [ftsQuery, matchCount]); + + return result.rows; + } catch (error) { + console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error); + return []; + } } async getChunkCount(): Promise { diff --git a/src/services/database/dimensionService.ts b/src/services/database/dimensionService.ts index 11fd2cb..dd7381a 100644 --- a/src/services/database/dimensionService.ts +++ b/src/services/database/dimensionService.ts @@ -1,7 +1,4 @@ import { getSQLiteClient } from './sqlite-client'; -import { openai as openaiProvider } from '@ai-sdk/openai'; -import { generateText } from 'ai'; -import { hasValidOpenAiKey } from '../storage/apiKeys'; export interface Dimension { name: string; @@ -18,85 +15,23 @@ export interface LockedDimension { export class DimensionService { /** - * Get all locked (priority) dimensions with their descriptions + * Legacy compatibility shim. Dimensions are now flat, so there is no locked subset. */ static async getLockedDimensions(): Promise { - const sqlite = getSQLiteClient(); - - const result = sqlite.query(` - WITH dimension_counts AS ( - SELECT nd.dimension, COUNT(*) AS count - FROM node_dimensions nd - GROUP BY nd.dimension - ) - SELECT - d.name, - d.description, - COALESCE(dc.count, 0) AS count - FROM dimensions d - LEFT JOIN dimension_counts dc ON dc.dimension = d.name - WHERE d.is_priority = 1 - ORDER BY d.name ASC - `); - - return result.rows.map((row: any) => ({ - name: row.name, - description: row.description, - count: Number(row.count) - })); + return []; } /** - * Automatically assign locked dimensions + suggest keyword dimensions - * Returns { locked: string[], keywords: string[] } - * - * IMPORTANT: Returns empty result immediately if no valid API key is configured. - * This prevents slow node creation when OpenAI is unavailable. + * Automatic special-dimension assignment has been removed. Callers must provide dimensions explicitly. */ static async assignDimensions(nodeData: { title: string; - notes?: string; + content?: string; link?: string; description?: string; }): Promise<{ locked: string[]; keywords: string[] }> { - // Fast path: skip AI if no valid API key - if (!hasValidOpenAiKey()) { - console.log(`[DimensionAssignment] No valid OpenAI key, skipping for: "${nodeData.title}"`); - return { locked: [], keywords: [] }; - } - - try { - const lockedDimensions = await this.getLockedDimensions(); - - if (lockedDimensions.length === 0) { - console.log('[DimensionAssignment] No locked dimensions available'); - return { locked: [], keywords: [] }; - } - - const prompt = this.buildAssignmentPrompt(nodeData, lockedDimensions); - - console.log(`[DimensionAssignment] Processing: "${nodeData.title}"`); - - const response = await generateText({ - model: openaiProvider('gpt-4o-mini'), - prompt, - maxOutputTokens: 300, // Increased to accommodate more dimensions - temperature: 0.1, - }); - - console.log(`[DimensionAssignment] AI Response:\n${response.text}`); - - const result = this.parseAssignmentResponse(response.text, lockedDimensions); - - console.log(`[DimensionAssignment] Locked: ${result.locked.join(', ') || 'none'}`); - console.log(`[DimensionAssignment] Keywords: ${result.keywords.join(', ') || 'none'}`); - - return result; - - } catch (error) { - console.error('[DimensionAssignment] Error:', error); - return { locked: [], keywords: [] }; - } + console.log(`[DimensionAssignment] Skipped for "${nodeData.title}" — flat dimensions require explicit assignment.`); + return { locked: [], keywords: [] }; } /** @@ -153,7 +88,7 @@ export class DimensionService { } /** - * Build AI prompt for dimension assignment (locked dimensions only) + * Legacy no-op prompt builder retained only for backward compatibility. */ private static buildAssignmentPrompt( nodeData: { title: string; notes?: string; link?: string; description?: string }, @@ -162,13 +97,13 @@ export class DimensionService { // Use description as primary context, content as fallback let nodeContextSection: string; if (nodeData.description) { - const contentPreview = nodeData.notes?.slice(0, 500) || ''; + const notesPreview = nodeData.notes?.slice(0, 500) || ''; nodeContextSection = `DESCRIPTION: ${nodeData.description} -NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`; +NOTES PREVIEW: ${notesPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`; } else { - const contentPreview = nodeData.notes?.slice(0, 2000) || ''; - nodeContextSection = `NOTES: ${contentPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`; + const notesPreview = nodeData.notes?.slice(0, 2000) || ''; + nodeContextSection = `NOTES: ${notesPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`; } // Include ALL locked dimensions, using fallback text for those without descriptions @@ -181,7 +116,7 @@ NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500 }) .join('\n---\n'); - return `You are categorizing a knowledge node into locked dimensions. + return `Dimensions are now flat categories with no locked subset. === NODE TO CATEGORIZE === Title: ${nodeData.title} @@ -203,7 +138,7 @@ LOCKED: } /** - * Parse AI response and extract locked dimensions + * Legacy no-op parser retained only for backward compatibility. */ private static parseAssignmentResponse( response: string, @@ -251,4 +186,4 @@ LOCKED: } } -export const dimensionService = new DimensionService(); \ No newline at end of file +export const dimensionService = new DimensionService(); diff --git a/src/services/database/edges.ts b/src/services/database/edges.ts index 6b8d182..009f05a 100644 --- a/src/services/database/edges.ts +++ b/src/services/database/edges.ts @@ -2,9 +2,11 @@ import { getSQLiteClient } from './sqlite-client'; import { Edge, EdgeContext, EdgeData, EdgeCreatedVia, NodeConnection, Node } from '@/types/database'; import { eventBroadcaster } from '../events'; import { nodeService } from './nodes'; +import { getOpenAiKey } from '../storage/apiKeys'; import { generateText } from 'ai'; import { createOpenAI } from '@ai-sdk/openai'; import { z } from 'zod'; +import { validateEdgeExplanation } from './quality'; const inferredEdgeContextSchema = z.object({ type: z.enum(['created_by', 'part_of', 'source_of', 'related_to']), @@ -53,7 +55,7 @@ async function inferEdgeContext(params: { // If no API key is configured, degrade gracefully. // We still enforce explanation, but fall back to "related_to" classification. - const apiKey = process.env.OPENAI_API_KEY; + const apiKey = getOpenAiKey(); if (!apiKey) { return { type: 'related_to', confidence: 0.0, swap_direction: false }; } @@ -118,11 +120,11 @@ async function autoInferEdge(params: { }): Promise<{ explanation: string; type: EdgeContext['type']; confidence: number; swap_direction: boolean }> { const { fromNode, toNode } = params; - const apiKey = process.env.OPENAI_API_KEY; + const apiKey = getOpenAiKey(); if (!apiKey) { // Fallback without AI return { - explanation: `Related to ${toNode.title}`, + explanation: `Connection to ${toNode.title}; exact relationship uncertain.`, type: 'related_to', confidence: 0.0, swap_direction: false, @@ -181,8 +183,8 @@ async function autoInferEdge(params: { const parsed = schema.safeParse(parsedJson); if (!parsed.success) { - return { - explanation: `Related to ${toNode.title}`, + return { + explanation: `Connection to ${toNode.title}; exact relationship uncertain.`, type: 'related_to', confidence: 0.2, swap_direction: false, @@ -193,7 +195,7 @@ async function autoInferEdge(params: { } catch (error) { console.warn('[edges] autoInferEdge failed; falling back', error); return { - explanation: `Related to ${toNode.title}`, + explanation: `Connection to ${toNode.title}; exact relationship uncertain.`, type: 'related_to', confidence: 0.2, swap_direction: false, @@ -205,13 +207,33 @@ export class EdgeService { async getEdges(): Promise { const sqlite = getSQLiteClient(); const result = sqlite.query('SELECT * FROM edges ORDER BY created_at DESC'); - return result.rows; + return result.rows.map((row: any) => { + let context: any = row.context; + if (typeof context === 'string') { + try { + context = JSON.parse(context); + } catch { + // Keep raw context string if JSON parsing fails. + } + } + return { ...row, context }; + }); } async getEdgeById(id: number): Promise { const sqlite = getSQLiteClient(); const result = sqlite.query('SELECT * FROM edges WHERE id = ?', [id]); - return result.rows[0] || null; + const row: any = result.rows[0]; + if (!row) return null; + let context: any = row.context; + if (typeof context === 'string') { + try { + context = JSON.parse(context); + } catch { + // Keep raw context string if JSON parsing fails. + } + } + return { ...row, context }; } async createEdge(edgeData: EdgeData): Promise { @@ -249,8 +271,12 @@ export class EdgeService { }; } else if (edgeData.skip_inference) { inferred = { type: 'related_to' as const, confidence: 0.0, swap_direction: false }; - if (!explanation) explanation = `Related to ${toNode.title}`; + if (!explanation) explanation = `Connection to ${toNode.title}; exact relationship uncertain.`; } else { + const explanationError = validateEdgeExplanation(explanation); + if (explanationError) { + throw new Error(explanationError); + } inferred = await inferEdgeContext({ explanation, fromNode, toNode }); } @@ -267,14 +293,15 @@ export class EdgeService { }; const result = sqlite.prepare(` - INSERT INTO edges (from_node_id, to_node_id, context, source, created_at) - VALUES (?, ?, ?, ?, ?) + INSERT INTO edges (from_node_id, to_node_id, context, source, created_at, explanation) + VALUES (?, ?, ?, ?, ?, ?) `).run( finalFromId, finalToId, JSON.stringify(context), edgeData.source, - now + now, + explanation ); const edgeId = Number(result.lastInsertRowid); @@ -316,6 +343,10 @@ export class EdgeService { if (!explanation) { throw new Error('Edge explanation is required'); } + const explanationError = validateEdgeExplanation(explanation); + if (explanationError) { + throw new Error(explanationError); + } const existingEdge = await this.getEdgeById(id); if (!existingEdge) { @@ -342,8 +373,13 @@ export class EdgeService { (existingContext?.created_via as EdgeCreatedVia) || 'ui'; - // Note: On update, we don't swap direction - the edge already exists with its direction. - // We only update the type and confidence based on the new explanation. + const nextFromId = inferred.swap_direction ? existingEdge.to_node_id : existingEdge.from_node_id; + const nextToId = inferred.swap_direction ? existingEdge.from_node_id : existingEdge.to_node_id; + if (inferred.swap_direction) { + updates.from_node_id = nextFromId; + updates.to_node_id = nextToId; + } + updates.context = { ...existingContext, ...incomingContext, @@ -368,6 +404,18 @@ export class EdgeService { } }); + if (Object.prototype.hasOwnProperty.call(updates, 'explanation')) { + const rawExplanation = (updates as any).explanation; + if (typeof rawExplanation === 'string') { + const explanationError = validateEdgeExplanation(rawExplanation); + if (explanationError) { + throw new Error(explanationError); + } + updateFields.push('explanation = ?'); + params.push(rawExplanation.trim()); + } + } + if (updateFields.length === 0) { throw new Error('No valid fields to update'); } @@ -429,7 +477,7 @@ export class EdgeService { WHEN e.from_node_id = ? THEN n_to.title ELSE n_from.title END as connected_node_title, - CASE + CASE WHEN e.from_node_id = ? THEN n_to.notes ELSE n_from.notes END as connected_node_notes, diff --git a/src/services/database/nodes.ts b/src/services/database/nodes.ts index e1694f9..e15a339 100644 --- a/src/services/database/nodes.ts +++ b/src/services/database/nodes.ts @@ -1,6 +1,44 @@ import { getSQLiteClient } from './sqlite-client'; import { Node, NodeFilters } from '@/types/database'; import { eventBroadcaster } from '../events'; +import { EmbeddingService } from '@/services/embeddings'; + +type NodeRow = Node & { dimensions_json: string }; +type NodeSearchRow = NodeRow & { rank?: number; similarity?: number }; + +function sanitizeFtsQuery(input: string): string { + return input + .replace(/['"()*:^~{}[\]]/g, ' ') + .trim() + .split(/\s+/) + .filter(word => word.length > 0 && !/^(AND|OR|NOT|NEAR)$/i.test(word)) + .join(' '); +} + +function reciprocalRankFuse( + rankedLists: T[][], + limit: number, +): T[] { + const scores = new Map(); + const k = 60; + + rankedLists.forEach((list) => { + list.forEach((item, index) => { + const existing = scores.get(item.id); + const score = 1 / (k + index + 1); + if (existing) { + existing.score += score; + } else { + scores.set(item.id, { score, item }); + } + }); + }); + + return Array.from(scores.values()) + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .map(entry => entry.item); +} export class NodeService { async getNodes(filters: NodeFilters = {}): Promise { @@ -10,6 +48,11 @@ export class NodeService { async countNodes(filters: NodeFilters = {}): Promise { const { dimensions, search, dimensionsMatch = 'any', createdAfter, createdBefore, eventAfter, eventBefore } = filters; + + if (search?.trim()) { + return this.countSearchNodesSQLite(filters); + } + const sqlite = getSQLiteClient(); let query = `SELECT COUNT(*) as total FROM nodes n WHERE 1=1`; @@ -52,6 +95,11 @@ export class NodeService { private async getNodesSQLite(filters: NodeFilters = {}): Promise { const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any', createdAfter, createdBefore, eventAfter, eventBefore } = filters; + + if (search?.trim()) { + return this.searchNodesSQLite(filters); + } + const sqlite = getSQLiteClient(); // Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance) @@ -135,6 +183,7 @@ export class NodeService { } else if (sortBy === 'created') { query += ' ORDER BY n.created_at DESC'; } else if (sortBy === 'event_date') { + // Nodes with event_date first (DESC), then by updated_at for nulls query += ' ORDER BY n.event_date IS NULL, n.event_date DESC, n.updated_at DESC'; } else { query += ' ORDER BY n.updated_at DESC'; @@ -150,14 +199,10 @@ export class NodeService { params.push(offset); } - const result = sqlite.query(query, params); + const result = sqlite.query(query, params); // Parse dimensions_json and metadata back for compatibility - return result.rows.map(row => ({ - ...row, - dimensions: JSON.parse(row.dimensions_json || '[]'), - metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null, - })); + return result.rows.map(row => this.mapNodeRow(row)); } async getNodeById(id: number): Promise { @@ -177,16 +222,12 @@ export class NodeService { FROM nodes n WHERE n.id = ? `; - const result = sqlite.query(query, [id]); + const result = sqlite.query(query, [id]); if (result.rows.length === 0) return null; const row = result.rows[0]; - return { - ...row, - dimensions: JSON.parse(row.dimensions_json || '[]'), - metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null, - }; + return this.mapNodeRow(row); } async createNode(nodeData: Partial): Promise { @@ -363,6 +404,264 @@ export class NodeService { return this.getNodes({ search: searchTerm, limit }); } + private mapNodeRow(row: NodeRow): Node { + return { + ...row, + dimensions: JSON.parse(row.dimensions_json || '[]'), + metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null, + }; + } + + private buildNodeFilterClauses(filters: NodeFilters, alias = 'n'): { clauses: string[]; params: any[] } { + const { + dimensions, + dimensionsMatch = 'any', + createdAfter, + createdBefore, + eventAfter, + eventBefore, + } = filters; + + const clauses: string[] = []; + const params: any[] = []; + + if (dimensions && dimensions.length > 0) { + if (dimensionsMatch === 'all' && dimensions.length > 1) { + clauses.push(`( + SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd + WHERE nd.node_id = ${alias}.id + AND nd.dimension IN (${dimensions.map(() => '?').join(',')}) + ) = ?`); + params.push(...dimensions, dimensions.length); + } else { + clauses.push(`EXISTS ( + SELECT 1 FROM node_dimensions nd + WHERE nd.node_id = ${alias}.id + AND nd.dimension IN (${dimensions.map(() => '?').join(',')}) + )`); + params.push(...dimensions); + } + } + + if (createdAfter) { clauses.push(`${alias}.created_at >= ?`); params.push(createdAfter); } + if (createdBefore) { clauses.push(`${alias}.created_at < ?`); params.push(createdBefore); } + if (eventAfter) { clauses.push(`${alias}.event_date >= ?`); params.push(eventAfter); } + if (eventBefore) { clauses.push(`${alias}.event_date < ?`); params.push(eventBefore); } + + return { clauses, params }; + } + + private async searchNodesSQLite(filters: NodeFilters): Promise { + const sqlite = getSQLiteClient(); + const search = filters.search?.trim(); + const limit = Math.min(Math.max(filters.limit ?? 100, 1), 100); + const offset = Math.max(filters.offset ?? 0, 0); + + if (!search) { + return []; + } + + const searchLimit = Math.max(limit + offset, Math.min(limit * 5, 100)); + let rankedRows = this.searchNodesFts(sqlite, search, filters, searchLimit); + + if (rankedRows.length === 0) { + rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit); + } + + if ((filters.searchMode ?? 'standard') === 'hybrid') { + const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit); + if (vectorRows.length > 0) { + rankedRows = reciprocalRankFuse([rankedRows, vectorRows], searchLimit); + } + } + + return rankedRows + .slice(offset, offset + limit) + .map(row => this.mapNodeRow(row)); + } + + private countSearchNodesSQLite(filters: NodeFilters): number { + const sqlite = getSQLiteClient(); + const search = filters.search?.trim(); + if (!search) return 0; + + const ftsQuery = sanitizeFtsQuery(search); + const ftsExists = sqlite.prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'" + ).get(); + const { clauses, params } = this.buildNodeFilterClauses(filters); + + if (ftsExists && ftsQuery) { + const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : ''; + const result = sqlite.query<{ total: number }>(` + WITH matched_nodes AS ( + SELECT rowid + FROM nodes_fts + WHERE nodes_fts MATCH ? + ) + SELECT COUNT(*) as total + FROM matched_nodes mn + JOIN nodes n ON n.id = mn.rowid + ${whereClauses} + `, [ftsQuery, ...params]); + + return Number(result.rows[0]?.total ?? 0); + } + + const words = search.split(/\s+/).filter(Boolean); + let query = `SELECT COUNT(*) as total FROM nodes n WHERE 1=1`; + const queryParams = [...params]; + + if (clauses.length > 0) { + query += ` AND ${clauses.join(' AND ')}`; + } + + for (const word of words) { + query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; + queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`); + } + + const result = sqlite.query<{ total: number }>(query, queryParams); + return Number(result.rows[0]?.total ?? 0); + } + + private searchNodesFts( + sqlite: ReturnType, + search: string, + filters: NodeFilters, + limit: number, + ): NodeSearchRow[] { + const ftsQuery = sanitizeFtsQuery(search); + if (!ftsQuery) return []; + + const ftsExists = sqlite.prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'" + ).get(); + if (!ftsExists) return []; + + const { clauses, params } = this.buildNodeFilterClauses(filters); + const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : ''; + + try { + const result = sqlite.query(` + WITH fts_matches AS ( + SELECT rowid, rank + FROM nodes_fts + WHERE nodes_fts MATCH ? + LIMIT ? + ) + SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + n.chunk_status, n.embedding_updated_at, n.embedding_text, + n.created_at, n.updated_at, + COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) + FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json, + fm.rank + FROM fts_matches fm + JOIN nodes n ON n.id = fm.rowid + ${whereClauses} + ORDER BY fm.rank + LIMIT ? + `, [ftsQuery, Math.max(limit * 2, 50), ...params, limit]); + + return result.rows; + } catch (error) { + console.warn('[NodeSearch] FTS search failed, falling back to LIKE:', error); + return []; + } + } + + private searchNodesLike( + sqlite: ReturnType, + search: string, + filters: NodeFilters, + limit: number, + ): NodeSearchRow[] { + const words = search.split(/\s+/).filter(Boolean); + const { clauses, params } = this.buildNodeFilterClauses(filters); + let query = ` + SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + n.chunk_status, n.embedding_updated_at, n.embedding_text, + n.created_at, n.updated_at, + COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) + FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json + FROM nodes n + WHERE 1=1 + `; + const queryParams = [...params]; + + if (clauses.length > 0) { + query += ` AND ${clauses.join(' AND ')}`; + } + + for (const word of words) { + query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; + queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`); + } + + query += ` ORDER BY + CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 6 END, + CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END, + CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END, + CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END, + CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END, + n.updated_at DESC + LIMIT ?`; + + queryParams.push(search, `${search}%`, `%${search}%`, `%${search}%`, `%${search}%`, limit); + + const result = sqlite.query(query, queryParams); + return result.rows; + } + + private async searchNodesVector( + sqlite: ReturnType, + search: string, + filters: NodeFilters, + limit: number, + ): Promise { + try { + const embedding = await EmbeddingService.generateQueryEmbedding(search); + if (!EmbeddingService.validateEmbedding(embedding)) { + return []; + } + + const vecExists = sqlite.prepare( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='vec_nodes'" + ).get(); + if (!vecExists) return []; + + const vectorString = `[${embedding.join(',')}]`; + const { clauses, params } = this.buildNodeFilterClauses(filters); + const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : ''; + + const result = sqlite.query(` + WITH vector_matches AS ( + SELECT node_id, distance + FROM vec_nodes + WHERE embedding MATCH ? + ORDER BY distance + LIMIT ? + ) + SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + n.chunk_status, n.embedding_updated_at, n.embedding_text, + n.created_at, n.updated_at, + COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) + FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json, + (1.0 / (1.0 + vm.distance)) AS similarity + FROM vector_matches vm + JOIN nodes n ON n.id = vm.node_id + ${whereClauses} + ORDER BY vm.distance + LIMIT ? + `, [vectorString, Math.max(limit * 2, 50), ...params, limit]); + + return result.rows; + } catch (error) { + console.warn('[NodeSearch] Vector search unavailable, continuing without it:', error); + return []; + } + } + async getNodeCount(): Promise { const sqlite = getSQLiteClient(); const result = sqlite.query('SELECT COUNT(*) as count FROM nodes'); diff --git a/src/services/database/quality.ts b/src/services/database/quality.ts new file mode 100644 index 0000000..54b9f95 --- /dev/null +++ b/src/services/database/quality.ts @@ -0,0 +1,64 @@ +const WEAK_DESCRIPTION_PATTERNS = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i; +const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|conversation|dataset|decision|dimension|document|episode|essay|guide|idea|insight|interview|node|note|paper|person|plan|placeholder|podcast|presentation|project|question|record|research|skill|source|summary|talk|target|test node|thread|tool|transcript|tweet|video|website|workflow)\b/i; +const UNCERTAINTY_PATTERNS = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i; +const GENERIC_EDGE_PATTERNS = /^(related|related to|connected|connected to|association|associated with)$/i; + +export function normalizeDimensionName(value: string): string { + return value.trim().replace(/\s+/g, ' '); +} + +export function normalizeDimensions(values: unknown, max = 5): string[] { + if (!Array.isArray(values)) return []; + + const seen = new Set(); + const normalized: string[] = []; + + for (const value of values) { + if (typeof value !== 'string') continue; + const trimmed = normalizeDimensionName(value); + if (!trimmed) continue; + const key = trimmed.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + normalized.push(trimmed); + if (normalized.length >= max) break; + } + + return normalized; +} + +export function validateExplicitDescription(description: string): string | null { + const text = description.trim(); + if (text.length < 24) { + return 'Description must be explicit and substantial (at least 24 characters).'; + } + if (WEAK_DESCRIPTION_PATTERNS.test(text)) { + return 'Description is too vague. State exactly what this is and why it matters.'; + } + if (!EXPLICIT_ENTITY_PATTERNS.test(text) && !UNCERTAINTY_PATTERNS.test(text)) { + return 'Description must explicitly identify what this thing is, or state uncertainty explicitly.'; + } + return null; +} + +export function validateEdgeExplanation(explanation: string): string | null { + const text = explanation.trim(); + if (text.length < 8) { + return 'Edge explanation must be explicit enough to describe the relationship.'; + } + if (GENERIC_EDGE_PATTERNS.test(text)) { + return 'Edge explanation is too generic. State the actual relationship or explicitly note uncertainty.'; + } + return null; +} + +export function validateDimensionDescription(description: string): string | null { + const text = description.trim(); + if (!text) { + return 'Dimension description is required.'; + } + if (text.length > 500) { + return 'Description must be 500 characters or less.'; + } + return null; +} diff --git a/src/services/database/sqlite-client.ts b/src/services/database/sqlite-client.ts index ae9117d..c4f61e0 100644 --- a/src/services/database/sqlite-client.ts +++ b/src/services/database/sqlite-client.ts @@ -19,12 +19,10 @@ class SQLiteClient { private db: Database.Database; private config: SQLiteConfig; private readonly readOnly: boolean; - private readonly embeddingsDisabled: boolean; private constructor() { this.config = this.getSQLiteConfig(); this.readOnly = process.env.SQLITE_READONLY === 'true'; - this.embeddingsDisabled = process.env.DISABLE_EMBEDDINGS === 'true'; // Initialize database connection const dbDirectory = path.dirname(this.config.dbPath); @@ -35,15 +33,13 @@ class SQLiteClient { ? new Database(this.config.dbPath, { readonly: true, fileMustExist: true }) : new Database(this.config.dbPath); - // Load sqlite-vec extension (skip entirely if embeddings are disabled) - if (!this.embeddingsDisabled) { - try { - this.db.loadExtension(this.config.vecExtensionPath); - console.log('SQLite vector extension loaded successfully'); - } catch (error) { - // Do not fail hard — allow the app to run without vector features - console.error('Warning: Failed to load vector extension:', error); - } + // Load sqlite-vec extension + try { + this.db.loadExtension(this.config.vecExtensionPath); + console.log('SQLite vector extension loaded successfully'); + } catch (error) { + // Do not fail hard — allow the app to run without vector features + console.error('Warning: Failed to load vector extension:', error); } // Configure SQLite settings @@ -61,11 +57,9 @@ class SQLiteClient { this.db.pragma('temp_store = memory'); this.db.pragma('busy_timeout = 5000'); - // Ensure vector virtual tables are present and healthy (skip if disabled) - if (!this.embeddingsDisabled) { - this.ensureVectorTables(); - this.healVectorTablesIfCorrupt(); - } + // Ensure vector virtual tables are present and healthy + this.ensureVectorTables(); + this.healVectorTablesIfCorrupt(); // Ensure logging schema (rename memory->logs if needed, create triggers/views) this.ensureLoggingAndMemorySchema(); @@ -140,9 +134,7 @@ class SQLiteClient { } as DatabaseError; } // Proactively validate/repair vec vtables before any write transaction - if (!this.embeddingsDisabled) { - this.healVectorTablesIfCorrupt(); - } + this.healVectorTablesIfCorrupt(); const txn = this.db.transaction(callback); try { return txn(); @@ -162,9 +154,6 @@ class SQLiteClient { } public async checkVectorExtension(): Promise { - if (this.embeddingsDisabled) { - return false; - } try { const result = this.query('SELECT vec_version() as version'); return result.rows.length > 0; @@ -266,7 +255,6 @@ class SQLiteClient { } }; ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;"); - // type column removed in final schema pass } catch (nodeErr) { console.warn('Failed to ensure nodes columns:', nodeErr); } @@ -363,6 +351,17 @@ class SQLiteClient { 'cache_hit', COALESCE(json_extract(NEW.metadata, '$.cache_hit'), 0), 'model', COALESCE(json_extract(NEW.metadata, '$.model_used'), ''), 'tools_count', COALESCE(json_extract(NEW.metadata, '$.tool_calls_count'), 0), + 'tools_used', COALESCE(json_extract(NEW.metadata, '$.tools_used'), json('[]')), + 'latency_ms', COALESCE(json_extract(NEW.metadata, '$.latency_ms'), 0), + 'prompt_build_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.promptBuildMs'), 0), + 'tools_build_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.toolsBuildMs'), 0), + 'model_resolve_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.modelResolveMs'), 0), + 'message_assembly_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.messageAssemblyMs'), 0), + 'stream_setup_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.streamSetupMs'), 0), + 'tool_loop_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.toolLoopMs'), 0), + 'first_token_latency_ms', COALESCE(json_extract(NEW.metadata, '$.first_token_latency_ms'), 0), + 'first_chunk_latency_ms', COALESCE(json_extract(NEW.metadata, '$.first_chunk_latency_ms'), 0), + 'tool_timings', COALESCE(json_extract(NEW.metadata, '$.tool_timings'), json('[]')), 'trace_id', COALESCE(json_extract(NEW.metadata, '$.trace_id'), ''), 'voice_tts_chars', COALESCE(json_extract(NEW.metadata, '$.voice_tts_chars'), 0), 'voice_tts_cost_usd', COALESCE(json_extract(NEW.metadata, '$.voice_tts_cost_usd'), 0), @@ -435,26 +434,18 @@ class SQLiteClient { } // Do not recreate memory_v; alias has been removed. - // 6) Drop orphaned chat_memory_state table (removed in final schema pass) - this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`); - - // Agent delegation table for orchestrator/worker coordination + // 6) Clean up removed chat_memory_state table try { - this.db.exec(` - CREATE TABLE IF NOT EXISTS agent_delegations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id TEXT UNIQUE NOT NULL, - task TEXT NOT NULL, - context TEXT, - expected_outcome TEXT, - status TEXT NOT NULL DEFAULT 'queued', - summary TEXT, - created_at TEXT DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT DEFAULT CURRENT_TIMESTAMP - ); - `); + this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`); } catch (e) { - console.warn('Failed to ensure agent_delegations table:', e); + // Ignore if table doesn't exist + } + + // Clean up removed agent_delegations table + try { + this.db.exec(`DROP TABLE IF EXISTS agent_delegations;`); + } catch (e) { + console.warn('Failed to drop agent_delegations table:', e); } // 8) Logs retention trigger (~10k most recent rows) @@ -623,77 +614,62 @@ class SQLiteClient { } } - // 10) Final schema pass migrations (content→notes, event_date, icon, drop dead columns) + // 10) Final schema pass migrations (content→notes, event_date, dimensions.icon, drop dead columns) try { const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>; - const nodeColNames = nodeCols2.map((c: any) => c.name); + const nodeColNames = nodeCols2.map(c => c.name); - // Rename content → notes + // Rename content → notes (additive first) if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) { - console.log('Renaming nodes.content → nodes.notes'); + console.log('Migrating nodes.content → nodes.notes...'); this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;'); } - // Add event_date with backfill from metadata + // Add event_date if (!nodeColNames.includes('event_date')) { - console.log('Adding nodes.event_date column'); this.db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;'); - this.db.exec(` - UPDATE nodes SET event_date = json_extract(metadata, '$.published_date') - WHERE metadata IS NOT NULL - AND json_extract(metadata, '$.published_date') IS NOT NULL - AND json_extract(metadata, '$.published_date') != ''; - `); + // Backfill from metadata.published_date where available + try { + this.db.exec(` + UPDATE nodes SET event_date = json_extract(metadata, '$.published_date') + WHERE event_date IS NULL AND json_extract(metadata, '$.published_date') IS NOT NULL; + `); + } catch {} } // Add dimensions.icon const dimCols2 = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>; - if (!dimCols2.some((c: any) => c.name === 'icon')) { - console.log('Adding dimensions.icon column'); + if (!dimCols2.some(c => c.name === 'icon')) { this.db.exec('ALTER TABLE dimensions ADD COLUMN icon TEXT;'); } - // Drop dead columns (SQLite 3.35+) + // Drop dead columns (requires SQLite 3.35+) + // nodes.type if (nodeColNames.includes('type')) { - console.log('Dropping nodes.type column'); - try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_type;'); } catch {} - try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch (e) { - console.warn('Could not drop nodes.type (SQLite < 3.35?):', e); - } + try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch {} } + // nodes.is_pinned if (nodeColNames.includes('is_pinned')) { - console.log('Dropping nodes.is_pinned column'); - try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_pinned;'); } catch {} - try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch (e) { - console.warn('Could not drop nodes.is_pinned:', e); - } + try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch {} } - - // Drop edges.user_feedback + // edges.user_feedback const edgeCols = this.db.prepare('PRAGMA table_info(edges)').all() as Array<{ name: string }>; - if (edgeCols.some((c: any) => c.name === 'user_feedback')) { - console.log('Dropping edges.user_feedback column'); - try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch (e) { - console.warn('Could not drop edges.user_feedback:', e); - } + if (edgeCols.some(c => c.name === 'user_feedback')) { + try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch {} } - // Rebuild FTS if it references 'content' instead of 'notes' + // Recreate nodes_fts to index title + description + notes try { - const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as any; - if (ftsCheck && ftsCheck.sql && ftsCheck.sql.includes('content')) { - console.log('Rebuilding nodes_fts to reference notes instead of content'); + const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined; + if (ftsCheck?.sql && (!ftsCheck.sql.includes('description') || ftsCheck.sql.includes('content'))) { this.db.exec('DROP TABLE IF EXISTS nodes_fts;'); - this.db.exec(` - CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content=nodes, content_rowid=id); - INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild'); - `); + this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content='nodes', content_rowid='id');"); + // Rebuild FTS index + this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');"); } } catch (ftsErr) { - console.warn('FTS rebuild skipped:', ftsErr); + console.warn('Failed to rebuild nodes_fts:', ftsErr); } - - console.log('Final schema pass migrations complete'); } catch (schemaErr) { console.warn('Final schema pass migration error:', schemaErr); } diff --git a/src/services/evals/evalsLogger.ts b/src/services/evals/evalsLogger.ts index 94fd971..cfe5473 100644 --- a/src/services/evals/evalsLogger.ts +++ b/src/services/evals/evalsLogger.ts @@ -34,6 +34,16 @@ type EvalChatLog = { workflowKey?: string | null; workflowNodeId?: number | null; latencyMs?: number; + firstChunkLatencyMs?: number | null; + firstTokenLatencyMs?: number | null; + promptBuildMs?: number | null; + toolsBuildMs?: number | null; + modelResolveMs?: number | null; + messageAssemblyMs?: number | null; + streamSetupMs?: number | null; + toolLoopMs?: number | null; + toolsUsed?: string[] | null; + toolCallsCount?: number | null; success?: boolean; error?: string | null; }; @@ -99,6 +109,16 @@ function ensureSchema(db: Database.Database) { workflow_key TEXT, workflow_node_id INTEGER, latency_ms INTEGER, + first_chunk_latency_ms INTEGER, + first_token_latency_ms INTEGER, + prompt_build_ms INTEGER, + tools_build_ms INTEGER, + model_resolve_ms INTEGER, + message_assembly_ms INTEGER, + stream_setup_ms INTEGER, + tool_loop_ms INTEGER, + tools_used_json TEXT, + tool_calls_count INTEGER, success INTEGER, error TEXT, dataset_id TEXT, @@ -140,6 +160,36 @@ function ensureSchema(db: Database.Database) { if (!columnNames.has('workflow_node_id')) { db.exec(`ALTER TABLE llm_chats ADD COLUMN workflow_node_id INTEGER;`); } + if (!columnNames.has('first_chunk_latency_ms')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN first_chunk_latency_ms INTEGER;`); + } + if (!columnNames.has('first_token_latency_ms')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN first_token_latency_ms INTEGER;`); + } + if (!columnNames.has('prompt_build_ms')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN prompt_build_ms INTEGER;`); + } + if (!columnNames.has('tools_build_ms')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN tools_build_ms INTEGER;`); + } + if (!columnNames.has('model_resolve_ms')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN model_resolve_ms INTEGER;`); + } + if (!columnNames.has('message_assembly_ms')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN message_assembly_ms INTEGER;`); + } + if (!columnNames.has('stream_setup_ms')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN stream_setup_ms INTEGER;`); + } + if (!columnNames.has('tool_loop_ms')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN tool_loop_ms INTEGER;`); + } + if (!columnNames.has('tools_used_json')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN tools_used_json TEXT;`); + } + if (!columnNames.has('tool_calls_count')) { + db.exec(`ALTER TABLE llm_chats ADD COLUMN tool_calls_count INTEGER;`); + } } function getDb() { @@ -236,8 +286,11 @@ export function logEvalChat(entry: EvalChatLog) { user_message, assistant_message, input_tokens, output_tokens, total_tokens, cache_write_tokens, cache_read_tokens, cache_hit, cache_savings_pct, estimated_cost_usd, provider, mode, workflow_key, workflow_node_id, - latency_ms, success, error, dataset_id, scenario_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + latency_ms, first_chunk_latency_ms, first_token_latency_ms, + prompt_build_ms, tools_build_ms, model_resolve_ms, message_assembly_ms, + stream_setup_ms, tool_loop_ms, tools_used_json, tool_calls_count, + success, error, dataset_id, scenario_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( now, traceId, @@ -261,6 +314,16 @@ export function logEvalChat(entry: EvalChatLog) { entry.workflowKey ?? null, entry.workflowNodeId ?? null, entry.latencyMs ?? null, + entry.firstChunkLatencyMs ?? null, + entry.firstTokenLatencyMs ?? null, + entry.promptBuildMs ?? null, + entry.toolsBuildMs ?? null, + entry.modelResolveMs ?? null, + entry.messageAssemblyMs ?? null, + entry.streamSetupMs ?? null, + entry.toolLoopMs ?? null, + stringifySafe(entry.toolsUsed ?? null), + entry.toolCallsCount ?? null, typeof entry.success === 'boolean' ? (entry.success ? 1 : 0) : null, entry.error ?? null, datasetId, diff --git a/src/services/evals/evalsStore.ts b/src/services/evals/evalsStore.ts index c817235..8303279 100644 --- a/src/services/evals/evalsStore.ts +++ b/src/services/evals/evalsStore.ts @@ -26,6 +26,16 @@ export type EvalChatRow = { workflow_key: string | null; workflow_node_id: number | null; latency_ms: number | null; + first_chunk_latency_ms: number | null; + first_token_latency_ms: number | null; + prompt_build_ms: number | null; + tools_build_ms: number | null; + model_resolve_ms: number | null; + message_assembly_ms: number | null; + stream_setup_ms: number | null; + tool_loop_ms: number | null; + tools_used_json: string | null; + tool_calls_count: number | null; success: number | null; error: string | null; dataset_id: string | null; diff --git a/src/services/runtime/apiBase.ts b/src/services/runtime/apiBase.ts new file mode 100644 index 0000000..488eed8 --- /dev/null +++ b/src/services/runtime/apiBase.ts @@ -0,0 +1,12 @@ +import { RequestContext } from '@/services/context/requestContext'; + +export function getInternalApiBaseUrl(): string { + const requestOrigin = (RequestContext.get() as { requestOrigin?: string }).requestOrigin; + if (requestOrigin) { + return requestOrigin; + } + + return process.env.NEXT_PUBLIC_BASE_URL + || process.env.NEXT_PUBLIC_APP_URL + || 'http://localhost:3000'; +} diff --git a/src/tools/database/createDimension.ts b/src/tools/database/createDimension.ts index 57a5b06..13ddaf5 100644 --- a/src/tools/database/createDimension.ts +++ b/src/tools/database/createDimension.ts @@ -1,12 +1,12 @@ import { tool } from 'ai'; import { z } from 'zod'; +import { getInternalApiBaseUrl } from '@/services/runtime/apiBase'; export const createDimensionTool = tool({ - description: 'Create a new dimension or update existing dimension. IMPORTANT: Always ask the user for a description explaining what belongs in this dimension before creating it. Dimensions without descriptions cannot be auto-assigned.', + description: 'Create a new dimension. Always provide a description explaining what belongs in this category.', inputSchema: z.object({ name: z.string().describe('Dimension name'), - description: z.string().min(1).max(500).describe('Dimension description explaining what content belongs in this dimension (required, max 500 characters)'), - isPriority: z.boolean().optional().describe('Whether to lock this dimension for auto-assignment (default: false)') + description: z.string().min(1).max(500).describe('Dimension description explaining what content belongs in this dimension (required, max 500 characters)') }), execute: async (params) => { console.log('📁 CreateDimension tool called with params:', JSON.stringify(params, null, 2)); @@ -21,13 +21,12 @@ export const createDimensionTool = tool({ } // Call POST /api/dimensions - const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/dimensions`, { + const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ name: trimmedName, - description: params.description.trim(), - isPriority: params.isPriority || false + description: params.description.trim() }) }); @@ -52,7 +51,7 @@ export const createDimensionTool = tool({ return { success: true, data: result.data, - message: `Created dimension "${trimmedName}"${params.isPriority ? ' (locked)' : ''}${params.description ? ' with description' : ''}` + message: `Created dimension "${trimmedName}"${params.description ? ' with description' : ''}` }; } catch (error) { return { @@ -63,4 +62,3 @@ export const createDimensionTool = tool({ } } }); - diff --git a/src/tools/database/createEdge.ts b/src/tools/database/createEdge.ts index 9c65809..9428305 100644 --- a/src/tools/database/createEdge.ts +++ b/src/tools/database/createEdge.ts @@ -3,33 +3,24 @@ import { z } from 'zod'; import { edgeService } from '@/services/database/edges'; import { nodeService } from '@/services/database/nodes'; import { formatNodeForChat } from '../infrastructure/nodeFormatter'; +import { validateEdgeExplanation } from '@/services/database/quality'; export const createEdgeTool = tool({ description: - 'Create directed relationship between nodes.\n\n' + - 'Direction rule: FROM node → TO node should read correctly.\n' + - 'Prefer the 4 core relations unless the user clearly wants an advanced intellectual relation:\n' + - '- Made by → created_by (attribution)\n' + - '- Part of → part_of (attribution)\n' + - '- Came from → source_of (intellectual)\n' + - '- Related → related_to (intellectual fallback)\n\n' + - 'Examples:\n' + - '- Episode → Podcast: "Episode of this podcast"\n' + - '- Book → Author: "Written by"\n' + - '- Company → Founder: "Founded by"\n' + - '- Insight → Source: "Came from / inspired by"\n', + 'Create a relationship between two nodes. Provide an explanation and the system will infer the type and direction.\n\n' + + 'Examples of explanations:\n' + + '- "Written by" (book → author)\n' + + '- "Episode of this podcast" (episode → podcast)\n' + + '- "Inspired this insight" (source → derivative)\n' + + '- "Related concept" (general relationship)\n', inputSchema: z.object({ - from_node_id: z.number().describe('The ID of the source node (where the connection originates)'), - to_node_id: z.number().describe('The ID of the target node (where the connection points to)'), + from_node_id: z.number().describe('The ID of the source node'), + to_node_id: z.number().describe('The ID of the target node'), explanation: z.string().describe( - 'REQUIRED: Why does this connection exist? Be specific. ' + - 'Write it as a relationship that reads FROM → TO. ' + - 'Examples: "Author of this book", "Guest on this podcast", ' + - '"Episode of this podcast", "This insight came from this podcast episode", "Extends the concept introduced here"' + 'REQUIRED: Why does this connection exist? The system will infer the relationship type from your explanation.' ), source: z.enum(['user', 'ai', 'ai_similarity', 'helper_name']).default('ai').describe( - 'Source of this edge. Use "ai" (or "helper_name") for AI-created connections, ' + - '"user" for manual connections, "ai_similarity" for similarity-based connections.' + 'Source of this edge. Use "ai" for AI-created, "user" for manual, "ai_similarity" for similarity-based.' ) }), execute: async (params) => { @@ -69,6 +60,14 @@ export const createEdgeTool = tool({ data: null }; } + const explanationError = validateEdgeExplanation(explanation); + if (explanationError) { + return { + success: false, + error: explanationError, + data: null + }; + } const [fromNode, toNode] = await Promise.all([ nodeService.getNodeById(params.from_node_id), diff --git a/src/tools/database/createNode.ts b/src/tools/database/createNode.ts index 10d451e..ba1443c 100644 --- a/src/tools/database/createNode.ts +++ b/src/tools/database/createNode.ts @@ -1,34 +1,41 @@ import { tool } from 'ai'; import { z } from 'zod'; +import { getInternalApiBaseUrl } from '@/services/runtime/apiBase'; import { formatNodeForChat } from '../infrastructure/nodeFormatter'; +import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality'; export const createNodeTool = tool({ - description: 'Create node with title/content/link and optional dimensions (locked dimensions auto-assigned)', + description: 'Create node. Description is REQUIRED and must be explicit about what the thing is (podcast, chat summary, idea, etc).', inputSchema: z.object({ title: z.string().describe('The title of the node'), - description: z.string().max(280).optional().describe('WHAT this is + WHY it matters. Extremely concise. No "discusses/explores". Auto-generated if omitted.'), - notes: z.string().optional().describe('The main notes or content for this node'), + notes: z.string().optional().describe('User notes, analysis, or thoughts about this node'), + description: z.string().max(280).describe('REQUIRED. Explicitly state WHAT this is (e.g. podcast episode, conversation summary, user insight) + WHY it matters for context grounding.'), link: z.string().optional().describe('A URL link to the source'), - event_date: z.string().optional().describe('ISO date string for time-anchored nodes (e.g. meetings, events)'), + event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'), dimensions: z .array(z.string()) .max(5) .optional() - .describe('Optional dimension tags to apply to this node (0-5 items). Locked dimensions will be auto-assigned.'), + .describe('Optional dimension tags to apply to this node (0-5 items).'), chunk: z.string().optional().describe('Raw content for later processing - CRITICAL for extracted content from URLs'), metadata: z.record(z.any()).optional().describe('Additional metadata like source info, extraction details, etc.') }), execute: async (params) => { console.log('🎯 CreateNode tool called with params:', JSON.stringify(params, null, 2)); try { - const rawDimensions = params.dimensions || []; - const trimmedDimensions = rawDimensions - .map(d => typeof d === 'string' ? d.trim() : '') - .filter(Boolean) - .slice(0, 5); // Limit to 5 dimensions max + const descriptionError = validateExplicitDescription(params.description); + if (descriptionError) { + return { + success: false, + error: `${descriptionError} Do not retry with minor rephrasing in the same turn. Rewrite the description so it explicitly names the entity type, such as note, node, person, episode, article, project, test node, or skill, and states why it matters.`, + data: null + }; + } + + const trimmedDimensions = normalizeDimensions(params.dimensions || [], 5); // Call the nodes API endpoint - const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, { + const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ ...params, dimensions: trimmedDimensions }) @@ -57,7 +64,7 @@ export const createNodeTool = tool({ ...result.data, formatted_display: formattedDisplay }, - message: `Created node ${formattedDisplay} with dimensions: ${result.data.dimensions ? result.data.dimensions.join(', ') : 'auto-assigned'}` + message: `Created node ${formattedDisplay} with dimensions: ${result.data.dimensions ? result.data.dimensions.join(', ') : 'none'}` }; } catch (error) { return { diff --git a/src/tools/database/getDimension.ts b/src/tools/database/getDimension.ts index f5c4c3f..0348c47 100644 --- a/src/tools/database/getDimension.ts +++ b/src/tools/database/getDimension.ts @@ -4,7 +4,7 @@ import { DimensionService } from '@/services/database/dimensionService'; import { getSQLiteClient } from '@/services/database/sqlite-client'; export const getDimensionTool = tool({ - description: 'Get detailed information about a specific dimension by name, including its description, priority status, and node count.', + description: 'Get dimension details: description and node count.', inputSchema: z.object({ name: z.string().describe('The exact name of the dimension to retrieve') }), @@ -43,7 +43,6 @@ export const getDimensionTool = tool({ data: { name: trimmedName, description: null, - isPriority: false, nodeCount, exists: true, hasMetadata: false @@ -62,7 +61,6 @@ export const getDimensionTool = tool({ const result = { name: dimension.name, description: dimension.description, - isPriority: dimension.is_priority, nodeCount, updatedAt: dimension.updated_at, exists: true, @@ -72,7 +70,6 @@ export const getDimensionTool = tool({ // Build descriptive message const parts: string[] = []; parts.push(`Dimension: ${result.name}`); - if (result.isPriority) parts.push('Status: 🔒 Priority (locked)'); parts.push(`Nodes: ${result.nodeCount}`); if (result.description) parts.push(`Description: ${result.description}`); parts.push(`Last updated: ${result.updatedAt}`); diff --git a/src/tools/database/queryDimensions.ts b/src/tools/database/queryDimensions.ts index 4cbbbc2..f6361cb 100644 --- a/src/tools/database/queryDimensions.ts +++ b/src/tools/database/queryDimensions.ts @@ -1,12 +1,12 @@ import { tool } from 'ai'; import { z } from 'zod'; +import { getInternalApiBaseUrl } from '@/services/runtime/apiBase'; export const queryDimensionsTool = tool({ - description: 'Query dimensions with optional filtering by name, priority status, or search term. Returns dimensions with their node counts.', + description: 'List dimensions with node counts. Use this to inspect the user\'s organizational categories.', inputSchema: z.object({ filters: z.object({ search: z.string().describe('Search term to match against dimension names').optional(), - isPriority: z.boolean().describe('Filter by priority (locked) status').optional(), limit: z.number().min(1).max(100).default(50).describe('Maximum number of results to return') }).optional() }), @@ -14,7 +14,7 @@ export const queryDimensionsTool = tool({ console.log('📁 QueryDimensions tool called with filters:', JSON.stringify(filters, null, 2)); try { const limit = filters.limit || 50; - const baseUrl = process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'; + const baseUrl = getInternalApiBaseUrl(); // Use existing API endpoint for dimension listing const response = await fetch(`${baseUrl}/api/dimensions/popular`); @@ -48,7 +48,6 @@ export const queryDimensionsTool = tool({ let dimensions = result.data as Array<{ dimension: string; count: number; - isPriority: boolean; description: string | null; }>; @@ -60,11 +59,6 @@ export const queryDimensionsTool = tool({ ); } - // Filter by priority status - if (filters.isPriority !== undefined) { - dimensions = dimensions.filter(d => d.isPriority === filters.isPriority); - } - // Apply limit const limitedDimensions = dimensions.slice(0, limit); @@ -72,18 +66,16 @@ export const queryDimensionsTool = tool({ const formattedDimensions = limitedDimensions.map(d => ({ name: d.dimension, count: d.count, - isPriority: d.isPriority, description: d.description })); // Build message const filterParts: string[] = []; if (filters.search) filterParts.push(`matching "${filters.search}"`); - if (filters.isPriority !== undefined) filterParts.push(filters.isPriority ? 'priority only' : 'non-priority only'); const filterDesc = filterParts.length > 0 ? ` (${filterParts.join(', ')})` : ''; const dimensionList = formattedDimensions - .map(d => `• ${d.name}${d.isPriority ? ' 🔒' : ''} (${d.count} nodes)${d.description ? ` - ${d.description}` : ''}`) + .map(d => `• ${d.name} (${d.count} nodes)${d.description ? ` - ${d.description}` : ''}`) .join('\n'); return { diff --git a/src/tools/database/queryEdge.ts b/src/tools/database/queryEdge.ts index 1732cf8..fdc4d9f 100644 --- a/src/tools/database/queryEdge.ts +++ b/src/tools/database/queryEdge.ts @@ -3,6 +3,15 @@ import { z } from 'zod'; import { edgeService } from '@/services/database/edges'; import { formatNodeForChat } from '../infrastructure/nodeFormatter'; +function truncateText(value: unknown, maxLength = 180): string | null { + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + if (!trimmed) return null; + if (trimmed.length <= maxLength) return trimmed; + if (maxLength <= 3) return trimmed.slice(0, maxLength); + return `${trimmed.slice(0, maxLength - 3)}...`; +} + export const queryEdgeTool = tool({ description: 'Find edges by node/direction/source/ID', inputSchema: z.object({ @@ -38,6 +47,7 @@ export const queryEdgeTool = tool({ // Handle node connections (most common use case) if (filters.node_id) { + const effectiveLimit = Math.min(filters.limit || 20, 12); const connections = await edgeService.getNodeConnections(filters.node_id); const edges = connections.map(conn => conn.edge); @@ -48,33 +58,60 @@ export const queryEdgeTool = tool({ } // Apply limit and format connected nodes - const limitedConnections = connections.slice(0, filters.limit || 20); + const limitedConnections = connections.slice(0, effectiveLimit); const formattedConnections = limitedConnections.map(connection => { const formattedNode = formatNodeForChat({ id: connection.connected_node.id, title: connection.connected_node.title, dimensions: connection.connected_node.dimensions || [] }); + + const context = connection.edge.context as Record | undefined; return { - ...connection, + edge: { + id: connection.edge.id, + from_node_id: connection.edge.from_node_id, + to_node_id: connection.edge.to_node_id, + source: connection.edge.source, + created_at: connection.edge.created_at, + context: { + type: typeof context?.type === 'string' ? context.type : null, + explanation: truncateText(context?.explanation), + confidence: typeof context?.confidence === 'number' ? context.confidence : null, + } + }, connected_node: { - ...connection.connected_node, + id: connection.connected_node.id, + title: connection.connected_node.title, + description: truncateText(connection.connected_node.description, 140), + dimensions: connection.connected_node.dimensions || [], formatted_display: formattedNode } }; }); + + const summarizedEdges = formattedConnections.map(connection => ({ + id: connection.edge.id, + from_node_id: connection.edge.from_node_id, + to_node_id: connection.edge.to_node_id, + source: connection.edge.source, + created_at: connection.edge.created_at, + context: connection.edge.context, + connected_node: connection.connected_node.formatted_display, + })); // Create message with formatted connected nodes const connectedNodeLabels = formattedConnections.map(conn => conn.connected_node.formatted_display).join(', '); - const message = `Found ${filteredEdges.length} edges for node ${filters.node_id}${connectedNodeLabels ? `. Connected nodes: ${connectedNodeLabels}` : ''}`; + const message = `Found ${filteredEdges.length} edges for node ${filters.node_id}. Showing ${formattedConnections.length}${connectedNodeLabels ? `. Connected nodes: ${connectedNodeLabels}` : ''}`; return { success: true, data: { - edges: filteredEdges.slice(0, filters.limit || 20), + edges: summarizedEdges, connections: formattedConnections, count: filteredEdges.length, + returned_count: formattedConnections.length, filters_applied: filters }, message: message diff --git a/src/tools/database/queryNodes.ts b/src/tools/database/queryNodes.ts index 4009809..a4e248f 100644 --- a/src/tools/database/queryNodes.ts +++ b/src/tools/database/queryNodes.ts @@ -5,11 +5,11 @@ import { formatNodeForChat } from '../infrastructure/nodeFormatter'; import type { Node } from '@/types/database'; export const queryNodesTool = tool({ - description: 'Search nodes by title/notes/dimensions', + description: 'Search nodes across title, description, notes, and dimensions. Multi-word queries use FTS/tokenized fallback, and agent calls can add node-vector retrieval.', inputSchema: z.object({ filters: z.object({ dimensions: z.array(z.string()).describe('Filter by dimensions (e.g., ["research", "ai", "technology"]). Replaces old type/stage filtering.').optional(), - search: z.string().describe('Search term to match against title or notes').optional(), + search: z.string().describe('Search term to match against node title, description, or notes').optional(), limit: z.number().min(1).max(50).default(10).describe('Maximum number of results to return'), createdAfter: z.string().optional().describe('ISO date (YYYY-MM-DD). Only return nodes created on or after this date.'), createdBefore: z.string().optional().describe('ISO date (YYYY-MM-DD). Only return nodes created before this date.'), @@ -73,6 +73,7 @@ export const queryNodesTool = tool({ limit, dimensions: filters.dimensions, search: filters.search, + searchMode: searchTerm ? 'hybrid' : 'standard', createdAfter: filters.createdAfter, createdBefore: filters.createdBefore, eventAfter: filters.eventAfter, diff --git a/src/tools/database/updateDimension.ts b/src/tools/database/updateDimension.ts index f94e801..6d88871 100644 --- a/src/tools/database/updateDimension.ts +++ b/src/tools/database/updateDimension.ts @@ -1,22 +1,22 @@ import { tool } from 'ai'; import { z } from 'zod'; +import { getInternalApiBaseUrl } from '@/services/runtime/apiBase'; export const updateDimensionTool = tool({ - description: 'Update dimension name, description, or lock status', + description: 'Update a dimension name or description.', inputSchema: z.object({ currentName: z.string().describe('Current dimension name'), newName: z.string().optional().describe('New dimension name (if renaming)'), - description: z.string().max(500).optional().describe('New description (max 500 characters)'), - isPriority: z.boolean().optional().describe('Lock/unlock status (true = locked, false = unlocked)') + description: z.string().max(500).optional().describe('New description (max 500 characters)') }), execute: async (params) => { console.log('📝 UpdateDimension tool called with params:', JSON.stringify(params, null, 2)); try { // Validate at least one update field - if (!params.newName && params.description === undefined && params.isPriority === undefined) { + if (!params.newName && params.description === undefined) { return { success: false, - error: 'At least one update field (newName, description, or isPriority) must be provided', + error: 'At least one update field (newName or description) must be provided', data: null }; } @@ -31,11 +31,7 @@ export const updateDimensionTool = tool({ body.newName = params.newName.trim(); } - if (params.isPriority !== undefined) { - body.isPriority = params.isPriority; - } - - const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/dimensions`, { + const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) @@ -62,7 +58,6 @@ export const updateDimensionTool = tool({ const updates = []; if (params.newName) updates.push(`renamed to "${params.newName}"`); if (params.description !== undefined) updates.push('description updated'); - if (params.isPriority !== undefined) updates.push(params.isPriority ? 'locked' : 'unlocked'); return { success: true, @@ -78,4 +73,3 @@ export const updateDimensionTool = tool({ } } }); - diff --git a/src/tools/database/updateEdge.ts b/src/tools/database/updateEdge.ts index cac3cdc..7ea9aff 100644 --- a/src/tools/database/updateEdge.ts +++ b/src/tools/database/updateEdge.ts @@ -1,12 +1,14 @@ import { tool } from 'ai'; import { z } from 'zod'; import { edgeService } from '@/services/database/edges'; +import { validateEdgeExplanation } from '@/services/database/quality'; export const updateEdgeTool = tool({ - description: 'Update edge context/source', + description: 'Update an edge explanation and/or source. Explanations must explicitly state the relationship.', inputSchema: z.object({ edge_id: z.number().describe('The ID of the edge to update'), updates: z.object({ + explanation: z.string().optional().describe('Updated relationship explanation'), context: z.record(z.any()).optional().describe('Updated context information for this edge - can include explanation, relationship type, strength, notes, etc.'), source: z.enum(['user', 'ai_similarity', 'helper_name']).optional().describe('Updated source classification for this edge'), }).describe('Fields to update on the edge') @@ -38,6 +40,34 @@ export const updateEdgeTool = tool({ }; } + if (typeof cleanUpdates.explanation === 'string') { + const explanationError = validateEdgeExplanation(cleanUpdates.explanation); + if (explanationError) { + return { + success: false, + error: explanationError, + data: existingEdge + }; + } + } + + if ( + !cleanUpdates.explanation && + cleanUpdates.context && + typeof cleanUpdates.context === 'object' && + !Array.isArray(cleanUpdates.context) && + typeof cleanUpdates.context.explanation === 'string' + ) { + const explanationError = validateEdgeExplanation(cleanUpdates.context.explanation); + if (explanationError) { + return { + success: false, + error: explanationError, + data: existingEdge + }; + } + } + // Update the edge const updatedEdge = await edgeService.updateEdge(params.edge_id, cleanUpdates); @@ -45,7 +75,7 @@ export const updateEdgeTool = tool({ const updateDescriptions = []; if (cleanUpdates.context) updateDescriptions.push('context'); if (cleanUpdates.source) updateDescriptions.push(`source to ${cleanUpdates.source}`); - + return { success: true, data: updatedEdge, diff --git a/src/tools/database/updateNode.ts b/src/tools/database/updateNode.ts index 513d484..b088aec 100644 --- a/src/tools/database/updateNode.ts +++ b/src/tools/database/updateNode.ts @@ -1,20 +1,22 @@ import { tool } from 'ai'; import { z } from 'zod'; +import { getInternalApiBaseUrl } from '@/services/runtime/apiBase'; +import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality'; export const updateNodeTool = tool({ - description: 'Update node fields', + description: 'Update node fields. Description is REQUIRED on every update and must explicitly state WHAT this is + WHY it matters.', inputSchema: z.object({ id: z.number().describe('The ID of the node to update'), updates: z.object({ title: z.string().optional().describe('New title'), - description: z.string().max(280).optional().describe('New description (overwrites existing). WHAT this is + WHY it matters. No "discusses/explores".'), - notes: z.string().optional().describe('New notes (appended to existing)'), + notes: z.string().optional().describe('User notes/analysis to append. USE THIS for workflow outputs, briefs, research notes, etc.'), + description: z.string().max(280).describe('REQUIRED on every update. Explicitly state WHAT this is + WHY it matters. No "discusses/explores".'), link: z.string().optional().describe('New link'), - event_date: z.string().optional().describe('ISO date string for time-anchored nodes'), + event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'), dimensions: z.array(z.string()).optional().describe('New dimension tags - completely replaces existing dimensions'), - chunk: z.string().optional().describe('New chunk content'), + chunk: z.string().optional().describe('DO NOT USE - raw source text that triggers re-embedding. Only for source corrections.'), metadata: z.record(z.any()).optional().describe('New metadata - completely replaces existing metadata') - }).describe('Object containing the fields to update') + }).describe('Object containing the fields to update. For adding notes/analysis, always use "notes" field.') }), execute: async ({ id, updates }) => { try { @@ -26,24 +28,40 @@ export const updateNodeTool = tool({ }; } - // FORCE APPEND for content field - fetch existing and append new content + if (!updates.description) { + return { + success: false, + error: 'Every node update requires an explicit description (WHAT this is + WHY it matters).', + data: null + }; + } + const descriptionError = validateExplicitDescription(updates.description); + if (descriptionError) { + return { + success: false, + error: descriptionError, + data: null + }; + } + + // FORCE APPEND for notes field - fetch existing and append new notes if (updates.notes) { - const fetchResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes/${id}`); + const fetchResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes/${id}`); if (fetchResponse.ok) { const { node } = await fetchResponse.json(); const existingNotes = (node?.notes || '').trim(); const newNotes = updates.notes.trim(); - - // Skip if new content is identical to existing (model sent duplicate) + + // Skip if new notes are identical to existing (model sent duplicate) if (existingNotes === newNotes) { - console.log(`[updateNode] ERROR - new content identical to existing (${existingNotes.length} chars). Model should NOT call updateNode again.`); + console.log(`[updateNode] ERROR - new notes identical to existing (${existingNotes.length} chars). Model should NOT call updateNode again.`); return { success: false, error: 'Notes already up to date - do not call updateNode again. Move to next step.', data: null }; } - + // Detect if adding a section that already exists (e.g., ## Integration Analysis) const newSectionMatch = newNotes.match(/^##\s+(.+)$/m); if (newSectionMatch && existingNotes) { @@ -57,12 +75,12 @@ export const updateNodeTool = tool({ }; } } - - // Detect if model included existing content + new content + + // Detect if model included existing notes + new notes if (existingNotes && newNotes.startsWith(existingNotes)) { // Extract only the new part const actualNewNotes = newNotes.substring(existingNotes.length).trim(); - console.log(`[updateNode] Model included existing content - extracting new part only (${actualNewNotes.length} chars)`); + console.log(`[updateNode] Model included existing notes - extracting new part only (${actualNewNotes.length} chars)`); const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n'; updates.notes = `${existingNotes}${separator}${actualNewNotes}`; } else if (existingNotes) { @@ -71,15 +89,17 @@ export const updateNodeTool = tool({ updates.notes = `${existingNotes}${separator}${newNotes}`; console.log(`[updateNode] Appended notes: ${existingNotes.length} + ${newNotes.length} = ${updates.notes.length} chars`); } else { - console.log(`[updateNode] No existing content, using new content as-is (${newNotes.length} chars)`); + console.log(`[updateNode] No existing notes, using new notes as-is (${newNotes.length} chars)`); } } } - // No dimension validation - user has full control over dimensions + if (Array.isArray(updates.dimensions)) { + updates.dimensions = normalizeDimensions(updates.dimensions, 5); + } // Call the nodes API endpoint - const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes/${id}`, { + const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes/${id}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(updates) diff --git a/src/tools/other/searchContentEmbeddings.ts b/src/tools/other/searchContentEmbeddings.ts index 2e0fec4..37fd707 100644 --- a/src/tools/other/searchContentEmbeddings.ts +++ b/src/tools/other/searchContentEmbeddings.ts @@ -4,7 +4,7 @@ import { chunkService } from '@/services/database/chunks'; import { EmbeddingService } from '@/services/embeddings'; export const searchContentEmbeddingsTool = tool({ - description: 'Semantic search over node chunks with text fallback for reliability', + description: 'Search source chunks with hybrid retrieval: vector similarity plus FTS/keyword fallback merged for reliability.', inputSchema: z.object({ query: z.string().describe('The search query to find semantically similar content'), limit: z.number().min(1).max(20).default(5).describe('Maximum number of results to return (default: 5)'), @@ -107,7 +107,7 @@ export const searchContentEmbeddingsTool = tool({ searched_nodes: searchNodeIds || 'all', count: chunks.length, similarity_threshold, - search_method: 'vector_search', + search_method: query ? 'hybrid_vector_fts' : 'vector_search', search_time_ms: searchTime, suggestions: suggestions.length > 0 ? suggestions : undefined } diff --git a/src/types/database.ts b/src/types/database.ts index eb9d4fa..0e8b30a 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -67,6 +67,7 @@ export interface EdgeContext { export interface NodeFilters { dimensions?: string[]; // Filter by dimensions (replaces stage/type filtering) search?: string; // Text search in title/content + searchMode?: 'standard' | 'hybrid'; // standard = FTS/LIKE, hybrid = add node-vector retrieval limit?: number; offset?: number; sortBy?: 'updated' | 'edges' | 'created' | 'event_date'; // Sort by updated_at, edge count, created_at, or event_date diff --git a/tests/evals/dataset.json b/tests/evals/dataset.json index 94afd9d..e7e13a5 100644 --- a/tests/evals/dataset.json +++ b/tests/evals/dataset.json @@ -1,5 +1,13 @@ { - "id": "golden-v1", - "name": "Golden Dataset v1", - "description": "Baseline eval scenarios for core RA-H helper behavior." + "id": "golden-ra-h-core-v2", + "name": "Golden RA-H Core v2", + "description": "Slim 5-scenario eval set for core RA-H usage: focused graph writes, skill-guided writes, indexed node search, chunk-grounded insight creation, and hub traversal.", + "inherits": "golden-v1", + "version": 2, + "focus": [ + "tool-call correctness", + "skill usage only when appropriate", + "core retrieval/write behavior", + "latency/tokens/cost guards" + ] } diff --git a/tests/evals/runner.ts b/tests/evals/runner.ts index b9eaa36..dc9caaa 100644 --- a/tests/evals/runner.ts +++ b/tests/evals/runner.ts @@ -13,10 +13,12 @@ type EvalChatRow = { input_tokens: number | null; output_tokens: number | null; total_tokens: number | null; + estimated_cost_usd: number | null; }; type EvalToolCallRow = { tool_name: string; + args_json: string | null; }; type EvalResult = { @@ -25,10 +27,19 @@ type EvalResult = { failures: string[]; warnings: string[]; latencyMs?: number | null; + totalTokens?: number | null; + estimatedCostUsd?: number | null; + triggerStats?: { + tp: number; + fp: number; + fn: number; + }; }; const BASE_URL = process.env.RAH_EVALS_BASE_URL || 'http://localhost:3000'; const DATASET_ENV = process.env.RAH_EVALS_DATASET_ID; +const SUITE_ENV = (process.env.RAH_EVALS_SUITE || 'all').toLowerCase(); +const WAIT_TIMEOUT_MS = Number(process.env.RAH_EVALS_TIMEOUT_MS || 60000); const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite'); const RAH_DB_PATH = process.env.SQLITE_DB_PATH || path.join( process.env.HOME || '~', @@ -43,6 +54,39 @@ function loadDatasetId() { return parsed.id || 'default'; } +type EvalSuite = 'all' | 'tools' | 'skills' | 'traversal' | 'internal' | 'external'; + +type FocusedNodeContext = { + id: number; + title: string; + description: string | null; + link: string | null; + chunk_status: string | null; + chunk: string | null; + metadata: string | null; +}; + +function getDefaultScenarioSuites(scenario: Scenario): string[] { + const suites = ['internal']; + if (scenario.id.startsWith('skill-trigger-')) { + suites.push('skills'); + } else { + suites.push('tools'); + } + if (scenario.id.includes('traverse')) { + suites.push('traversal'); + } + return suites; +} + +function shouldRunScenario(scenario: Scenario, suite: EvalSuite) { + if (suite === 'all') return true; + const suites = scenario.suites && scenario.suites.length > 0 + ? scenario.suites + : getDefaultScenarioSuites(scenario); + return suites.includes(suite); +} + function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): number | null { if (!query) return null; if (!fs.existsSync(RAH_DB_PATH)) return null; @@ -70,6 +114,18 @@ function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): num return null; } +function loadFocusedNodeContext(nodeId: number | null): FocusedNodeContext[] { + if (!nodeId || !fs.existsSync(RAH_DB_PATH)) return []; + const db = new Database(RAH_DB_PATH, { readonly: true, fileMustExist: true }); + const row = db.prepare(` + SELECT id, title, description, link, chunk_status, chunk, metadata + FROM nodes + WHERE id = ? + LIMIT 1 + `).get(nodeId) as FocusedNodeContext | undefined; + return row ? [row] : []; +} + async function drainResponse(response: Response) { if (!response.body) return; const reader = response.body.getReader(); @@ -90,7 +146,7 @@ function openEvalDb() { function getEvalChatRow(db: Database.Database, traceId: string, scenarioId: string) { return db.prepare(` - SELECT trace_id, scenario_id, assistant_message, latency_ms, input_tokens, output_tokens, total_tokens + SELECT trace_id, scenario_id, assistant_message, latency_ms, input_tokens, output_tokens, total_tokens, estimated_cost_usd FROM llm_chats WHERE trace_id = ? AND scenario_id = ? ORDER BY id DESC @@ -100,7 +156,7 @@ function getEvalChatRow(db: Database.Database, traceId: string, scenarioId: stri function getEvalToolCalls(db: Database.Database, traceId: string, scenarioId: string) { return db.prepare(` - SELECT tool_name + SELECT tool_name, args_json FROM tool_calls WHERE trace_id = ? AND scenario_id = ? `).all(traceId, scenarioId) as EvalToolCallRow[]; @@ -119,10 +175,83 @@ async function waitForEvalRow(traceId: string, scenarioId: string, timeoutMs = 1 return { row: undefined, db: null }; } +async function ensureServerReady() { + try { + const response = await fetch(`${BASE_URL}/api/health/ping`); + if (!response.ok) { + throw new Error(`Health check returned HTTP ${response.status}`); + } + } catch (error) { + throw new Error( + `RA-H dev server is not reachable at ${BASE_URL}. Start it with "npm run dev:evals".` + + ` ${error instanceof Error ? error.message : String(error)}` + ); + } +} + +async function ensureEvalsEnabled() { + try { + const response = await fetch(`${BASE_URL}/evals`, { + redirect: 'manual', + }); + + if (response.status === 404) { + throw new Error('The /evals route returned 404'); + } + + if (!response.ok && response.status !== 307 && response.status !== 308) { + throw new Error(`The /evals route returned HTTP ${response.status}`); + } + } catch (error) { + throw new Error( + `Eval logging does not appear to be enabled on the dev server. Start the server with "npm run dev:evals" before running "npm run evals".` + + ` ${error instanceof Error ? error.message : String(error)}` + ); + } +} + function normalizeContains(text: string) { return text.toLowerCase(); } +function normalizeSkillId(value: string) { + return value + .trim() + .toLowerCase() + .replace(/[\s_]+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); +} + +function safeJsonParse(value: string | null): Record | null { + if (!value) return null; + try { + const parsed = JSON.parse(value); + if (parsed && typeof parsed === 'object') { + return parsed as Record; + } + return null; + } catch { + return null; + } +} + +function getSkillsRead(toolCalls: EvalToolCallRow[]) { + const names = new Set(); + for (const call of toolCalls) { + if (call.tool_name !== 'readSkill' && call.tool_name !== 'rah_read_skill') { + continue; + } + const parsed = safeJsonParse(call.args_json); + const rawName = parsed?.name; + if (typeof rawName === 'string' && rawName.trim().length > 0) { + names.add(normalizeSkillId(rawName)); + } + } + return names; +} + function checkScenario( scenario: Scenario, chatRow: EvalChatRow, @@ -133,6 +262,12 @@ function checkScenario( const expect = scenario.expect || {}; const toolNames = toolCalls.map(call => call.tool_name); const responseText = chatRow.assistant_message || ''; + const skillsRead = getSkillsRead(toolCalls); + const requiredSkills = new Set((expect.skillsRead || []).map(normalizeSkillId)); + const requiredSkillsSoft = new Set((expect.skillsReadSoft || []).map(normalizeSkillId)); + const forbiddenSkills = new Set((expect.skillsNotRead || []).map(normalizeSkillId)); + const forbiddenSkillsSoft = new Set((expect.skillsNotReadSoft || []).map(normalizeSkillId)); + const triggerStats = { tp: 0, fp: 0, fn: 0 }; (expect.toolsCalled || []).forEach(tool => { if (!toolNames.includes(tool)) { @@ -146,6 +281,54 @@ function checkScenario( } }); + (expect.toolsNotCalled || []).forEach(tool => { + if (toolNames.includes(tool)) { + failures.push(`Tool "${tool}" should not be called`); + } + }); + + (expect.toolsNotCalledSoft || []).forEach(tool => { + if (toolNames.includes(tool)) { + warnings.push(`(soft) Tool "${tool}" should not be called`); + } + }); + + requiredSkills.forEach(skill => { + if (!skillsRead.has(skill)) { + failures.push(`Expected skill "${skill}" not read`); + triggerStats.fn += 1; + } else { + triggerStats.tp += 1; + } + }); + + requiredSkillsSoft.forEach(skill => { + if (!skillsRead.has(skill)) { + warnings.push(`(soft) Expected skill "${skill}" not read`); + } + }); + + forbiddenSkills.forEach(skill => { + if (skillsRead.has(skill)) { + failures.push(`Skill "${skill}" should not be read`); + triggerStats.fp += 1; + } + }); + + forbiddenSkillsSoft.forEach(skill => { + if (skillsRead.has(skill)) { + warnings.push(`(soft) Skill "${skill}" should not be read`); + } + }); + + if (requiredSkills.size > 0) { + skillsRead.forEach(skill => { + if (!requiredSkills.has(skill)) { + triggerStats.fp += 1; + } + }); + } + (expect.responseContains || []).forEach(text => { if (!normalizeContains(responseText).includes(normalizeContains(text))) { failures.push(`Response missing "${text}"`); @@ -170,12 +353,27 @@ function checkScenario( } } + if (typeof expect.maxTotalTokens === 'number' && chatRow.total_tokens !== null) { + if (chatRow.total_tokens > expect.maxTotalTokens) { + failures.push(`Total tokens ${chatRow.total_tokens} exceeded ${expect.maxTotalTokens}`); + } + } + + if (typeof expect.maxEstimatedCostUsd === 'number' && chatRow.estimated_cost_usd !== null) { + if (chatRow.estimated_cost_usd > expect.maxEstimatedCostUsd) { + failures.push(`Estimated cost ${chatRow.estimated_cost_usd} exceeded ${expect.maxEstimatedCostUsd}`); + } + } + return { scenario: scenario.name, passed: failures.length === 0, failures, warnings, latencyMs: chatRow.latency_ms, + totalTokens: chatRow.total_tokens, + estimatedCostUsd: chatRow.estimated_cost_usd, + triggerStats: triggerStats.tp || triggerStats.fp || triggerStats.fn ? triggerStats : undefined, }; } @@ -183,15 +381,15 @@ async function runScenario(scenario: Scenario, datasetId: string): Promise s.enabled !== false && shouldRunScenario(s, suite)); + console.log(`Running ${runnable.length} scenarios (dataset: ${datasetId}, suite: ${suite})...\n`); const results: EvalResult[] = []; - for (const scenario of scenarios.filter(s => s.enabled !== false)) { + for (const scenario of runnable) { const result = await runScenario(scenario, datasetId); results.push(result); const icon = result.passed ? '✓' : '✗'; - const latency = result.latencyMs ? ` (${result.latencyMs}ms)` : ''; - console.log(`${icon} ${result.scenario}${latency}`); + const parts: string[] = []; + if (typeof result.latencyMs === 'number') parts.push(`${result.latencyMs}ms`); + if (typeof result.totalTokens === 'number') parts.push(`${result.totalTokens} tok`); + if (typeof result.estimatedCostUsd === 'number') parts.push(`$${result.estimatedCostUsd.toFixed(4)}`); + const details = parts.length > 0 ? ` (${parts.join(', ')})` : ''; + console.log(`${icon} ${result.scenario}${details}`); result.failures.forEach(failure => console.log(` - ${failure}`)); result.warnings.forEach(warning => console.log(` - ${warning}`)); } const failed = results.filter(result => !result.passed); const warnings = results.filter(result => result.warnings.length > 0); + const triggerTotals = results.reduce( + (acc, result) => { + if (result.triggerStats) { + acc.tp += result.triggerStats.tp; + acc.fp += result.triggerStats.fp; + acc.fn += result.triggerStats.fn; + } + return acc; + }, + { tp: 0, fp: 0, fn: 0 } + ); + const triggerPrecision = triggerTotals.tp + triggerTotals.fp > 0 + ? triggerTotals.tp / (triggerTotals.tp + triggerTotals.fp) + : null; + const triggerRecall = triggerTotals.tp + triggerTotals.fn > 0 + ? triggerTotals.tp / (triggerTotals.tp + triggerTotals.fn) + : null; console.log('\nSummary'); console.log(`- Passed: ${results.length - failed.length}`); console.log(`- Failed: ${failed.length}`); console.log(`- With warnings: ${warnings.length}`); + if (triggerPrecision !== null || triggerRecall !== null) { + console.log(`- Trigger TP/FP/FN: ${triggerTotals.tp}/${triggerTotals.fp}/${triggerTotals.fn}`); + console.log(`- Trigger precision: ${triggerPrecision === null ? 'n/a' : triggerPrecision.toFixed(3)}`); + console.log(`- Trigger recall: ${triggerRecall === null ? 'n/a' : triggerRecall.toFixed(3)}`); + } if (failed.length > 0) { process.exit(1); diff --git a/tests/evals/scenarios/create-edge.ts b/tests/evals/scenarios/archive/create-edge.ts similarity index 100% rename from tests/evals/scenarios/create-edge.ts rename to tests/evals/scenarios/archive/create-edge.ts diff --git a/tests/evals/scenarios/create-node.ts b/tests/evals/scenarios/archive/create-node.ts similarity index 100% rename from tests/evals/scenarios/create-node.ts rename to tests/evals/scenarios/archive/create-node.ts diff --git a/tests/evals/scenarios/dimension-lifecycle.ts b/tests/evals/scenarios/archive/dimension-lifecycle.ts similarity index 100% rename from tests/evals/scenarios/dimension-lifecycle.ts rename to tests/evals/scenarios/archive/dimension-lifecycle.ts diff --git a/tests/evals/scenarios/get-dimension.ts b/tests/evals/scenarios/archive/get-dimension.ts similarity index 100% rename from tests/evals/scenarios/get-dimension.ts rename to tests/evals/scenarios/archive/get-dimension.ts diff --git a/tests/evals/scenarios/hard-mode-query.ts b/tests/evals/scenarios/archive/hard-mode-query.ts similarity index 100% rename from tests/evals/scenarios/hard-mode-query.ts rename to tests/evals/scenarios/archive/hard-mode-query.ts diff --git a/tests/evals/scenarios/paper-extract.ts b/tests/evals/scenarios/archive/paper-extract.ts similarity index 100% rename from tests/evals/scenarios/paper-extract.ts rename to tests/evals/scenarios/archive/paper-extract.ts diff --git a/tests/evals/scenarios/query-dimensions.ts b/tests/evals/scenarios/archive/query-dimensions.ts similarity index 100% rename from tests/evals/scenarios/query-dimensions.ts rename to tests/evals/scenarios/archive/query-dimensions.ts diff --git a/tests/evals/scenarios/search-embeddings.ts b/tests/evals/scenarios/archive/search-embeddings.ts similarity index 100% rename from tests/evals/scenarios/search-embeddings.ts rename to tests/evals/scenarios/archive/search-embeddings.ts diff --git a/tests/evals/scenarios/simple-query.ts b/tests/evals/scenarios/archive/simple-query.ts similarity index 100% rename from tests/evals/scenarios/simple-query.ts rename to tests/evals/scenarios/archive/simple-query.ts diff --git a/tests/evals/scenarios/archive/types.ts b/tests/evals/scenarios/archive/types.ts new file mode 100644 index 0000000..90c2895 --- /dev/null +++ b/tests/evals/scenarios/archive/types.ts @@ -0,0 +1 @@ +export * from '../../types'; diff --git a/tests/evals/scenarios/update-node.ts b/tests/evals/scenarios/archive/update-node.ts similarity index 100% rename from tests/evals/scenarios/update-node.ts rename to tests/evals/scenarios/archive/update-node.ts diff --git a/tests/evals/scenarios/website-extract.ts b/tests/evals/scenarios/archive/website-extract.ts similarity index 100% rename from tests/evals/scenarios/website-extract.ts rename to tests/evals/scenarios/archive/website-extract.ts diff --git a/tests/evals/scenarios/youtube-extract.ts b/tests/evals/scenarios/archive/youtube-extract.ts similarity index 100% rename from tests/evals/scenarios/youtube-extract.ts rename to tests/evals/scenarios/archive/youtube-extract.ts diff --git a/tests/evals/scenarios/chunk-quote-insight.ts b/tests/evals/scenarios/chunk-quote-insight.ts new file mode 100644 index 0000000..20d7ab3 --- /dev/null +++ b/tests/evals/scenarios/chunk-quote-insight.ts @@ -0,0 +1,19 @@ +import { Scenario } from '../types'; + +export const scenario: Scenario = { + id: 'chunk-quote-insight', + name: 'Chunk quote to insight', + description: 'Search a focused transcript for a specific quote, then create a grounded insight node from it.', + tools: ['searchContentEmbeddings', 'createNode', 'createEdge'], + input: { + message: 'Search inside this focused transcript for a quote about verification being harder than generating solutions. Quote it briefly, then create a new insight node titled "Lange on verification difficulty" and connect it back to this transcript with explanation "Insight extracted from quoted passage."', + focusedNodeQuery: { titleContains: 'When AI Discovers the Next Transformer' }, + }, + expect: { + toolsCalledSoft: ['searchContentEmbeddings', 'createNode', 'createEdge'], + responseContainsSoft: ['Lange on verification difficulty'], + maxLatencyMs: 35000, + maxTotalTokens: 14000, + maxEstimatedCostUsd: 0.14, + }, +}; diff --git a/tests/evals/scenarios/focused-graph-write.ts b/tests/evals/scenarios/focused-graph-write.ts new file mode 100644 index 0000000..cd8ca99 --- /dev/null +++ b/tests/evals/scenarios/focused-graph-write.ts @@ -0,0 +1,20 @@ +import { Scenario } from '../types'; + +export const scenario: Scenario = { + id: 'focused-graph-write', + name: 'Focused graph write', + description: 'Create one new node from the focused transcript and connect it back without unnecessary retrieval.', + tools: ['createNode', 'createEdge'], + input: { + message: 'Create a new node titled "Lange: Verification bottleneck" for the claim that generating many solutions is easier than verifying them, then connect it to this focused transcript with explanation "Claim extracted from this transcript source."', + focusedNodeQuery: { titleContains: 'When AI Discovers the Next Transformer' }, + }, + expect: { + toolsCalledSoft: ['createNode', 'createEdge'], + toolsNotCalledSoft: ['readSkill', 'queryNodes', 'searchContentEmbeddings'], + responseContainsSoft: ['Lange: Verification bottleneck'], + maxLatencyMs: 25000, + maxTotalTokens: 9000, + maxEstimatedCostUsd: 0.08, + }, +}; diff --git a/tests/evals/scenarios/hub-traversal.ts b/tests/evals/scenarios/hub-traversal.ts new file mode 100644 index 0000000..c899bfe --- /dev/null +++ b/tests/evals/scenarios/hub-traversal.ts @@ -0,0 +1,19 @@ +import { Scenario } from '../types'; + +export const scenario: Scenario = { + id: 'hub-traversal', + name: 'Hub traversal', + description: 'Traverse from core hubs and connected nodes to synthesize what the user should focus on next.', + tools: ['queryEdge'], + suites: ['traversal', 'internal'], + input: { + message: 'Traverse from my hub nodes "Building RA-H — Personal Knowledge Graph" and "Nature of Intelligence & Consciousness" and tell me what I should focus on next and why.', + }, + expect: { + toolsCalledSoft: ['queryEdge'], + responseContainsSoft: ['focus', 'why'], + maxLatencyMs: 45000, + maxTotalTokens: 16000, + maxEstimatedCostUsd: 0.18, + }, +}; diff --git a/tests/evals/scenarios/index.ts b/tests/evals/scenarios/index.ts index 7026aa4..9868d1d 100644 --- a/tests/evals/scenarios/index.ts +++ b/tests/evals/scenarios/index.ts @@ -1,27 +1,13 @@ -import { scenario as simpleQuery } from './simple-query'; -import { scenario as searchEmbeddings } from './search-embeddings'; -import { scenario as createNode } from './create-node'; -import { scenario as hardModeQuery } from './hard-mode-query'; -import { scenario as updateNode } from './update-node'; -import { scenario as createEdge } from './create-edge'; -import { scenario as queryDimensions } from './query-dimensions'; -import { scenario as getDimension } from './get-dimension'; -import { scenario as dimensionLifecycle } from './dimension-lifecycle'; -import { scenario as youtubeExtract } from './youtube-extract'; -import { scenario as websiteExtract } from './website-extract'; -import { scenario as paperExtract } from './paper-extract'; +import { scenario as focusedGraphWrite } from './focused-graph-write'; +import { scenario as skillGuidedWrite } from './skill-guided-write'; +import { scenario as nodeIndexSearch } from './node-index-search'; +import { scenario as chunkQuoteInsight } from './chunk-quote-insight'; +import { scenario as hubTraversal } from './hub-traversal'; export const scenarios = [ - simpleQuery, - searchEmbeddings, - createNode, - updateNode, - createEdge, - queryDimensions, - getDimension, - dimensionLifecycle, - hardModeQuery, - youtubeExtract, - websiteExtract, - paperExtract, + focusedGraphWrite, + skillGuidedWrite, + nodeIndexSearch, + chunkQuoteInsight, + hubTraversal, ]; diff --git a/tests/evals/scenarios/node-index-search.ts b/tests/evals/scenarios/node-index-search.ts new file mode 100644 index 0000000..635e915 --- /dev/null +++ b/tests/evals/scenarios/node-index-search.ts @@ -0,0 +1,19 @@ +import { Scenario } from '../types'; + +export const scenario: Scenario = { + id: 'node-index-search', + name: 'Node index search', + description: 'Simple lookup should stay in node search and avoid chunk retrieval.', + tools: ['queryNodes'], + input: { + message: 'Find me the node about Plaintext Productivity. Just return the matching node.', + }, + expect: { + toolsCalledSoft: ['queryNodes'], + toolsNotCalledSoft: ['searchContentEmbeddings', 'readSkill'], + responseContainsSoft: ['Plaintext Productivity'], + maxLatencyMs: 15000, + maxTotalTokens: 7000, + maxEstimatedCostUsd: 0.06, + }, +}; diff --git a/tests/evals/scenarios/skill-guided-write.ts b/tests/evals/scenarios/skill-guided-write.ts new file mode 100644 index 0000000..2d80a6a --- /dev/null +++ b/tests/evals/scenarios/skill-guided-write.ts @@ -0,0 +1,20 @@ +import { Scenario } from '../types'; + +export const scenario: Scenario = { + id: 'skill-guided-write', + name: 'Skill-guided graph write', + description: 'Explicit policy-guided graph work should read the DB policy skill, then either create the requested node+edge or correctly reuse the existing node+edge without duplicating them.', + tools: ['readSkill', 'queryNodes', 'createNode', 'createEdge'], + suites: ['skills', 'internal'], + input: { + message: 'Using your DB operations policy, create a node titled "Eval: SQLite-first retrieval audit" with an explicit description and connect it to "Building RA-H — Personal Knowledge Graph" with explanation "Improves RA-H retrieval architecture."', + }, + expect: { + skillsReadSoft: ['db-operations'], + toolsCalledSoft: ['readSkill'], + responseContainsSoft: ['SQLite-first retrieval audit'], + maxLatencyMs: 35000, + maxTotalTokens: 12000, + maxEstimatedCostUsd: 0.12, + }, +}; diff --git a/tests/evals/scenarios/types.ts b/tests/evals/scenarios/types.ts new file mode 100644 index 0000000..0fc0c30 --- /dev/null +++ b/tests/evals/scenarios/types.ts @@ -0,0 +1 @@ +export type { Scenario, ScenarioExpectations, ScenarioInput } from '../types'; diff --git a/tests/evals/types.ts b/tests/evals/types.ts index 243a968..8517c4f 100644 --- a/tests/evals/types.ts +++ b/tests/evals/types.ts @@ -1,10 +1,18 @@ export type ScenarioExpectations = { + skillsRead?: string[]; + skillsReadSoft?: string[]; + skillsNotRead?: string[]; + skillsNotReadSoft?: string[]; toolsCalled?: string[]; toolsCalledSoft?: string[]; + toolsNotCalled?: string[]; + toolsNotCalledSoft?: string[]; responseContains?: string[]; responseContainsSoft?: string[]; responseNotContains?: string[]; maxLatencyMs?: number; + maxTotalTokens?: number; + maxEstimatedCostUsd?: number; }; export type ScenarioInput = { @@ -24,6 +32,7 @@ export type Scenario = { expect?: ScenarioExpectations; description?: string; tools?: string[]; + suites?: string[]; enabled?: boolean; notes?: string; };