diff --git a/app/api/extract/pdf/upload/route.ts b/app/api/extract/pdf/upload/route.ts index 75ef942..b404435 100644 --- a/app/api/extract/pdf/upload/route.ts +++ b/app/api/extract/pdf/upload/route.ts @@ -57,10 +57,8 @@ export async function POST(request: NextRequest) { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title, - // Give the description generator and UI a readable summary plus a short excerpt. - notes: `Imported PDF document: ${file.name} (${extraction.metadata.pages} pages, ${Math.round(extraction.metadata.text_length / 1000)}k characters).\n\nPreview:\n${extraction.chunk.slice(0, 1200)}`, - // Full extracted text goes in chunk for universal chunking/embedding - chunk: extraction.chunk, + description: `PDF document imported from ${file.name} — ${extraction.metadata.pages} pages of extracted text. Useful as a searchable local source.`, + source: extraction.chunk, metadata: { source: 'pdf_upload', original_filename: file.name, diff --git a/app/api/mcp/route.ts b/app/api/mcp/route.ts index 9ba9379..711dc8d 100644 --- a/app/api/mcp/route.ts +++ b/app/api/mcp/route.ts @@ -134,7 +134,7 @@ function createRAHServer(): McpServer { id: node.id, title: node.title, description: node.description ?? null, - notes: node.notes ?? null, + source: node.source ?? node.notes ?? null, link: node.link ?? null, dimensions: node.dimensions || [], metadata: node.metadata || {}, @@ -263,21 +263,22 @@ function createRAHServer(): McpServer { description: 'Create a new node in the knowledge graph.', inputSchema: { title: z.string().min(1).max(160).describe('Node title'), - content: z.string().max(20000).optional().describe('Node content/notes'), + content: z.string().max(20000).optional().describe('Legacy content field; mapped to source'), + source: z.string().max(50000).optional().describe('Full source text'), link: z.string().url().optional().describe('Source URL'), description: z.string().max(2000).optional().describe('Short description'), dimensions: z.array(z.string()).min(1).max(5).describe('Categories/tags (at least 1)'), metadata: z.record(z.any()).optional().describe('Additional metadata'), }, }, - async ({ title, content, link, description, dimensions, metadata }) => { + async ({ title, content, source, link, description, dimensions, metadata }) => { // Call the nodes API internally const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title: title.trim(), - content: content?.trim(), + source: source?.trim() || content?.trim(), link: link?.trim(), description: description?.trim(), dimensions, diff --git a/app/api/nodes/[id]/regenerate-description/route.ts b/app/api/nodes/[id]/regenerate-description/route.ts index c5f0c0e..f286240 100644 --- a/app/api/nodes/[id]/regenerate-description/route.ts +++ b/app/api/nodes/[id]/regenerate-description/route.ts @@ -85,7 +85,7 @@ export async function POST( // Generate new description using the description service const newDescription = await generateDescription({ title: node.title, - notes: node.notes || undefined, + notes: node.source || node.description || undefined, link: node.link || undefined, metadata: enrichedMetadata, diff --git a/app/api/nodes/[id]/route.ts b/app/api/nodes/[id]/route.ts index 003680b..99b934e 100644 --- a/app/api/nodes/[id]/route.ts +++ b/app/api/nodes/[id]/route.ts @@ -85,13 +85,15 @@ export async function PUT( updates.dimensions = normalizeDimensions(body.dimensions, 5); } - const incomingChunk = typeof body.chunk === 'string' ? body.chunk : undefined; - const incomingNotes = typeof body.notes === 'string' ? body.notes : undefined; - const existingChunk = existingNode.chunk ?? ''; + delete updates.notes; + delete updates.chunk; - if (incomingChunk !== undefined) { - const trimmedIncoming = incomingChunk.trim(); - const trimmedExisting = existingChunk.trim(); + const incomingSource = typeof body.source === 'string' ? body.source : undefined; + const existingSource = existingNode.source ?? ''; + + if (incomingSource !== undefined) { + const trimmedIncoming = incomingSource.trim(); + const trimmedExisting = existingSource.trim(); if (!trimmedIncoming) { updates.chunk_status = null; @@ -101,10 +103,6 @@ export async function PUT( } else { delete updates.chunk_status; } - } else if (!existingChunk.trim() && hasSufficientContent(incomingNotes)) { - updates.chunk = incomingNotes; - updates.chunk_status = 'not_chunked'; - shouldQueueEmbed = true; } const node = await nodeService.updateNode(nodeId, updates); diff --git a/app/api/nodes/route.ts b/app/api/nodes/route.ts index 5e9aff6..76335ec 100644 --- a/app/api/nodes/route.ts +++ b/app/api/nodes/route.ts @@ -92,15 +92,15 @@ export async function POST(request: NextRequest) { // Sanitize title (strip extraction artifacts) body.title = sanitizeTitle(body.title); - const rawNotes = typeof body.notes === 'string' ? body.notes : null; - const rawChunk = typeof body.chunk === 'string' ? body.chunk : null; + const rawSource = typeof body.source === 'string' ? body.source.trim() : null; const eventDate = typeof body.event_date === 'string' ? body.event_date : null; // Process provided dimensions first (needed for description generation) const trimmedProvidedDimensions = normalizeDimensions(body.dimensions, 5); // Use provided description if present, otherwise auto-generate - let nodeDescription: string | undefined = typeof body.description === 'string' && body.description.trim() + const isUserSuppliedDescription = typeof body.description === 'string' && body.description.trim().length > 0; + let nodeDescription: string | undefined = isUserSuppliedDescription ? body.description.trim().slice(0, 280) : undefined; @@ -108,7 +108,7 @@ export async function POST(request: NextRequest) { try { nodeDescription = await generateDescription({ title: body.title, - notes: rawNotes || rawChunk?.slice(0, 2000) || undefined, + notes: rawSource?.slice(0, 2000) || undefined, link: body.link || undefined, metadata: body.metadata, dimensions: trimmedProvidedDimensions @@ -127,7 +127,6 @@ export async function POST(request: NextRequest) { const descriptionError = validateExplicitDescription(finalDescription); if (descriptionError) { - const isUserSuppliedDescription = typeof body.description === 'string' && body.description.trim().length > 0; if (isUserSuppliedDescription) { return NextResponse.json({ success: false, @@ -148,33 +147,20 @@ export async function POST(request: NextRequest) { // Use only provided dimensions (no auto-assignment) const finalDimensions = trimmedProvidedDimensions; - let chunkToStore = rawChunk; + const sourceToStore = rawSource || [body.title, nodeDescription].filter(Boolean).join('\n\n').trim() || null; let chunkStatus: Node['chunk_status']; - if (chunkToStore && chunkToStore.trim().length > 0) { + if (sourceToStore && sourceToStore.trim().length > 0) { chunkStatus = 'not_chunked'; - } else { - // Build chunk from all available notes if not provided - // This ensures every node gets at least one chunk for search - const fallbackContent = [body.title, nodeDescription, rawNotes] - .filter(Boolean) - .join('\n\n') - .trim(); - - if (fallbackContent) { - chunkToStore = fallbackContent; - chunkStatus = 'not_chunked'; - } } const node = await nodeService.createNode({ title: body.title, description: finalDescription, - notes: rawNotes ?? undefined, + source: sourceToStore ?? undefined, event_date: eventDate ?? undefined, link: body.link, dimensions: finalDimensions, - chunk: chunkToStore ?? undefined, chunk_status: chunkStatus, metadata: body.metadata || {} }); diff --git a/apps/mcp-server-standalone/index.js b/apps/mcp-server-standalone/index.js index 1e38f40..636688f 100644 --- a/apps/mcp-server-standalone/index.js +++ b/apps/mcp-server-standalone/index.js @@ -77,7 +77,8 @@ All data stays on this device.`; // Tool schemas const addNodeInputSchema = { title: z.string().min(1).max(160).describe('Clear, descriptive title'), - content: z.string().max(20000).optional().describe('Node content/notes'), + content: z.string().max(20000).optional().describe('Legacy content field; mapped to source'), + source: z.string().max(50000).optional().describe('Full source text'), link: z.string().url().optional().describe('Source URL'), description: z.string().min(24).max(280).describe('REQUIRED. One-sentence summary: WHAT this is (explicit, concrete) + WHY it matters. No weak verbs (discusses, explores, examines). Example: "Podcast — Lex Fridman interviews Sam Altman on AGI timelines. First public comments since board drama."'), dimensions: z.array(z.string()).min(1).max(5).describe('1-5 categories. Call queryDimensions first to use existing ones.'), @@ -354,7 +355,7 @@ async function main() { // Note: MCP schema uses "content" for external API compat; mapped to "notes" internally inputSchema: addNodeInputSchema }, - async ({ title, content, link, description, dimensions, metadata, chunk }) => { + async ({ title, content, source, link, description, dimensions, metadata, chunk }) => { const normalizedDimensions = sanitizeDimensions(dimensions); if (normalizedDimensions.length === 0) { throw new Error('At least one dimension is required.'); @@ -366,12 +367,11 @@ async function main() { const node = nodeService.createNode({ title: title.trim(), - notes: content?.trim(), + source: source?.trim() || chunk?.trim() || content?.trim(), link: link?.trim(), description: description?.trim(), dimensions: normalizedDimensions, - metadata: metadata || {}, - chunk: chunk?.trim() + metadata: metadata || {} }); const summary = `Created node #${node.id}: ${node.title} [${node.dimensions.join(', ')}]`; @@ -425,7 +425,7 @@ async function main() { WITH fts_matches AS ( SELECT rowid, rank FROM nodes_fts WHERE nodes_fts MATCH ? LIMIT 100 ) - SELECT n.id, n.title, n.description, n.notes, n.link, + SELECT n.id, n.title, n.description, n.source, n.link, n.created_at, n.updated_at, n.event_date, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -446,7 +446,7 @@ async function main() { WITH fts_matches AS ( SELECT rowid, rank FROM nodes_fts WHERE nodes_fts MATCH ? LIMIT ? ) - SELECT n.id, n.title, n.description, n.notes, n.link, + SELECT n.id, n.title, n.description, n.source, n.link, n.created_at, n.updated_at, n.event_date, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -462,7 +462,7 @@ async function main() { nodes = rows.map(row => ({ id: row.id, title: row.title, - notes: row.notes ?? null, + source: row.source ?? null, description: row.description ?? null, link: row.link ?? null, dimensions: JSON.parse(row.dimensions_json || '[]'), @@ -482,7 +482,7 @@ async function main() { const words = trimmedQuery.split(/\s+/).filter(w => w.length > 0); let sql = ` - SELECT n.id, n.title, n.description, n.notes, n.link, + SELECT n.id, n.title, n.description, n.source, n.link, n.created_at, n.updated_at, n.event_date, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -492,7 +492,7 @@ async function main() { const params = []; for (const word of words) { - sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; + sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`; params.push(`%${word}%`, `%${word}%`, `%${word}%`); } @@ -518,7 +518,7 @@ async function main() { nodes = rows.map(row => ({ id: row.id, title: row.title, - notes: row.notes ?? null, + source: row.source ?? null, description: row.description ?? null, link: row.link ?? null, dimensions: JSON.parse(row.dimensions_json || '[]'), @@ -560,13 +560,13 @@ async function main() { for (const id of uniqueIds) { const node = nodeService.getNodeById(id); if (node) { - const rawChunk = node.chunk ?? null; + const rawChunk = node.source ?? node.chunk ?? null; const chunkTruncated = rawChunk ? rawChunk.length > CHUNK_LIMIT : false; nodes.push({ id: node.id, title: node.title, - notes: node.notes ?? null, + source: node.source ?? node.notes ?? null, description: node.description ?? null, link: node.link ?? null, chunk: chunkTruncated ? rawChunk.substring(0, CHUNK_LIMIT) : rawChunk, @@ -610,14 +610,18 @@ async function main() { throw new Error(descriptionError); } - // Map MCP 'content' field → internal 'notes' field + // Map MCP legacy fields to canonical source const mappedUpdates = { ...updates }; if (mappedUpdates.content !== undefined) { - mappedUpdates.notes = mappedUpdates.content; - delete mappedUpdates.content; + mappedUpdates.source = mappedUpdates.content; } + if (mappedUpdates.chunk !== undefined && mappedUpdates.source === undefined) { + mappedUpdates.source = mappedUpdates.chunk; + } + delete mappedUpdates.content; + delete mappedUpdates.chunk; - const node = nodeService.updateNode(id, mappedUpdates, { appendNotes: true }); + const node = nodeService.updateNode(id, mappedUpdates, { appendNotes: false }); return { content: [{ type: 'text', text: `Updated node #${id}` }], diff --git a/apps/mcp-server-standalone/services/nodeService.js b/apps/mcp-server-standalone/services/nodeService.js index 8472c29..52b31b4 100644 --- a/apps/mcp-server-standalone/services/nodeService.js +++ b/apps/mcp-server-standalone/services/nodeService.js @@ -9,7 +9,7 @@ function getNodes(filters = {}) { const { dimensions, search, limit = 100, offset = 0 } = filters; let sql = ` - SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -30,7 +30,7 @@ function getNodes(filters = {}) { // Text search if (search) { - sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; + sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`; params.push(`%${search}%`, `%${search}%`, `%${search}%`); } @@ -70,7 +70,7 @@ function getNodes(filters = {}) { */ function getNodeById(id) { const sql = ` - SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -108,11 +108,10 @@ function createNode(nodeData) { const { title: rawTitle, description, - notes, + source, link, event_date, dimensions = [], - chunk, metadata = {} } = nodeData; @@ -121,30 +120,23 @@ function createNode(nodeData) { const now = new Date().toISOString(); const db = getDb(); - // Build chunk fallback from available fields when no explicit chunk provided - let chunkToStore = chunk ?? null; - if (!chunkToStore || !chunkToStore.trim()) { - const fallbackParts = [title, description, notes].filter(Boolean); - const fallback = fallbackParts.join('\n\n').trim(); - if (fallback) { - chunkToStore = fallback; - } - } + const sourceToStore = source && source.trim() + ? source + : [title, description].filter(Boolean).join('\n\n').trim() || null; const nodeId = transaction(() => { const stmt = db.prepare(` - INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO nodes (title, description, source, link, event_date, metadata, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) `); const result = stmt.run( title, description ?? null, - notes ?? null, + sourceToStore, link ?? null, event_date ?? null, JSON.stringify(metadata), - chunkToStore, now, now ); @@ -169,11 +161,11 @@ function createNode(nodeData) { /** * Update an existing node. - * Note: notes is APPENDED by default (MCP tool behavior), not replaced. + * Source-first update path. */ function updateNode(id, updates, options = {}) { const { appendNotes = true } = options; - const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates; + const { title, description, source, notes, link, event_date, dimensions, chunk, metadata } = updates; const now = new Date().toISOString(); const db = getDb(); @@ -195,15 +187,15 @@ function updateNode(id, updates, options = {}) { setFields.push('description = ?'); params.push(description); } - if (notes !== undefined) { - if (appendNotes && existing.notes) { - // Append to existing notes - setFields.push('notes = ?'); - params.push(existing.notes + '\n\n' + notes); - } else { - setFields.push('notes = ?'); - params.push(notes); - } + if (source !== undefined) { + setFields.push('source = ?'); + params.push(source); + } else if (notes !== undefined) { + const nextSource = appendNotes && existing.source + ? `${existing.source}\n\n${notes}` + : notes; + setFields.push('source = ?'); + params.push(nextSource); } if (link !== undefined) { setFields.push('link = ?'); @@ -213,8 +205,8 @@ function updateNode(id, updates, options = {}) { setFields.push('event_date = ?'); params.push(event_date); } - if (chunk !== undefined) { - setFields.push('chunk = ?'); + if (chunk !== undefined && source === undefined) { + setFields.push('source = ?'); params.push(chunk); } if (metadata !== undefined) { diff --git a/apps/mcp-server/server.js b/apps/mcp-server/server.js index 3620782..4201a77 100644 --- a/apps/mcp-server/server.js +++ b/apps/mcp-server/server.js @@ -86,6 +86,7 @@ const sanitizeDimensions = (raw) => { const addNodeInputSchema = { title: z.string().min(1).max(160), content: z.string().max(20000).optional(), + source: z.string().max(50000).optional(), link: z.string().url().optional(), description: z.string().max(2000).optional(), dimensions: z.array(z.string()).min(1).max(5), @@ -112,7 +113,7 @@ const searchNodesOutputSchema = { z.object({ id: z.number(), title: z.string(), - notes: z.string().nullable(), + source: z.string().nullable(), description: z.string().nullable(), link: z.string().nullable(), dimensions: z.array(z.string()), @@ -151,7 +152,7 @@ const getNodesOutputSchema = { z.object({ id: z.number(), title: z.string(), - notes: z.string().nullable(), + source: z.string().nullable(), link: z.string().nullable(), dimensions: z.array(z.string()), updated_at: z.string() @@ -349,7 +350,7 @@ mcpServer.registerTool( inputSchema: addNodeInputSchema, outputSchema: addNodeOutputSchema }, - async ({ title, content, link, description, dimensions, metadata, chunk }) => { + async ({ title, content, source, link, description, dimensions, metadata, chunk }) => { const normalizedDimensions = sanitizeDimensions(dimensions); if (normalizedDimensions.length === 0) { throw new McpError( @@ -360,12 +361,11 @@ mcpServer.registerTool( const payload = { title: title.trim(), - notes: content?.trim() || undefined, + source: source?.trim() || chunk?.trim() || content?.trim() || undefined, link: link?.trim() || undefined, description: description?.trim() || undefined, dimensions: normalizedDimensions, - metadata: metadata || {}, - chunk: chunk?.trim() || undefined + metadata: metadata || {} }; const result = await callRaHApi('/api/nodes', { @@ -422,7 +422,7 @@ mcpServer.registerTool( nodes: nodes.map((node) => ({ id: node.id, title: node.title, - notes: node.notes ?? null, + source: node.source ?? node.notes ?? null, description: node.description ?? null, link: node.link ?? null, dimensions: node.dimensions || [], @@ -446,12 +446,16 @@ mcpServer.registerTool( throw new McpError(ErrorCode.InvalidParams, 'At least one field must be provided in updates.'); } - // Map MCP 'content' field → internal 'notes' field + // Map MCP legacy fields to canonical source const mappedUpdates = { ...updates }; if (mappedUpdates.content !== undefined) { - mappedUpdates.notes = mappedUpdates.content; - delete mappedUpdates.content; + mappedUpdates.source = mappedUpdates.content; } + if (mappedUpdates.chunk !== undefined && mappedUpdates.source === undefined) { + mappedUpdates.source = mappedUpdates.chunk; + } + delete mappedUpdates.content; + delete mappedUpdates.chunk; const result = await callRaHApi(`/api/nodes/${id}`, { method: 'PUT', @@ -492,7 +496,7 @@ mcpServer.registerTool( nodes.push({ id: result.node.id, title: result.node.title, - notes: result.node.notes ?? null, + source: result.node.source ?? result.node.notes ?? null, link: result.node.link ?? null, dimensions: result.node.dimensions || [], updated_at: result.node.updated_at diff --git a/apps/mcp-server/stdio-server.js b/apps/mcp-server/stdio-server.js index 1a71642..217fb57 100644 --- a/apps/mcp-server/stdio-server.js +++ b/apps/mcp-server/stdio-server.js @@ -35,6 +35,7 @@ const STATUS_PATH = path.join( const addNodeInputSchema = { title: z.string().min(1).max(160), content: z.string().max(20000).optional(), + source: z.string().max(50000).optional(), link: z.string().url().optional(), description: z.string().max(2000).optional(), dimensions: z.array(z.string()).min(1).max(5), @@ -322,7 +323,7 @@ server.registerTool( inputSchema: addNodeInputSchema, outputSchema: addNodeOutputSchema }, - async ({ title, content, link, description, dimensions, metadata, chunk }) => { + async ({ title, content, source, link, description, dimensions, metadata, chunk }) => { const normalizedDimensions = sanitizeDimensions(dimensions); if (normalizedDimensions.length === 0) { throw new Error('At least one dimension/tag is required when creating a node.'); @@ -330,12 +331,11 @@ server.registerTool( const payload = { title: title.trim(), - notes: content?.trim() || undefined, + source: source?.trim() || chunk?.trim() || content?.trim() || undefined, link: link?.trim() || undefined, description: description?.trim() || undefined, dimensions: normalizedDimensions, - metadata: metadata || {}, - chunk: chunk?.trim() || undefined + metadata: metadata || {} }; const result = await callRaHApi('/api/nodes', { @@ -393,7 +393,7 @@ server.registerTool( nodes: nodes.map((node) => ({ id: node.id, title: node.title, - notes: node.notes ?? null, + source: node.source ?? node.notes ?? null, description: node.description ?? null, link: node.link ?? null, dimensions: node.dimensions || [], @@ -417,12 +417,16 @@ server.registerTool( throw new Error('At least one field must be provided in updates.'); } - // Map MCP 'content' field → internal 'notes' field + // Map MCP legacy fields to canonical source const mappedUpdates = { ...updates }; if (mappedUpdates.content !== undefined) { - mappedUpdates.notes = mappedUpdates.content; - delete mappedUpdates.content; + mappedUpdates.source = mappedUpdates.content; } + if (mappedUpdates.chunk !== undefined && mappedUpdates.source === undefined) { + mappedUpdates.source = mappedUpdates.chunk; + } + delete mappedUpdates.content; + delete mappedUpdates.chunk; const result = await callRaHApi(`/api/nodes/${id}`, { method: 'PUT', @@ -463,7 +467,7 @@ server.registerTool( nodes.push({ id: result.node.id, title: result.node.title, - notes: result.node.notes ?? null, + source: result.node.source ?? result.node.notes ?? null, link: result.node.link ?? null, dimensions: result.node.dimensions || [], updated_at: result.node.updated_at diff --git a/docs/2_schema.md b/docs/2_schema.md index d31b66b..5953b8f 100644 --- a/docs/2_schema.md +++ b/docs/2_schema.md @@ -1,5 +1,7 @@ # Database Schema +This page describes the database as it exists on disk, including some legacy tables and columns that remain for compatibility and historical records. The current shipped product uses a single RA-H assistant; old delegation-era fields should not be read as active UX concepts. + ## Entity Relationship Diagram ```mermaid @@ -8,13 +10,16 @@ erDiagram nodes ||--o{ edges : "from" nodes ||--o{ edges : "to" nodes ||--o{ chunks : "contains" + nodes ||--o{ chats : "focused_on" dimensions ||--o{ node_dimensions : "tagged_with" + chats }o--|| agent_delegations : "belongs_to" nodes { INTEGER id PK TEXT title - TEXT content - TEXT type + TEXT source + TEXT description + TEXT event_date BLOB embedding } @@ -29,6 +34,7 @@ erDiagram dimensions { TEXT name PK INTEGER is_priority + TEXT icon } node_dimensions { @@ -41,6 +47,19 @@ erDiagram INTEGER node_id FK TEXT text } + + chats { + INTEGER id PK + INTEGER focused_node_id FK + TEXT user_message + TEXT assistant_message + } + + agent_delegations { + INTEGER id PK + TEXT task + TEXT status + } ``` --- @@ -80,25 +99,24 @@ Primary knowledge storage. Each row is a discrete knowledge item. **Columns:** - `id` (INTEGER PK) - Unique identifier - `title` (TEXT) - Node title -- `content` (TEXT) - Full content -- `type` (TEXT) - Node type (memory, paper, idea, person, etc) +- `description` (TEXT) - WHAT this is + WHY it matters (primary identity field) +- `source` (TEXT) - Canonical source content used for chunking and embedding - `link` (TEXT) - External source URL (only for source nodes, not derived ideas) -- `description` (TEXT) - Brief summary +- `event_date` (TEXT) - When the thing actually happened (vs `created_at` = when it entered the graph) - `metadata` (TEXT) - JSON metadata -- `chunk` (TEXT) - Source text for chunking - `chunk_status` (TEXT) - Chunking status (not_chunked, chunked) - `embedding` (BLOB) - Node-level embedding vector - `embedding_text` (TEXT) - Text that was embedded - `embedding_updated_at` (TEXT) - Embedding timestamp -- `is_pinned` (INTEGER) - Legacy pin flag (kept for migration; not surfaced in UI) - `created_at`, `updated_at` (TEXT) - Timestamps -**Indexes:** -- `idx_nodes_type` - Fast type filtering -- `idx_nodes_pinned` - Legacy partial index (no longer recreated, safe to drop later) +**Temporal dimensions:** Each node has three timestamps: +- `created_at` - When the node entered the graph (transaction time) +- `updated_at` - When the node was last modified +- `event_date` - When the thing actually happened (valid time) **FTS:** -- `nodes_fts` - Full-text search on title + content +- `nodes_fts` - Full-text search on title + source + description ### edges Directed relationships between nodes (knowledge graph). @@ -117,7 +135,6 @@ Directed relationships between nodes (knowledge graph). - `created_at` (TEXT) - `context` (TEXT) — JSON blob (canonical; see `EdgeContext` below) - `explanation` (TEXT) — legacy column (currently not the canonical source of truth) -- `user_feedback` (INTEGER) — user rating (not currently used in core flows) **Indexes:** - `idx_edges_from` — fast “outgoing edges” queries @@ -200,7 +217,8 @@ Master list of categorization tags. **Columns:** - `name` (TEXT PK) - Dimension name -- `is_priority` (INTEGER) - Priority dimension flag +- `is_priority` (INTEGER) - Legacy compatibility field retained in schema +- `icon` (TEXT) - Icon identifier (persisted in database) - `updated_at` (TEXT) ### node_dimensions @@ -218,12 +236,14 @@ Many-to-many junction table (nodes ↔ dimensions). ### chats Conversation history with token/cost tracking. +The chat schema still carries some legacy multi-agent fields. Current product framing is simpler: one RA-H assistant, with these columns mainly retained for older records, analytics, and backwards compatibility. + **Columns:** - `id` (INTEGER PK) - `chat_type` (TEXT) - Conversation type -- `helper_name` (TEXT) - Agent key (ra-h, ra-h-easy, mini-rah, wise-rah) -- `agent_type` (TEXT) - Role category (orchestrator, executor, planner) -- `delegation_id` (INTEGER FK) +- `helper_name` (TEXT) - Legacy runtime label; current app sessions use `ra-h` +- `agent_type` (TEXT) - Legacy role field retained for historical rows/analytics +- `delegation_id` (INTEGER FK) - Legacy link to delegation records - `user_message` (TEXT) - `assistant_message` (TEXT) - `thread_id` (TEXT) - Conversation thread @@ -235,7 +255,7 @@ Conversation history with token/cost tracking. - `idx_chats_thread` - Fast thread retrieval ### agent_delegations -Task queue for mini-rah workers. +Legacy delegation queue from the older multi-agent runtime. Retained so old rows and analytics remain readable; not an active user-facing feature in the current product. **Columns:** - `id` (INTEGER PK) @@ -248,18 +268,6 @@ Task queue for mini-rah workers. - `summary` (TEXT) - Result summary - `created_at`, `updated_at` (TEXT) -### chat_memory_state -Checkpoint tracker for memory pipeline. - -**Columns:** -- `thread_id` (TEXT PK) -- `helper_name` (TEXT) - Agent that owns the thread -- `last_processed_chat_id` (INTEGER) - Last chat processed -- `last_processed_at` (TEXT) - -**Indexes:** -- `idx_chat_memory_thread` - Fast state lookup - ### logs Activity audit trail (auto-pruned to last 10k). @@ -316,8 +324,8 @@ Nodes with dimensions aggregated as JSON array. ```sql SELECT - n.id, n.title, n.content, n.link, n.metadata, n.chunk, - n.created_at, n.updated_at, + n.id, n.title, n.description, n.source, n.link, n.metadata, + n.event_date, n.created_at, n.updated_at, COALESCE(JSON_GROUP_ARRAY(d.dimension), '[]') AS dimensions_json FROM nodes n LEFT JOIN node_dimensions d ON d.node_id = n.id diff --git a/scripts/database/sqlite-ensure-app-schema.sh b/scripts/database/sqlite-ensure-app-schema.sh index f5d6e20..0303af6 100755 --- a/scripts/database/sqlite-ensure-app-schema.sh +++ b/scripts/database/sqlite-ensure-app-schema.sh @@ -118,13 +118,12 @@ CREATE TABLE nodes ( id INTEGER PRIMARY KEY, title TEXT, description TEXT, - notes TEXT, + source TEXT, link TEXT, event_date TEXT, created_at TEXT, updated_at TEXT, metadata TEXT, - chunk TEXT, embedding BLOB, embedding_updated_at TEXT, embedding_text TEXT, @@ -281,9 +280,9 @@ if has_table nodes; then echo "Adding nodes.metadata" "$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN metadata TEXT;" fi - if ! has_col nodes chunk; then - echo "Adding nodes.chunk" - "$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN chunk TEXT;" + if ! has_col nodes source; then + echo "Adding nodes.source" + "$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN source TEXT;" fi # is_pinned removed in final schema pass fi diff --git a/src/components/focus/FocusPanel.tsx b/src/components/focus/FocusPanel.tsx index fd627cf..2ac7dd3 100644 --- a/src/components/focus/FocusPanel.tsx +++ b/src/components/focus/FocusPanel.tsx @@ -9,10 +9,10 @@ import { parseNodeMarkers } from '@/tools/infrastructure/nodeFormatter'; import { Node, NodeConnection, Chunk } from '@/types/database'; import DimensionTags from './dimensions/DimensionTags'; import { getNodeIcon } from '@/utils/nodeIcons'; -import { openExternalUrl, shouldOpenExternally } from '@/utils/openExternalUrl'; import { useDimensionIcons } from '@/context/DimensionIconsContext'; import ConfirmDialog from '../common/ConfirmDialog'; import { SourceReader } from './source'; +import { openExternalUrl, shouldOpenExternally } from '@/utils/openExternalUrl'; interface NodeSearchResult { @@ -36,8 +36,8 @@ interface FocusPanelProps { } export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeClick, onTabClose, refreshTrigger, onOpenInOtherSlot, hideTabBar, onTextSelect, highlightedPassage }: FocusPanelProps) { - const { dimensionIcons } = useDimensionIcons(); const [nodesData, setNodesData] = useState>({}); + const { dimensionIcons } = useDimensionIcons(); // Context menu state const [contextMenu, setContextMenu] = useState<{ x: number; y: number; tabId: number } | null>(null); @@ -97,10 +97,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const [chunkExpanded, setChunkExpanded] = useState<{ [key: number]: boolean }>({}); // Edges expand/collapse state - // edgesExpanded removed — all connections shown in Edges tab - - // Connections section collapsed state (default closed) - const [edgeSearchOpen, setEdgeSearchOpen] = useState(false); + const [edgesExpanded, setEdgesExpanded] = useState<{ [key: number]: boolean }>({}); // Title expanded state for click-to-expand full title const [titleExpanded, setTitleExpanded] = useState<{ [key: number]: boolean }>({}); @@ -108,7 +105,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli // Description regeneration state const [regeneratingDescription, setRegeneratingDescription] = useState(null); - // Content tab state: 'notes', 'desc', or 'source' + // Content tab state: 'desc', 'notes', 'edges', or 'source' const [activeContentTab, setActiveContentTab] = useState<'notes' | 'desc' | 'edges' | 'source'>('desc'); // Desc (description) edit mode state @@ -135,6 +132,9 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const [loadingChunks, setLoadingChunks] = useState>(new Set()); const [chunksExpanded, setChunksExpanded] = useState>({}); + // Edge creation search toggle + const [edgeSearchOpen, setEdgeSearchOpen] = useState(false); + // Helper: preview edge type based on heuristics (mirrors backend logic) const previewEdgeType = (explanation: string): { type: string; label: string } | null => { const norm = (explanation || '').trim().toLowerCase(); @@ -230,13 +230,16 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli setEditingField(null); setEditingValue(''); setEditingNodeId(null); - // Also clear notes/desc/source edit modes + // Also clear notes/desc/source/edges edit modes setNotesEditMode(false); setNotesEditValue(''); setDescEditMode(false); setDescEditValue(''); setSourceEditMode(false); setSourceEditValue(''); + setEdgeSearchOpen(false); + setNodeSearchQuery(''); + setNodeSearchSuggestions([]); }, [activeTab]); const fetchNodeData = async (id: number) => { @@ -356,7 +359,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const updateData: Record = {}; if (editingField === 'notes') { - updateData.content = editingValue; + updateData.notes = editingValue; } else { updateData[editingField] = editingValue; } @@ -378,7 +381,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli setNodesData(prev => ({ ...prev, [nodeId]: result.node })); } - // Safety net: ensure edges exist for any tokens present in saved content + // Safety net: ensure edges exist for any tokens present in saved notes if (editingField === 'notes' && typeof editingValue === 'string') { try { const tokens = parseNodeMarkers(editingValue); @@ -421,7 +424,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const response = await fetch(`/api/nodes/${nodeId}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ notes: contentValue }), + body: JSON.stringify({ content: contentValue }), }); if (!response.ok) throw new Error('Failed to save'); const result = await response.json(); @@ -497,16 +500,16 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli setDescEditMode(true); }; - // Sync description to source (chunk) and re-embed + // Sync description to source and re-embed const syncDescToSource = async () => { if (!activeTab) return; setDescSaving(true); try { - // Save description to chunk field + // Save description to source field const response = await fetch(`/api/nodes/${activeTab}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ chunk: descEditValue }), + body: JSON.stringify({ source: descEditValue }), }); if (!response.ok) throw new Error('Failed to sync'); const result = await response.json(); @@ -533,7 +536,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli } }; - // Save notes (content) with explicit Save button + // Save notes-tab content into canonical source const saveNotes = async () => { if (!activeTab) return; setNotesSaving(true); @@ -541,7 +544,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const response = await fetch(`/api/nodes/${activeTab}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ notes: notesEditValue }), + body: JSON.stringify({ source: notesEditValue }), }); if (!response.ok) throw new Error('Failed to save'); const result = await response.json(); @@ -571,8 +574,8 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli setNotesEditMode(false); setNotesEditValue(''); } catch (e) { - console.error('Error saving notes:', e); - alert('Failed to save notes. Please try again.'); + console.error('Error saving source content:', e); + alert('Failed to save content. Please try again.'); } finally { setNotesSaving(false); } @@ -587,11 +590,11 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli // Start editing notes const startNotesEdit = () => { if (!activeTab || !nodesData[activeTab]) return; - setNotesEditValue(nodesData[activeTab].notes || ''); + setNotesEditValue(nodesData[activeTab].source || nodesData[activeTab].notes || ''); setNotesEditMode(true); }; - // Save source (chunk) with explicit Save button + // Save source with explicit Save button const saveSource = async () => { if (!activeTab) return; setSourceSaving(true); @@ -599,7 +602,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const response = await fetch(`/api/nodes/${activeTab}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ chunk: sourceEditValue }), + body: JSON.stringify({ source: sourceEditValue }), }); if (!response.ok) throw new Error('Failed to save'); const result = await response.json(); @@ -625,7 +628,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli // Start editing source const startSourceEdit = () => { if (!activeTab || !nodesData[activeTab]) return; - setSourceEditValue(nodesData[activeTab].chunk || ''); + setSourceEditValue(nodesData[activeTab].source || nodesData[activeTab].chunk || ''); setSourceEditMode(true); }; @@ -641,11 +644,11 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli setSyncing(true); setShowSyncConfirm(false); try { - // First, save notes content to chunk field + // First, save notes content to source field const response = await fetch(`/api/nodes/${activeTab}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ chunk: notesEditValue }), + body: JSON.stringify({ source: notesEditValue }), }); if (!response.ok) throw new Error('Failed to sync'); const result = await response.json(); @@ -997,14 +1000,14 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const embedContent = async (nodeId: number) => { const node = nodesData[nodeId]; const hasNotes = node?.notes?.trim(); - const hasChunk = node?.chunk?.trim(); - // If chunk is empty but content exists, auto-populate chunk from content - if (!hasChunk && hasNotes) { + const hasSource = (node?.source || node?.chunk)?.trim(); + // If source is empty but notes exist, auto-populate source from notes + if (!hasSource && hasNotes) { try { const response = await fetch(`/api/nodes/${nodeId}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ chunk: hasNotes }) + body: JSON.stringify({ source: hasNotes }) }); if (response.ok) { @@ -1014,13 +1017,13 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli } } } catch (error) { - console.error('Failed to auto-populate chunk for embedding:', error); + console.error('Failed to auto-populate source for embedding:', error); return; } } - // If neither content nor chunk exist, require content - if (!hasNotes && !hasChunk) { - startEdit('content', ''); + // If neither notes nor source exist, require notes + if (!hasNotes && !hasSource) { + startEdit('notes', ''); return; } setEmbeddingNode(nodeId); @@ -1119,7 +1122,6 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli setAddingEdge(null); setEdgeExplanation(''); setPendingEdgeTarget(null); - setEdgeSearchOpen(false); } catch (error) { console.error('Error creating edge:', error); @@ -1157,7 +1159,6 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli setPendingEdgeTarget(null); setNodeSearchQuery(''); setNodeSearchSuggestions([]); - setEdgeSearchOpen(false); } catch (error) { console.error('Error creating edge:', error); @@ -1277,63 +1278,55 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli borderRadius: '8px', maxHeight: '200px', overflowY: 'auto', - zIndex: 10, - boxShadow: '0 8px 24px rgba(0, 0, 0, 0.4)' + zIndex: 100, + boxShadow: '0 8px 24px rgba(0, 0, 0, 0.5)', }}> {nodeSearchSuggestions.map((suggestion, index) => (
handleSelectNodeSuggestion(suggestion)} + onClick={() => { + handleSelectNodeSuggestion(suggestion); + setNodeSearchQuery(''); + setNodeSearchSuggestions([]); + setEdgeSearchOpen(false); + }} style={{ - padding: '10px 14px', + padding: '8px 12px', cursor: 'pointer', - borderBottom: index < nodeSearchSuggestions.length - 1 ? '1px solid var(--rah-bg-active)' : 'none', + borderBottom: index < nodeSearchSuggestions.length - 1 ? '1px solid var(--rah-border)' : 'none', background: index === selectedSearchIndex ? 'var(--rah-bg-active)' : 'transparent', transition: 'background 100ms ease', display: 'flex', alignItems: 'center', - gap: '10px' - }} - onMouseEnter={(e) => { - if (index !== selectedSearchIndex) { - e.currentTarget.style.background = 'var(--rah-bg-active)'; - } - }} - onMouseLeave={(e) => { - if (index !== selectedSearchIndex) { - e.currentTarget.style.background = 'transparent'; - } + gap: '8px', }} + onMouseEnter={(e) => { e.currentTarget.style.background = 'var(--rah-bg-active)'; }} + onMouseLeave={(e) => { if (index !== selectedSearchIndex) e.currentTarget.style.background = 'transparent'; }} > {suggestion.id} {suggestion.title} {index === selectedSearchIndex && ( - ↵ + ↵ )}
))} @@ -1342,10 +1335,10 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli )} - {/* Connections List */} + {/* Connections list */}
{loadingEdges.has(activeTab) ? ( -
Loading connections…
+
Loading connections…
) : (() => { const list = edgesData[activeTab] || []; if (list.length === 0) { @@ -1358,18 +1351,18 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli key={connection.id} style={{ padding: '6px 0', - borderBottom: '1px solid var(--rah-bg-panel)', + borderBottom: '1px solid var(--rah-border)', display: 'flex', flexDirection: 'column', gap: '2px', minWidth: 0, }} > - {/* Row 1: arrow + ID + icon + title + delete */} + {/* Row 1: arrow + ID + title + delete */}
{ e.currentTarget.style.color = '#22c55e'; }} + onMouseEnter={(e) => { e.currentTarget.style.color = 'var(--rah-accent-green)'; }} onMouseLeave={(e) => { e.currentTarget.style.color = 'var(--rah-text-base)'; }} > {connection.connected_node.title} @@ -1414,7 +1407,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli onClick={() => deleteEdge(connection.edge.id)} disabled={deletingEdge === connection.edge.id} style={{ - color: 'var(--rah-border-stronger)', + color: 'var(--rah-text-muted)', fontSize: '14px', background: 'transparent', border: 'none', @@ -1424,7 +1417,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli lineHeight: 1, }} onMouseEnter={(e) => { if (deletingEdge !== connection.edge.id) e.currentTarget.style.color = '#dc2626'; }} - onMouseLeave={(e) => { if (deletingEdge !== connection.edge.id) e.currentTarget.style.color = 'var(--rah-border-stronger)'; }} + onMouseLeave={(e) => { if (deletingEdge !== connection.edge.id) e.currentTarget.style.color = 'var(--rah-text-muted)'; }} > {deletingEdge === connection.edge.id ? '...' : '×'} @@ -1464,8 +1457,8 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli disabled={edgeSavingId === connection.edge.id} style={{ fontSize: '10px', - color: '#000', - background: '#22c55e', + color: 'var(--rah-text-inverse)', + background: 'var(--rah-accent-green)', border: 'none', borderRadius: '4px', padding: '3px 8px', @@ -1508,9 +1501,9 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli {typeof connection.edge.context?.type === 'string' && ( ) : !currentNode ? ( -
Node not found.
+
Node not found.
) : nodesData[activeTab] ? (
{/* URL Row - Above Title */} @@ -1791,8 +1784,8 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli return; } - const link = nodesData[activeTab].link; - if (!link || !shouldOpenExternally(link)) { + const link = nodesData[activeTab].link || ''; + if (!shouldOpenExternally(link)) { return; } @@ -1819,7 +1812,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli startEdit('link', '')} style={{ - color: 'var(--rah-text-muted)', + color: '#555', fontSize: '11px', cursor: 'pointer', fontStyle: 'italic' @@ -1850,8 +1843,8 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli }} style={{ display: 'inline-block', - background: '#22c55e', - color: 'var(--rah-bg-base)', + background: 'var(--rah-accent-green)', + color: 'var(--rah-text-inverse)', fontSize: '10px', fontWeight: 600, padding: '2px 6px', @@ -1864,7 +1857,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli {activeTab} - {/* Node type icon */} + {/* Node icon */} {nodesData[activeTab] && ( {getNodeIcon(nodesData[activeTab], dimensionIcons, 18)} @@ -1938,7 +1931,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli title={titleExpanded[activeTab] ? undefined : (nodesData[activeTab].title || 'Untitled')} > {nodesData[activeTab].title || 'Untitled'} - {savingField === 'title' && saving...} + {savingField === 'title' && saving...} )} @@ -1948,14 +1941,14 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli style={{ display: 'flex', alignItems: 'center', - gap: '4px', + gap: '6px', padding: '4px 8px', fontSize: '10px', fontWeight: 500, - color: activeContentTab === 'edges' ? '#22c55e' : 'var(--rah-text-muted)', + color: activeContentTab === 'edges' ? 'var(--rah-accent-green)' : 'var(--rah-text-muted)', background: activeContentTab === 'edges' ? '#0f2417' : 'transparent', border: '1px solid', - borderColor: activeContentTab === 'edges' ? '#22c55e' : 'var(--rah-border-strong)', + borderColor: activeContentTab === 'edges' ? 'var(--rah-accent-green)' : 'var(--rah-border-strong)', borderRadius: '4px', cursor: 'pointer', transition: 'all 0.2s', @@ -1963,7 +1956,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli }} onMouseEnter={(e) => { if (activeContentTab !== 'edges') { - e.currentTarget.style.color = '#22c55e'; + e.currentTarget.style.color = 'var(--rah-accent-green)'; e.currentTarget.style.borderColor = '#166534'; } }} @@ -1983,18 +1976,18 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli style={{ minWidth: '18px', height: '18px', + padding: '0 6px', borderRadius: '999px', display: 'inline-flex', alignItems: 'center', justifyContent: 'center', - padding: '0 6px', - background: activeContentTab === 'edges' ? 'rgba(34, 197, 94, 0.18)' : 'var(--rah-bg-active)', - color: activeContentTab === 'edges' ? '#a7f3b8' : 'var(--rah-text-muted)', + background: activeContentTab === 'edges' ? 'var(--rah-accent-green-soft)' : 'var(--rah-bg-active)', + color: activeContentTab === 'edges' ? 'var(--rah-accent-green)' : 'var(--rah-text-soft)', fontSize: '10px', lineHeight: 1, }} > - {(edgesData[activeTab] || []).length} + {edgesData[activeTab]?.length ?? 0} )} @@ -2084,7 +2077,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli borderBottom: '1px solid var(--rah-border)' }}>
@@ -2719,7 +2712,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
@@ -236,8 +236,8 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb style={{ display: 'flex', alignItems: 'center', gap: '4px', padding: '4px 7px', background: 'transparent', - border: '1px solid var(--rah-border-strong)', borderRadius: '5px', - color: 'var(var(--rah-text-muted))', fontSize: '11px', cursor: 'pointer', + border: '1px solid var(--rah-border)', borderRadius: '5px', + color: 'var(--rah-text-soft)', fontSize: '11px', cursor: 'pointer', }} onMouseEnter={(e) => { e.currentTarget.style.background = 'rgba(255,255,255,0.04)'; }} onMouseLeave={(e) => { e.currentTarget.style.background = 'transparent'; }} @@ -249,7 +249,7 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb {showFilterPicker && (
@@ -260,13 +260,13 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb placeholder="Search dimensions..." autoFocus style={{ - width: '100%', padding: '7px 10px', background: 'var(var(--rah-bg-surface))', + width: '100%', padding: '7px 10px', background: 'var(--rah-bg-base)', border: '1px solid transparent', borderRadius: '6px', - color: 'var(var(--rah-text-active))', fontSize: '12px', marginBottom: '4px', outline: 'none', + color: 'var(--rah-text-active)', fontSize: '12px', marginBottom: '4px', outline: 'none', }} /> {filterPickerDimensions.length === 0 ? ( -
+
No matching dimensions
) : ( @@ -283,14 +283,14 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', width: '100%', padding: '7px 10px', background: 'transparent', - border: 'none', borderRadius: '5px', color: 'var(var(--rah-text-secondary))', + border: 'none', borderRadius: '5px', color: 'var(--rah-text-secondary)', fontSize: '12px', cursor: 'pointer', textAlign: 'left', }} onMouseEnter={(e) => { e.currentTarget.style.background = 'rgba(255,255,255,0.04)'; }} onMouseLeave={(e) => { e.currentTarget.style.background = 'transparent'; }} > {d.dimension} - + {d.count} @@ -303,9 +303,9 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb {selectedFilters.length > 0 && ( @@ -319,8 +319,8 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb style={{ display: 'flex', alignItems: 'center', gap: '4px', padding: '4px 7px', background: 'transparent', - border: '1px solid var(--rah-border-strong)', borderRadius: '5px', - color: 'var(var(--rah-text-muted))', fontSize: '11px', cursor: 'pointer', whiteSpace: 'nowrap', + border: '1px solid var(--rah-border)', borderRadius: '5px', + color: 'var(--rah-text-soft)', fontSize: '11px', cursor: 'pointer', whiteSpace: 'nowrap', }} onMouseEnter={(e) => { e.currentTarget.style.background = 'rgba(255,255,255,0.04)'; }} onMouseLeave={(e) => { e.currentTarget.style.background = 'transparent'; }} @@ -333,7 +333,7 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb {showSortDropdown && (
@@ -346,13 +346,13 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb width: '100%', padding: '7px 10px', background: sortOrder === key ? 'rgba(255,255,255,0.04)' : 'transparent', border: 'none', borderRadius: '5px', - color: sortOrder === key ? '#f0f0f0' : '#999', + color: sortOrder === key ? 'var(--rah-text-active)' : 'var(--rah-text-soft)', fontSize: '12px', cursor: 'pointer', textAlign: 'left', }} onMouseEnter={(e) => { e.currentTarget.style.background = 'rgba(255,255,255,0.04)'; }} onMouseLeave={(e) => { e.currentTarget.style.background = sortOrder === key ? 'rgba(255,255,255,0.04)' : 'transparent'; }} > - {sortOrder === key && ✓} + {sortOrder === key && ✓} {SORT_LABELS[key]} ))} @@ -363,15 +363,15 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb {/* Pagination */}
{total > 0 ? `${startItem}-${endItem} of ${total}` : '0 nodes'}
{/* Description or Content Preview */} - {(node.description || node.notes) && ( + {(node.description || node.source || node.notes) && (
- {node.description || truncateContent(node.notes)} + {node.description || truncateContent(node.source || node.notes)}
)} diff --git a/src/components/views/ListView.tsx b/src/components/views/ListView.tsx index 24d2c51..078df18 100644 --- a/src/components/views/ListView.tsx +++ b/src/components/views/ListView.tsx @@ -105,7 +105,7 @@ export default function ListView({ nodes, onNodeClick }: ListViewProps) {
{/* Description or Content Preview */} - {(node.description || node.notes) && ( + {(node.description || node.source || node.notes) && (
- {node.description || truncateContent(node.notes)} + {node.description || truncateContent(node.source || node.notes)}
)} diff --git a/src/config/skills/db-operations.md b/src/config/skills/db-operations.md index 24c265e..ae03da3 100644 --- a/src/config/skills/db-operations.md +++ b/src/config/skills/db-operations.md @@ -1,9 +1,6 @@ --- name: DB Operations -description: "Use this for all graph read/write operations with strict data quality standards." -when_to_use: "Any request to read, create, update, connect, classify, or traverse graph data." -when_not_to_use: "Pure conversation with no graph interaction needed." -success_criteria: "Writes are explicit and correct; descriptions are concrete; edges and dimensions are high-signal." +description: "Use for graph read, write, connect, classify, or traverse operations with strict data quality standards." --- # DB Operations @@ -20,8 +17,9 @@ success_criteria: "Writes are explicit and correct; descriptions are concrete; e - `title`: clear and specific. - `description`: concrete object-level description, not vague summaries. -- `notes/content`: extra context, analysis, supporting detail. +- `source`: full verbatim or canonical content of the node (transcript, article text, book passage, user's thoughts). This is what gets chunked and embedded for semantic search. - `link`: external source URL only. +- Derived analysis, briefs, and research notes should be stored in a separate linked node, not appended to the source node. ## Execution Pattern diff --git a/src/services/agents/quickAdd.ts b/src/services/agents/quickAdd.ts index 7c1cdbd..76d9162 100644 --- a/src/services/agents/quickAdd.ts +++ b/src/services/agents/quickAdd.ts @@ -236,7 +236,7 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin const title = deriveFallbackLinkTitle(url); const description = `Link record for this source. RA-H could not correctly process the URL during ingestion because ${message}. Stored so the source is not lost and can be revisited later.`; - const notes = [ + const source = [ `Original URL: ${url}`, `Ingestion failure: ${message}`, `Attempted pipeline: ${type}`, @@ -248,7 +248,7 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin body: JSON.stringify({ title, description, - notes, + source, link: url, metadata: { source: 'quick-add-link-fallback', @@ -292,7 +292,7 @@ async function handleNoteQuickAdd(rawInput: string, task: string, userDescriptio const title = deriveNoteTitle(content); const nodePayload: Record = { title, - notes: content, + source: content, metadata: { source: 'quick-add-note', refined_at: new Date().toISOString(), @@ -396,8 +396,8 @@ async function handleChatTranscriptQuickAdd(rawInput: string, task: string): Pro headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title, - notes: content, - chunk: transcript, + source: transcript, + description: content, metadata, }), }); diff --git a/src/services/database/edges.ts b/src/services/database/edges.ts index 009f05a..4b673f0 100644 --- a/src/services/database/edges.ts +++ b/src/services/database/edges.ts @@ -478,17 +478,13 @@ export class EdgeService { ELSE n_from.title END as connected_node_title, CASE - WHEN e.from_node_id = ? THEN n_to.notes - ELSE n_from.notes - END as connected_node_notes, - CASE WHEN e.from_node_id = ? THEN n_to.link ELSE n_from.link END as connected_node_link, CASE - WHEN e.from_node_id = ? THEN n_to.chunk - ELSE n_from.chunk - END as connected_node_chunk, + WHEN e.from_node_id = ? THEN n_to.source + ELSE n_from.source + END as connected_node_source, CASE WHEN e.from_node_id = ? THEN n_to.metadata ELSE n_from.metadata @@ -547,11 +543,10 @@ export class EdgeService { const connected_node: Node = { id: row.connected_node_id, title: row.connected_node_title, - notes: row.connected_node_notes, link: row.connected_node_link, dimensions: row.connected_node_dimensions, embedding: undefined, // Not needed for display - chunk: row.connected_node_chunk, + source: row.connected_node_source, metadata: row.connected_node_metadata, created_at: row.connected_node_created_at, updated_at: row.connected_node_updated_at @@ -592,11 +587,10 @@ export class EdgeService { const connected_node: Node = { id: row.connected_node_id, title: row.connected_node_title, - notes: row.connected_node_notes, link: row.connected_node_link, dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'), embedding: undefined, // Not needed for display - chunk: row.connected_node_chunk, + source: row.connected_node_source, metadata: typeof row.connected_node_metadata === 'string' ? JSON.parse(row.connected_node_metadata) : row.connected_node_metadata, created_at: row.connected_node_created_at, updated_at: row.connected_node_updated_at diff --git a/src/services/database/nodes.ts b/src/services/database/nodes.ts index 7246865..8efeea3 100644 --- a/src/services/database/nodes.ts +++ b/src/services/database/nodes.ts @@ -16,6 +16,38 @@ function sanitizeFtsQuery(input: string): string { .join(' '); } +function extractRelaxedSearchTerms(query: string): string[] { + const stopWords = new Set([ + 'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'can', 'do', 'find', + 'for', 'from', 'hello', 'i', 'in', 'is', 'it', 'me', 'my', 'of', 'on', + 'or', 'recent', 'stuff', 'term', 'that', 'the', 'this', 'to', 'with', 'you' + ]); + + const rawTerms = query + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' ') + .split(/\s+/) + .map(term => term.trim()) + .filter(Boolean); + + const expanded = new Set(); + + for (const term of rawTerms) { + if (!stopWords.has(term) && term.length >= 3) { + expanded.add(term); + } + + const alphaParts = term.replace(/\d+/g, ' ').split(/\s+/).filter(Boolean); + for (const part of alphaParts) { + if (!stopWords.has(part) && part.length >= 3) { + expanded.add(part); + } + } + } + + return Array.from(expanded).slice(0, 8); +} + function reciprocalRankFuse( rankedLists: T[][], limit: number, @@ -48,7 +80,7 @@ export class NodeService { async countNodes(filters: NodeFilters = {}): Promise { const { dimensions, search, dimensionsMatch = 'any', - createdAfter, createdBefore, eventAfter, eventBefore } = filters; + createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters; if (search?.trim()) { return this.countSearchNodesSQLite(filters); @@ -78,7 +110,7 @@ export class NodeService { } if (search) { - query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; + query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`; params.push(`%${search}%`, `%${search}%`, `%${search}%`); } @@ -86,6 +118,7 @@ export class NodeService { if (createdBefore) { query += ` AND n.created_at < ?`; params.push(createdBefore); } if (eventAfter) { query += ` AND n.event_date >= ?`; params.push(eventAfter); } if (eventBefore) { query += ` AND n.event_date < ?`; params.push(eventBefore); } + if (chunkStatus) { query += ` AND n.chunk_status = ?`; params.push(chunkStatus); } const result = sqlite.query<{ total: number }>(query, params); return result.rows[0]?.total ?? 0; @@ -95,7 +128,7 @@ export class NodeService { private async getNodesSQLite(filters: NodeFilters = {}): Promise { const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any', - createdAfter, createdBefore, eventAfter, eventBefore } = filters; + createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters; if (search?.trim()) { return this.searchNodesSQLite(filters); @@ -105,7 +138,7 @@ export class NodeService { // Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance) let query = ` - SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -137,9 +170,9 @@ export class NodeService { } } - // Text search in title, description, and notes (SQLite LIKE with COLLATE NOCASE) + // Text search in title, description, and source (SQLite LIKE with COLLATE NOCASE) if (search) { - query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; + query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`; params.push(`%${search}%`, `%${search}%`, `%${search}%`); } @@ -160,23 +193,27 @@ export class NodeService { query += ` AND n.event_date < ?`; params.push(eventBefore); } + if (chunkStatus) { + query += ` AND n.chunk_status = ?`; + params.push(chunkStatus); + } // Sorting logic if (search) { - // For search queries, prioritize by relevance: exact title → starts with → contains in title → description → notes + // For search queries, prioritize by relevance: exact title → starts with → contains in title → description → source query += ` ORDER BY CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 6 END, CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END, CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END, CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END, - CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END, + CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END, n.updated_at DESC`; params.push( search, // Exact match (case-insensitive) `${search}%`, // Starts with search term `%${search}%`, // Contains in title `%${search}%`, // Contains in description - `%${search}%` // Contains in notes + `%${search}%` // Contains in source ); } else if (sortBy === 'edges') { // Sort by edge count (most connected first) @@ -215,7 +252,7 @@ export class NodeService { private async getNodeByIdSQLite(id: number): Promise { const sqlite = getSQLiteClient(); const query = ` - SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -241,11 +278,10 @@ export class NodeService { const { title, description, - notes, + source, link, event_date, dimensions = [], - chunk, chunk_status, metadata = {} } = nodeData; @@ -255,16 +291,15 @@ export class NodeService { const nodeId = sqlite.transaction(() => { // Insert node using prepare/run for lastInsertRowid access const nodeResult = sqlite.prepare(` - INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, chunk_status, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO nodes (title, description, source, link, event_date, metadata, chunk_status, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( title, description ?? null, - notes ?? null, + source ?? null, link ?? null, event_date ?? null, JSON.stringify(metadata), - chunk ?? null, chunk_status ?? null, now, now @@ -308,7 +343,7 @@ export class NodeService { // PostgreSQL path removed in SQLite-only consolidation private async updateNodeSQLite(id: number, updates: Partial): Promise { - const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates; + const { title, description, source, link, event_date, dimensions, metadata } = updates; const now = new Date().toISOString(); const sqlite = getSQLiteClient(); @@ -327,10 +362,9 @@ export class NodeService { if (title !== undefined) { setFields.push('title = ?'); params.push(title); } if (description !== undefined) { setFields.push('description = ?'); params.push(description); } - if (notes !== undefined) { setFields.push('notes = ?'); params.push(notes); } + if (source !== undefined) { setFields.push('source = ?'); params.push(source); } if (link !== undefined) { setFields.push('link = ?'); params.push(link); } if (event_date !== undefined) { setFields.push('event_date = ?'); params.push(event_date); } - if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); } if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) { setFields.push('chunk_status = ?'); params.push(updates.chunk_status ?? null); @@ -469,6 +503,10 @@ export class NodeService { rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit); } + if (rankedRows.length === 0) { + rankedRows = this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit); + } + if ((filters.searchMode ?? 'standard') === 'hybrid') { const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit); if (vectorRows.length > 0) { @@ -518,7 +556,7 @@ export class NodeService { } for (const word of words) { - query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; + query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`; queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`); } @@ -551,7 +589,7 @@ export class NodeService { WHERE nodes_fts MATCH ? LIMIT ? ) - SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -580,7 +618,7 @@ export class NodeService { const words = search.split(/\s+/).filter(Boolean); const { clauses, params } = this.buildNodeFilterClauses(filters); let query = ` - SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -595,7 +633,7 @@ export class NodeService { } for (const word of words) { - query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; + query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`; queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`); } @@ -604,7 +642,7 @@ export class NodeService { CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END, CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END, CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END, - CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END, + CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END, n.updated_at DESC LIMIT ?`; @@ -614,6 +652,62 @@ export class NodeService { return result.rows; } + private searchNodesLikeRelaxed( + sqlite: ReturnType, + search: string, + filters: NodeFilters, + limit: number, + ): NodeSearchRow[] { + const terms = extractRelaxedSearchTerms(search); + if (terms.length === 0) return []; + + const { clauses, params } = this.buildNodeFilterClauses(filters); + let query = ` + SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + n.chunk_status, n.embedding_updated_at, n.embedding_text, + n.created_at, n.updated_at, + COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) + FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json + FROM nodes n + WHERE 1=1 + `; + const queryParams = [...params]; + + if (clauses.length > 0) { + query += ` AND ${clauses.join(' AND ')}`; + } + + const termClauses: string[] = []; + for (const term of terms) { + termClauses.push(`n.title LIKE ? COLLATE NOCASE`); + termClauses.push(`n.description LIKE ? COLLATE NOCASE`); + termClauses.push(`n.source LIKE ? COLLATE NOCASE`); + queryParams.push(`%${term}%`, `%${term}%`, `%${term}%`); + } + + query += ` AND (${termClauses.join(' OR ')})`; + + const scoreClauses: string[] = []; + const scoreParams: string[] = []; + for (const term of terms) { + scoreClauses.push(`CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 0 END`); + scoreClauses.push(`CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 2 ELSE 0 END`); + scoreClauses.push(`CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 1 ELSE 0 END`); + scoreParams.push(`%${term}%`, `%${term}%`, `%${term}%`); + } + + query += ` ORDER BY + (${scoreClauses.join(' + ')}) DESC, + CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 0 ELSE 1 END, + n.updated_at DESC + LIMIT ?`; + + queryParams.push(...scoreParams, `%${search}%`, limit); + + const result = sqlite.query(query, queryParams); + return result.rows; + } + private async searchNodesVector( sqlite: ReturnType, search: string, @@ -643,7 +737,7 @@ export class NodeService { ORDER BY distance LIMIT ? ) - SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) diff --git a/src/services/database/sqlite-client.ts b/src/services/database/sqlite-client.ts index 7c99f8f..f3c0407 100644 --- a/src/services/database/sqlite-client.ts +++ b/src/services/database/sqlite-client.ts @@ -616,13 +616,60 @@ class SQLiteClient { // 10) Final schema pass migrations (content→notes, event_date, dimensions.icon, drop dead columns) try { - const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>; - const nodeColNames = nodeCols2.map(c => c.name); + let nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>; + let nodeColNames = nodeCols2.map(c => c.name); // Rename content → notes (additive first) if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) { console.log('Migrating nodes.content → nodes.notes...'); this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;'); + nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>; + nodeColNames = nodeCols2.map(c => c.name); + } + + if (!nodeColNames.includes('source')) { + console.log('Adding nodes.source column...'); + this.db.exec('ALTER TABLE nodes ADD COLUMN source TEXT;'); + nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>; + nodeColNames = nodeCols2.map(c => c.name); + } + + if (nodeColNames.includes('source')) { + this.db.exec(` + UPDATE nodes + SET source = chunk + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND chunk IS NOT NULL + AND LENGTH(TRIM(chunk)) > 0; + `); + + this.db.exec(` + UPDATE nodes + SET source = notes, + chunk_status = 'not_chunked' + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND notes IS NOT NULL + AND LENGTH(TRIM(notes)) > 0; + `); + + this.db.exec(` + UPDATE nodes + SET source = title || CASE + WHEN description IS NOT NULL AND LENGTH(TRIM(description)) > 0 + THEN char(10) || char(10) || description + ELSE '' + END, + chunk_status = 'not_chunked' + WHERE source IS NULL OR LENGTH(TRIM(source)) = 0; + `); + + this.db.exec(` + UPDATE nodes + SET chunk_status = 'not_chunked' + WHERE source IS NOT NULL + AND LENGTH(TRIM(source)) > 0 + AND (chunk_status IS NULL OR chunk_status != 'chunked'); + `); } // Add event_date @@ -657,8 +704,10 @@ class SQLiteClient { if (edgeCols.some(c => c.name === 'user_feedback')) { try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch {} } + // edges.explanation (top-level column added alongside context JSON) if (!edgeCols.some(c => c.name === 'explanation')) { this.db.exec('ALTER TABLE edges ADD COLUMN explanation TEXT;'); + // Backfill from context JSON where available try { this.db.exec(` UPDATE edges SET explanation = json_extract(context, '$.explanation') @@ -667,13 +716,13 @@ class SQLiteClient { } catch {} } - // Recreate nodes_fts to index title + description + notes + // Recreate nodes_fts to index title + source + description try { const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined; - if (ftsCheck?.sql && (!ftsCheck.sql.includes('description') || ftsCheck.sql.includes('content'))) { + const needsRebuild = !ftsCheck?.sql || !ftsCheck.sql.includes('source') || ftsCheck.sql.includes('notes') || ftsCheck.sql.includes('content'); + if (needsRebuild) { this.db.exec('DROP TABLE IF EXISTS nodes_fts;'); - this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content='nodes', content_rowid='id');"); - // Rebuild FTS index + this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');"); this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');"); } } catch (ftsErr) { diff --git a/src/services/embedding/autoEmbedQueue.ts b/src/services/embedding/autoEmbedQueue.ts index b67d88e..0392486 100644 --- a/src/services/embedding/autoEmbedQueue.ts +++ b/src/services/embedding/autoEmbedQueue.ts @@ -16,12 +16,15 @@ export class AutoEmbedQueue { private readonly lastRunAt = new Map(); private readonly maxConcurrent = 1; private readonly cooldownMs = DEFAULT_COOLDOWN_MS; - private readonly embeddingsDisabled = process.env.DISABLE_EMBEDDINGS === 'true'; + + async recoverStuckNodes(): Promise { + const stuckNodes = await nodeService.getNodes({ chunkStatus: 'not_chunked', limit: 1000 }); + for (const node of stuckNodes) { + this.enqueue(node.id, { reason: 'startup_recovery' }); + } + } enqueue(nodeId: number, task: Omit = {}): boolean { - if (this.embeddingsDisabled && !task.force) { - return false; - } const existing = this.pendingTasks.get(nodeId); if (!existing) { this.pendingTasks.set(nodeId, { nodeId, ...task }); @@ -77,21 +80,12 @@ export class AutoEmbedQueue { } private async executeTask(task: AutoEmbedTask) { - if (this.embeddingsDisabled && !task.force) { - return; - } const node = await nodeService.getNodeById(task.nodeId); if (!node) { console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId); return; } - const chunkText = node.chunk?.trim(); - if (!chunkText) { - console.warn('[AutoEmbedQueue] Node has no chunk content, skipping', task.nodeId); - return; - } - if (!task.force && node.chunk_status === 'chunked') { return; } @@ -117,4 +111,7 @@ declare global { export const autoEmbedQueue = globalThis.autoEmbedQueue ?? new AutoEmbedQueue(); if (!globalThis.autoEmbedQueue) { globalThis.autoEmbedQueue = autoEmbedQueue; + autoEmbedQueue.recoverStuckNodes().catch(error => { + console.error('[AutoEmbedQueue] Startup recovery failed', error); + }); } diff --git a/src/services/embedding/ingestion.ts b/src/services/embedding/ingestion.ts index c5ddc48..bcaabab 100644 --- a/src/services/embedding/ingestion.ts +++ b/src/services/embedding/ingestion.ts @@ -103,10 +103,10 @@ export async function embedNodeContent(nodeId: number): Promise 0) { + // Add AI analysis if source exists + const sourceText = node.source || node.notes || ''; + if (sourceText.trim().length > 0) { const analysis = await this.analyzeNodeWithAI(node); if (analysis) { embeddingText += `\n\nAI Analysis: ${analysis}`; @@ -176,8 +178,8 @@ Focus on the main concepts, key relationships, and practical implications.`; if (nodeId) { // Single node - query = ` - SELECT n.id, n.title, n.content, n.description, + query = ` + SELECT n.id, n.title, n.source, n.notes, n.description, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json, n.embedding, n.embedding_updated_at @@ -188,7 +190,7 @@ Focus on the main concepts, key relationships, and practical implications.`; } else if (forceReEmbed) { // All nodes query = ` - SELECT n.id, n.title, n.content, n.description, + SELECT n.id, n.title, n.source, n.notes, n.description, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json, n.embedding, n.embedding_updated_at @@ -198,7 +200,7 @@ Focus on the main concepts, key relationships, and practical implications.`; } else { // Only nodes without embeddings query = ` - SELECT n.id, n.title, n.content, n.description, + SELECT n.id, n.title, n.source, n.notes, n.description, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json, n.embedding, n.embedding_updated_at diff --git a/src/services/typescript/embed-universal.ts b/src/services/typescript/embed-universal.ts index 43c95cd..158b1b2 100644 --- a/src/services/typescript/embed-universal.ts +++ b/src/services/typescript/embed-universal.ts @@ -1,6 +1,6 @@ /** * Universal chunking and embedding service for RA-H knowledge management system - * Takes a node_id, reads chunk content from nodes table, chunks it, and stores in chunks table + * Takes a node_id, reads source content from nodes table, chunks it, and stores in chunks table */ import OpenAI from 'openai'; @@ -14,7 +14,7 @@ import { interface Node { id: number; title: string; - chunk: string | null; + source: string | null; chunk_status?: string | null; } @@ -157,7 +157,7 @@ export class UniversalEmbedder { // Get node data const stmt = this.db.prepare(` - SELECT id, title, chunk, chunk_status + SELECT id, title, source, chunk_status FROM nodes WHERE id = ? `); @@ -168,8 +168,8 @@ export class UniversalEmbedder { throw new Error(`Node ${nodeId} not found`); } - if (!node.chunk || node.chunk.trim().length === 0) { - console.log(`Node ${nodeId} has no chunk content to process`); + if (!node.source || node.source.trim().length === 0) { + console.log(`Node ${nodeId} has no source content to process`); return { chunks: 0 }; } @@ -179,7 +179,7 @@ export class UniversalEmbedder { this.deleteExistingChunks(nodeId); // Split text into chunks - const chunks = await this.textSplitter.splitText(node.chunk); + const chunks = await this.textSplitter.splitText(node.source); if (verbose) { console.log(`Split into ${chunks.length} chunks`); diff --git a/src/tools/database/createNode.ts b/src/tools/database/createNode.ts index ba1443c..ee05a0f 100644 --- a/src/tools/database/createNode.ts +++ b/src/tools/database/createNode.ts @@ -8,8 +8,8 @@ export const createNodeTool = tool({ description: 'Create node. Description is REQUIRED and must be explicit about what the thing is (podcast, chat summary, idea, etc).', inputSchema: z.object({ title: z.string().describe('The title of the node'), - notes: z.string().optional().describe('User notes, analysis, or thoughts about this node'), description: z.string().max(280).describe('REQUIRED. Explicitly state WHAT this is (e.g. podcast episode, conversation summary, user insight) + WHY it matters for context grounding.'), + source: z.string().optional().describe('Raw content for embedding: transcript, article text, book passages, or user thoughts. If omitted, falls back to title + description.'), link: z.string().optional().describe('A URL link to the source'), event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'), dimensions: z @@ -17,7 +17,6 @@ export const createNodeTool = tool({ .max(5) .optional() .describe('Optional dimension tags to apply to this node (0-5 items).'), - chunk: z.string().optional().describe('Raw content for later processing - CRITICAL for extracted content from URLs'), metadata: z.record(z.any()).optional().describe('Additional metadata like source info, extraction details, etc.') }), execute: async (params) => { diff --git a/src/tools/database/getNodesById.ts b/src/tools/database/getNodesById.ts index f0388d0..4de61cf 100644 --- a/src/tools/database/getNodesById.ts +++ b/src/tools/database/getNodesById.ts @@ -6,9 +6,9 @@ export const getNodesByIdTool = tool({ description: 'Load full node records by IDs', inputSchema: z.object({ nodeIds: z.array(z.number().int().positive()).min(1).max(10).describe('List of node IDs to load'), - includeNotesPreview: z.boolean().default(true).describe('Whether to return a trimmed content preview for each node'), + includeSourcePreview: z.boolean().default(true).describe('Whether to return a trimmed source preview for each node'), }), - execute: async ({ nodeIds, includeNotesPreview }) => { + execute: async ({ nodeIds, includeSourcePreview }) => { const uniqueIds = Array.from(new Set(nodeIds.filter(id => Number.isFinite(id) && id > 0))); if (uniqueIds.length === 0) { return { @@ -23,8 +23,8 @@ export const getNodesByIdTool = tool({ try { const node = await nodeService.getNodeById(id); if (!node) return null; - const preview = includeNotesPreview - ? (node.notes || node.description || '') + const preview = includeSourcePreview + ? (node.source || node.description || '') .split(/\s+/) .slice(0, 80) .join(' ') @@ -35,10 +35,12 @@ export const getNodesByIdTool = tool({ id: node.id, title: node.title, link: node.link, + event_date: node.event_date ?? null, dimensions: node.dimensions || [], chunk_status: node.chunk_status || 'unknown', + created_at: node.created_at, updated_at: node.updated_at, - notes_preview: preview || null, + source_preview: preview || null, metadata: node.metadata ?? null, }; } catch (error) { diff --git a/src/tools/database/queryDimensionNodes.ts b/src/tools/database/queryDimensionNodes.ts index d86de47..06fba7d 100644 --- a/src/tools/database/queryDimensionNodes.ts +++ b/src/tools/database/queryDimensionNodes.ts @@ -48,11 +48,12 @@ export const queryDimensionNodesTool = tool({ event_date: node.event_date ?? null, }; - if (includeContent && node.notes) { + if (includeContent && (node.source || node.notes)) { + const previewSource = node.source || node.notes || ''; // Truncate to ~100 chars - formatted.notesPreview = node.notes.length > 100 - ? node.notes.substring(0, 100) + '...' - : node.notes; + formatted.sourcePreview = previewSource.length > 100 + ? previewSource.substring(0, 100) + '...' + : previewSource; } return formatted; diff --git a/src/tools/database/queryNodes.ts b/src/tools/database/queryNodes.ts index 67f9394..8ca0305 100644 --- a/src/tools/database/queryNodes.ts +++ b/src/tools/database/queryNodes.ts @@ -44,7 +44,7 @@ function scoreNodeForSearch(node: Node, searchTerm: string): number { const normalizedSearch = searchTerm.toLowerCase(); const title = (node.title || '').toLowerCase(); const description = (node.description || '').toLowerCase(); - const notes = (node.notes || '').toLowerCase(); + const source = (node.source || '').toLowerCase(); const terms = extractSearchTerms(searchTerm); let score = 0; @@ -52,23 +52,23 @@ function scoreNodeForSearch(node: Node, searchTerm: string): number { if (title === normalizedSearch) score += 100; if (title.includes(normalizedSearch)) score += 40; if (description.includes(normalizedSearch)) score += 20; - if (notes.includes(normalizedSearch)) score += 10; + if (source.includes(normalizedSearch)) score += 10; for (const term of terms) { if (title.includes(term)) score += 8; if (description.includes(term)) score += 3; - if (notes.includes(term)) score += 2; + if (source.includes(term)) score += 2; } return score; } export const queryNodesTool = tool({ - description: 'Search nodes across title, description, and notes. For free-text lookups, search the graph broadly and prioritize title/description matches. Do not use dimensions to constrain keyword search unless the user is explicitly asking about a known dimension.', + description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Do not use dimensions to constrain keyword search unless the user is explicitly asking about a known dimension.', inputSchema: z.object({ filters: z.object({ dimensions: z.array(z.string()).describe('Filter by dimensions (e.g., ["research", "ai", "technology"]). Replaces old type/stage filtering.').optional(), - search: z.string().describe('Search term to match against node title, description, or notes').optional(), + search: z.string().describe('Search term to match against node title, description, or source').optional(), limit: z.number().min(1).max(50).default(10).describe('Maximum number of results to return'), createdAfter: z.string().optional().describe('ISO date (YYYY-MM-DD). Only return nodes created on or after this date.'), createdBefore: z.string().optional().describe('ISO date (YYYY-MM-DD). Only return nodes created before this date.'), diff --git a/src/tools/database/updateNode.ts b/src/tools/database/updateNode.ts index b088aec..c4abed8 100644 --- a/src/tools/database/updateNode.ts +++ b/src/tools/database/updateNode.ts @@ -9,14 +9,13 @@ export const updateNodeTool = tool({ id: z.number().describe('The ID of the node to update'), updates: z.object({ title: z.string().optional().describe('New title'), - notes: z.string().optional().describe('User notes/analysis to append. USE THIS for workflow outputs, briefs, research notes, etc.'), description: z.string().max(280).describe('REQUIRED on every update. Explicitly state WHAT this is + WHY it matters. No "discusses/explores".'), + source: z.string().optional().describe('Canonical source content for embedding. Use this only to set or correct the raw source text.'), link: z.string().optional().describe('New link'), event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'), dimensions: z.array(z.string()).optional().describe('New dimension tags - completely replaces existing dimensions'), - chunk: z.string().optional().describe('DO NOT USE - raw source text that triggers re-embedding. Only for source corrections.'), metadata: z.record(z.any()).optional().describe('New metadata - completely replaces existing metadata') - }).describe('Object containing the fields to update. For adding notes/analysis, always use "notes" field.') + }).describe('Object containing the fields to update. Derived analysis should be stored in a separate linked node, not appended to the source node.') }), execute: async ({ id, updates }) => { try { @@ -44,56 +43,6 @@ export const updateNodeTool = tool({ }; } - // FORCE APPEND for notes field - fetch existing and append new notes - if (updates.notes) { - const fetchResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes/${id}`); - if (fetchResponse.ok) { - const { node } = await fetchResponse.json(); - const existingNotes = (node?.notes || '').trim(); - const newNotes = updates.notes.trim(); - - // Skip if new notes are identical to existing (model sent duplicate) - if (existingNotes === newNotes) { - console.log(`[updateNode] ERROR - new notes identical to existing (${existingNotes.length} chars). Model should NOT call updateNode again.`); - return { - success: false, - error: 'Notes already up to date - do not call updateNode again. Move to next step.', - data: null - }; - } - - // Detect if adding a section that already exists (e.g., ## Integration Analysis) - const newSectionMatch = newNotes.match(/^##\s+(.+)$/m); - if (newSectionMatch && existingNotes) { - const sectionHeader = newSectionMatch[0]; // e.g., "## Integration Analysis" - if (existingNotes.includes(sectionHeader)) { - console.log(`[updateNode] ERROR - Section "${sectionHeader}" already exists in node`); - return { - success: false, - error: `Section "${sectionHeader}" already exists in this node. Cannot append duplicate section.`, - data: null - }; - } - } - - // Detect if model included existing notes + new notes - if (existingNotes && newNotes.startsWith(existingNotes)) { - // Extract only the new part - const actualNewNotes = newNotes.substring(existingNotes.length).trim(); - console.log(`[updateNode] Model included existing notes - extracting new part only (${actualNewNotes.length} chars)`); - const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n'; - updates.notes = `${existingNotes}${separator}${actualNewNotes}`; - } else if (existingNotes) { - // Normal append - const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n'; - updates.notes = `${existingNotes}${separator}${newNotes}`; - console.log(`[updateNode] Appended notes: ${existingNotes.length} + ${newNotes.length} = ${updates.notes.length} chars`); - } else { - console.log(`[updateNode] No existing notes, using new notes as-is (${newNotes.length} chars)`); - } - } - } - if (Array.isArray(updates.dimensions)) { updates.dimensions = normalizeDimensions(updates.dimensions, 5); } diff --git a/src/tools/other/paperExtract.ts b/src/tools/other/paperExtract.ts index b990de1..94795bd 100644 --- a/src/tools/other/paperExtract.ts +++ b/src/tools/other/paperExtract.ts @@ -69,7 +69,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks): } export const paperExtractTool = tool({ - description: 'Extract a PDF or research paper into a node with summary, metadata, and full-text chunk', + description: 'Extract a PDF or research paper into a node with summary, metadata, and full-text source', inputSchema: z.object({ url: z.string().describe('The PDF URL to add to inbox'), title: z.string().optional().describe('Custom title (auto-generated if not provided)'), @@ -95,14 +95,13 @@ export const paperExtractTool = tool({ }; } - let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string }; + let result: { success: boolean; source?: string; metadata?: any; error?: string }; try { const extractionResult = await extractPaper(url); result = { success: true, - notes: extractionResult.content, - chunk: extractionResult.chunk, + source: extractionResult.chunk || extractionResult.content, metadata: { title: extractionResult.metadata.title, pages: extractionResult.metadata.pages, @@ -119,7 +118,7 @@ export const paperExtractTool = tool({ }; } - if (!result.success || (!result.notes && !result.chunk)) { + if (!result.success || !result.source) { return { success: false, error: result.error || 'Failed to extract PDF content', @@ -132,7 +131,7 @@ export const paperExtractTool = tool({ // Step 2: AI Analysis for enhanced metadata const aiAnalysis = await analyzeContentWithAI( result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`, - result.notes?.substring(0, 2000) || 'PDF document content', + result.source.substring(0, 2000) || 'PDF document content', 'pdf' ); @@ -153,17 +152,16 @@ export const paperExtractTool = tool({ body: JSON.stringify({ title: nodeTitle, description: aiAnalysis?.nodeDescription, - notes: enhancedDescription, + source: result.source, link: url, dimensions: trimmedDimensions, - chunk: result.chunk || result.notes, metadata: { source: 'pdf', hostname: new URL(url).hostname, author: result.metadata?.author || result.metadata?.info?.Author, pages: result.metadata?.pages, file_size: result.metadata?.file_size, - content_length: (result.chunk || result.notes)?.length, + content_length: result.source.length, extraction_method: result.metadata?.extraction_method || 'python_pdfplumber', ai_analysis: aiAnalysis?.reasoning, enhanced_description: enhancedDescription, @@ -197,7 +195,7 @@ export const paperExtractTool = tool({ data: { nodeId: createResult.data?.id, title: nodeTitle, - contentLength: (result.chunk || result.notes || '').length, + contentLength: result.source.length, url: url, dimensions: actualDimensions } diff --git a/src/tools/other/websiteExtract.ts b/src/tools/other/websiteExtract.ts index 825a808..2354554 100644 --- a/src/tools/other/websiteExtract.ts +++ b/src/tools/other/websiteExtract.ts @@ -149,7 +149,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks): } export const websiteExtractTool = tool({ - description: 'Extract website content and metadata into a node with summary, tags, and raw chunk', + description: 'Extract website content and metadata into a node with summary, tags, and raw source text', inputSchema: z.object({ url: z.string().describe('The website URL to add to knowledge base'), title: z.string().optional().describe('Custom title (auto-generated if not provided)'), @@ -166,14 +166,13 @@ export const websiteExtractTool = tool({ }; } - let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string }; + let result: { success: boolean; source?: string; metadata?: any; error?: string }; try { const extractionResult = await extractWebsite(url); result = { success: true, - notes: extractionResult.content, - chunk: extractionResult.chunk, + source: extractionResult.chunk || extractionResult.content, metadata: { title: extractionResult.metadata.title, author: extractionResult.metadata.author, @@ -191,7 +190,7 @@ export const websiteExtractTool = tool({ }; } - if (!result.success || (!result.notes && !result.chunk)) { + if (!result.success || !result.source) { return { success: false, error: result.error || 'Failed to extract website content', @@ -206,7 +205,7 @@ export const websiteExtractTool = tool({ const contentType = inferWebsiteContentType(url); const aiAnalysis = await analyzeContentWithAI( result.metadata?.title || `Website: ${new URL(url).hostname}`, - result.notes?.substring(0, 2000) || 'Website content', + result.source.substring(0, 2000) || 'Website content', contentType, existingDimensions ); @@ -231,17 +230,16 @@ export const websiteExtractTool = tool({ body: JSON.stringify({ title: nodeTitle, description: aiAnalysis?.nodeDescription, - notes: enhancedDescription, + source: result.source, link: url, event_date: result.metadata?.published_date || result.metadata?.date || null, dimensions: finalDimensions, - chunk: result.chunk || result.notes, metadata: { source: contentType, hostname: new URL(url).hostname, author: result.metadata?.author, published_date: result.metadata?.published_date || result.metadata?.date, - content_length: (result.chunk || result.notes)?.length, + content_length: result.source.length, extraction_method: result.metadata?.extraction_method || 'python_beautifulsoup', ai_analysis: aiAnalysis?.reasoning, enhanced_description: enhancedDescription, @@ -275,7 +273,7 @@ export const websiteExtractTool = tool({ data: { nodeId: createResult.data?.id, title: nodeTitle, - contentLength: (result.chunk || result.notes || '').length, + contentLength: result.source.length, url: url, dimensions: actualDimensions } diff --git a/src/tools/other/youtubeExtract.ts b/src/tools/other/youtubeExtract.ts index fd0e523..719dda3 100644 --- a/src/tools/other/youtubeExtract.ts +++ b/src/tools/other/youtubeExtract.ts @@ -193,15 +193,14 @@ export const youtubeExtractTool = tool({ }; } - let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string }; + let result: { success: boolean; source?: string; metadata?: any; error?: string }; console.log('📝 Using TypeScript yt-dlp extractor'); try { const extractionResult = await extractYouTube(url); result = { success: extractionResult.success, - notes: extractionResult.content, - chunk: extractionResult.chunk, + source: extractionResult.chunk || extractionResult.content, metadata: { video_title: extractionResult.metadata.video_title, channel_name: extractionResult.metadata.channel_name, @@ -222,7 +221,7 @@ export const youtubeExtractTool = tool({ }; } - if (!result.success || (!result.notes && !result.chunk)) { + if (!result.success || !result.source) { return { success: false, error: result.error || 'Failed to extract YouTube content', @@ -243,8 +242,7 @@ export const youtubeExtractTool = tool({ // Step 3: Create node with extracted content and AI analysis const nodeTitle = title || result.metadata?.video_title || `YouTube Video ${url.split('/').pop()?.split('?')[0]}`; - const transcriptSummary = await summariseTranscript(nodeTitle, result.chunk || result.notes || ''); - const nodeNotes = transcriptSummary || aiAnalysis?.enhancedDescription || `YouTube video by ${result.metadata?.channel_name || 'Unknown Channel'}`; + const transcriptSummary = await summariseTranscript(nodeTitle, result.source); const suppliedDimensions = Array.isArray(dimensions) ? dimensions : []; let trimmedDimensions = suppliedDimensions @@ -262,10 +260,9 @@ export const youtubeExtractTool = tool({ body: JSON.stringify({ title: nodeTitle, description: aiAnalysis?.nodeDescription, - notes: nodeNotes, + source: result.source, link: url, dimensions: finalDimensions, - chunk: result.chunk || result.notes, metadata: { source: 'youtube', video_id: result.metadata?.video_id, @@ -308,7 +305,7 @@ export const youtubeExtractTool = tool({ data: { nodeId: createResult.data?.id, title: nodeTitle, - contentLength: (result.chunk || result.notes || '').length, + contentLength: result.source.length, url: url, dimensions: actualDimensions } diff --git a/src/types/database.ts b/src/types/database.ts index 0e8b30a..eacefd0 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -3,12 +3,13 @@ export interface Node { id: number; title: string; description?: string; - notes?: string; // User's notes/thoughts about this node + source?: string; // Canonical embeddable content + notes?: string; // Deprecated legacy field - do not write link?: string; event_date?: string | null; // When the thing actually happened (ISO 8601) dimensions: string[]; // Flexible dimensions replacing type + stage + segment + tags embedding?: Buffer; // Node-level embedding (BLOB data) - chunk?: string; + chunk?: string; // Deprecated legacy field - do not write metadata?: any; // Flexible metadata storage created_at: string; updated_at: string; @@ -68,6 +69,7 @@ export interface NodeFilters { dimensions?: string[]; // Filter by dimensions (replaces stage/type filtering) search?: string; // Text search in title/content searchMode?: 'standard' | 'hybrid'; // standard = FTS/LIKE, hybrid = add node-vector retrieval + chunkStatus?: 'not_chunked' | 'chunking' | 'chunked' | 'error'; limit?: number; offset?: number; sortBy?: 'updated' | 'edges' | 'created' | 'event_date'; // Sort by updated_at, edge count, created_at, or event_date