diff --git a/app/api/mcp/route.ts b/app/api/mcp/route.ts index 711dc8d..ba72547 100644 --- a/app/api/mcp/route.ts +++ b/app/api/mcp/route.ts @@ -134,7 +134,7 @@ function createRAHServer(): McpServer { id: node.id, title: node.title, description: node.description ?? null, - source: node.source ?? node.notes ?? null, + source: node.source ?? null, link: node.link ?? null, dimensions: node.dimensions || [], metadata: node.metadata || {}, @@ -309,22 +309,29 @@ function createRAHServer(): McpServer { 'rah_update_node', { title: 'Update RA-H node', - description: 'Update an existing node. Content is APPENDED, dimensions are replaced.', + description: 'Update an existing node. Source content is canonical and dimensions are replaced.', inputSchema: { id: z.number().int().positive().describe('Node ID to update'), updates: z.object({ title: z.string().optional().describe('New title'), - content: z.string().optional().describe('Content to APPEND'), + content: z.string().optional().describe('Legacy alias for source'), + source: z.string().optional().describe('Canonical source text'), link: z.string().optional().describe('New link'), dimensions: z.array(z.string()).optional().describe('New dimensions (replaces existing)'), }).describe('Fields to update'), }, }, async ({ id, updates }) => { + const mappedUpdates = { ...updates } as Record; + if (mappedUpdates.content !== undefined && mappedUpdates.source === undefined) { + mappedUpdates.source = mappedUpdates.content; + } + delete mappedUpdates.content; + const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes/${id}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(updates), + body: JSON.stringify(mappedUpdates), }); const result = await response.json(); diff --git a/app/api/nodes/[id]/regenerate-description/route.ts b/app/api/nodes/[id]/regenerate-description/route.ts index f286240..2426fc3 100644 --- a/app/api/nodes/[id]/regenerate-description/route.ts +++ b/app/api/nodes/[id]/regenerate-description/route.ts @@ -85,7 +85,7 @@ export async function POST( // Generate new description using the description service const newDescription = await generateDescription({ title: node.title, - notes: node.source || node.description || undefined, + source: node.source || node.description || undefined, link: node.link || undefined, metadata: enrichedMetadata, diff --git a/app/api/nodes/route.ts b/app/api/nodes/route.ts index 76335ec..a5f7573 100644 --- a/app/api/nodes/route.ts +++ b/app/api/nodes/route.ts @@ -108,7 +108,7 @@ export async function POST(request: NextRequest) { try { nodeDescription = await generateDescription({ title: body.title, - notes: rawSource?.slice(0, 2000) || undefined, + source: rawSource?.slice(0, 2000) || undefined, link: body.link || undefined, metadata: body.metadata, dimensions: trimmedProvidedDimensions diff --git a/apps/mcp-server-standalone/index.js b/apps/mcp-server-standalone/index.js index 636688f..8291b55 100644 --- a/apps/mcp-server-standalone/index.js +++ b/apps/mcp-server-standalone/index.js @@ -351,8 +351,7 @@ async function main() { 'createNode', { title: 'Add RA-H node', - description: 'Create a new node. Always search first (queryNodes) to avoid duplicates. Title: max 160 chars, clear and descriptive. Description is REQUIRED and must be explicit about what the thing is and why it matters for contextual grounding. Use "link" ONLY for external content (URL, video, article) — omit for synthesis/ideas derived from existing nodes. "content" = your notes/analysis. "chunk" = verbatim source text. Assign 1-5 dimensions — call queryDimensions first to use existing ones.', - // Note: MCP schema uses "content" for external API compat; mapped to "notes" internally + description: 'Create a new node. Always search first (queryNodes) to avoid duplicates. Title: max 160 chars, clear and descriptive. Description is REQUIRED and must be explicit about what the thing is and why it matters for contextual grounding. Use "link" ONLY for external content (URL, video, article) — omit for synthesis/ideas derived from existing nodes. "source" = verbatim or canonical content for embedding. Legacy "content" and "chunk" are mapped to source for compatibility. Assign 1-5 dimensions — call queryDimensions first to use existing ones.', inputSchema: addNodeInputSchema }, async ({ title, content, source, link, description, dimensions, metadata, chunk }) => { @@ -392,8 +391,7 @@ async function main() { 'queryNodes', { title: 'Search RA-H nodes', - description: 'Search nodes by keyword across title, description, and content fields. Multi-word queries find nodes containing all words (not exact phrases). Returns up to 25 results (default 10). Call before creating nodes to check for duplicates. Optionally filter by dimensions. NOT for searching source documents (transcripts, articles) — use searchContentEmbeddings for that.', - // Note: searches across title, description, and notes columns + description: 'Search nodes by keyword across title, description, and source fields. Multi-word queries find nodes containing all words (not exact phrases). Returns up to 25 results (default 10). Call before creating nodes to check for duplicates. Optionally filter by dimensions. NOT for searching source documents (transcripts, articles) — use searchContentEmbeddings for that.', inputSchema: searchNodesInputSchema }, async ({ query: searchQuery, limit = 10, dimensions, created_after, created_before, event_after, event_before }) => { @@ -560,13 +558,13 @@ async function main() { for (const id of uniqueIds) { const node = nodeService.getNodeById(id); if (node) { - const rawChunk = node.source ?? node.chunk ?? null; + const rawChunk = node.source ?? null; const chunkTruncated = rawChunk ? rawChunk.length > CHUNK_LIMIT : false; nodes.push({ id: node.id, title: node.title, - source: node.source ?? node.notes ?? null, + source: node.source ?? null, description: node.description ?? null, link: node.link ?? null, chunk: chunkTruncated ? rawChunk.substring(0, CHUNK_LIMIT) : rawChunk, @@ -595,7 +593,7 @@ async function main() { 'updateNode', { title: 'Update RA-H node', - description: 'Update an existing node. Description is REQUIRED on every update and must explicitly state WHAT this thing is + WHY it matters for contextual grounding. Content is APPENDED to existing notes (not replaced). Dimensions are REPLACED entirely with the new array. Title, description, and link are overwritten. Call getNodesById first to verify current state before updating.', + description: 'Update an existing node. Description is REQUIRED on every update and must explicitly state WHAT this thing is + WHY it matters for contextual grounding. Source content lives in "source". Legacy "content" and "chunk" are mapped to source for compatibility. Dimensions are REPLACED entirely with the new array. Title, description, and link are overwritten. Call getNodesById first to verify current state before updating.', inputSchema: updateNodeInputSchema }, async ({ id, updates }) => { @@ -621,7 +619,7 @@ async function main() { delete mappedUpdates.content; delete mappedUpdates.chunk; - const node = nodeService.updateNode(id, mappedUpdates, { appendNotes: false }); + const node = nodeService.updateNode(id, mappedUpdates); return { content: [{ type: 'text', text: `Updated node #${id}` }], @@ -916,7 +914,7 @@ async function main() { const tableCheck = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='chunks'").get(); if (!tableCheck) { return { - content: [{ type: 'text', text: 'No chunks table found. Source content has not been chunked yet. Use getNodesById to read the raw chunk field instead.' }], + content: [{ type: 'text', text: 'No chunks table found. Source content has not been chunked yet. Use getNodesById to read the raw source text instead.' }], structuredContent: { count: 0, chunks: [], note: 'chunks table does not exist' } }; } diff --git a/apps/mcp-server-standalone/services/nodeService.js b/apps/mcp-server-standalone/services/nodeService.js index 52b31b4..3168422 100644 --- a/apps/mcp-server-standalone/services/nodeService.js +++ b/apps/mcp-server-standalone/services/nodeService.js @@ -9,7 +9,7 @@ function getNodes(filters = {}) { const { dimensions, search, limit = 100, offset = 0 } = filters; let sql = ` - SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -70,7 +70,7 @@ function getNodes(filters = {}) { */ function getNodeById(id) { const sql = ` - SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -164,8 +164,7 @@ function createNode(nodeData) { * Source-first update path. */ function updateNode(id, updates, options = {}) { - const { appendNotes = true } = options; - const { title, description, source, notes, link, event_date, dimensions, chunk, metadata } = updates; + const { title, description, source, link, event_date, dimensions, metadata } = updates; const now = new Date().toISOString(); const db = getDb(); @@ -190,12 +189,6 @@ function updateNode(id, updates, options = {}) { if (source !== undefined) { setFields.push('source = ?'); params.push(source); - } else if (notes !== undefined) { - const nextSource = appendNotes && existing.source - ? `${existing.source}\n\n${notes}` - : notes; - setFields.push('source = ?'); - params.push(nextSource); } if (link !== undefined) { setFields.push('link = ?'); @@ -205,10 +198,6 @@ function updateNode(id, updates, options = {}) { setFields.push('event_date = ?'); params.push(event_date); } - if (chunk !== undefined && source === undefined) { - setFields.push('source = ?'); - params.push(chunk); - } if (metadata !== undefined) { setFields.push('metadata = ?'); params.push(JSON.stringify(metadata)); diff --git a/apps/mcp-server-standalone/services/sqlite-client.js b/apps/mcp-server-standalone/services/sqlite-client.js index b4cf84a..3668d32 100644 --- a/apps/mcp-server-standalone/services/sqlite-client.js +++ b/apps/mcp-server-standalone/services/sqlite-client.js @@ -50,13 +50,12 @@ function initDatabase() { id INTEGER PRIMARY KEY, title TEXT, description TEXT, - notes TEXT, + source TEXT, link TEXT, event_date TEXT, created_at TEXT, updated_at TEXT, metadata TEXT, - chunk TEXT, embedding BLOB, embedding_updated_at TEXT, embedding_text TEXT, @@ -113,6 +112,38 @@ function initDatabase() { db.pragma('cache_size = 5000'); db.pragma('busy_timeout = 5000'); + const nodeCols = db.prepare('PRAGMA table_info(nodes)').all().map(c => c.name); + if (!nodeCols.includes('source')) { + db.exec('ALTER TABLE nodes ADD COLUMN source TEXT;'); + } + if (nodeCols.includes('content')) { + db.exec(` + UPDATE nodes + SET source = content + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND content IS NOT NULL + AND LENGTH(TRIM(content)) > 0; + `); + } + if (nodeCols.includes('notes')) { + db.exec(` + UPDATE nodes + SET source = notes + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND notes IS NOT NULL + AND LENGTH(TRIM(notes)) > 0; + `); + } + if (nodeCols.includes('chunk')) { + db.exec(` + UPDATE nodes + SET source = chunk + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND chunk IS NOT NULL + AND LENGTH(TRIM(chunk)) > 0; + `); + } + const edgeCols = db.prepare('PRAGMA table_info(edges)').all().map(c => c.name); if (!edgeCols.includes('explanation')) { db.exec('ALTER TABLE edges ADD COLUMN explanation TEXT;'); diff --git a/apps/mcp-server/server.js b/apps/mcp-server/server.js index 4201a77..ebc90fd 100644 --- a/apps/mcp-server/server.js +++ b/apps/mcp-server/server.js @@ -422,7 +422,7 @@ mcpServer.registerTool( nodes: nodes.map((node) => ({ id: node.id, title: node.title, - source: node.source ?? node.notes ?? null, + source: node.source ?? null, description: node.description ?? null, link: node.link ?? null, dimensions: node.dimensions || [], @@ -496,7 +496,7 @@ mcpServer.registerTool( nodes.push({ id: result.node.id, title: result.node.title, - source: result.node.source ?? result.node.notes ?? null, + source: result.node.source ?? null, link: result.node.link ?? null, dimensions: result.node.dimensions || [], updated_at: result.node.updated_at @@ -729,7 +729,7 @@ mcpServer.registerTool( results: results.map(r => ({ nodeId: r.node_id || r.nodeId || r.id, title: r.title || 'Untitled', - chunkPreview: (r.chunk || r.notes || '').slice(0, 200), + chunkPreview: (r.source || '').slice(0, 200), similarity: r.similarity || r.score || 0 })) } diff --git a/apps/mcp-server/stdio-server.js b/apps/mcp-server/stdio-server.js index 217fb57..042fc93 100644 --- a/apps/mcp-server/stdio-server.js +++ b/apps/mcp-server/stdio-server.js @@ -393,7 +393,7 @@ server.registerTool( nodes: nodes.map((node) => ({ id: node.id, title: node.title, - source: node.source ?? node.notes ?? null, + source: node.source ?? null, description: node.description ?? null, link: node.link ?? null, dimensions: node.dimensions || [], @@ -467,7 +467,7 @@ server.registerTool( nodes.push({ id: result.node.id, title: result.node.title, - source: result.node.source ?? result.node.notes ?? null, + source: result.node.source ?? null, link: result.node.link ?? null, dimensions: result.node.dimensions || [], updated_at: result.node.updated_at @@ -700,7 +700,7 @@ server.registerTool( results: results.map(r => ({ nodeId: r.node_id || r.nodeId || r.id, title: r.title || 'Untitled', - chunkPreview: (r.chunk || r.notes || '').slice(0, 200), + chunkPreview: (r.source || '').slice(0, 200), similarity: r.similarity || r.score || 0 })) } diff --git a/docs/8_mcp.md b/docs/8_mcp.md index 0a624c5..b746529 100644 --- a/docs/8_mcp.md +++ b/docs/8_mcp.md @@ -2,7 +2,7 @@ > Connect Claude Code and other AI assistants to your knowledge base. -**How it works:** RA-OS includes an MCP (Model Context Protocol) server. This lets any MCP-compatible assistant — like Claude Code — search your notes, add new knowledge, and manage your knowledge graph. Everything stays local. +**How it works:** RA-OS includes an MCP (Model Context Protocol) server. This lets any MCP-compatible assistant — like Claude Code — search your knowledge graph, add new knowledge, and manage your graph. Everything stays local. --- @@ -77,7 +77,7 @@ If you want real-time UI updates when nodes are created: | Tool | Description | |------|-------------| | `getContext` | Get graph overview — stats, hub nodes, dimensions, recent activity. Called first automatically. | -| `createNode` | Create a new node (title/content/dimensions) | +| `createNode` | Create a new node (title/source/dimensions) | | `queryNodes` | Search existing nodes by keyword | | `updateNode` | Update an existing node | | `getNodesById` | Get nodes by ID | diff --git a/scripts/database/sqlite-ensure-app-schema.sh b/scripts/database/sqlite-ensure-app-schema.sh index 0303af6..fbd2a14 100755 --- a/scripts/database/sqlite-ensure-app-schema.sh +++ b/scripts/database/sqlite-ensure-app-schema.sh @@ -287,6 +287,65 @@ if has_table nodes; then # is_pinned removed in final schema pass fi +if has_table nodes && has_col nodes source; then + if has_col nodes content; then + echo "Backfilling nodes.source from nodes.content" + "$SQLITE_BIN" "$DB_PATH" <<'SQL' +UPDATE nodes +SET source = content, + chunk_status = 'not_chunked' +WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND content IS NOT NULL + AND LENGTH(TRIM(content)) > 0; +SQL + fi + + if has_col nodes notes; then + echo "Backfilling nodes.source from nodes.notes" + "$SQLITE_BIN" "$DB_PATH" <<'SQL' +UPDATE nodes +SET source = notes, + chunk_status = 'not_chunked' +WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND notes IS NOT NULL + AND LENGTH(TRIM(notes)) > 0; +SQL + fi + + if has_col nodes chunk; then + echo "Backfilling nodes.source from nodes.chunk" + "$SQLITE_BIN" "$DB_PATH" <<'SQL' +UPDATE nodes +SET source = chunk, + chunk_status = 'not_chunked' +WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND chunk IS NOT NULL + AND LENGTH(TRIM(chunk)) > 0; +SQL + fi + + echo "Filling empty nodes.source from title/description fallback" + "$SQLITE_BIN" "$DB_PATH" <<'SQL' +UPDATE nodes +SET source = title || CASE + WHEN description IS NOT NULL AND LENGTH(TRIM(description)) > 0 + THEN char(10) || char(10) || description + ELSE '' +END, +chunk_status = 'not_chunked' +WHERE source IS NULL OR LENGTH(TRIM(source)) = 0; +SQL + + echo "Marking nodes with source for rechunking" + "$SQLITE_BIN" "$DB_PATH" <<'SQL' +UPDATE nodes +SET chunk_status = 'not_chunked' +WHERE source IS NOT NULL + AND LENGTH(TRIM(source)) > 0 + AND (chunk_status IS NULL OR chunk_status != 'chunked'); +SQL +fi + if has_table chunks; then if ! has_col chunks embedding_type; then echo "Adding chunks.embedding_type" @@ -329,7 +388,7 @@ CREATE VIEW nodes_v AS SELECT n.id, n.title, n.description, - n.notes, + n.source, n.link, n.event_date, n.metadata, diff --git a/scripts/dev/bootstrap-local.mjs b/scripts/dev/bootstrap-local.mjs index 4fdfdec..aa5bb51 100644 --- a/scripts/dev/bootstrap-local.mjs +++ b/scripts/dev/bootstrap-local.mjs @@ -248,6 +248,90 @@ function ensureCoreSchema(db) { '[{"id":"p_seed_0","name":"Summary of Focus","content":"Summarize the primary focused node clearly. Include 3–5 key points and cite [NODE:id:\\"title\\"]."},{"id":"p_seed_1","name":"Next Steps","content":"Propose 3 concrete next actions based on the focused nodes with references to [NODE:id:\\"title\\"]."}]' ); } + + const nodeCols = db.prepare('PRAGMA table_info(nodes)').all().map(col => col.name); + const hasNodeCol = (name) => nodeCols.includes(name); + + if (!hasNodeCol('description')) { + db.exec('ALTER TABLE nodes ADD COLUMN description TEXT;'); + } + if (!hasNodeCol('metadata')) { + db.exec('ALTER TABLE nodes ADD COLUMN metadata TEXT;'); + } + if (!hasNodeCol('source')) { + db.exec('ALTER TABLE nodes ADD COLUMN source TEXT;'); + } + if (!hasNodeCol('event_date')) { + db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;'); + } + + if (hasNodeCol('content')) { + db.exec(` + UPDATE nodes + SET source = content, + chunk_status = 'not_chunked' + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND content IS NOT NULL + AND LENGTH(TRIM(content)) > 0; + `); + } + if (hasNodeCol('notes')) { + db.exec(` + UPDATE nodes + SET source = notes, + chunk_status = 'not_chunked' + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND notes IS NOT NULL + AND LENGTH(TRIM(notes)) > 0; + `); + } + if (hasNodeCol('chunk')) { + db.exec(` + UPDATE nodes + SET source = chunk, + chunk_status = 'not_chunked' + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND chunk IS NOT NULL + AND LENGTH(TRIM(chunk)) > 0; + `); + } + + db.exec(` + UPDATE nodes + SET source = title || CASE + WHEN description IS NOT NULL AND LENGTH(TRIM(description)) > 0 + THEN char(10) || char(10) || description + ELSE '' + END, + chunk_status = 'not_chunked' + WHERE source IS NULL OR LENGTH(TRIM(source)) = 0; + `); + + db.exec(` + UPDATE nodes + SET chunk_status = 'not_chunked' + WHERE source IS NOT NULL + AND LENGTH(TRIM(source)) > 0 + AND (chunk_status IS NULL OR chunk_status != 'chunked'); + `); + + db.exec('DROP VIEW IF EXISTS nodes_v;'); + db.exec(` + CREATE VIEW nodes_v AS + SELECT n.id, + n.title, + n.description, + n.source, + n.link, + n.event_date, + n.metadata, + n.created_at, + n.updated_at, + COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) + FROM node_dimensions d + WHERE d.node_id = n.id), '[]') AS dimensions_json + FROM nodes n; + `); } function tryInitVectorTables(db, dbPath) { diff --git a/src/components/focus/FocusPanel.tsx b/src/components/focus/FocusPanel.tsx index bdeb714..7ee9a5c 100644 --- a/src/components/focus/FocusPanel.tsx +++ b/src/components/focus/FocusPanel.tsx @@ -291,7 +291,7 @@ export default function FocusPanel({ }; const startSourceEdit = () => { - setSourceEditValue(currentNode?.source || currentNode?.chunk || ''); + setSourceEditValue(currentNode?.source || ''); setSourceEditMode(true); }; @@ -598,7 +598,7 @@ export default function FocusPanel({ }; const renderSourceSection = () => { - const sourceContent = currentNode?.source || currentNode?.chunk || ''; + const sourceContent = currentNode?.source || ''; return (
- {/* Notes */} + {/* Source */}
- {(node.source || node.notes) ? (node.source || node.notes || '').slice(0, 120) : '\u2014'} + {node.source ? node.source.slice(0, 120) : '\u2014'}
@@ -576,7 +576,7 @@ export default function DatabaseTableView({ onNodeClick, refreshToken = 0, toolb
- {(node.source || node.chunk) ? (node.source || node.chunk || '').slice(0, 100) : '\u2014'} + {node.source ? node.source.slice(0, 100) : '\u2014'}
diff --git a/src/components/views/GridView.tsx b/src/components/views/GridView.tsx index ef8f41a..3a35fae 100644 --- a/src/components/views/GridView.tsx +++ b/src/components/views/GridView.tsx @@ -104,7 +104,7 @@ export default function GridView({ nodes, onNodeClick }: GridViewProps) { {/* Description or Content Preview */} - {(node.description || node.source || node.notes) && ( + {(node.description || node.source) && (
- {node.description || truncateContent(node.source || node.notes)} + {node.description || truncateContent(node.source)}
)} diff --git a/src/components/views/ListView.tsx b/src/components/views/ListView.tsx index 078df18..5153aa5 100644 --- a/src/components/views/ListView.tsx +++ b/src/components/views/ListView.tsx @@ -105,7 +105,7 @@ export default function ListView({ nodes, onNodeClick }: ListViewProps) { {/* Description or Content Preview */} - {(node.description || node.source || node.notes) && ( + {(node.description || node.source) && (
- {node.description || truncateContent(node.source || node.notes)} + {node.description || truncateContent(node.source)}
)} diff --git a/src/services/database/descriptionService.ts b/src/services/database/descriptionService.ts index 0cd5c88..dd7746e 100644 --- a/src/services/database/descriptionService.ts +++ b/src/services/database/descriptionService.ts @@ -4,7 +4,7 @@ import { hasValidOpenAiKey } from '../storage/apiKeys'; export interface DescriptionInput { title: string; - notes?: string; + source?: string; link?: string; metadata?: { source?: string; @@ -61,7 +61,7 @@ export async function generateDescription(input: DescriptionInput): Promise 800 ? '...' : ''}`); + const contentPreview = input.source?.slice(0, 800) || ''; + if (contentPreview) lines.push(`Source excerpt: ${contentPreview}${input.source && input.source.length > 800 ? '...' : ''}`); return `Write a description for this knowledge node. Max 280 characters. diff --git a/src/services/database/nodes.ts b/src/services/database/nodes.ts index 8efeea3..4832b0a 100644 --- a/src/services/database/nodes.ts +++ b/src/services/database/nodes.ts @@ -138,7 +138,7 @@ export class NodeService { // Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance) let query = ` - SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -252,7 +252,7 @@ export class NodeService { private async getNodeByIdSQLite(id: number): Promise { const sqlite = getSQLiteClient(); const query = ` - SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -589,7 +589,7 @@ export class NodeService { WHERE nodes_fts MATCH ? LIMIT ? ) - SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -618,7 +618,7 @@ export class NodeService { const words = search.split(/\s+/).filter(Boolean); const { clauses, params } = this.buildNodeFilterClauses(filters); let query = ` - SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -663,7 +663,7 @@ export class NodeService { const { clauses, params } = this.buildNodeFilterClauses(filters); let query = ` - SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -737,7 +737,7 @@ export class NodeService { ORDER BY distance LIMIT ? ) - SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) diff --git a/src/services/database/sqlite-client.ts b/src/services/database/sqlite-client.ts index cda87c2..6d6944d 100644 --- a/src/services/database/sqlite-client.ts +++ b/src/services/database/sqlite-client.ts @@ -616,19 +616,11 @@ class SQLiteClient { } } - // 10) Final schema pass migrations (content→notes, event_date, dimensions.icon, drop dead columns) + // 10) Final schema pass migrations (source canonicalization, event_date, dimensions.icon, drop dead columns) try { let nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>; let nodeColNames = nodeCols2.map(c => c.name); - // Rename content → notes (additive first) - if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) { - console.log('Migrating nodes.content → nodes.notes...'); - this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;'); - nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>; - nodeColNames = nodeCols2.map(c => c.name); - } - if (!nodeColNames.includes('source')) { console.log('Adding nodes.source column...'); this.db.exec('ALTER TABLE nodes ADD COLUMN source TEXT;'); @@ -637,22 +629,38 @@ class SQLiteClient { } if (nodeColNames.includes('source')) { - this.db.exec(` - UPDATE nodes - SET source = chunk - WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) - AND chunk IS NOT NULL - AND LENGTH(TRIM(chunk)) > 0; - `); + if (nodeColNames.includes('content')) { + this.db.exec(` + UPDATE nodes + SET source = content, + chunk_status = 'not_chunked' + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND content IS NOT NULL + AND LENGTH(TRIM(content)) > 0; + `); + } - this.db.exec(` - UPDATE nodes - SET source = notes, - chunk_status = 'not_chunked' - WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) - AND notes IS NOT NULL - AND LENGTH(TRIM(notes)) > 0; - `); + if (nodeColNames.includes('notes')) { + this.db.exec(` + UPDATE nodes + SET source = notes, + chunk_status = 'not_chunked' + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND notes IS NOT NULL + AND LENGTH(TRIM(notes)) > 0; + `); + } + + if (nodeColNames.includes('chunk')) { + this.db.exec(` + UPDATE nodes + SET source = chunk, + chunk_status = 'not_chunked' + WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0) + AND chunk IS NOT NULL + AND LENGTH(TRIM(chunk)) > 0; + `); + } this.db.exec(` UPDATE nodes diff --git a/src/services/typescript/embed-nodes.ts b/src/services/typescript/embed-nodes.ts index 73e0426..adcca61 100644 --- a/src/services/typescript/embed-nodes.ts +++ b/src/services/typescript/embed-nodes.ts @@ -19,7 +19,6 @@ interface NodeRecord { id: number; title: string; source: string | null; - notes: string | null; description: string | null; dimensions_json: string; embedding?: Buffer | null; @@ -63,7 +62,7 @@ export class NodeEmbedder { const prompt = `Analyze this content and provide 2-3 key insights or themes in a concise paragraph (max 100 words): Title: ${node.title} -Source: ${node.source || node.notes || 'No source'} +Source: ${node.source || 'No source'} Dimensions: ${dimensionsText} Focus on the main concepts, key relationships, and practical implications.`; @@ -111,13 +110,13 @@ Focus on the main concepts, key relationships, and practical implications.`; // Create base embedding text let embeddingText = formatEmbeddingText( node.title, - node.source || node.notes || '', + node.source || '', dimensions, node.description ); // Add AI analysis if source exists - const sourceText = node.source || node.notes || ''; + const sourceText = node.source || ''; if (sourceText.trim().length > 0) { const analysis = await this.analyzeNodeWithAI(node); if (analysis) { @@ -178,7 +177,7 @@ Focus on the main concepts, key relationships, and practical implications.`; if (nodeId) { // Single node query = ` - SELECT n.id, n.title, n.source, n.notes, n.description, + SELECT n.id, n.title, n.source, n.description, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json, n.embedding, n.embedding_updated_at @@ -189,7 +188,7 @@ Focus on the main concepts, key relationships, and practical implications.`; } else if (forceReEmbed) { // All nodes query = ` - SELECT n.id, n.title, n.source, n.notes, n.description, + SELECT n.id, n.title, n.source, n.description, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json, n.embedding, n.embedding_updated_at @@ -199,7 +198,7 @@ Focus on the main concepts, key relationships, and practical implications.`; } else { // Only nodes without embeddings query = ` - SELECT n.id, n.title, n.source, n.notes, n.description, + SELECT n.id, n.title, n.source, n.description, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json, n.embedding, n.embedding_updated_at diff --git a/src/tools/database/queryDimensionNodes.ts b/src/tools/database/queryDimensionNodes.ts index 06fba7d..0cd28ff 100644 --- a/src/tools/database/queryDimensionNodes.ts +++ b/src/tools/database/queryDimensionNodes.ts @@ -48,8 +48,8 @@ export const queryDimensionNodesTool = tool({ event_date: node.event_date ?? null, }; - if (includeContent && (node.source || node.notes)) { - const previewSource = node.source || node.notes || ''; + if (includeContent && node.source) { + const previewSource = node.source; // Truncate to ~100 chars formatted.sourcePreview = previewSource.length > 100 ? previewSource.substring(0, 100) + '...'