From 86f43c997595781d232594f183dc4ff138de89cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CBeeRad=E2=80=9D?= Date: Sun, 15 Feb 2026 12:53:52 +1100 Subject: [PATCH] =?UTF-8?q?feat:=20final=20schema=20pass=20=E2=80=94=20syn?= =?UTF-8?q?c=209=20schema=20changes=20from=20main=20repo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename nodes.content → nodes.notes across all services, tools, API routes, and UI - Add dimensions.icon column support (GET/POST routes, types) - Add nodes.event_date column (create/update flows, types) - Drop nodes.type and nodes.is_pinned references - Drop edges.user_feedback references - Drop chat_memory_state table (replaced with DROP IF EXISTS in migration) - Update schema guide files (system/schema.md) - Add runtime migration block in sqlite-client.ts for existing databases - Fix MCP tool schemas (external API keeps 'content' param, maps to 'notes' internally) - Update standalone MCP server (nodeService.js, sqlite-client.js, index.js) - Update HTTP MCP server (server.js, stdio-server.js) - Update all extraction tools (paper, website, youtube) - Update all UI components (FocusPanel, ThreePanelLayout, GridView, ListView, FolderViewOverlay) - TypeScript: 0 errors Co-Authored-By: Claude Opus 4.6 --- app/api/dimensions/route.ts | 14 +-- app/api/mcp/route.ts | 2 +- .../[id]/regenerate-description/route.ts | 4 +- app/api/nodes/[id]/route.ts | 11 +-- app/api/nodes/route.ts | 13 ++- .../guides/system/schema.md | 16 +++- apps/mcp-server-standalone/index.js | 24 +++-- .../services/nodeService.js | 18 ++-- .../services/sqlite-client.js | 6 +- apps/mcp-server/server.js | 6 +- apps/mcp-server/stdio-server.js | 6 +- scripts/database/sqlite-ensure-app-schema.sh | 33 ++----- src/components/focus/FocusPanel.tsx | 22 ++--- src/components/layout/ThreePanelLayout.tsx | 2 +- src/components/nodes/FolderViewOverlay.tsx | 16 ++-- src/components/views/GridView.tsx | 4 +- src/components/views/ListView.tsx | 4 +- src/config/guides/system/schema.md | 45 +++++++--- src/services/context/autoContext.ts | 2 +- src/services/database/descriptionService.ts | 15 ++-- src/services/database/dimensionService.ts | 12 +-- src/services/database/edges.ts | 10 +-- src/services/database/nodes.ts | 24 ++--- src/services/database/sqlite-client.ts | 89 ++++++++++++++++--- src/services/settings/autoContextSettings.ts | 2 +- src/services/typescript/embed-nodes.ts | 8 +- src/tools/database/createNode.ts | 2 +- src/tools/database/getNodesById.ts | 10 +-- src/tools/database/queryDimensionNodes.ts | 8 +- src/tools/database/queryNodes.ts | 4 +- src/tools/database/updateEdge.ts | 8 +- src/tools/database/updateNode.ts | 42 ++++----- src/tools/other/paperExtract.ts | 16 ++-- src/tools/other/websiteExtract.ts | 16 ++-- src/tools/other/youtubeExtract.ts | 12 +-- src/types/database.ts | 6 +- 36 files changed, 303 insertions(+), 229 deletions(-) diff --git a/app/api/dimensions/route.ts b/app/api/dimensions/route.ts index 8d82ea0..3867860 100644 --- a/app/api/dimensions/route.ts +++ b/app/api/dimensions/route.ts @@ -19,6 +19,7 @@ export async function GET() { SELECT d.name AS dimension, d.description, + d.icon, d.is_priority AS isPriority, COALESCE(dc.count, 0) AS count FROM dimensions d @@ -31,6 +32,7 @@ export async function GET() { data: result.rows.map((row: any) => ({ dimension: row.dimension, description: row.description, + icon: row.icon || null, isPriority: Boolean(row.isPriority), count: Number(row.count) })) @@ -49,6 +51,7 @@ export async function POST(request: NextRequest) { const body = await request.json(); const rawName = typeof body?.name === 'string' ? body.name.trim() : ''; const description = typeof body?.description === 'string' ? body.description.trim() : null; + const icon = typeof body?.icon === 'string' ? body.icon.trim() : null; const isPriority = typeof body?.isPriority === 'boolean' ? body.isPriority : false; if (!rawName) { @@ -67,14 +70,15 @@ export async function POST(request: NextRequest) { const sqlite = getSQLiteClient(); const result = sqlite.query(` - INSERT INTO dimensions(name, description, is_priority, updated_at) - VALUES (?, ?, ?, CURRENT_TIMESTAMP) - ON CONFLICT(name) DO UPDATE SET + INSERT INTO dimensions(name, description, icon, is_priority, updated_at) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(name) DO UPDATE SET description = COALESCE(?, description), + icon = COALESCE(?, icon), is_priority = COALESCE(?, is_priority), updated_at = CURRENT_TIMESTAMP - RETURNING name, description, is_priority - `, [rawName, description, isPriority ? 1 : 0, description, isPriority ? 1 : 0]); + RETURNING name, description, icon, is_priority + `, [rawName, description, icon, isPriority ? 1 : 0, description, icon, isPriority ? 1 : 0]); if (result.rows.length === 0) { throw new Error('Failed to create dimension'); diff --git a/app/api/mcp/route.ts b/app/api/mcp/route.ts index 1fae575..9ba9379 100644 --- a/app/api/mcp/route.ts +++ b/app/api/mcp/route.ts @@ -134,7 +134,7 @@ function createRAHServer(): McpServer { id: node.id, title: node.title, description: node.description ?? null, - content: node.content ?? null, + notes: node.notes ?? null, link: node.link ?? null, dimensions: node.dimensions || [], metadata: node.metadata || {}, diff --git a/app/api/nodes/[id]/regenerate-description/route.ts b/app/api/nodes/[id]/regenerate-description/route.ts index 2207a87..c5f0c0e 100644 --- a/app/api/nodes/[id]/regenerate-description/route.ts +++ b/app/api/nodes/[id]/regenerate-description/route.ts @@ -85,10 +85,10 @@ export async function POST( // Generate new description using the description service const newDescription = await generateDescription({ title: node.title, - content: node.content || undefined, + notes: node.notes || undefined, link: node.link || undefined, metadata: enrichedMetadata, - type: enrichedMetadata?.type, + dimensions: node.dimensions || [] }); diff --git a/app/api/nodes/[id]/route.ts b/app/api/nodes/[id]/route.ts index d6f019b..cb34b80 100644 --- a/app/api/nodes/[id]/route.ts +++ b/app/api/nodes/[id]/route.ts @@ -68,16 +68,11 @@ export async function PUT( }, { status: 404 }); } - if (body && Object.prototype.hasOwnProperty.call(body, 'is_pinned')) { - console.warn(`[nodes/${nodeId}] Ignoring legacy is_pinned payload`); - delete body.is_pinned; - } - const updates: Record = { ...body }; let shouldQueueEmbed = false; const incomingChunk = typeof body.chunk === 'string' ? body.chunk : undefined; - const incomingContent = typeof body.content === 'string' ? body.content : undefined; + const incomingNotes = typeof body.notes === 'string' ? body.notes : undefined; const existingChunk = existingNode.chunk ?? ''; if (incomingChunk !== undefined) { @@ -92,8 +87,8 @@ export async function PUT( } else { delete updates.chunk_status; } - } else if (!existingChunk.trim() && hasSufficientContent(incomingContent)) { - updates.chunk = incomingContent; + } else if (!existingChunk.trim() && hasSufficientContent(incomingNotes)) { + updates.chunk = incomingNotes; updates.chunk_status = 'not_chunked'; shouldQueueEmbed = true; } diff --git a/app/api/nodes/route.ts b/app/api/nodes/route.ts index 3204e5a..0890059 100644 --- a/app/api/nodes/route.ts +++ b/app/api/nodes/route.ts @@ -60,7 +60,7 @@ export async function POST(request: NextRequest) { }, { status: 400 }); } - const rawContent = typeof body.content === 'string' ? body.content : null; + const rawNotes = typeof body.notes === 'string' ? body.notes : null; // Process provided dimensions first (needed for description generation) const providedDimensions = Array.isArray(body.dimensions) ? body.dimensions : []; @@ -74,10 +74,9 @@ export async function POST(request: NextRequest) { try { nodeDescription = await generateDescription({ title: body.title, - content: rawContent || undefined, + notes: rawNotes || undefined, link: body.link || undefined, metadata: body.metadata, - type: body.type, dimensions: trimmedProvidedDimensions }); } catch (error) { @@ -88,7 +87,7 @@ export async function POST(request: NextRequest) { // Auto-assign locked dimensions + keyword dimensions for all new nodes const { locked, keywords } = await DimensionService.assignDimensions({ title: body.title, - content: rawContent || undefined, + notes: rawNotes || undefined, link: body.link, description: nodeDescription }); @@ -107,15 +106,15 @@ export async function POST(request: NextRequest) { if (chunkToStore && chunkToStore.trim().length > 0) { chunkStatus = 'not_chunked'; - } else if (!chunkToStore && hasSufficientContent(rawContent)) { - chunkToStore = rawContent; + } else if (!chunkToStore && hasSufficientContent(rawNotes)) { + chunkToStore = rawNotes; chunkStatus = 'not_chunked'; } const node = await nodeService.createNode({ title: body.title, description: nodeDescription, - content: rawContent ?? undefined, + notes: rawNotes ?? undefined, link: body.link, dimensions: finalDimensions, chunk: chunkToStore ?? undefined, diff --git a/apps/mcp-server-standalone/guides/system/schema.md b/apps/mcp-server-standalone/guides/system/schema.md index 61589f8..db29938 100644 --- a/apps/mcp-server-standalone/guides/system/schema.md +++ b/apps/mcp-server-standalone/guides/system/schema.md @@ -14,13 +14,12 @@ immutable: true | id | INTEGER | Primary key, auto-increment | | title | TEXT | Required | | description | TEXT | AI-generated grounding context (~1 sentence) | -| content | TEXT | User's notes/thoughts (not source content) | +| notes | TEXT | User's notes/thoughts (not source content) | | chunk | TEXT | Full verbatim source content | | chunk_status | TEXT | 'pending', 'chunked', 'failed' | | link | TEXT | External URL (only for nodes representing external content) | -| type | TEXT | Nullable (reserved for future use) | +| event_date | TEXT | When the content happened (ISO date) — distinct from created_at | | metadata | TEXT | JSON blob (map_position, transcript_length, etc.) | -| is_pinned | INTEGER | Legacy — use hub node queries instead | | created_at | TEXT | ISO timestamp | | updated_at | TEXT | ISO timestamp | @@ -40,6 +39,7 @@ immutable: true |--------|------|-------| | name | TEXT | Primary key | | description | TEXT | Purpose description | +| icon | TEXT | Emoji or icon identifier for UI display | | is_priority | INTEGER | 1 = priority dimension (auto-assigns to new nodes) | | updated_at | TEXT | ISO timestamp | @@ -60,9 +60,17 @@ immutable: true | embedding_type | TEXT | Embedding model used | | metadata | TEXT | JSON blob | +### voice_usage (daily tracking) +| Column | Type | Notes | +|--------|------|-------| +| id | INTEGER | Primary key | +| date | TEXT | ISO date (UNIQUE) | +| minutes_used | REAL | Minutes consumed | +| updated_at | TEXT | ISO timestamp | + ### FTS Tables - `chunks_fts` — full-text search on chunk text -- `nodes_fts` — full-text search on node title + content +- `nodes_fts` — full-text search on node title + notes ## Common Query Patterns diff --git a/apps/mcp-server-standalone/index.js b/apps/mcp-server-standalone/index.js index c42e54e..48efa5e 100755 --- a/apps/mcp-server-standalone/index.js +++ b/apps/mcp-server-standalone/index.js @@ -310,7 +310,7 @@ async function main() { const node = nodeService.createNode({ title: title.trim(), - content: content?.trim(), + notes: content?.trim(), link: link?.trim(), description: description?.trim(), dimensions: normalizedDimensions, @@ -359,7 +359,7 @@ async function main() { WITH fts_matches AS ( SELECT rowid, rank FROM nodes_fts WHERE nodes_fts MATCH ? LIMIT 100 ) - SELECT n.id, n.title, n.description, n.content, n.link, n.updated_at, + SELECT n.id, n.title, n.description, n.notes, n.link, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json FROM fts_matches fm @@ -378,7 +378,7 @@ async function main() { WITH fts_matches AS ( SELECT rowid, rank FROM nodes_fts WHERE nodes_fts MATCH ? LIMIT ? ) - SELECT n.id, n.title, n.description, n.content, n.link, n.updated_at, + SELECT n.id, n.title, n.description, n.notes, n.link, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json FROM fts_matches fm @@ -392,7 +392,7 @@ async function main() { nodes = rows.map(row => ({ id: row.id, title: row.title, - content: row.content ?? null, + notes: row.notes ?? null, description: row.description ?? null, link: row.link ?? null, dimensions: JSON.parse(row.dimensions_json || '[]'), @@ -410,7 +410,7 @@ async function main() { const words = trimmedQuery.split(/\s+/).filter(w => w.length > 0); let sql = ` - SELECT n.id, n.title, n.description, n.content, n.link, n.updated_at, + SELECT n.id, n.title, n.description, n.notes, n.link, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json FROM nodes n @@ -419,7 +419,7 @@ async function main() { const params = []; for (const word of words) { - sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.content LIKE ? COLLATE NOCASE)`; + sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; params.push(`%${word}%`, `%${word}%`, `%${word}%`); } @@ -439,7 +439,7 @@ async function main() { nodes = rows.map(row => ({ id: row.id, title: row.title, - content: row.content ?? null, + notes: row.notes ?? null, description: row.description ?? null, link: row.link ?? null, dimensions: JSON.parse(row.dimensions_json || '[]'), @@ -485,7 +485,7 @@ async function main() { nodes.push({ id: node.id, title: node.title, - content: node.content ?? null, + notes: node.notes ?? null, description: node.description ?? null, link: node.link ?? null, chunk: chunkTruncated ? rawChunk.substring(0, CHUNK_LIMIT) : rawChunk, @@ -520,7 +520,13 @@ async function main() { throw new Error('At least one field must be provided in updates.'); } - const node = nodeService.updateNode(id, updates, { appendContent: true }); + // Map external 'content' param to internal 'notes' + const mappedUpdates = { ...updates }; + if (mappedUpdates.content !== undefined) { + mappedUpdates.notes = mappedUpdates.content; + delete mappedUpdates.content; + } + const node = nodeService.updateNode(id, mappedUpdates, { appendNotes: true }); return { content: [{ type: 'text', text: `Updated node #${id}` }], diff --git a/apps/mcp-server-standalone/services/nodeService.js b/apps/mcp-server-standalone/services/nodeService.js index 8d3a8b0..7cca795 100644 --- a/apps/mcp-server-standalone/services/nodeService.js +++ b/apps/mcp-server-standalone/services/nodeService.js @@ -9,7 +9,7 @@ function getNodes(filters = {}) { const { dimensions, search, limit = 100, offset = 0 } = filters; let sql = ` - SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -30,7 +30,7 @@ function getNodes(filters = {}) { // Text search if (search) { - sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.content LIKE ? COLLATE NOCASE)`; + sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; params.push(`%${search}%`, `%${search}%`, `%${search}%`); } @@ -70,7 +70,7 @@ function getNodes(filters = {}) { */ function getNodeById(id) { const sql = ` - SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json @@ -97,7 +97,7 @@ function createNode(nodeData) { const { title, description, - content, + notes, link, type, dimensions = [], @@ -110,7 +110,7 @@ function createNode(nodeData) { const nodeId = transaction(() => { const stmt = db.prepare(` - INSERT INTO nodes (title, description, content, link, type, metadata, chunk, created_at, updated_at) + INSERT INTO nodes (title, description, notes, link, type, metadata, chunk, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `); @@ -149,8 +149,8 @@ function createNode(nodeData) { * Note: content is APPENDED by default (MCP tool behavior), not replaced. */ function updateNode(id, updates, options = {}) { - const { appendContent = true } = options; - const { title, description, content, link, type, dimensions, chunk, metadata } = updates; + const { appendNotes = true } = options; + const { title, description, notes, link, type, dimensions, chunk, metadata } = updates; const now = new Date().toISOString(); const db = getDb(); @@ -173,10 +173,10 @@ function updateNode(id, updates, options = {}) { params.push(description); } if (content !== undefined) { - if (appendContent && existing.content) { + if (appendNotes && existing.notes) { // Append to existing content setFields.push('content = ?'); - params.push(existing.content + '\n\n' + content); + params.push(existing.notes + '\n\n' + content); } else { setFields.push('content = ?'); params.push(content); diff --git a/apps/mcp-server-standalone/services/sqlite-client.js b/apps/mcp-server-standalone/services/sqlite-client.js index 469618e..ed17dab 100644 --- a/apps/mcp-server-standalone/services/sqlite-client.js +++ b/apps/mcp-server-standalone/services/sqlite-client.js @@ -50,9 +50,9 @@ function initDatabase() { id INTEGER PRIMARY KEY, title TEXT, description TEXT, - content TEXT, + notes TEXT, link TEXT, - type TEXT, + event_date TEXT, created_at TEXT, updated_at TEXT, metadata TEXT, @@ -71,7 +71,7 @@ function initDatabase() { source TEXT, created_at TEXT, context TEXT, - user_feedback INTEGER, + FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE, FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE ); diff --git a/apps/mcp-server/server.js b/apps/mcp-server/server.js index c1622eb..d90f303 100644 --- a/apps/mcp-server/server.js +++ b/apps/mcp-server/server.js @@ -360,7 +360,7 @@ mcpServer.registerTool( const payload = { title: title.trim(), - content: content?.trim() || undefined, + notes: content?.trim() || undefined, link: link?.trim() || undefined, description: description?.trim() || undefined, dimensions: normalizedDimensions, @@ -422,7 +422,7 @@ mcpServer.registerTool( nodes: nodes.map((node) => ({ id: node.id, title: node.title, - content: node.content ?? null, + notes: node.notes ?? null, description: node.description ?? null, link: node.link ?? null, dimensions: node.dimensions || [], @@ -718,7 +718,7 @@ mcpServer.registerTool( results: results.map(r => ({ nodeId: r.node_id || r.nodeId || r.id, title: r.title || 'Untitled', - chunkPreview: (r.chunk || r.content || '').slice(0, 200), + chunkPreview: (r.chunk || r.notes || '').slice(0, 200), similarity: r.similarity || r.score || 0 })) } diff --git a/apps/mcp-server/stdio-server.js b/apps/mcp-server/stdio-server.js index 07555b1..cff62ad 100644 --- a/apps/mcp-server/stdio-server.js +++ b/apps/mcp-server/stdio-server.js @@ -288,7 +288,7 @@ server.registerTool( const payload = { title: title.trim(), - content: content?.trim() || undefined, + notes: content?.trim() || undefined, link: link?.trim() || undefined, description: description?.trim() || undefined, dimensions: normalizedDimensions, @@ -351,7 +351,7 @@ server.registerTool( nodes: nodes.map((node) => ({ id: node.id, title: node.title, - content: node.content ?? null, + notes: node.notes ?? null, description: node.description ?? null, link: node.link ?? null, dimensions: node.dimensions || [], @@ -647,7 +647,7 @@ server.registerTool( results: results.map(r => ({ nodeId: r.node_id || r.nodeId || r.id, title: r.title || 'Untitled', - chunkPreview: (r.chunk || r.content || '').slice(0, 200), + chunkPreview: (r.chunk || r.notes || '').slice(0, 200), similarity: r.similarity || r.score || 0 })) } diff --git a/scripts/database/sqlite-ensure-app-schema.sh b/scripts/database/sqlite-ensure-app-schema.sh index ac15325..f5d6e20 100755 --- a/scripts/database/sqlite-ensure-app-schema.sh +++ b/scripts/database/sqlite-ensure-app-schema.sh @@ -118,9 +118,9 @@ CREATE TABLE nodes ( id INTEGER PRIMARY KEY, title TEXT, description TEXT, - content TEXT, + notes TEXT, link TEXT, - type TEXT, + event_date TEXT, created_at TEXT, updated_at TEXT, metadata TEXT, @@ -159,7 +159,6 @@ CREATE TABLE edges ( source TEXT, created_at TEXT, context TEXT, - user_feedback INTEGER, FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE, FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE ); @@ -252,17 +251,8 @@ if has_table chats && ! has_col chats delegation_id; then "$SQLITE_BIN" "$DB_PATH" "ALTER TABLE chats ADD COLUMN delegation_id INTEGER;" fi -if ! has_table chat_memory_state; then - "$SQLITE_BIN" "$DB_PATH" <<'SQL' -CREATE TABLE chat_memory_state ( - thread_id TEXT PRIMARY KEY, - helper_name TEXT, - last_processed_chat_id INTEGER DEFAULT 0, - last_processed_at TEXT -); -CREATE INDEX IF NOT EXISTS idx_chat_memory_thread ON chat_memory_state(thread_id); -SQL -fi +# chat_memory_state removed in final schema pass +"$SQLITE_BIN" "$DB_PATH" "DROP TABLE IF EXISTS chat_memory_state;" echo "Dropping legacy agent_delegations table if present..." "$SQLITE_BIN" "$DB_PATH" "DROP TABLE IF EXISTS agent_delegations;" @@ -283,10 +273,6 @@ fi echo "Checking/adding missing columns..." if has_table nodes; then - if ! has_col nodes type; then - echo "Adding nodes.type" - "$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN type TEXT;" - fi if ! has_col nodes description; then echo "Adding nodes.description" "$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN description TEXT;" @@ -299,10 +285,7 @@ if has_table nodes; then echo "Adding nodes.chunk" "$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN chunk TEXT;" fi - if ! has_col nodes is_pinned; then - echo "Adding nodes.is_pinned" - "$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN is_pinned INTEGER DEFAULT 0;" - fi + # is_pinned removed in final schema pass fi if has_table chunks; then @@ -321,6 +304,8 @@ if ! has_table dimensions; then "$SQLITE_BIN" "$DB_PATH" <<'SQL' CREATE TABLE dimensions ( name TEXT PRIMARY KEY, + description TEXT, + icon TEXT, is_priority INTEGER DEFAULT 0, updated_at TEXT DEFAULT CURRENT_TIMESTAMP ); @@ -345,9 +330,9 @@ CREATE VIEW nodes_v AS SELECT n.id, n.title, n.description, - n.content, + n.notes, n.link, - n.type, + n.event_date, n.metadata, n.created_at, n.updated_at, diff --git a/src/components/focus/FocusPanel.tsx b/src/components/focus/FocusPanel.tsx index d1f758b..477afbc 100644 --- a/src/components/focus/FocusPanel.tsx +++ b/src/components/focus/FocusPanel.tsx @@ -382,7 +382,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli try { const updateData: Record = {}; - if (editingField === 'content') { + if (editingField === 'notes') { updateData.content = editingValue; } else { updateData[editingField] = editingValue; @@ -406,7 +406,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli } // Safety net: ensure edges exist for any tokens present in saved content - if (editingField === 'content' && typeof editingValue === 'string') { + if (editingField === 'notes' && typeof editingValue === 'string') { try { const tokens = parseNodeMarkers(editingValue); const uniqueTargets = Array.from(new Set(tokens.map(t => t.id))).filter(id => id !== nodeId); @@ -448,7 +448,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const response = await fetch(`/api/nodes/${nodeId}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ content: contentValue }), + body: JSON.stringify({ notes: contentValue }), }); if (!response.ok) throw new Error('Failed to save'); const result = await response.json(); @@ -568,7 +568,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const response = await fetch(`/api/nodes/${activeTab}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ content: notesEditValue }), + body: JSON.stringify({ notes: notesEditValue }), }); if (!response.ok) throw new Error('Failed to save'); const result = await response.json(); @@ -614,7 +614,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli // Start editing notes const startNotesEdit = () => { if (!activeTab || !nodesData[activeTab]) return; - setNotesEditValue(nodesData[activeTab].content || ''); + setNotesEditValue(nodesData[activeTab].notes || ''); setNotesEditMode(true); }; @@ -1023,15 +1023,15 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli const embedContent = async (nodeId: number) => { const node = nodesData[nodeId]; - const hasContent = node?.content?.trim(); + const hasNotes = node?.notes?.trim(); const hasChunk = node?.chunk?.trim(); // If chunk is empty but content exists, auto-populate chunk from content - if (!hasChunk && hasContent) { + if (!hasChunk && hasNotes) { try { const response = await fetch(`/api/nodes/${nodeId}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ chunk: hasContent }) + body: JSON.stringify({ chunk: hasNotes }) }); if (response.ok) { @@ -1046,7 +1046,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli } } // If neither content nor chunk exist, require content - if (!hasContent && !hasChunk) { + if (!hasNotes && !hasChunk) { startEdit('content', ''); return; } @@ -2722,7 +2722,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli )} - ) : nodesData[activeTab]?.content ? ( + ) : nodesData[activeTab]?.notes ? (
diff --git a/src/components/layout/ThreePanelLayout.tsx b/src/components/layout/ThreePanelLayout.tsx index ef18fad..f053471 100644 --- a/src/components/layout/ThreePanelLayout.tsx +++ b/src/components/layout/ThreePanelLayout.tsx @@ -149,7 +149,7 @@ export default function ThreePanelLayout() { id: node.id, title: node.title, link: node.link, - content: node.content, + notes: node.notes, dimensions: node.dimensions, created_at: node.created_at, updated_at: node.updated_at, diff --git a/src/components/nodes/FolderViewOverlay.tsx b/src/components/nodes/FolderViewOverlay.tsx index 0514091..da7bdb3 100644 --- a/src/components/nodes/FolderViewOverlay.tsx +++ b/src/components/nodes/FolderViewOverlay.tsx @@ -1240,7 +1240,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o )} - {node.content && ( + {node.notes && (
- {getContentPreview(node.content)} + {getContentPreview(node.notes)}
)} {node.dimensions && node.dimensions.length > 0 && ( @@ -1376,7 +1376,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o {node.title || 'Untitled'} - {node.content && ( + {node.notes && (
- {getContentPreview(node.content)} + {getContentPreview(node.notes)}
)} @@ -1846,7 +1846,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o {node.title || 'Untitled'} - {node.content && ( + {node.notes && (
- {getContentPreview(node.content)} + {getContentPreview(node.notes)}
)} @@ -2862,7 +2862,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o {node.title || 'Untitled'} - {node.content && ( + {node.notes && (
- {getContentPreview(node.content)} + {getContentPreview(node.notes)}
)} {node.dimensions && node.dimensions.length > 1 && ( diff --git a/src/components/views/GridView.tsx b/src/components/views/GridView.tsx index 9f953fa..205686d 100644 --- a/src/components/views/GridView.tsx +++ b/src/components/views/GridView.tsx @@ -104,7 +104,7 @@ export default function GridView({ nodes, onNodeClick }: GridViewProps) { {/* Description or Content Preview */} - {(node.description || node.content) && ( + {(node.description || node.notes) && (
- {node.description || truncateContent(node.content)} + {node.description || truncateContent(node.notes)}
)} diff --git a/src/components/views/ListView.tsx b/src/components/views/ListView.tsx index 73442fd..24d2c51 100644 --- a/src/components/views/ListView.tsx +++ b/src/components/views/ListView.tsx @@ -105,7 +105,7 @@ export default function ListView({ nodes, onNodeClick }: ListViewProps) { {/* Description or Content Preview */} - {(node.description || node.content) && ( + {(node.description || node.notes) && (
- {node.description || truncateContent(node.content)} + {node.description || truncateContent(node.notes)}
)} diff --git a/src/config/guides/system/schema.md b/src/config/guides/system/schema.md index 5c0fa80..db29938 100644 --- a/src/config/guides/system/schema.md +++ b/src/config/guides/system/schema.md @@ -14,13 +14,12 @@ immutable: true | id | INTEGER | Primary key, auto-increment | | title | TEXT | Required | | description | TEXT | AI-generated grounding context (~1 sentence) | -| content | TEXT | User's notes/thoughts (not source content) | +| notes | TEXT | User's notes/thoughts (not source content) | | chunk | TEXT | Full verbatim source content | | chunk_status | TEXT | 'pending', 'chunked', 'failed' | | link | TEXT | External URL (only for nodes representing external content) | -| type | TEXT | Nullable (reserved for future use) | +| event_date | TEXT | When the content happened (ISO date) — distinct from created_at | | metadata | TEXT | JSON blob (map_position, transcript_length, etc.) | -| is_pinned | INTEGER | Legacy — use hub node queries instead | | created_at | TEXT | ISO timestamp | | updated_at | TEXT | ISO timestamp | @@ -38,29 +37,40 @@ immutable: true ### dimensions | Column | Type | Notes | |--------|------|-------| -| id | INTEGER | Primary key | -| name | TEXT | Unique, case-insensitive | +| name | TEXT | Primary key | | description | TEXT | Purpose description | -| is_locked | INTEGER | 1 = priority dimension (auto-assigns to new nodes) | +| icon | TEXT | Emoji or icon identifier for UI display | +| is_priority | INTEGER | 1 = priority dimension (auto-assigns to new nodes) | +| updated_at | TEXT | ISO timestamp | ### node_dimensions (junction) | Column | Type | |--------|------| | node_id | INTEGER FK → nodes.id | -| dimension_id | INTEGER FK → dimensions.id | +| dimension | TEXT (dimension name) | ### chunks (for semantic search) | Column | Type | Notes | |--------|------|-------| | id | INTEGER | Primary key | | node_id | INTEGER | FK → nodes.id | -| chunk_index | INTEGER | Position in sequence | +| chunk_idx | INTEGER | Position in sequence | | text | TEXT | Chunk content | -| embedding | BLOB | Vector (via sqlite-vec) | +| created_at | TEXT | ISO timestamp | +| embedding_type | TEXT | Embedding model used | +| metadata | TEXT | JSON blob | + +### voice_usage (daily tracking) +| Column | Type | Notes | +|--------|------|-------| +| id | INTEGER | Primary key | +| date | TEXT | ISO date (UNIQUE) | +| minutes_used | REAL | Minutes consumed | +| updated_at | TEXT | ISO timestamp | ### FTS Tables - `chunks_fts` — full-text search on chunk text -- `nodes_fts` — full-text search on node title + content +- `nodes_fts` — full-text search on node title + notes ## Common Query Patterns @@ -76,8 +86,7 @@ GROUP BY n.id ORDER BY edge_count DESC LIMIT 5 ```sql SELECT n.* FROM nodes n JOIN node_dimensions nd ON n.id = nd.node_id -JOIN dimensions d ON nd.dimension_id = d.id -WHERE d.name = ? +WHERE nd.dimension = ? ``` **Edges for a node (both directions):** @@ -89,4 +98,14 @@ JOIN nodes n2 ON e.to_node_id = n2.id WHERE e.from_node_id = ? OR e.to_node_id = ? ``` -**Use sqliteQuery for any read operation not covered by structured tools.** +**Search source content (chunks):** +```sql +SELECT c.id, c.node_id, c.chunk_idx, c.text, n.title +FROM chunks c +JOIN nodes n ON c.node_id = n.id +WHERE c.text LIKE '%search term%' COLLATE NOCASE +ORDER BY c.chunk_idx ASC +LIMIT 10 +``` + +**Use `rah_search_content` to search chunks by keyword, or `rah_sqlite_query` for any read operation not covered by structured tools.** diff --git a/src/services/context/autoContext.ts b/src/services/context/autoContext.ts index 2e38000..0ca96d7 100644 --- a/src/services/context/autoContext.ts +++ b/src/services/context/autoContext.ts @@ -27,7 +27,7 @@ function fetchAutoContextRows(limit: number): AutoContextSummary[] { FROM nodes n LEFT JOIN edges e ON (e.from_node_id = n.id OR e.to_node_id = n.id) - WHERE n.type IS NULL OR n.type != 'memory' + WHERE 1=1 GROUP BY n.id ORDER BY edge_count DESC, n.updated_at DESC LIMIT ? diff --git a/src/services/database/descriptionService.ts b/src/services/database/descriptionService.ts index 2ec9d26..9c3a67b 100644 --- a/src/services/database/descriptionService.ts +++ b/src/services/database/descriptionService.ts @@ -4,7 +4,7 @@ import { hasValidOpenAiKey } from '../storage/apiKeys'; export interface DescriptionInput { title: string; - content?: string; + notes?: string; link?: string; metadata?: { source?: string; @@ -12,7 +12,6 @@ export interface DescriptionInput { author?: string; site_name?: string; }; - type?: string; dimensions?: string[]; } @@ -24,7 +23,7 @@ export { hasValidOpenAiKey } from '../storage/apiKeys'; * Used when no API key is available or for simple inputs. */ export function generateFallbackDescription(input: DescriptionInput): string { - const { title, type, metadata, dimensions } = input; + const { title, metadata, dimensions } = input; // Build a contextual fallback const parts: string[] = []; @@ -33,10 +32,6 @@ export function generateFallbackDescription(input: DescriptionInput): string { parts.push(`By ${metadata.author || metadata.channel_name}`); } - if (type) { - parts.push(type.charAt(0).toUpperCase() + type.slice(1)); - } - if (dimensions?.length) { parts.push(`in ${dimensions.slice(0, 2).join(', ')}`); } @@ -63,7 +58,7 @@ export async function generateDescription(input: DescriptionInput): Promise 800 ? '...' : ''}`); + const contentPreview = input.notes?.slice(0, 800) || ''; + if (contentPreview) lines.push(`Notes: ${contentPreview}${input.notes && input.notes.length > 800 ? '...' : ''}`); return `Your job is to answer: "what is this?" in one short line. Max 280 characters. diff --git a/src/services/database/dimensionService.ts b/src/services/database/dimensionService.ts index 52c1ed3..11fd2cb 100644 --- a/src/services/database/dimensionService.ts +++ b/src/services/database/dimensionService.ts @@ -55,7 +55,7 @@ export class DimensionService { */ static async assignDimensions(nodeData: { title: string; - content?: string; + notes?: string; link?: string; description?: string; }): Promise<{ locked: string[]; keywords: string[] }> { @@ -156,19 +156,19 @@ export class DimensionService { * Build AI prompt for dimension assignment (locked dimensions only) */ private static buildAssignmentPrompt( - nodeData: { title: string; content?: string; link?: string; description?: string }, + nodeData: { title: string; notes?: string; link?: string; description?: string }, lockedDimensions: LockedDimension[] ): string { // Use description as primary context, content as fallback let nodeContextSection: string; if (nodeData.description) { - const contentPreview = nodeData.content?.slice(0, 500) || ''; + const contentPreview = nodeData.notes?.slice(0, 500) || ''; nodeContextSection = `DESCRIPTION: ${nodeData.description} -CONTENT PREVIEW: ${contentPreview}${nodeData.content && nodeData.content.length > 500 ? '...' : ''}`; +NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`; } else { - const contentPreview = nodeData.content?.slice(0, 2000) || ''; - nodeContextSection = `CONTENT: ${contentPreview}${nodeData.content && nodeData.content.length > 2000 ? '...' : ''}`; + const contentPreview = nodeData.notes?.slice(0, 2000) || ''; + nodeContextSection = `NOTES: ${contentPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`; } // Include ALL locked dimensions, using fallback text for those without descriptions diff --git a/src/services/database/edges.ts b/src/services/database/edges.ts index 5f1aebf..6b8d182 100644 --- a/src/services/database/edges.ts +++ b/src/services/database/edges.ts @@ -430,9 +430,9 @@ export class EdgeService { ELSE n_from.title END as connected_node_title, CASE - WHEN e.from_node_id = ? THEN n_to.content - ELSE n_from.content - END as connected_node_content, + WHEN e.from_node_id = ? THEN n_to.notes + ELSE n_from.notes + END as connected_node_notes, CASE WHEN e.from_node_id = ? THEN n_to.link ELSE n_from.link @@ -499,7 +499,7 @@ export class EdgeService { const connected_node: Node = { id: row.connected_node_id, title: row.connected_node_title, - content: row.connected_node_content, + notes: row.connected_node_notes, link: row.connected_node_link, dimensions: row.connected_node_dimensions, embedding: undefined, // Not needed for display @@ -544,7 +544,7 @@ export class EdgeService { const connected_node: Node = { id: row.connected_node_id, title: row.connected_node_title, - content: row.connected_node_content, + notes: row.connected_node_notes, link: row.connected_node_link, dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'), embedding: undefined, // Not needed for display diff --git a/src/services/database/nodes.ts b/src/services/database/nodes.ts index 2d1616c..68909d7 100644 --- a/src/services/database/nodes.ts +++ b/src/services/database/nodes.ts @@ -15,7 +15,7 @@ export class NodeService { // Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance) let query = ` - SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -38,7 +38,7 @@ export class NodeService { // Text search in title, description, and content (SQLite LIKE with COLLATE NOCASE) if (search) { - query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.content LIKE ? COLLATE NOCASE)`; + query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`; params.push(`%${search}%`, `%${search}%`, `%${search}%`); } @@ -50,7 +50,7 @@ export class NodeService { CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END, CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END, CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END, - CASE WHEN n.content LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END, + CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END, n.updated_at DESC`; params.push( search, // Exact match (case-insensitive) @@ -95,7 +95,7 @@ export class NodeService { private async getNodeByIdSQLite(id: number): Promise { const sqlite = getSQLiteClient(); const query = ` - SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk, + SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk, n.chunk_status, n.embedding_updated_at, n.embedding_text, n.created_at, n.updated_at, COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension) @@ -125,9 +125,9 @@ export class NodeService { const { title, description, - content, + notes, link, - type, + event_date, dimensions = [], chunk, chunk_status, @@ -139,14 +139,14 @@ export class NodeService { const nodeId = sqlite.transaction(() => { // Insert node using prepare/run for lastInsertRowid access const nodeResult = sqlite.prepare(` - INSERT INTO nodes (title, description, content, link, type, metadata, chunk, chunk_status, created_at, updated_at) + INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, chunk_status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( title, description ?? null, - content ?? null, + notes ?? null, link ?? null, - type ?? null, + event_date ?? null, JSON.stringify(metadata), chunk ?? null, chunk_status ?? null, @@ -192,7 +192,7 @@ export class NodeService { // PostgreSQL path removed in SQLite-only consolidation private async updateNodeSQLite(id: number, updates: Partial): Promise { - const { title, description, content, link, type, dimensions, chunk, metadata } = updates; + const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates; const now = new Date().toISOString(); const sqlite = getSQLiteClient(); @@ -211,9 +211,9 @@ export class NodeService { if (title !== undefined) { setFields.push('title = ?'); params.push(title); } if (description !== undefined) { setFields.push('description = ?'); params.push(description); } - if (content !== undefined) { setFields.push('content = ?'); params.push(content); } + if (notes !== undefined) { setFields.push('notes = ?'); params.push(notes); } if (link !== undefined) { setFields.push('link = ?'); params.push(link); } - if (type !== undefined) { setFields.push('type = ?'); params.push(type); } + if (event_date !== undefined) { setFields.push('event_date = ?'); params.push(event_date); } if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); } if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) { setFields.push('chunk_status = ?'); diff --git a/src/services/database/sqlite-client.ts b/src/services/database/sqlite-client.ts index 90323eb..ae9117d 100644 --- a/src/services/database/sqlite-client.ts +++ b/src/services/database/sqlite-client.ts @@ -266,7 +266,7 @@ class SQLiteClient { } }; ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;"); - ensureNodeCol('type', "ALTER TABLE nodes ADD COLUMN type TEXT;"); + // type column removed in final schema pass } catch (nodeErr) { console.warn('Failed to ensure nodes columns:', nodeErr); } @@ -435,16 +435,8 @@ class SQLiteClient { } // Do not recreate memory_v; alias has been removed. - // 6) Chat memory state tracking for chat-triggered memories - this.db.exec(` - CREATE TABLE IF NOT EXISTS chat_memory_state ( - thread_id TEXT PRIMARY KEY, - helper_name TEXT, - last_processed_chat_id INTEGER DEFAULT 0, - last_processed_at TEXT - ); - CREATE INDEX IF NOT EXISTS idx_chat_memory_thread ON chat_memory_state(thread_id); - `); + // 6) Drop orphaned chat_memory_state table (removed in final schema pass) + this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`); // Agent delegation table for orchestrator/worker coordination try { @@ -631,6 +623,81 @@ class SQLiteClient { } } + // 10) Final schema pass migrations (content→notes, event_date, icon, drop dead columns) + try { + const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>; + const nodeColNames = nodeCols2.map((c: any) => c.name); + + // Rename content → notes + if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) { + console.log('Renaming nodes.content → nodes.notes'); + this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;'); + } + + // Add event_date with backfill from metadata + if (!nodeColNames.includes('event_date')) { + console.log('Adding nodes.event_date column'); + this.db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;'); + this.db.exec(` + UPDATE nodes SET event_date = json_extract(metadata, '$.published_date') + WHERE metadata IS NOT NULL + AND json_extract(metadata, '$.published_date') IS NOT NULL + AND json_extract(metadata, '$.published_date') != ''; + `); + } + + // Add dimensions.icon + const dimCols2 = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>; + if (!dimCols2.some((c: any) => c.name === 'icon')) { + console.log('Adding dimensions.icon column'); + this.db.exec('ALTER TABLE dimensions ADD COLUMN icon TEXT;'); + } + + // Drop dead columns (SQLite 3.35+) + if (nodeColNames.includes('type')) { + console.log('Dropping nodes.type column'); + try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_type;'); } catch {} + try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch (e) { + console.warn('Could not drop nodes.type (SQLite < 3.35?):', e); + } + } + if (nodeColNames.includes('is_pinned')) { + console.log('Dropping nodes.is_pinned column'); + try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_pinned;'); } catch {} + try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch (e) { + console.warn('Could not drop nodes.is_pinned:', e); + } + } + + // Drop edges.user_feedback + const edgeCols = this.db.prepare('PRAGMA table_info(edges)').all() as Array<{ name: string }>; + if (edgeCols.some((c: any) => c.name === 'user_feedback')) { + console.log('Dropping edges.user_feedback column'); + try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch (e) { + console.warn('Could not drop edges.user_feedback:', e); + } + } + + // Rebuild FTS if it references 'content' instead of 'notes' + try { + const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as any; + if (ftsCheck && ftsCheck.sql && ftsCheck.sql.includes('content')) { + console.log('Rebuilding nodes_fts to reference notes instead of content'); + this.db.exec('DROP TABLE IF EXISTS nodes_fts;'); + this.db.exec(` + CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content=nodes, content_rowid=id); + INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild'); + `); + } + } catch (ftsErr) { + console.warn('FTS rebuild skipped:', ftsErr); + } + + console.log('Final schema pass migrations complete'); + } catch (schemaErr) { + console.warn('Final schema pass migration error:', schemaErr); + } + console.log('Logging + memory schema ensured'); } catch (error) { console.error('Failed to ensure logging/memory schema:', error); diff --git a/src/services/settings/autoContextSettings.ts b/src/services/settings/autoContextSettings.ts index 18a06c8..9c10adc 100644 --- a/src/services/settings/autoContextSettings.ts +++ b/src/services/settings/autoContextSettings.ts @@ -68,7 +68,7 @@ function bootstrapFromLegacyPins(): void { try { const db = getSQLiteClient(); const countRow = db - .query<{ count: number }>('SELECT COUNT(*) as count FROM nodes WHERE is_pinned = 1') + .query<{ count: number }>('SELECT COUNT(*) as count FROM nodes WHERE 1=0 /* /* is_pinned removed */ removed */') .rows[0]; const pinnedCount = Number(countRow?.count ?? 0); if (pinnedCount > 0) { diff --git a/src/services/typescript/embed-nodes.ts b/src/services/typescript/embed-nodes.ts index b44375c..6ac5f42 100644 --- a/src/services/typescript/embed-nodes.ts +++ b/src/services/typescript/embed-nodes.ts @@ -16,7 +16,7 @@ import { interface NodeRecord { id: number; title: string; - content: string | null; + notes: string | null; description: string | null; dimensions_json: string; embedding?: Buffer | null; @@ -58,7 +58,7 @@ export class NodeEmbedder { const prompt = `Analyze this content and provide 2-3 key insights or themes in a concise paragraph (max 100 words): Title: ${node.title} -Content: ${node.content || 'No content'} +Content: ${node.notes || 'No content'} Dimensions: ${dimensionsText} Focus on the main concepts, key relationships, and practical implications.`; @@ -106,13 +106,13 @@ Focus on the main concepts, key relationships, and practical implications.`; // Create base embedding text let embeddingText = formatEmbeddingText( node.title, - node.content || '', + node.notes || '', dimensions, node.description ); // Add AI analysis if content exists - if (node.content && node.content.trim().length > 0) { + if (node.notes && node.notes.trim().length > 0) { const analysis = await this.analyzeNodeWithAI(node); if (analysis) { embeddingText += `\n\nAI Analysis: ${analysis}`; diff --git a/src/tools/database/createNode.ts b/src/tools/database/createNode.ts index e55c479..ed29ce8 100644 --- a/src/tools/database/createNode.ts +++ b/src/tools/database/createNode.ts @@ -6,7 +6,7 @@ export const createNodeTool = tool({ description: 'Create node with title/content/link and optional dimensions (locked dimensions auto-assigned)', inputSchema: z.object({ title: z.string().describe('The title of the node'), - content: z.string().optional().describe('The main content, description, or notes for this node'), + notes: z.string().optional().describe('The main notes, description, or notes for this node'), link: z.string().optional().describe('A URL link to the source'), dimensions: z .array(z.string()) diff --git a/src/tools/database/getNodesById.ts b/src/tools/database/getNodesById.ts index 0c30692..f0388d0 100644 --- a/src/tools/database/getNodesById.ts +++ b/src/tools/database/getNodesById.ts @@ -6,9 +6,9 @@ export const getNodesByIdTool = tool({ description: 'Load full node records by IDs', inputSchema: z.object({ nodeIds: z.array(z.number().int().positive()).min(1).max(10).describe('List of node IDs to load'), - includeContentPreview: z.boolean().default(true).describe('Whether to return a trimmed content preview for each node'), + includeNotesPreview: z.boolean().default(true).describe('Whether to return a trimmed content preview for each node'), }), - execute: async ({ nodeIds, includeContentPreview }) => { + execute: async ({ nodeIds, includeNotesPreview }) => { const uniqueIds = Array.from(new Set(nodeIds.filter(id => Number.isFinite(id) && id > 0))); if (uniqueIds.length === 0) { return { @@ -23,8 +23,8 @@ export const getNodesByIdTool = tool({ try { const node = await nodeService.getNodeById(id); if (!node) return null; - const preview = includeContentPreview - ? (node.content || node.description || '') + const preview = includeNotesPreview + ? (node.notes || node.description || '') .split(/\s+/) .slice(0, 80) .join(' ') @@ -38,7 +38,7 @@ export const getNodesByIdTool = tool({ dimensions: node.dimensions || [], chunk_status: node.chunk_status || 'unknown', updated_at: node.updated_at, - content_preview: preview || null, + notes_preview: preview || null, metadata: node.metadata ?? null, }; } catch (error) { diff --git a/src/tools/database/queryDimensionNodes.ts b/src/tools/database/queryDimensionNodes.ts index 741acce..cc11be6 100644 --- a/src/tools/database/queryDimensionNodes.ts +++ b/src/tools/database/queryDimensionNodes.ts @@ -45,11 +45,11 @@ export const queryDimensionNodesTool = tool({ dimensions: node.dimensions || [], }; - if (includeContent && node.content) { + if (includeContent && node.notes) { // Truncate to ~100 chars - formatted.contentPreview = node.content.length > 100 - ? node.content.substring(0, 100) + '...' - : node.content; + formatted.notesPreview = node.notes.length > 100 + ? node.notes.substring(0, 100) + '...' + : node.notes; } return formatted; diff --git a/src/tools/database/queryNodes.ts b/src/tools/database/queryNodes.ts index cba096b..729e017 100644 --- a/src/tools/database/queryNodes.ts +++ b/src/tools/database/queryNodes.ts @@ -5,11 +5,11 @@ import { formatNodeForChat } from '../infrastructure/nodeFormatter'; import type { Node } from '@/types/database'; export const queryNodesTool = tool({ - description: 'Search nodes by title/content/dimensions', + description: 'Search nodes by title/notes/dimensions', inputSchema: z.object({ filters: z.object({ dimensions: z.array(z.string()).describe('Filter by dimensions (e.g., ["research", "ai", "technology"]). Replaces old type/stage filtering.').optional(), - search: z.string().describe('Search term to match against title or content').optional(), + search: z.string().describe('Search term to match against title or notes').optional(), limit: z.number().min(1).max(50).default(10).describe('Maximum number of results to return') }).optional() }), diff --git a/src/tools/database/updateEdge.ts b/src/tools/database/updateEdge.ts index 57336fa..cac3cdc 100644 --- a/src/tools/database/updateEdge.ts +++ b/src/tools/database/updateEdge.ts @@ -3,13 +3,12 @@ import { z } from 'zod'; import { edgeService } from '@/services/database/edges'; export const updateEdgeTool = tool({ - description: 'Update edge context/source/feedback', + description: 'Update edge context/source', inputSchema: z.object({ edge_id: z.number().describe('The ID of the edge to update'), updates: z.object({ context: z.record(z.any()).optional().describe('Updated context information for this edge - can include explanation, relationship type, strength, notes, etc.'), source: z.enum(['user', 'ai_similarity', 'helper_name']).optional().describe('Updated source classification for this edge'), - user_feedback: z.boolean().optional().describe('User feedback on this edge connection (true = positive, false = negative)') }).describe('Fields to update on the edge') }), execute: async (params) => { @@ -46,10 +45,7 @@ export const updateEdgeTool = tool({ const updateDescriptions = []; if (cleanUpdates.context) updateDescriptions.push('context'); if (cleanUpdates.source) updateDescriptions.push(`source to ${cleanUpdates.source}`); - if (cleanUpdates.user_feedback !== undefined) { - updateDescriptions.push(`user feedback to ${cleanUpdates.user_feedback ? 'positive' : 'negative'}`); - } - + return { success: true, data: updatedEdge, diff --git a/src/tools/database/updateNode.ts b/src/tools/database/updateNode.ts index 1e68e56..e0324ee 100644 --- a/src/tools/database/updateNode.ts +++ b/src/tools/database/updateNode.ts @@ -7,7 +7,7 @@ export const updateNodeTool = tool({ id: z.number().describe('The ID of the node to update'), updates: z.object({ title: z.string().optional().describe('New title'), - content: z.string().optional().describe('New content/description/notes'), + notes: z.string().optional().describe('New content/description/notes'), link: z.string().optional().describe('New link'), dimensions: z.array(z.string()).optional().describe('New dimension tags - completely replaces existing dimensions'), chunk: z.string().optional().describe('New chunk content'), @@ -25,28 +25,28 @@ export const updateNodeTool = tool({ } // FORCE APPEND for content field - fetch existing and append new content - if (updates.content) { + if (updates.notes) { const fetchResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes/${id}`); if (fetchResponse.ok) { const { node } = await fetchResponse.json(); - const existingContent = (node?.content || '').trim(); - const newContent = updates.content.trim(); + const existingNotes = (node?.notes || '').trim(); + const newNotes = updates.notes.trim(); // Skip if new content is identical to existing (model sent duplicate) - if (existingContent === newContent) { - console.log(`[updateNode] ERROR - new content identical to existing (${existingContent.length} chars). Model should NOT call updateNode again.`); + if (existingNotes === newNotes) { + console.log(`[updateNode] ERROR - new content identical to existing (${existingNotes.length} chars). Model should NOT call updateNode again.`); return { success: false, - error: 'Content already up to date - do not call updateNode again. Move to next step.', + error: 'Notes already up to date - do not call updateNode again. Move to next step.', data: null }; } // Detect if adding a section that already exists (e.g., ## Integration Analysis) - const newSectionMatch = newContent.match(/^##\s+(.+)$/m); - if (newSectionMatch && existingContent) { + const newSectionMatch = newNotes.match(/^##\s+(.+)$/m); + if (newSectionMatch && existingNotes) { const sectionHeader = newSectionMatch[0]; // e.g., "## Integration Analysis" - if (existingContent.includes(sectionHeader)) { + if (existingNotes.includes(sectionHeader)) { console.log(`[updateNode] ERROR - Section "${sectionHeader}" already exists in node`); return { success: false, @@ -57,19 +57,19 @@ export const updateNodeTool = tool({ } // Detect if model included existing content + new content - if (existingContent && newContent.startsWith(existingContent)) { + if (existingNotes && newNotes.startsWith(existingNotes)) { // Extract only the new part - const actualNewContent = newContent.substring(existingContent.length).trim(); - console.log(`[updateNode] Model included existing content - extracting new part only (${actualNewContent.length} chars)`); - const separator = existingContent.endsWith('\n\n') ? '' : '\n\n'; - updates.content = `${existingContent}${separator}${actualNewContent}`; - } else if (existingContent) { + const actualNewNotes = newNotes.substring(existingNotes.length).trim(); + console.log(`[updateNode] Model included existing content - extracting new part only (${actualNewNotes.length} chars)`); + const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n'; + updates.notes = `${existingNotes}${separator}${actualNewNotes}`; + } else if (existingNotes) { // Normal append - const separator = existingContent.endsWith('\n\n') ? '' : '\n\n'; - updates.content = `${existingContent}${separator}${newContent}`; - console.log(`[updateNode] Appended content: ${existingContent.length} + ${newContent.length} = ${updates.content.length} chars`); + const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n'; + updates.notes = `${existingNotes}${separator}${newNotes}`; + console.log(`[updateNode] Appended notes: ${existingNotes.length} + ${newNotes.length} = ${updates.notes.length} chars`); } else { - console.log(`[updateNode] No existing content, using new content as-is (${newContent.length} chars)`); + console.log(`[updateNode] No existing content, using new content as-is (${newNotes.length} chars)`); } } } @@ -79,7 +79,7 @@ export const updateNodeTool = tool({ // Call the nodes API endpoint const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes/${id}`, { method: 'PUT', - headers: { 'Content-Type': 'application/json' }, + headers: { 'Notes-Type': 'application/json' }, body: JSON.stringify(updates) }); diff --git a/src/tools/other/paperExtract.ts b/src/tools/other/paperExtract.ts index 3671aaf..463fcbf 100644 --- a/src/tools/other/paperExtract.ts +++ b/src/tools/other/paperExtract.ts @@ -86,13 +86,13 @@ export const paperExtractTool = tool({ }; } - let result: { success: boolean; content?: string; chunk?: string; metadata?: any; error?: string }; + let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string }; try { const extractionResult = await extractPaper(url); result = { success: true, - content: extractionResult.content, + notes: extractionResult.content, chunk: extractionResult.chunk, metadata: { title: extractionResult.metadata.title, @@ -110,7 +110,7 @@ export const paperExtractTool = tool({ }; } - if (!result.success || (!result.content && !result.chunk)) { + if (!result.success || (!result.notes && !result.chunk)) { return { success: false, error: result.error || 'Failed to extract PDF content', @@ -123,7 +123,7 @@ export const paperExtractTool = tool({ // Step 2: AI Analysis for enhanced metadata const aiAnalysis = await analyzeContentWithAI( result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`, - result.content?.substring(0, 2000) || 'PDF document content', + result.notes?.substring(0, 2000) || 'PDF document content', 'pdf' ); @@ -143,17 +143,17 @@ export const paperExtractTool = tool({ headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title: nodeTitle, - content: enhancedDescription, + notes: enhancedDescription, link: url, dimensions: trimmedDimensions, - chunk: result.chunk || result.content, + chunk: result.chunk || result.notes, metadata: { source: 'pdf', hostname: new URL(url).hostname, author: result.metadata?.author || result.metadata?.info?.Author, pages: result.metadata?.pages, file_size: result.metadata?.file_size, - content_length: (result.chunk || result.content)?.length, + content_length: (result.chunk || result.notes)?.length, extraction_method: result.metadata?.extraction_method || 'python_pdfplumber', ai_analysis: aiAnalysis?.reasoning, enhanced_description: enhancedDescription, @@ -187,7 +187,7 @@ export const paperExtractTool = tool({ data: { nodeId: createResult.data?.id, title: nodeTitle, - contentLength: (result.chunk || result.content || '').length, + contentLength: (result.chunk || result.notes || '').length, url: url, dimensions: actualDimensions } diff --git a/src/tools/other/websiteExtract.ts b/src/tools/other/websiteExtract.ts index f293e09..7085b05 100644 --- a/src/tools/other/websiteExtract.ts +++ b/src/tools/other/websiteExtract.ts @@ -76,13 +76,13 @@ export const websiteExtractTool = tool({ }; } - let result: { success: boolean; content?: string; chunk?: string; metadata?: any; error?: string }; + let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string }; try { const extractionResult = await extractWebsite(url); result = { success: true, - content: extractionResult.content, + notes: extractionResult.content, chunk: extractionResult.chunk, metadata: { title: extractionResult.metadata.title, @@ -101,7 +101,7 @@ export const websiteExtractTool = tool({ }; } - if (!result.success || (!result.content && !result.chunk)) { + if (!result.success || (!result.notes && !result.chunk)) { return { success: false, error: result.error || 'Failed to extract website content', @@ -114,7 +114,7 @@ export const websiteExtractTool = tool({ // Step 2: AI Analysis for enhanced metadata const aiAnalysis = await analyzeContentWithAI( result.metadata?.title || `Website: ${new URL(url).hostname}`, - result.content?.substring(0, 2000) || 'Website content', + result.notes?.substring(0, 2000) || 'Website content', 'website' ); @@ -134,16 +134,16 @@ export const websiteExtractTool = tool({ headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title: nodeTitle, - content: enhancedDescription, + notes: enhancedDescription, link: url, dimensions: trimmedDimensions, - chunk: result.chunk || result.content, + chunk: result.chunk || result.notes, metadata: { source: 'website', hostname: new URL(url).hostname, author: result.metadata?.author, published_date: result.metadata?.published_date || result.metadata?.date, - content_length: (result.chunk || result.content)?.length, + content_length: (result.chunk || result.notes)?.length, extraction_method: result.metadata?.extraction_method || 'python_beautifulsoup', ai_analysis: aiAnalysis?.reasoning, enhanced_description: enhancedDescription, @@ -177,7 +177,7 @@ export const websiteExtractTool = tool({ data: { nodeId: createResult.data?.id, title: nodeTitle, - contentLength: (result.chunk || result.content || '').length, + contentLength: (result.chunk || result.notes || '').length, url: url, dimensions: actualDimensions } diff --git a/src/tools/other/youtubeExtract.ts b/src/tools/other/youtubeExtract.ts index f309772..468eff8 100644 --- a/src/tools/other/youtubeExtract.ts +++ b/src/tools/other/youtubeExtract.ts @@ -130,14 +130,14 @@ export const youtubeExtractTool = tool({ }; } - let result: { success: boolean; content?: string; chunk?: string; metadata?: any; error?: string }; + let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string }; console.log('📝 Using TypeScript yt-dlp extractor'); try { const extractionResult = await extractYouTube(url); result = { success: extractionResult.success, - content: extractionResult.content, + notes: extractionResult.content, chunk: extractionResult.chunk, metadata: { video_title: extractionResult.metadata.video_title, @@ -159,7 +159,7 @@ export const youtubeExtractTool = tool({ }; } - if (!result.success || (!result.content && !result.chunk)) { + if (!result.success || (!result.notes && !result.chunk)) { return { success: false, error: result.error || 'Failed to extract YouTube content', @@ -178,7 +178,7 @@ export const youtubeExtractTool = tool({ // Step 3: Create node with extracted content and AI analysis const nodeTitle = title || result.metadata?.video_title || `YouTube Video ${url.split('/').pop()?.split('?')[0]}`; - const transcriptSummary = await summariseTranscript(nodeTitle, result.chunk || result.content || ''); + const transcriptSummary = await summariseTranscript(nodeTitle, result.chunk || result.notes || ''); const content = transcriptSummary || aiAnalysis?.enhancedDescription || `YouTube video by ${result.metadata?.channel_name || 'Unknown Channel'}`; const suppliedDimensions = Array.isArray(dimensions) ? dimensions : []; @@ -196,7 +196,7 @@ export const youtubeExtractTool = tool({ content, link: url, dimensions: trimmedDimensions, - chunk: result.chunk || result.content, + chunk: result.chunk || result.notes, metadata: { source: 'youtube', video_id: result.metadata?.video_id, @@ -239,7 +239,7 @@ export const youtubeExtractTool = tool({ data: { nodeId: createResult.data?.id, title: nodeTitle, - contentLength: (result.chunk || result.content || '').length, + contentLength: (result.chunk || result.notes || '').length, url: url, dimensions: actualDimensions } diff --git a/src/types/database.ts b/src/types/database.ts index 3b068fe..5b9a099 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -3,16 +3,16 @@ export interface Node { id: number; title: string; description?: string; - content?: string; // Consolidated content from description + abstract + notes + notes?: string; // Consolidated content from description + abstract + notes link?: string; - type?: string; + event_date?: string; dimensions: string[]; // Flexible dimensions replacing type + stage + segment + tags embedding?: Buffer; // Node-level embedding (BLOB data) chunk?: string; metadata?: any; // Flexible metadata storage from extras + chunk_status + sub_type created_at: string; updated_at: string; - is_pinned?: number; // Legacy pin flag (read-only, slated for removal) + // Legacy pin flag (read-only, slated for removal) edge_count?: number; // Derived count of edges, included in some queries // Optional embedding fields (restored from migration)