SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: final schema pass — sync 9 schema changes from main repo

Browse Source 
- Rename nodes.content → nodes.notes across all services, tools, API routes, and UI
- Add dimensions.icon column support (GET/POST routes, types)
- Add nodes.event_date column (create/update flows, types)
- Drop nodes.type and nodes.is_pinned references
- Drop edges.user_feedback references
- Drop chat_memory_state table (replaced with DROP IF EXISTS in migration)
- Update schema guide files (system/schema.md)
- Add runtime migration block in sqlite-client.ts for existing databases
- Fix MCP tool schemas (external API keeps 'content' param, maps to 'notes' internally)
- Update standalone MCP server (nodeService.js, sqlite-client.js, index.js)
- Update HTTP MCP server (server.js, stdio-server.js)
- Update all extraction tools (paper, website, youtube)
- Update all UI components (FocusPanel, ThreePanelLayout, GridView, ListView, FolderViewOverlay)
- TypeScript: 0 errors

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
“BeeRad”
2026-02-15 12:53:52 +11:00
co-authored by Claude Opus 4.6
parent 1d323de11a
commit 86f43c9975
36 changed files with 303 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/focus/FocusPanel.tsx
+11 -11
View File
@@ -382,7 +382,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
try {
const updateData: Record<string, string> = {};
if (editingField === 'content') {
if (editingField === 'notes') {
updateData.content = editingValue;
} else {
updateData[editingField] = editingValue;
@@ -406,7 +406,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
}
// Safety net: ensure edges exist for any tokens present in saved content
if (editingField === 'content' && typeof editingValue === 'string') {
if (editingField === 'notes' && typeof editingValue === 'string') {
try {
const tokens = parseNodeMarkers(editingValue);
const uniqueTargets = Array.from(new Set(tokens.map(t => t.id))).filter(id => id !== nodeId);
@@ -448,7 +448,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
const response = await fetch(`/api/nodes/${nodeId}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ content: contentValue }),
body: JSON.stringify({ notes: contentValue }),
});
if (!response.ok) throw new Error('Failed to save');
const result = await response.json();
@@ -568,7 +568,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
const response = await fetch(`/api/nodes/${activeTab}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ content: notesEditValue }),
body: JSON.stringify({ notes: notesEditValue }),
});
if (!response.ok) throw new Error('Failed to save');
const result = await response.json();
@@ -614,7 +614,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
// Start editing notes
const startNotesEdit = () => {
if (!activeTab || !nodesData[activeTab]) return;
setNotesEditValue(nodesData[activeTab].content || '');
setNotesEditValue(nodesData[activeTab].notes || '');
setNotesEditMode(true);
};
@@ -1023,15 +1023,15 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
const embedContent = async (nodeId: number) => {
const node = nodesData[nodeId];
const hasContent = node?.content?.trim();
const hasNotes = node?.notes?.trim();
const hasChunk = node?.chunk?.trim();
// If chunk is empty but content exists, auto-populate chunk from content
if (!hasChunk && hasContent) {
if (!hasChunk && hasNotes) {
try {
const response = await fetch(`/api/nodes/${nodeId}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ chunk: hasContent })
body: JSON.stringify({ chunk: hasNotes })
});
if (response.ok) {
@@ -1046,7 +1046,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
}
}
// If neither content nor chunk exist, require content
if (!hasContent && !hasChunk) {
if (!hasNotes && !hasChunk) {
startEdit('content', '');
return;
}
@@ -2722,7 +2722,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
</div>
)}
</div>
) : nodesData[activeTab]?.content ? (
) : nodesData[activeTab]?.notes ? (
<div
style={{
color: '#e5e5e5',
@@ -2734,7 +2734,7 @@ export default function FocusPanel({ openTabs, activeTab, onTabSelect, onNodeCli
}}
>
<MarkdownWithNodeTokens
content={nodesData[activeTab].content}
content={nodesData[activeTab].notes || ''}
onNodeClick={onNodeClick || onTabSelect}
/>
</div>
src/components/layout/ThreePanelLayout.tsx
+1 -1
View File
@@ -149,7 +149,7 @@ export default function ThreePanelLayout() {
id: node.id,
title: node.title,
link: node.link,
content: node.content,
notes: node.notes,
dimensions: node.dimensions,
created_at: node.created_at,
updated_at: node.updated_at,
src/components/nodes/FolderViewOverlay.tsx
+8 -8
View File
@@ -1240,7 +1240,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o
</span>
)}
</div>
{node.content && (
{node.notes && (
<div style={{
fontSize: '12px',
color: '#94a3b8',
@@ -1251,7 +1251,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o
WebkitBoxOrient: 'vertical',
fontWeight: 400
}}>
{getContentPreview(node.content)}
{getContentPreview(node.notes)}
</div>
)}
{node.dimensions && node.dimensions.length > 0 && (
@@ -1376,7 +1376,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o
{node.title || 'Untitled'}
</div>
{node.content && (
{node.notes && (
<div style={{
fontSize: '12px',
color: '#94a3b8',
@@ -1387,7 +1387,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o
WebkitBoxOrient: 'vertical',
overflow: 'hidden'
}}>
{getContentPreview(node.content)}
{getContentPreview(node.notes)}
</div>
)}
@@ -1846,7 +1846,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o
{node.title || 'Untitled'}
</div>
{node.content && (
{node.notes && (
<div style={{
fontSize: '12px',
color: '#94a3b8',
@@ -1857,7 +1857,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o
WebkitBoxOrient: 'vertical',
overflow: 'hidden'
}}>
{getContentPreview(node.content)}
{getContentPreview(node.notes)}
</div>
)}
@@ -2862,7 +2862,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o
{node.title || 'Untitled'}
</div>
</div>
{node.content && (
{node.notes && (
<div style={{
fontSize: '11px',
color: '#666',
@@ -2872,7 +2872,7 @@ export default function FolderViewOverlay({ onClose, onNodeOpen, refreshToken, o
WebkitLineClamp: 2,
WebkitBoxOrient: 'vertical'
}}>
{getContentPreview(node.content)}
{getContentPreview(node.notes)}
</div>
)}
{node.dimensions && node.dimensions.length > 1 && (
src/components/views/GridView.tsx
+2 -2
View File
@@ -104,7 +104,7 @@ export default function GridView({ nodes, onNodeClick }: GridViewProps) {
</div>
{/* Description or Content Preview */}
{(node.description || node.content) && (
{(node.description || node.notes) && (
<div style={{
flex: 1,
fontSize: '11px',
@@ -116,7 +116,7 @@ export default function GridView({ nodes, onNodeClick }: GridViewProps) {
WebkitBoxOrient: 'vertical',
marginBottom: '10px'
}}>
{node.description || truncateContent(node.content)}
{node.description || truncateContent(node.notes)}
</div>
)}
src/components/views/ListView.tsx
+2 -2
View File
@@ -105,7 +105,7 @@ export default function ListView({ nodes, onNodeClick }: ListViewProps) {
</div>
{/* Description or Content Preview */}
{(node.description || node.content) && (
{(node.description || node.notes) && (
<div style={{
fontSize: '12px',
color: '#666',
@@ -116,7 +116,7 @@ export default function ListView({ nodes, onNodeClick }: ListViewProps) {
WebkitBoxOrient: 'vertical',
overflow: 'hidden'
}}>
{node.description || truncateContent(node.content)}
{node.description || truncateContent(node.notes)}
</div>
)}
src/config/guides/system/schema.md
+32 -13
View File
@@ -14,13 +14,12 @@ immutable: true
| id | INTEGER | Primary key, auto-increment |
| title | TEXT | Required |
| description | TEXT | AI-generated grounding context (~1 sentence) |
| content | TEXT | User's notes/thoughts (not source content) |
| notes | TEXT | User's notes/thoughts (not source content) |
| chunk | TEXT | Full verbatim source content |
| chunk_status | TEXT | 'pending', 'chunked', 'failed' |
| link | TEXT | External URL (only for nodes representing external content) |
| type | TEXT | Nullable (reserved for future use) |
| event_date | TEXT | When the content happened (ISO date) — distinct from created_at |
| metadata | TEXT | JSON blob (map_position, transcript_length, etc.) |
| is_pinned | INTEGER | Legacy — use hub node queries instead |
| created_at | TEXT | ISO timestamp |
| updated_at | TEXT | ISO timestamp |
@@ -38,29 +37,40 @@ immutable: true
### dimensions
| Column | Type | Notes |
|--------|------|-------|
| id | INTEGER | Primary key |
| name | TEXT | Unique, case-insensitive |
| name | TEXT | Primary key |
| description | TEXT | Purpose description |
| is_locked | INTEGER | 1 = priority dimension (auto-assigns to new nodes) |
| icon | TEXT | Emoji or icon identifier for UI display |
| is_priority | INTEGER | 1 = priority dimension (auto-assigns to new nodes) |
| updated_at | TEXT | ISO timestamp |
### node_dimensions (junction)
| Column | Type |
|--------|------|
| node_id | INTEGER FK → nodes.id |
| dimension_id | INTEGER FK → dimensions.id |
| dimension | TEXT (dimension name) |
### chunks (for semantic search)
| Column | Type | Notes |
|--------|------|-------|
| id | INTEGER | Primary key |
| node_id | INTEGER | FK → nodes.id |
| chunk_index | INTEGER | Position in sequence |
| chunk_idx | INTEGER | Position in sequence |
| text | TEXT | Chunk content |
| embedding | BLOB | Vector (via sqlite-vec) |
| created_at | TEXT | ISO timestamp |
| embedding_type | TEXT | Embedding model used |
| metadata | TEXT | JSON blob |
### voice_usage (daily tracking)
| Column | Type | Notes |
|--------|------|-------|
| id | INTEGER | Primary key |
| date | TEXT | ISO date (UNIQUE) |
| minutes_used | REAL | Minutes consumed |
| updated_at | TEXT | ISO timestamp |
### FTS Tables
- `chunks_fts` — full-text search on chunk text
- `nodes_fts` — full-text search on node title + content
- `nodes_fts` — full-text search on node title + notes
## Common Query Patterns
@@ -76,8 +86,7 @@ GROUP BY n.id ORDER BY edge_count DESC LIMIT 5
```sql
SELECT n.* FROM nodes n
JOIN node_dimensions nd ON n.id = nd.node_id
JOIN dimensions d ON nd.dimension_id = d.id
WHERE d.name = ?
WHERE nd.dimension = ?
```
**Edges for a node (both directions):**
@@ -89,4 +98,14 @@ JOIN nodes n2 ON e.to_node_id = n2.id
WHERE e.from_node_id = ? OR e.to_node_id = ?
```
**Use sqliteQuery for any read operation not covered by structured tools.**
**Search source content (chunks):**
```sql
SELECT c.id, c.node_id, c.chunk_idx, c.text, n.title
FROM chunks c
JOIN nodes n ON c.node_id = n.id
WHERE c.text LIKE '%search term%' COLLATE NOCASE
ORDER BY c.chunk_idx ASC
LIMIT 10
```
**Use `rah_search_content` to search chunks by keyword, or `rah_sqlite_query` for any read operation not covered by structured tools.**
src/services/context/autoContext.ts
+1 -1
View File
@@ -27,7 +27,7 @@ function fetchAutoContextRows(limit: number): AutoContextSummary[] {
FROM nodes n
LEFT JOIN edges e
ON (e.from_node_id = n.id OR e.to_node_id = n.id)
WHERE n.type IS NULL OR n.type != 'memory'
WHERE 1=1
GROUP BY n.id
ORDER BY edge_count DESC, n.updated_at DESC
LIMIT ?
src/services/database/descriptionService.ts
+5 -10
View File
@@ -4,7 +4,7 @@ import { hasValidOpenAiKey } from '../storage/apiKeys';
export interface DescriptionInput {
title: string;
content?: string;
notes?: string;
link?: string;
metadata?: {
source?: string;
@@ -12,7 +12,6 @@ export interface DescriptionInput {
author?: string;
site_name?: string;
};
type?: string;
dimensions?: string[];
}
@@ -24,7 +23,7 @@ export { hasValidOpenAiKey } from '../storage/apiKeys';
* Used when no API key is available or for simple inputs.
*/
export function generateFallbackDescription(input: DescriptionInput): string {
const { title, type, metadata, dimensions } = input;
const { title, metadata, dimensions } = input;
// Build a contextual fallback
const parts: string[] = [];
@@ -33,10 +32,6 @@ export function generateFallbackDescription(input: DescriptionInput): string {
parts.push(`By ${metadata.author || metadata.channel_name}`);
}
if (type) {
parts.push(type.charAt(0).toUpperCase() + type.slice(1));
}
if (dimensions?.length) {
parts.push(`in ${dimensions.slice(0, 2).join(', ')}`);
}
@@ -63,7 +58,7 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
}
// Fast path: skip AI for very short inputs (likely just notes)
if (!input.content && !input.link && input.title.length < 30) {
if (!input.notes && !input.link && input.title.length < 30) {
console.log(`[DescriptionService] Short input, using fallback for: "${input.title}"`);
return generateFallbackDescription(input);
}
@@ -135,8 +130,8 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
if (publisherHint) lines.push(`Publisher hint: ${publisherHint}`);
lines.push(`Likely user-authored: ${likelyUserAuthored ? 'yes' : 'no'}`);
const contentPreview = input.content?.slice(0, 800) || '';
if (contentPreview) lines.push(`Content: ${contentPreview}${input.content && input.content.length > 800 ? '...' : ''}`);
const contentPreview = input.notes?.slice(0, 800) || '';
if (contentPreview) lines.push(`Notes: ${contentPreview}${input.notes && input.notes.length > 800 ? '...' : ''}`);
return `Your job is to answer: "what is this?" in one short line. Max 280 characters.
src/services/database/dimensionService.ts
+6 -6
View File
@@ -55,7 +55,7 @@ export class DimensionService {
*/
static async assignDimensions(nodeData: {
title: string;
content?: string;
notes?: string;
link?: string;
description?: string;
}): Promise<{ locked: string[]; keywords: string[] }> {
@@ -156,19 +156,19 @@ export class DimensionService {
* Build AI prompt for dimension assignment (locked dimensions only)
*/
private static buildAssignmentPrompt(
nodeData: { title: string; content?: string; link?: string; description?: string },
nodeData: { title: string; notes?: string; link?: string; description?: string },
lockedDimensions: LockedDimension[]
): string {
// Use description as primary context, content as fallback
let nodeContextSection: string;
if (nodeData.description) {
const contentPreview = nodeData.content?.slice(0, 500) || '';
const contentPreview = nodeData.notes?.slice(0, 500) || '';
nodeContextSection = `DESCRIPTION: ${nodeData.description}
CONTENT PREVIEW: ${contentPreview}${nodeData.content && nodeData.content.length > 500 ? '...' : ''}`;
NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`;
} else {
const contentPreview = nodeData.content?.slice(0, 2000) || '';
nodeContextSection = `CONTENT: ${contentPreview}${nodeData.content && nodeData.content.length > 2000 ? '...' : ''}`;
const contentPreview = nodeData.notes?.slice(0, 2000) || '';
nodeContextSection = `NOTES: ${contentPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`;
}
// Include ALL locked dimensions, using fallback text for those without descriptions
src/services/database/edges.ts
+5 -5
View File
@@ -430,9 +430,9 @@ export class EdgeService {
ELSE n_from.title
END as connected_node_title,
CASE
WHEN e.from_node_id = ? THEN n_to.content
ELSE n_from.content
END as connected_node_content,
WHEN e.from_node_id = ? THEN n_to.notes
ELSE n_from.notes
END as connected_node_notes,
CASE
WHEN e.from_node_id = ? THEN n_to.link
ELSE n_from.link
@@ -499,7 +499,7 @@ export class EdgeService {
const connected_node: Node = {
id: row.connected_node_id,
title: row.connected_node_title,
content: row.connected_node_content,
notes: row.connected_node_notes,
link: row.connected_node_link,
dimensions: row.connected_node_dimensions,
embedding: undefined, // Not needed for display
@@ -544,7 +544,7 @@ export class EdgeService {
const connected_node: Node = {
id: row.connected_node_id,
title: row.connected_node_title,
content: row.connected_node_content,
notes: row.connected_node_notes,
link: row.connected_node_link,
dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'),
embedding: undefined, // Not needed for display
src/services/database/nodes.ts
+12 -12
View File
@@ -15,7 +15,7 @@ export class NodeService {
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
let query = `
SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -38,7 +38,7 @@ export class NodeService {
// Text search in title, description, and content (SQLite LIKE with COLLATE NOCASE)
if (search) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.content LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
}
@@ -50,7 +50,7 @@ export class NodeService {
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
CASE WHEN n.content LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
n.updated_at DESC`;
params.push(
search, // Exact match (case-insensitive)
@@ -95,7 +95,7 @@ export class NodeService {
private async getNodeByIdSQLite(id: number): Promise<Node | null> {
const sqlite = getSQLiteClient();
const query = `
SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -125,9 +125,9 @@ export class NodeService {
const {
title,
description,
content,
notes,
link,
type,
event_date,
dimensions = [],
chunk,
chunk_status,
@@ -139,14 +139,14 @@ export class NodeService {
const nodeId = sqlite.transaction(() => {
// Insert node using prepare/run for lastInsertRowid access
const nodeResult = sqlite.prepare(`
INSERT INTO nodes (title, description, content, link, type, metadata, chunk, chunk_status, created_at, updated_at)
INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, chunk_status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
title,
description ?? null,
content ?? null,
notes ?? null,
link ?? null,
type ?? null,
event_date ?? null,
JSON.stringify(metadata),
chunk ?? null,
chunk_status ?? null,
@@ -192,7 +192,7 @@ export class NodeService {
// PostgreSQL path removed in SQLite-only consolidation
private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
const { title, description, content, link, type, dimensions, chunk, metadata } = updates;
const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates;
const now = new Date().toISOString();
const sqlite = getSQLiteClient();
@@ -211,9 +211,9 @@ export class NodeService {
if (title !== undefined) { setFields.push('title = ?'); params.push(title); }
if (description !== undefined) { setFields.push('description = ?'); params.push(description); }
if (content !== undefined) { setFields.push('content = ?'); params.push(content); }
if (notes !== undefined) { setFields.push('notes = ?'); params.push(notes); }
if (link !== undefined) { setFields.push('link = ?'); params.push(link); }
if (type !== undefined) { setFields.push('type = ?'); params.push(type); }
if (event_date !== undefined) { setFields.push('event_date = ?'); params.push(event_date); }
if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); }
if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) {
setFields.push('chunk_status = ?');
src/services/database/sqlite-client.ts
+78 -11
View File
@@ -266,7 +266,7 @@ class SQLiteClient {
}
};
ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;");
ensureNodeCol('type', "ALTER TABLE nodes ADD COLUMN type TEXT;");
// type column removed in final schema pass
} catch (nodeErr) {
console.warn('Failed to ensure nodes columns:', nodeErr);
}
@@ -435,16 +435,8 @@ class SQLiteClient {
}
// Do not recreate memory_v; alias has been removed.
// 6) Chat memory state tracking for chat-triggered memories
this.db.exec(`
CREATE TABLE IF NOT EXISTS chat_memory_state (
thread_id TEXT PRIMARY KEY,
helper_name TEXT,
last_processed_chat_id INTEGER DEFAULT 0,
last_processed_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_chat_memory_thread ON chat_memory_state(thread_id);
`);
// 6) Drop orphaned chat_memory_state table (removed in final schema pass)
this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`);
// Agent delegation table for orchestrator/worker coordination
try {
@@ -631,6 +623,81 @@ class SQLiteClient {
}
}
// 10) Final schema pass migrations (content→notes, event_date, icon, drop dead columns)
try {
const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
const nodeColNames = nodeCols2.map((c: any) => c.name);
// Rename content → notes
if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) {
console.log('Renaming nodes.content → nodes.notes');
this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;');
}
// Add event_date with backfill from metadata
if (!nodeColNames.includes('event_date')) {
console.log('Adding nodes.event_date column');
this.db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;');
this.db.exec(`
UPDATE nodes SET event_date = json_extract(metadata, '$.published_date')
WHERE metadata IS NOT NULL
AND json_extract(metadata, '$.published_date') IS NOT NULL
AND json_extract(metadata, '$.published_date') != '';
`);
}
// Add dimensions.icon
const dimCols2 = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
if (!dimCols2.some((c: any) => c.name === 'icon')) {
console.log('Adding dimensions.icon column');
this.db.exec('ALTER TABLE dimensions ADD COLUMN icon TEXT;');
}
// Drop dead columns (SQLite 3.35+)
if (nodeColNames.includes('type')) {
console.log('Dropping nodes.type column');
try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_type;'); } catch {}
try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch (e) {
console.warn('Could not drop nodes.type (SQLite < 3.35?):', e);
}
}
if (nodeColNames.includes('is_pinned')) {
console.log('Dropping nodes.is_pinned column');
try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_pinned;'); } catch {}
try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch (e) {
console.warn('Could not drop nodes.is_pinned:', e);
}
}
// Drop edges.user_feedback
const edgeCols = this.db.prepare('PRAGMA table_info(edges)').all() as Array<{ name: string }>;
if (edgeCols.some((c: any) => c.name === 'user_feedback')) {
console.log('Dropping edges.user_feedback column');
try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch (e) {
console.warn('Could not drop edges.user_feedback:', e);
}
}
// Rebuild FTS if it references 'content' instead of 'notes'
try {
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as any;
if (ftsCheck && ftsCheck.sql && ftsCheck.sql.includes('content')) {
console.log('Rebuilding nodes_fts to reference notes instead of content');
this.db.exec('DROP TABLE IF EXISTS nodes_fts;');
this.db.exec(`
CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content=nodes, content_rowid=id);
INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');
`);
}
} catch (ftsErr) {
console.warn('FTS rebuild skipped:', ftsErr);
}
console.log('Final schema pass migrations complete');
} catch (schemaErr) {
console.warn('Final schema pass migration error:', schemaErr);
}
console.log('Logging + memory schema ensured');
} catch (error) {
console.error('Failed to ensure logging/memory schema:', error);
src/services/settings/autoContextSettings.ts
+1 -1
View File
@@ -68,7 +68,7 @@ function bootstrapFromLegacyPins(): void {
try {
const db = getSQLiteClient();
const countRow = db
.query<{ count: number }>('SELECT COUNT(*) as count FROM nodes WHERE is_pinned = 1')
.query<{ count: number }>('SELECT COUNT(*) as count FROM nodes WHERE 1=0 /* /* is_pinned removed */ removed */')
.rows[0];
const pinnedCount = Number(countRow?.count ?? 0);
if (pinnedCount > 0) {
src/services/typescript/embed-nodes.ts
+4 -4
View File
@@ -16,7 +16,7 @@ import {
interface NodeRecord {
id: number;
title: string;
content: string | null;
notes: string | null;
description: string | null;
dimensions_json: string;
embedding?: Buffer | null;
@@ -58,7 +58,7 @@ export class NodeEmbedder {
const prompt = `Analyze this content and provide 2-3 key insights or themes in a concise paragraph (max 100 words):
Title: ${node.title}
Content: ${node.content || 'No content'}
Content: ${node.notes || 'No content'}
Dimensions: ${dimensionsText}
Focus on the main concepts, key relationships, and practical implications.`;
@@ -106,13 +106,13 @@ Focus on the main concepts, key relationships, and practical implications.`;
// Create base embedding text
let embeddingText = formatEmbeddingText(
node.title,
node.content || '',
node.notes || '',
dimensions,
node.description
);
// Add AI analysis if content exists
if (node.content && node.content.trim().length > 0) {
if (node.notes && node.notes.trim().length > 0) {
const analysis = await this.analyzeNodeWithAI(node);
if (analysis) {
embeddingText += `\n\nAI Analysis: ${analysis}`;
src/tools/database/createNode.ts
+1 -1
View File
@@ -6,7 +6,7 @@ export const createNodeTool = tool({
description: 'Create node with title/content/link and optional dimensions (locked dimensions auto-assigned)',
inputSchema: z.object({
title: z.string().describe('The title of the node'),
content: z.string().optional().describe('The main content, description, or notes for this node'),
notes: z.string().optional().describe('The main notes, description, or notes for this node'),
link: z.string().optional().describe('A URL link to the source'),
dimensions: z
.array(z.string())
src/tools/database/getNodesById.ts
+5 -5
View File
@@ -6,9 +6,9 @@ export const getNodesByIdTool = tool({
description: 'Load full node records by IDs',
inputSchema: z.object({
nodeIds: z.array(z.number().int().positive()).min(1).max(10).describe('List of node IDs to load'),
includeContentPreview: z.boolean().default(true).describe('Whether to return a trimmed content preview for each node'),
includeNotesPreview: z.boolean().default(true).describe('Whether to return a trimmed content preview for each node'),
}),
execute: async ({ nodeIds, includeContentPreview }) => {
execute: async ({ nodeIds, includeNotesPreview }) => {
const uniqueIds = Array.from(new Set(nodeIds.filter(id => Number.isFinite(id) && id > 0)));
if (uniqueIds.length === 0) {
return {
@@ -23,8 +23,8 @@ export const getNodesByIdTool = tool({
try {
const node = await nodeService.getNodeById(id);
if (!node) return null;
const preview = includeContentPreview
? (node.content || node.description || '')
const preview = includeNotesPreview
? (node.notes || node.description || '')
.split(/\s+/)
.slice(0, 80)
.join(' ')
@@ -38,7 +38,7 @@ export const getNodesByIdTool = tool({
dimensions: node.dimensions || [],
chunk_status: node.chunk_status || 'unknown',
updated_at: node.updated_at,
content_preview: preview || null,
notes_preview: preview || null,
metadata: node.metadata ?? null,
};
} catch (error) {
src/tools/database/queryDimensionNodes.ts
+4 -4
View File
@@ -45,11 +45,11 @@ export const queryDimensionNodesTool = tool({
dimensions: node.dimensions || [],
};
if (includeContent && node.content) {
if (includeContent && node.notes) {
// Truncate to ~100 chars
formatted.contentPreview = node.content.length > 100
? node.content.substring(0, 100) + '...'
: node.content;
formatted.notesPreview = node.notes.length > 100
? node.notes.substring(0, 100) + '...'
: node.notes;
}
return formatted;
src/tools/database/queryNodes.ts
+2 -2
View File
@@ -5,11 +5,11 @@ import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import type { Node } from '@/types/database';
export const queryNodesTool = tool({
description: 'Search nodes by title/content/dimensions',
description: 'Search nodes by title/notes/dimensions',
inputSchema: z.object({
filters: z.object({
dimensions: z.array(z.string()).describe('Filter by dimensions (e.g., ["research", "ai", "technology"]). Replaces old type/stage filtering.').optional(),
search: z.string().describe('Search term to match against title or content').optional(),
search: z.string().describe('Search term to match against title or notes').optional(),
limit: z.number().min(1).max(50).default(10).describe('Maximum number of results to return')
}).optional()
}),
src/tools/database/updateEdge.ts
+2 -6
View File
@@ -3,13 +3,12 @@ import { z } from 'zod';
import { edgeService } from '@/services/database/edges';
export const updateEdgeTool = tool({
description: 'Update edge context/source/feedback',
description: 'Update edge context/source',
inputSchema: z.object({
edge_id: z.number().describe('The ID of the edge to update'),
updates: z.object({
context: z.record(z.any()).optional().describe('Updated context information for this edge - can include explanation, relationship type, strength, notes, etc.'),
source: z.enum(['user', 'ai_similarity', 'helper_name']).optional().describe('Updated source classification for this edge'),
user_feedback: z.boolean().optional().describe('User feedback on this edge connection (true = positive, false = negative)')
}).describe('Fields to update on the edge')
}),
execute: async (params) => {
@@ -46,10 +45,7 @@ export const updateEdgeTool = tool({
const updateDescriptions = [];
if (cleanUpdates.context) updateDescriptions.push('context');
if (cleanUpdates.source) updateDescriptions.push(`source to ${cleanUpdates.source}`);
if (cleanUpdates.user_feedback !== undefined) {
updateDescriptions.push(`user feedback to ${cleanUpdates.user_feedback ? 'positive' : 'negative'}`);
}
return {
success: true,
data: updatedEdge,
src/tools/database/updateNode.ts
+21 -21
View File
@@ -7,7 +7,7 @@ export const updateNodeTool = tool({
id: z.number().describe('The ID of the node to update'),
updates: z.object({
title: z.string().optional().describe('New title'),
content: z.string().optional().describe('New content/description/notes'),
notes: z.string().optional().describe('New content/description/notes'),
link: z.string().optional().describe('New link'),
dimensions: z.array(z.string()).optional().describe('New dimension tags - completely replaces existing dimensions'),
chunk: z.string().optional().describe('New chunk content'),
@@ -25,28 +25,28 @@ export const updateNodeTool = tool({
}
// FORCE APPEND for content field - fetch existing and append new content
if (updates.content) {
if (updates.notes) {
const fetchResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes/${id}`);
if (fetchResponse.ok) {
const { node } = await fetchResponse.json();
const existingContent = (node?.content || '').trim();
const newContent = updates.content.trim();
const existingNotes = (node?.notes || '').trim();
const newNotes = updates.notes.trim();
// Skip if new content is identical to existing (model sent duplicate)
if (existingContent === newContent) {
console.log(`[updateNode] ERROR - new content identical to existing (${existingContent.length} chars). Model should NOT call updateNode again.`);
if (existingNotes === newNotes) {
console.log(`[updateNode] ERROR - new content identical to existing (${existingNotes.length} chars). Model should NOT call updateNode again.`);
return {
success: false,
error: 'Content already up to date - do not call updateNode again. Move to next step.',
error: 'Notes already up to date - do not call updateNode again. Move to next step.',
data: null
};
}
// Detect if adding a section that already exists (e.g., ## Integration Analysis)
const newSectionMatch = newContent.match(/^##\s+(.+)$/m);
if (newSectionMatch && existingContent) {
const newSectionMatch = newNotes.match(/^##\s+(.+)$/m);
if (newSectionMatch && existingNotes) {
const sectionHeader = newSectionMatch[0]; // e.g., "## Integration Analysis"
if (existingContent.includes(sectionHeader)) {
if (existingNotes.includes(sectionHeader)) {
console.log(`[updateNode] ERROR - Section "${sectionHeader}" already exists in node`);
return {
success: false,
@@ -57,19 +57,19 @@ export const updateNodeTool = tool({
}
// Detect if model included existing content + new content
if (existingContent && newContent.startsWith(existingContent)) {
if (existingNotes && newNotes.startsWith(existingNotes)) {
// Extract only the new part
const actualNewContent = newContent.substring(existingContent.length).trim();
console.log(`[updateNode] Model included existing content - extracting new part only (${actualNewContent.length} chars)`);
const separator = existingContent.endsWith('\n\n') ? '' : '\n\n';
updates.content = `${existingContent}${separator}${actualNewContent}`;
} else if (existingContent) {
const actualNewNotes = newNotes.substring(existingNotes.length).trim();
console.log(`[updateNode] Model included existing content - extracting new part only (${actualNewNotes.length} chars)`);
const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n';
updates.notes = `${existingNotes}${separator}${actualNewNotes}`;
} else if (existingNotes) {
// Normal append
const separator = existingContent.endsWith('\n\n') ? '' : '\n\n';
updates.content = `${existingContent}${separator}${newContent}`;
console.log(`[updateNode] Appended content: ${existingContent.length} + ${newContent.length} = ${updates.content.length} chars`);
const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n';
updates.notes = `${existingNotes}${separator}${newNotes}`;
console.log(`[updateNode] Appended notes: ${existingNotes.length} + ${newNotes.length} = ${updates.notes.length} chars`);
} else {
console.log(`[updateNode] No existing content, using new content as-is (${newContent.length} chars)`);
console.log(`[updateNode] No existing content, using new content as-is (${newNotes.length} chars)`);
}
}
}
@@ -79,7 +79,7 @@ export const updateNodeTool = tool({
// Call the nodes API endpoint
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes/${id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
headers: { 'Notes-Type': 'application/json' },
body: JSON.stringify(updates)
});
src/tools/other/paperExtract.ts
+8 -8
View File
@@ -86,13 +86,13 @@ export const paperExtractTool = tool({
};
}
let result: { success: boolean; content?: string; chunk?: string; metadata?: any; error?: string };
let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractPaper(url);
result = {
success: true,
content: extractionResult.content,
notes: extractionResult.content,
chunk: extractionResult.chunk,
metadata: {
title: extractionResult.metadata.title,
@@ -110,7 +110,7 @@ export const paperExtractTool = tool({
};
}
if (!result.success || (!result.content && !result.chunk)) {
if (!result.success || (!result.notes && !result.chunk)) {
return {
success: false,
error: result.error || 'Failed to extract PDF content',
@@ -123,7 +123,7 @@ export const paperExtractTool = tool({
// Step 2: AI Analysis for enhanced metadata
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`,
result.content?.substring(0, 2000) || 'PDF document content',
result.notes?.substring(0, 2000) || 'PDF document content',
'pdf'
);
@@ -143,17 +143,17 @@ export const paperExtractTool = tool({
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
content: enhancedDescription,
notes: enhancedDescription,
link: url,
dimensions: trimmedDimensions,
chunk: result.chunk || result.content,
chunk: result.chunk || result.notes,
metadata: {
source: 'pdf',
hostname: new URL(url).hostname,
author: result.metadata?.author || result.metadata?.info?.Author,
pages: result.metadata?.pages,
file_size: result.metadata?.file_size,
content_length: (result.chunk || result.content)?.length,
content_length: (result.chunk || result.notes)?.length,
extraction_method: result.metadata?.extraction_method || 'python_pdfplumber',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: enhancedDescription,
@@ -187,7 +187,7 @@ export const paperExtractTool = tool({
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: (result.chunk || result.content || '').length,
contentLength: (result.chunk || result.notes || '').length,
url: url,
dimensions: actualDimensions
}
src/tools/other/websiteExtract.ts
+8 -8
View File
@@ -76,13 +76,13 @@ export const websiteExtractTool = tool({
};
}
let result: { success: boolean; content?: string; chunk?: string; metadata?: any; error?: string };
let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractWebsite(url);
result = {
success: true,
content: extractionResult.content,
notes: extractionResult.content,
chunk: extractionResult.chunk,
metadata: {
title: extractionResult.metadata.title,
@@ -101,7 +101,7 @@ export const websiteExtractTool = tool({
};
}
if (!result.success || (!result.content && !result.chunk)) {
if (!result.success || (!result.notes && !result.chunk)) {
return {
success: false,
error: result.error || 'Failed to extract website content',
@@ -114,7 +114,7 @@ export const websiteExtractTool = tool({
// Step 2: AI Analysis for enhanced metadata
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.content?.substring(0, 2000) || 'Website content',
result.notes?.substring(0, 2000) || 'Website content',
'website'
);
@@ -134,16 +134,16 @@ export const websiteExtractTool = tool({
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
content: enhancedDescription,
notes: enhancedDescription,
link: url,
dimensions: trimmedDimensions,
chunk: result.chunk || result.content,
chunk: result.chunk || result.notes,
metadata: {
source: 'website',
hostname: new URL(url).hostname,
author: result.metadata?.author,
published_date: result.metadata?.published_date || result.metadata?.date,
content_length: (result.chunk || result.content)?.length,
content_length: (result.chunk || result.notes)?.length,
extraction_method: result.metadata?.extraction_method || 'python_beautifulsoup',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: enhancedDescription,
@@ -177,7 +177,7 @@ export const websiteExtractTool = tool({
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: (result.chunk || result.content || '').length,
contentLength: (result.chunk || result.notes || '').length,
url: url,
dimensions: actualDimensions
}
src/tools/other/youtubeExtract.ts
+6 -6
View File
@@ -130,14 +130,14 @@ export const youtubeExtractTool = tool({
};
}
let result: { success: boolean; content?: string; chunk?: string; metadata?: any; error?: string };
let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string };
console.log('📝 Using TypeScript yt-dlp extractor');
try {
const extractionResult = await extractYouTube(url);
result = {
success: extractionResult.success,
content: extractionResult.content,
notes: extractionResult.content,
chunk: extractionResult.chunk,
metadata: {
video_title: extractionResult.metadata.video_title,
@@ -159,7 +159,7 @@ export const youtubeExtractTool = tool({
};
}
if (!result.success || (!result.content && !result.chunk)) {
if (!result.success || (!result.notes && !result.chunk)) {
return {
success: false,
error: result.error || 'Failed to extract YouTube content',
@@ -178,7 +178,7 @@ export const youtubeExtractTool = tool({
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.video_title || `YouTube Video ${url.split('/').pop()?.split('?')[0]}`;
const transcriptSummary = await summariseTranscript(nodeTitle, result.chunk || result.content || '');
const transcriptSummary = await summariseTranscript(nodeTitle, result.chunk || result.notes || '');
const content = transcriptSummary || aiAnalysis?.enhancedDescription || `YouTube video by ${result.metadata?.channel_name || 'Unknown Channel'}`;
const suppliedDimensions = Array.isArray(dimensions) ? dimensions : [];
@@ -196,7 +196,7 @@ export const youtubeExtractTool = tool({
content,
link: url,
dimensions: trimmedDimensions,
chunk: result.chunk || result.content,
chunk: result.chunk || result.notes,
metadata: {
source: 'youtube',
video_id: result.metadata?.video_id,
@@ -239,7 +239,7 @@ export const youtubeExtractTool = tool({
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: (result.chunk || result.content || '').length,
contentLength: (result.chunk || result.notes || '').length,
url: url,
dimensions: actualDimensions
}
src/types/database.ts
+3 -3
View File
@@ -3,16 +3,16 @@ export interface Node {
id: number;
title: string;
description?: string;
content?: string; // Consolidated content from description + abstract + notes
notes?: string; // Consolidated content from description + abstract + notes
link?: string;
type?: string;
event_date?: string;
dimensions: string[]; // Flexible dimensions replacing type + stage + segment + tags
embedding?: Buffer; // Node-level embedding (BLOB data)
chunk?: string;
metadata?: any; // Flexible metadata storage from extras + chunk_status + sub_type
created_at: string;
updated_at: string;
is_pinned?: number; // Legacy pin flag (read-only, slated for removal)
// Legacy pin flag (read-only, slated for removal)
edge_count?: number; // Derived count of edges, included in some queries
// Optional embedding fields (restored from migration)