feat: final schema pass — sync 9 schema changes from main repo
- Rename nodes.content → nodes.notes across all services, tools, API routes, and UI - Add dimensions.icon column support (GET/POST routes, types) - Add nodes.event_date column (create/update flows, types) - Drop nodes.type and nodes.is_pinned references - Drop edges.user_feedback references - Drop chat_memory_state table (replaced with DROP IF EXISTS in migration) - Update schema guide files (system/schema.md) - Add runtime migration block in sqlite-client.ts for existing databases - Fix MCP tool schemas (external API keeps 'content' param, maps to 'notes' internally) - Update standalone MCP server (nodeService.js, sqlite-client.js, index.js) - Update HTTP MCP server (server.js, stdio-server.js) - Update all extraction tools (paper, website, youtube) - Update all UI components (FocusPanel, ThreePanelLayout, GridView, ListView, FolderViewOverlay) - TypeScript: 0 errors Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
“BeeRad”
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
1d323de11a
commit
86f43c9975
@@ -27,7 +27,7 @@ function fetchAutoContextRows(limit: number): AutoContextSummary[] {
|
||||
FROM nodes n
|
||||
LEFT JOIN edges e
|
||||
ON (e.from_node_id = n.id OR e.to_node_id = n.id)
|
||||
WHERE n.type IS NULL OR n.type != 'memory'
|
||||
WHERE 1=1
|
||||
GROUP BY n.id
|
||||
ORDER BY edge_count DESC, n.updated_at DESC
|
||||
LIMIT ?
|
||||
|
||||
@@ -4,7 +4,7 @@ import { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
|
||||
export interface DescriptionInput {
|
||||
title: string;
|
||||
content?: string;
|
||||
notes?: string;
|
||||
link?: string;
|
||||
metadata?: {
|
||||
source?: string;
|
||||
@@ -12,7 +12,6 @@ export interface DescriptionInput {
|
||||
author?: string;
|
||||
site_name?: string;
|
||||
};
|
||||
type?: string;
|
||||
dimensions?: string[];
|
||||
}
|
||||
|
||||
@@ -24,7 +23,7 @@ export { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
* Used when no API key is available or for simple inputs.
|
||||
*/
|
||||
export function generateFallbackDescription(input: DescriptionInput): string {
|
||||
const { title, type, metadata, dimensions } = input;
|
||||
const { title, metadata, dimensions } = input;
|
||||
|
||||
// Build a contextual fallback
|
||||
const parts: string[] = [];
|
||||
@@ -33,10 +32,6 @@ export function generateFallbackDescription(input: DescriptionInput): string {
|
||||
parts.push(`By ${metadata.author || metadata.channel_name}`);
|
||||
}
|
||||
|
||||
if (type) {
|
||||
parts.push(type.charAt(0).toUpperCase() + type.slice(1));
|
||||
}
|
||||
|
||||
if (dimensions?.length) {
|
||||
parts.push(`in ${dimensions.slice(0, 2).join(', ')}`);
|
||||
}
|
||||
@@ -63,7 +58,7 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
|
||||
}
|
||||
|
||||
// Fast path: skip AI for very short inputs (likely just notes)
|
||||
if (!input.content && !input.link && input.title.length < 30) {
|
||||
if (!input.notes && !input.link && input.title.length < 30) {
|
||||
console.log(`[DescriptionService] Short input, using fallback for: "${input.title}"`);
|
||||
return generateFallbackDescription(input);
|
||||
}
|
||||
@@ -135,8 +130,8 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
|
||||
if (publisherHint) lines.push(`Publisher hint: ${publisherHint}`);
|
||||
lines.push(`Likely user-authored: ${likelyUserAuthored ? 'yes' : 'no'}`);
|
||||
|
||||
const contentPreview = input.content?.slice(0, 800) || '';
|
||||
if (contentPreview) lines.push(`Content: ${contentPreview}${input.content && input.content.length > 800 ? '...' : ''}`);
|
||||
const contentPreview = input.notes?.slice(0, 800) || '';
|
||||
if (contentPreview) lines.push(`Notes: ${contentPreview}${input.notes && input.notes.length > 800 ? '...' : ''}`);
|
||||
|
||||
return `Your job is to answer: "what is this?" in one short line. Max 280 characters.
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ export class DimensionService {
|
||||
*/
|
||||
static async assignDimensions(nodeData: {
|
||||
title: string;
|
||||
content?: string;
|
||||
notes?: string;
|
||||
link?: string;
|
||||
description?: string;
|
||||
}): Promise<{ locked: string[]; keywords: string[] }> {
|
||||
@@ -156,19 +156,19 @@ export class DimensionService {
|
||||
* Build AI prompt for dimension assignment (locked dimensions only)
|
||||
*/
|
||||
private static buildAssignmentPrompt(
|
||||
nodeData: { title: string; content?: string; link?: string; description?: string },
|
||||
nodeData: { title: string; notes?: string; link?: string; description?: string },
|
||||
lockedDimensions: LockedDimension[]
|
||||
): string {
|
||||
// Use description as primary context, content as fallback
|
||||
let nodeContextSection: string;
|
||||
if (nodeData.description) {
|
||||
const contentPreview = nodeData.content?.slice(0, 500) || '';
|
||||
const contentPreview = nodeData.notes?.slice(0, 500) || '';
|
||||
nodeContextSection = `DESCRIPTION: ${nodeData.description}
|
||||
|
||||
CONTENT PREVIEW: ${contentPreview}${nodeData.content && nodeData.content.length > 500 ? '...' : ''}`;
|
||||
NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`;
|
||||
} else {
|
||||
const contentPreview = nodeData.content?.slice(0, 2000) || '';
|
||||
nodeContextSection = `CONTENT: ${contentPreview}${nodeData.content && nodeData.content.length > 2000 ? '...' : ''}`;
|
||||
const contentPreview = nodeData.notes?.slice(0, 2000) || '';
|
||||
nodeContextSection = `NOTES: ${contentPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`;
|
||||
}
|
||||
|
||||
// Include ALL locked dimensions, using fallback text for those without descriptions
|
||||
|
||||
@@ -430,9 +430,9 @@ export class EdgeService {
|
||||
ELSE n_from.title
|
||||
END as connected_node_title,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.content
|
||||
ELSE n_from.content
|
||||
END as connected_node_content,
|
||||
WHEN e.from_node_id = ? THEN n_to.notes
|
||||
ELSE n_from.notes
|
||||
END as connected_node_notes,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.link
|
||||
ELSE n_from.link
|
||||
@@ -499,7 +499,7 @@ export class EdgeService {
|
||||
const connected_node: Node = {
|
||||
id: row.connected_node_id,
|
||||
title: row.connected_node_title,
|
||||
content: row.connected_node_content,
|
||||
notes: row.connected_node_notes,
|
||||
link: row.connected_node_link,
|
||||
dimensions: row.connected_node_dimensions,
|
||||
embedding: undefined, // Not needed for display
|
||||
@@ -544,7 +544,7 @@ export class EdgeService {
|
||||
const connected_node: Node = {
|
||||
id: row.connected_node_id,
|
||||
title: row.connected_node_title,
|
||||
content: row.connected_node_content,
|
||||
notes: row.connected_node_notes,
|
||||
link: row.connected_node_link,
|
||||
dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'),
|
||||
embedding: undefined, // Not needed for display
|
||||
|
||||
@@ -15,7 +15,7 @@ export class NodeService {
|
||||
|
||||
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
|
||||
let query = `
|
||||
SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk,
|
||||
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
@@ -38,7 +38,7 @@ export class NodeService {
|
||||
|
||||
// Text search in title, description, and content (SQLite LIKE with COLLATE NOCASE)
|
||||
if (search) {
|
||||
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.content LIKE ? COLLATE NOCASE)`;
|
||||
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
|
||||
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ export class NodeService {
|
||||
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
|
||||
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
|
||||
CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
|
||||
CASE WHEN n.content LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
|
||||
CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
|
||||
n.updated_at DESC`;
|
||||
params.push(
|
||||
search, // Exact match (case-insensitive)
|
||||
@@ -95,7 +95,7 @@ export class NodeService {
|
||||
private async getNodeByIdSQLite(id: number): Promise<Node | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk,
|
||||
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
@@ -125,9 +125,9 @@ export class NodeService {
|
||||
const {
|
||||
title,
|
||||
description,
|
||||
content,
|
||||
notes,
|
||||
link,
|
||||
type,
|
||||
event_date,
|
||||
dimensions = [],
|
||||
chunk,
|
||||
chunk_status,
|
||||
@@ -139,14 +139,14 @@ export class NodeService {
|
||||
const nodeId = sqlite.transaction(() => {
|
||||
// Insert node using prepare/run for lastInsertRowid access
|
||||
const nodeResult = sqlite.prepare(`
|
||||
INSERT INTO nodes (title, description, content, link, type, metadata, chunk, chunk_status, created_at, updated_at)
|
||||
INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, chunk_status, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
title,
|
||||
description ?? null,
|
||||
content ?? null,
|
||||
notes ?? null,
|
||||
link ?? null,
|
||||
type ?? null,
|
||||
event_date ?? null,
|
||||
JSON.stringify(metadata),
|
||||
chunk ?? null,
|
||||
chunk_status ?? null,
|
||||
@@ -192,7 +192,7 @@ export class NodeService {
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
|
||||
const { title, description, content, link, type, dimensions, chunk, metadata } = updates;
|
||||
const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates;
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
@@ -211,9 +211,9 @@ export class NodeService {
|
||||
|
||||
if (title !== undefined) { setFields.push('title = ?'); params.push(title); }
|
||||
if (description !== undefined) { setFields.push('description = ?'); params.push(description); }
|
||||
if (content !== undefined) { setFields.push('content = ?'); params.push(content); }
|
||||
if (notes !== undefined) { setFields.push('notes = ?'); params.push(notes); }
|
||||
if (link !== undefined) { setFields.push('link = ?'); params.push(link); }
|
||||
if (type !== undefined) { setFields.push('type = ?'); params.push(type); }
|
||||
if (event_date !== undefined) { setFields.push('event_date = ?'); params.push(event_date); }
|
||||
if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); }
|
||||
if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) {
|
||||
setFields.push('chunk_status = ?');
|
||||
|
||||
@@ -266,7 +266,7 @@ class SQLiteClient {
|
||||
}
|
||||
};
|
||||
ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;");
|
||||
ensureNodeCol('type', "ALTER TABLE nodes ADD COLUMN type TEXT;");
|
||||
// type column removed in final schema pass
|
||||
} catch (nodeErr) {
|
||||
console.warn('Failed to ensure nodes columns:', nodeErr);
|
||||
}
|
||||
@@ -435,16 +435,8 @@ class SQLiteClient {
|
||||
}
|
||||
// Do not recreate memory_v; alias has been removed.
|
||||
|
||||
// 6) Chat memory state tracking for chat-triggered memories
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS chat_memory_state (
|
||||
thread_id TEXT PRIMARY KEY,
|
||||
helper_name TEXT,
|
||||
last_processed_chat_id INTEGER DEFAULT 0,
|
||||
last_processed_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_memory_thread ON chat_memory_state(thread_id);
|
||||
`);
|
||||
// 6) Drop orphaned chat_memory_state table (removed in final schema pass)
|
||||
this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`);
|
||||
|
||||
// Agent delegation table for orchestrator/worker coordination
|
||||
try {
|
||||
@@ -631,6 +623,81 @@ class SQLiteClient {
|
||||
}
|
||||
}
|
||||
|
||||
// 10) Final schema pass migrations (content→notes, event_date, icon, drop dead columns)
|
||||
try {
|
||||
const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
|
||||
const nodeColNames = nodeCols2.map((c: any) => c.name);
|
||||
|
||||
// Rename content → notes
|
||||
if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) {
|
||||
console.log('Renaming nodes.content → nodes.notes');
|
||||
this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;');
|
||||
}
|
||||
|
||||
// Add event_date with backfill from metadata
|
||||
if (!nodeColNames.includes('event_date')) {
|
||||
console.log('Adding nodes.event_date column');
|
||||
this.db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;');
|
||||
this.db.exec(`
|
||||
UPDATE nodes SET event_date = json_extract(metadata, '$.published_date')
|
||||
WHERE metadata IS NOT NULL
|
||||
AND json_extract(metadata, '$.published_date') IS NOT NULL
|
||||
AND json_extract(metadata, '$.published_date') != '';
|
||||
`);
|
||||
}
|
||||
|
||||
// Add dimensions.icon
|
||||
const dimCols2 = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
|
||||
if (!dimCols2.some((c: any) => c.name === 'icon')) {
|
||||
console.log('Adding dimensions.icon column');
|
||||
this.db.exec('ALTER TABLE dimensions ADD COLUMN icon TEXT;');
|
||||
}
|
||||
|
||||
// Drop dead columns (SQLite 3.35+)
|
||||
if (nodeColNames.includes('type')) {
|
||||
console.log('Dropping nodes.type column');
|
||||
try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_type;'); } catch {}
|
||||
try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch (e) {
|
||||
console.warn('Could not drop nodes.type (SQLite < 3.35?):', e);
|
||||
}
|
||||
}
|
||||
if (nodeColNames.includes('is_pinned')) {
|
||||
console.log('Dropping nodes.is_pinned column');
|
||||
try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_pinned;'); } catch {}
|
||||
try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch (e) {
|
||||
console.warn('Could not drop nodes.is_pinned:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Drop edges.user_feedback
|
||||
const edgeCols = this.db.prepare('PRAGMA table_info(edges)').all() as Array<{ name: string }>;
|
||||
if (edgeCols.some((c: any) => c.name === 'user_feedback')) {
|
||||
console.log('Dropping edges.user_feedback column');
|
||||
try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch (e) {
|
||||
console.warn('Could not drop edges.user_feedback:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuild FTS if it references 'content' instead of 'notes'
|
||||
try {
|
||||
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as any;
|
||||
if (ftsCheck && ftsCheck.sql && ftsCheck.sql.includes('content')) {
|
||||
console.log('Rebuilding nodes_fts to reference notes instead of content');
|
||||
this.db.exec('DROP TABLE IF EXISTS nodes_fts;');
|
||||
this.db.exec(`
|
||||
CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content=nodes, content_rowid=id);
|
||||
INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');
|
||||
`);
|
||||
}
|
||||
} catch (ftsErr) {
|
||||
console.warn('FTS rebuild skipped:', ftsErr);
|
||||
}
|
||||
|
||||
console.log('Final schema pass migrations complete');
|
||||
} catch (schemaErr) {
|
||||
console.warn('Final schema pass migration error:', schemaErr);
|
||||
}
|
||||
|
||||
console.log('Logging + memory schema ensured');
|
||||
} catch (error) {
|
||||
console.error('Failed to ensure logging/memory schema:', error);
|
||||
|
||||
@@ -68,7 +68,7 @@ function bootstrapFromLegacyPins(): void {
|
||||
try {
|
||||
const db = getSQLiteClient();
|
||||
const countRow = db
|
||||
.query<{ count: number }>('SELECT COUNT(*) as count FROM nodes WHERE is_pinned = 1')
|
||||
.query<{ count: number }>('SELECT COUNT(*) as count FROM nodes WHERE 1=0 /* /* is_pinned removed */ removed */')
|
||||
.rows[0];
|
||||
const pinnedCount = Number(countRow?.count ?? 0);
|
||||
if (pinnedCount > 0) {
|
||||
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
interface NodeRecord {
|
||||
id: number;
|
||||
title: string;
|
||||
content: string | null;
|
||||
notes: string | null;
|
||||
description: string | null;
|
||||
dimensions_json: string;
|
||||
embedding?: Buffer | null;
|
||||
@@ -58,7 +58,7 @@ export class NodeEmbedder {
|
||||
const prompt = `Analyze this content and provide 2-3 key insights or themes in a concise paragraph (max 100 words):
|
||||
|
||||
Title: ${node.title}
|
||||
Content: ${node.content || 'No content'}
|
||||
Content: ${node.notes || 'No content'}
|
||||
Dimensions: ${dimensionsText}
|
||||
|
||||
Focus on the main concepts, key relationships, and practical implications.`;
|
||||
@@ -106,13 +106,13 @@ Focus on the main concepts, key relationships, and practical implications.`;
|
||||
// Create base embedding text
|
||||
let embeddingText = formatEmbeddingText(
|
||||
node.title,
|
||||
node.content || '',
|
||||
node.notes || '',
|
||||
dimensions,
|
||||
node.description
|
||||
);
|
||||
|
||||
// Add AI analysis if content exists
|
||||
if (node.content && node.content.trim().length > 0) {
|
||||
if (node.notes && node.notes.trim().length > 0) {
|
||||
const analysis = await this.analyzeNodeWithAI(node);
|
||||
if (analysis) {
|
||||
embeddingText += `\n\nAI Analysis: ${analysis}`;
|
||||
|
||||
Reference in New Issue
Block a user