feat: port source-first ingestion to os repo
- make source the canonical field across os routes, tools, ui, and mcp - align fresh-install schema, search, and embedding flows with source-first ingestion Generated with Claude Code
This commit is contained in:
“BeeRad”
@@ -77,7 +77,8 @@ All data stays on this device.`;
|
||||
// Tool schemas
|
||||
const addNodeInputSchema = {
|
||||
title: z.string().min(1).max(160).describe('Clear, descriptive title'),
|
||||
content: z.string().max(20000).optional().describe('Node content/notes'),
|
||||
content: z.string().max(20000).optional().describe('Legacy content field; mapped to source'),
|
||||
source: z.string().max(50000).optional().describe('Full source text'),
|
||||
link: z.string().url().optional().describe('Source URL'),
|
||||
description: z.string().min(24).max(280).describe('REQUIRED. One-sentence summary: WHAT this is (explicit, concrete) + WHY it matters. No weak verbs (discusses, explores, examines). Example: "Podcast — Lex Fridman interviews Sam Altman on AGI timelines. First public comments since board drama."'),
|
||||
dimensions: z.array(z.string()).min(1).max(5).describe('1-5 categories. Call queryDimensions first to use existing ones.'),
|
||||
@@ -354,7 +355,7 @@ async function main() {
|
||||
// Note: MCP schema uses "content" for external API compat; mapped to "notes" internally
|
||||
inputSchema: addNodeInputSchema
|
||||
},
|
||||
async ({ title, content, link, description, dimensions, metadata, chunk }) => {
|
||||
async ({ title, content, source, link, description, dimensions, metadata, chunk }) => {
|
||||
const normalizedDimensions = sanitizeDimensions(dimensions);
|
||||
if (normalizedDimensions.length === 0) {
|
||||
throw new Error('At least one dimension is required.');
|
||||
@@ -366,12 +367,11 @@ async function main() {
|
||||
|
||||
const node = nodeService.createNode({
|
||||
title: title.trim(),
|
||||
notes: content?.trim(),
|
||||
source: source?.trim() || chunk?.trim() || content?.trim(),
|
||||
link: link?.trim(),
|
||||
description: description?.trim(),
|
||||
dimensions: normalizedDimensions,
|
||||
metadata: metadata || {},
|
||||
chunk: chunk?.trim()
|
||||
metadata: metadata || {}
|
||||
});
|
||||
|
||||
const summary = `Created node #${node.id}: ${node.title} [${node.dimensions.join(', ')}]`;
|
||||
@@ -425,7 +425,7 @@ async function main() {
|
||||
WITH fts_matches AS (
|
||||
SELECT rowid, rank FROM nodes_fts WHERE nodes_fts MATCH ? LIMIT 100
|
||||
)
|
||||
SELECT n.id, n.title, n.description, n.notes, n.link,
|
||||
SELECT n.id, n.title, n.description, n.source, n.link,
|
||||
n.created_at, n.updated_at, n.event_date,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
@@ -446,7 +446,7 @@ async function main() {
|
||||
WITH fts_matches AS (
|
||||
SELECT rowid, rank FROM nodes_fts WHERE nodes_fts MATCH ? LIMIT ?
|
||||
)
|
||||
SELECT n.id, n.title, n.description, n.notes, n.link,
|
||||
SELECT n.id, n.title, n.description, n.source, n.link,
|
||||
n.created_at, n.updated_at, n.event_date,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
@@ -462,7 +462,7 @@ async function main() {
|
||||
nodes = rows.map(row => ({
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
notes: row.notes ?? null,
|
||||
source: row.source ?? null,
|
||||
description: row.description ?? null,
|
||||
link: row.link ?? null,
|
||||
dimensions: JSON.parse(row.dimensions_json || '[]'),
|
||||
@@ -482,7 +482,7 @@ async function main() {
|
||||
const words = trimmedQuery.split(/\s+/).filter(w => w.length > 0);
|
||||
|
||||
let sql = `
|
||||
SELECT n.id, n.title, n.description, n.notes, n.link,
|
||||
SELECT n.id, n.title, n.description, n.source, n.link,
|
||||
n.created_at, n.updated_at, n.event_date,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
@@ -492,7 +492,7 @@ async function main() {
|
||||
const params = [];
|
||||
|
||||
for (const word of words) {
|
||||
sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
|
||||
sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
|
||||
params.push(`%${word}%`, `%${word}%`, `%${word}%`);
|
||||
}
|
||||
|
||||
@@ -518,7 +518,7 @@ async function main() {
|
||||
nodes = rows.map(row => ({
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
notes: row.notes ?? null,
|
||||
source: row.source ?? null,
|
||||
description: row.description ?? null,
|
||||
link: row.link ?? null,
|
||||
dimensions: JSON.parse(row.dimensions_json || '[]'),
|
||||
@@ -560,13 +560,13 @@ async function main() {
|
||||
for (const id of uniqueIds) {
|
||||
const node = nodeService.getNodeById(id);
|
||||
if (node) {
|
||||
const rawChunk = node.chunk ?? null;
|
||||
const rawChunk = node.source ?? node.chunk ?? null;
|
||||
const chunkTruncated = rawChunk ? rawChunk.length > CHUNK_LIMIT : false;
|
||||
|
||||
nodes.push({
|
||||
id: node.id,
|
||||
title: node.title,
|
||||
notes: node.notes ?? null,
|
||||
source: node.source ?? node.notes ?? null,
|
||||
description: node.description ?? null,
|
||||
link: node.link ?? null,
|
||||
chunk: chunkTruncated ? rawChunk.substring(0, CHUNK_LIMIT) : rawChunk,
|
||||
@@ -610,14 +610,18 @@ async function main() {
|
||||
throw new Error(descriptionError);
|
||||
}
|
||||
|
||||
// Map MCP 'content' field → internal 'notes' field
|
||||
// Map MCP legacy fields to canonical source
|
||||
const mappedUpdates = { ...updates };
|
||||
if (mappedUpdates.content !== undefined) {
|
||||
mappedUpdates.notes = mappedUpdates.content;
|
||||
delete mappedUpdates.content;
|
||||
mappedUpdates.source = mappedUpdates.content;
|
||||
}
|
||||
if (mappedUpdates.chunk !== undefined && mappedUpdates.source === undefined) {
|
||||
mappedUpdates.source = mappedUpdates.chunk;
|
||||
}
|
||||
delete mappedUpdates.content;
|
||||
delete mappedUpdates.chunk;
|
||||
|
||||
const node = nodeService.updateNode(id, mappedUpdates, { appendNotes: true });
|
||||
const node = nodeService.updateNode(id, mappedUpdates, { appendNotes: false });
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: `Updated node #${id}` }],
|
||||
|
||||
@@ -9,7 +9,7 @@ function getNodes(filters = {}) {
|
||||
const { dimensions, search, limit = 100, offset = 0 } = filters;
|
||||
|
||||
let sql = `
|
||||
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
|
||||
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
@@ -30,7 +30,7 @@ function getNodes(filters = {}) {
|
||||
|
||||
// Text search
|
||||
if (search) {
|
||||
sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
|
||||
sql += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
|
||||
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ function getNodes(filters = {}) {
|
||||
*/
|
||||
function getNodeById(id) {
|
||||
const sql = `
|
||||
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
|
||||
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
@@ -108,11 +108,10 @@ function createNode(nodeData) {
|
||||
const {
|
||||
title: rawTitle,
|
||||
description,
|
||||
notes,
|
||||
source,
|
||||
link,
|
||||
event_date,
|
||||
dimensions = [],
|
||||
chunk,
|
||||
metadata = {}
|
||||
} = nodeData;
|
||||
|
||||
@@ -121,30 +120,23 @@ function createNode(nodeData) {
|
||||
const now = new Date().toISOString();
|
||||
const db = getDb();
|
||||
|
||||
// Build chunk fallback from available fields when no explicit chunk provided
|
||||
let chunkToStore = chunk ?? null;
|
||||
if (!chunkToStore || !chunkToStore.trim()) {
|
||||
const fallbackParts = [title, description, notes].filter(Boolean);
|
||||
const fallback = fallbackParts.join('\n\n').trim();
|
||||
if (fallback) {
|
||||
chunkToStore = fallback;
|
||||
}
|
||||
}
|
||||
const sourceToStore = source && source.trim()
|
||||
? source
|
||||
: [title, description].filter(Boolean).join('\n\n').trim() || null;
|
||||
|
||||
const nodeId = transaction(() => {
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO nodes (title, description, source, link, event_date, metadata, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const result = stmt.run(
|
||||
title,
|
||||
description ?? null,
|
||||
notes ?? null,
|
||||
sourceToStore,
|
||||
link ?? null,
|
||||
event_date ?? null,
|
||||
JSON.stringify(metadata),
|
||||
chunkToStore,
|
||||
now,
|
||||
now
|
||||
);
|
||||
@@ -169,11 +161,11 @@ function createNode(nodeData) {
|
||||
|
||||
/**
|
||||
* Update an existing node.
|
||||
* Note: notes is APPENDED by default (MCP tool behavior), not replaced.
|
||||
* Source-first update path.
|
||||
*/
|
||||
function updateNode(id, updates, options = {}) {
|
||||
const { appendNotes = true } = options;
|
||||
const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates;
|
||||
const { title, description, source, notes, link, event_date, dimensions, chunk, metadata } = updates;
|
||||
const now = new Date().toISOString();
|
||||
const db = getDb();
|
||||
|
||||
@@ -195,15 +187,15 @@ function updateNode(id, updates, options = {}) {
|
||||
setFields.push('description = ?');
|
||||
params.push(description);
|
||||
}
|
||||
if (notes !== undefined) {
|
||||
if (appendNotes && existing.notes) {
|
||||
// Append to existing notes
|
||||
setFields.push('notes = ?');
|
||||
params.push(existing.notes + '\n\n' + notes);
|
||||
} else {
|
||||
setFields.push('notes = ?');
|
||||
params.push(notes);
|
||||
}
|
||||
if (source !== undefined) {
|
||||
setFields.push('source = ?');
|
||||
params.push(source);
|
||||
} else if (notes !== undefined) {
|
||||
const nextSource = appendNotes && existing.source
|
||||
? `${existing.source}\n\n${notes}`
|
||||
: notes;
|
||||
setFields.push('source = ?');
|
||||
params.push(nextSource);
|
||||
}
|
||||
if (link !== undefined) {
|
||||
setFields.push('link = ?');
|
||||
@@ -213,8 +205,8 @@ function updateNode(id, updates, options = {}) {
|
||||
setFields.push('event_date = ?');
|
||||
params.push(event_date);
|
||||
}
|
||||
if (chunk !== undefined) {
|
||||
setFields.push('chunk = ?');
|
||||
if (chunk !== undefined && source === undefined) {
|
||||
setFields.push('source = ?');
|
||||
params.push(chunk);
|
||||
}
|
||||
if (metadata !== undefined) {
|
||||
|
||||
+15
-11
@@ -86,6 +86,7 @@ const sanitizeDimensions = (raw) => {
|
||||
const addNodeInputSchema = {
|
||||
title: z.string().min(1).max(160),
|
||||
content: z.string().max(20000).optional(),
|
||||
source: z.string().max(50000).optional(),
|
||||
link: z.string().url().optional(),
|
||||
description: z.string().max(2000).optional(),
|
||||
dimensions: z.array(z.string()).min(1).max(5),
|
||||
@@ -112,7 +113,7 @@ const searchNodesOutputSchema = {
|
||||
z.object({
|
||||
id: z.number(),
|
||||
title: z.string(),
|
||||
notes: z.string().nullable(),
|
||||
source: z.string().nullable(),
|
||||
description: z.string().nullable(),
|
||||
link: z.string().nullable(),
|
||||
dimensions: z.array(z.string()),
|
||||
@@ -151,7 +152,7 @@ const getNodesOutputSchema = {
|
||||
z.object({
|
||||
id: z.number(),
|
||||
title: z.string(),
|
||||
notes: z.string().nullable(),
|
||||
source: z.string().nullable(),
|
||||
link: z.string().nullable(),
|
||||
dimensions: z.array(z.string()),
|
||||
updated_at: z.string()
|
||||
@@ -349,7 +350,7 @@ mcpServer.registerTool(
|
||||
inputSchema: addNodeInputSchema,
|
||||
outputSchema: addNodeOutputSchema
|
||||
},
|
||||
async ({ title, content, link, description, dimensions, metadata, chunk }) => {
|
||||
async ({ title, content, source, link, description, dimensions, metadata, chunk }) => {
|
||||
const normalizedDimensions = sanitizeDimensions(dimensions);
|
||||
if (normalizedDimensions.length === 0) {
|
||||
throw new McpError(
|
||||
@@ -360,12 +361,11 @@ mcpServer.registerTool(
|
||||
|
||||
const payload = {
|
||||
title: title.trim(),
|
||||
notes: content?.trim() || undefined,
|
||||
source: source?.trim() || chunk?.trim() || content?.trim() || undefined,
|
||||
link: link?.trim() || undefined,
|
||||
description: description?.trim() || undefined,
|
||||
dimensions: normalizedDimensions,
|
||||
metadata: metadata || {},
|
||||
chunk: chunk?.trim() || undefined
|
||||
metadata: metadata || {}
|
||||
};
|
||||
|
||||
const result = await callRaHApi('/api/nodes', {
|
||||
@@ -422,7 +422,7 @@ mcpServer.registerTool(
|
||||
nodes: nodes.map((node) => ({
|
||||
id: node.id,
|
||||
title: node.title,
|
||||
notes: node.notes ?? null,
|
||||
source: node.source ?? node.notes ?? null,
|
||||
description: node.description ?? null,
|
||||
link: node.link ?? null,
|
||||
dimensions: node.dimensions || [],
|
||||
@@ -446,12 +446,16 @@ mcpServer.registerTool(
|
||||
throw new McpError(ErrorCode.InvalidParams, 'At least one field must be provided in updates.');
|
||||
}
|
||||
|
||||
// Map MCP 'content' field → internal 'notes' field
|
||||
// Map MCP legacy fields to canonical source
|
||||
const mappedUpdates = { ...updates };
|
||||
if (mappedUpdates.content !== undefined) {
|
||||
mappedUpdates.notes = mappedUpdates.content;
|
||||
delete mappedUpdates.content;
|
||||
mappedUpdates.source = mappedUpdates.content;
|
||||
}
|
||||
if (mappedUpdates.chunk !== undefined && mappedUpdates.source === undefined) {
|
||||
mappedUpdates.source = mappedUpdates.chunk;
|
||||
}
|
||||
delete mappedUpdates.content;
|
||||
delete mappedUpdates.chunk;
|
||||
|
||||
const result = await callRaHApi(`/api/nodes/${id}`, {
|
||||
method: 'PUT',
|
||||
@@ -492,7 +496,7 @@ mcpServer.registerTool(
|
||||
nodes.push({
|
||||
id: result.node.id,
|
||||
title: result.node.title,
|
||||
notes: result.node.notes ?? null,
|
||||
source: result.node.source ?? result.node.notes ?? null,
|
||||
link: result.node.link ?? null,
|
||||
dimensions: result.node.dimensions || [],
|
||||
updated_at: result.node.updated_at
|
||||
|
||||
@@ -35,6 +35,7 @@ const STATUS_PATH = path.join(
|
||||
const addNodeInputSchema = {
|
||||
title: z.string().min(1).max(160),
|
||||
content: z.string().max(20000).optional(),
|
||||
source: z.string().max(50000).optional(),
|
||||
link: z.string().url().optional(),
|
||||
description: z.string().max(2000).optional(),
|
||||
dimensions: z.array(z.string()).min(1).max(5),
|
||||
@@ -322,7 +323,7 @@ server.registerTool(
|
||||
inputSchema: addNodeInputSchema,
|
||||
outputSchema: addNodeOutputSchema
|
||||
},
|
||||
async ({ title, content, link, description, dimensions, metadata, chunk }) => {
|
||||
async ({ title, content, source, link, description, dimensions, metadata, chunk }) => {
|
||||
const normalizedDimensions = sanitizeDimensions(dimensions);
|
||||
if (normalizedDimensions.length === 0) {
|
||||
throw new Error('At least one dimension/tag is required when creating a node.');
|
||||
@@ -330,12 +331,11 @@ server.registerTool(
|
||||
|
||||
const payload = {
|
||||
title: title.trim(),
|
||||
notes: content?.trim() || undefined,
|
||||
source: source?.trim() || chunk?.trim() || content?.trim() || undefined,
|
||||
link: link?.trim() || undefined,
|
||||
description: description?.trim() || undefined,
|
||||
dimensions: normalizedDimensions,
|
||||
metadata: metadata || {},
|
||||
chunk: chunk?.trim() || undefined
|
||||
metadata: metadata || {}
|
||||
};
|
||||
|
||||
const result = await callRaHApi('/api/nodes', {
|
||||
@@ -393,7 +393,7 @@ server.registerTool(
|
||||
nodes: nodes.map((node) => ({
|
||||
id: node.id,
|
||||
title: node.title,
|
||||
notes: node.notes ?? null,
|
||||
source: node.source ?? node.notes ?? null,
|
||||
description: node.description ?? null,
|
||||
link: node.link ?? null,
|
||||
dimensions: node.dimensions || [],
|
||||
@@ -417,12 +417,16 @@ server.registerTool(
|
||||
throw new Error('At least one field must be provided in updates.');
|
||||
}
|
||||
|
||||
// Map MCP 'content' field → internal 'notes' field
|
||||
// Map MCP legacy fields to canonical source
|
||||
const mappedUpdates = { ...updates };
|
||||
if (mappedUpdates.content !== undefined) {
|
||||
mappedUpdates.notes = mappedUpdates.content;
|
||||
delete mappedUpdates.content;
|
||||
mappedUpdates.source = mappedUpdates.content;
|
||||
}
|
||||
if (mappedUpdates.chunk !== undefined && mappedUpdates.source === undefined) {
|
||||
mappedUpdates.source = mappedUpdates.chunk;
|
||||
}
|
||||
delete mappedUpdates.content;
|
||||
delete mappedUpdates.chunk;
|
||||
|
||||
const result = await callRaHApi(`/api/nodes/${id}`, {
|
||||
method: 'PUT',
|
||||
@@ -463,7 +467,7 @@ server.registerTool(
|
||||
nodes.push({
|
||||
id: result.node.id,
|
||||
title: result.node.title,
|
||||
notes: result.node.notes ?? null,
|
||||
source: result.node.source ?? result.node.notes ?? null,
|
||||
link: result.node.link ?? null,
|
||||
dimensions: result.node.dimensions || [],
|
||||
updated_at: result.node.updated_at
|
||||
|
||||
Reference in New Issue
Block a user