SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port source-first ingestion to os repo

Browse Source 
- make source the canonical field across os routes, tools, ui, and mcp
- align fresh-install schema, search, and embedding flows with source-first ingestion

Generated with Claude Code
This commit is contained in:
“BeeRad”
2026-03-20 12:36:26 +11:00
parent 41f8498d4a
commit 28e570696c
34 changed files with 588 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tools/database/createNode.ts
+1 -2
View File
@@ -8,8 +8,8 @@ export const createNodeTool = tool({
description: 'Create node. Description is REQUIRED and must be explicit about what the thing is (podcast, chat summary, idea, etc).',
inputSchema: z.object({
title: z.string().describe('The title of the node'),
notes: z.string().optional().describe('User notes, analysis, or thoughts about this node'),
description: z.string().max(280).describe('REQUIRED. Explicitly state WHAT this is (e.g. podcast episode, conversation summary, user insight) + WHY it matters for context grounding.'),
source: z.string().optional().describe('Raw content for embedding: transcript, article text, book passages, or user thoughts. If omitted, falls back to title + description.'),
link: z.string().optional().describe('A URL link to the source'),
event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'),
dimensions: z
@@ -17,7 +17,6 @@ export const createNodeTool = tool({
.max(5)
.optional()
.describe('Optional dimension tags to apply to this node (0-5 items).'),
chunk: z.string().optional().describe('Raw content for later processing - CRITICAL for extracted content from URLs'),
metadata: z.record(z.any()).optional().describe('Additional metadata like source info, extraction details, etc.')
}),
execute: async (params) => {
src/tools/database/getNodesById.ts
+7 -5
View File
@@ -6,9 +6,9 @@ export const getNodesByIdTool = tool({
description: 'Load full node records by IDs',
inputSchema: z.object({
nodeIds: z.array(z.number().int().positive()).min(1).max(10).describe('List of node IDs to load'),
includeNotesPreview: z.boolean().default(true).describe('Whether to return a trimmed content preview for each node'),
includeSourcePreview: z.boolean().default(true).describe('Whether to return a trimmed source preview for each node'),
}),
execute: async ({ nodeIds, includeNotesPreview }) => {
execute: async ({ nodeIds, includeSourcePreview }) => {
const uniqueIds = Array.from(new Set(nodeIds.filter(id => Number.isFinite(id) && id > 0)));
if (uniqueIds.length === 0) {
return {
@@ -23,8 +23,8 @@ export const getNodesByIdTool = tool({
try {
const node = await nodeService.getNodeById(id);
if (!node) return null;
const preview = includeNotesPreview
? (node.notes || node.description || '')
const preview = includeSourcePreview
? (node.source || node.description || '')
.split(/\s+/)
.slice(0, 80)
.join(' ')
@@ -35,10 +35,12 @@ export const getNodesByIdTool = tool({
id: node.id,
title: node.title,
link: node.link,
event_date: node.event_date ?? null,
dimensions: node.dimensions || [],
chunk_status: node.chunk_status || 'unknown',
created_at: node.created_at,
updated_at: node.updated_at,
notes_preview: preview || null,
source_preview: preview || null,
metadata: node.metadata ?? null,
};
} catch (error) {
src/tools/database/queryDimensionNodes.ts
+5 -4
View File
@@ -48,11 +48,12 @@ export const queryDimensionNodesTool = tool({
event_date: node.event_date ?? null,
};
if (includeContent && node.notes) {
if (includeContent && (node.source || node.notes)) {
const previewSource = node.source || node.notes || '';
// Truncate to ~100 chars
formatted.notesPreview = node.notes.length > 100
? node.notes.substring(0, 100) + '...'
: node.notes;
formatted.sourcePreview = previewSource.length > 100
? previewSource.substring(0, 100) + '...'
: previewSource;
}
return formatted;
src/tools/database/queryNodes.ts
+5 -5
View File
@@ -44,7 +44,7 @@ function scoreNodeForSearch(node: Node, searchTerm: string): number {
const normalizedSearch = searchTerm.toLowerCase();
const title = (node.title || '').toLowerCase();
const description = (node.description || '').toLowerCase();
const notes = (node.notes || '').toLowerCase();
const source = (node.source || '').toLowerCase();
const terms = extractSearchTerms(searchTerm);
let score = 0;
@@ -52,23 +52,23 @@ function scoreNodeForSearch(node: Node, searchTerm: string): number {
if (title === normalizedSearch) score += 100;
if (title.includes(normalizedSearch)) score += 40;
if (description.includes(normalizedSearch)) score += 20;
if (notes.includes(normalizedSearch)) score += 10;
if (source.includes(normalizedSearch)) score += 10;
for (const term of terms) {
if (title.includes(term)) score += 8;
if (description.includes(term)) score += 3;
if (notes.includes(term)) score += 2;
if (source.includes(term)) score += 2;
}
return score;
}
export const queryNodesTool = tool({
description: 'Search nodes across title, description, and notes. For free-text lookups, search the graph broadly and prioritize title/description matches. Do not use dimensions to constrain keyword search unless the user is explicitly asking about a known dimension.',
description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Do not use dimensions to constrain keyword search unless the user is explicitly asking about a known dimension.',
inputSchema: z.object({
filters: z.object({
dimensions: z.array(z.string()).describe('Filter by dimensions (e.g., ["research", "ai", "technology"]). Replaces old type/stage filtering.').optional(),
search: z.string().describe('Search term to match against node title, description, or notes').optional(),
search: z.string().describe('Search term to match against node title, description, or source').optional(),
limit: z.number().min(1).max(50).default(10).describe('Maximum number of results to return'),
createdAfter: z.string().optional().describe('ISO date (YYYY-MM-DD). Only return nodes created on or after this date.'),
createdBefore: z.string().optional().describe('ISO date (YYYY-MM-DD). Only return nodes created before this date.'),
src/tools/database/updateNode.ts
+2 -53
View File
@@ -9,14 +9,13 @@ export const updateNodeTool = tool({
id: z.number().describe('The ID of the node to update'),
updates: z.object({
title: z.string().optional().describe('New title'),
notes: z.string().optional().describe('User notes/analysis to append. USE THIS for workflow outputs, briefs, research notes, etc.'),
description: z.string().max(280).describe('REQUIRED on every update. Explicitly state WHAT this is + WHY it matters. No "discusses/explores".'),
source: z.string().optional().describe('Canonical source content for embedding. Use this only to set or correct the raw source text.'),
link: z.string().optional().describe('New link'),
event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'),
dimensions: z.array(z.string()).optional().describe('New dimension tags - completely replaces existing dimensions'),
chunk: z.string().optional().describe('DO NOT USE - raw source text that triggers re-embedding. Only for source corrections.'),
metadata: z.record(z.any()).optional().describe('New metadata - completely replaces existing metadata')
}).describe('Object containing the fields to update. For adding notes/analysis, always use "notes" field.')
}).describe('Object containing the fields to update. Derived analysis should be stored in a separate linked node, not appended to the source node.')
}),
execute: async ({ id, updates }) => {
try {
@@ -44,56 +43,6 @@ export const updateNodeTool = tool({
};
}
// FORCE APPEND for notes field - fetch existing and append new notes
if (updates.notes) {
const fetchResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes/${id}`);
if (fetchResponse.ok) {
const { node } = await fetchResponse.json();
const existingNotes = (node?.notes || '').trim();
const newNotes = updates.notes.trim();
// Skip if new notes are identical to existing (model sent duplicate)
if (existingNotes === newNotes) {
console.log(`[updateNode] ERROR - new notes identical to existing (${existingNotes.length} chars). Model should NOT call updateNode again.`);
return {
success: false,
error: 'Notes already up to date - do not call updateNode again. Move to next step.',
data: null
};
}
// Detect if adding a section that already exists (e.g., ## Integration Analysis)
const newSectionMatch = newNotes.match(/^##\s+(.+)$/m);
if (newSectionMatch && existingNotes) {
const sectionHeader = newSectionMatch[0]; // e.g., "## Integration Analysis"
if (existingNotes.includes(sectionHeader)) {
console.log(`[updateNode] ERROR - Section "${sectionHeader}" already exists in node`);
return {
success: false,
error: `Section "${sectionHeader}" already exists in this node. Cannot append duplicate section.`,
data: null
};
}
}
// Detect if model included existing notes + new notes
if (existingNotes && newNotes.startsWith(existingNotes)) {
// Extract only the new part
const actualNewNotes = newNotes.substring(existingNotes.length).trim();
console.log(`[updateNode] Model included existing notes - extracting new part only (${actualNewNotes.length} chars)`);
const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n';
updates.notes = `${existingNotes}${separator}${actualNewNotes}`;
} else if (existingNotes) {
// Normal append
const separator = existingNotes.endsWith('\n\n') ? '' : '\n\n';
updates.notes = `${existingNotes}${separator}${newNotes}`;
console.log(`[updateNode] Appended notes: ${existingNotes.length} + ${newNotes.length} = ${updates.notes.length} chars`);
} else {
console.log(`[updateNode] No existing notes, using new notes as-is (${newNotes.length} chars)`);
}
}
}
if (Array.isArray(updates.dimensions)) {
updates.dimensions = normalizeDimensions(updates.dimensions, 5);
}
src/tools/other/paperExtract.ts
+8 -10
View File
@@ -69,7 +69,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
}
export const paperExtractTool = tool({
description: 'Extract a PDF or research paper into a node with summary, metadata, and full-text chunk',
description: 'Extract a PDF or research paper into a node with summary, metadata, and full-text source',
inputSchema: z.object({
url: z.string().describe('The PDF URL to add to inbox'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
@@ -95,14 +95,13 @@ export const paperExtractTool = tool({
};
}
let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string };
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractPaper(url);
result = {
success: true,
notes: extractionResult.content,
chunk: extractionResult.chunk,
source: extractionResult.chunk || extractionResult.content,
metadata: {
title: extractionResult.metadata.title,
pages: extractionResult.metadata.pages,
@@ -119,7 +118,7 @@ export const paperExtractTool = tool({
};
}
if (!result.success || (!result.notes && !result.chunk)) {
if (!result.success || !result.source) {
return {
success: false,
error: result.error || 'Failed to extract PDF content',
@@ -132,7 +131,7 @@ export const paperExtractTool = tool({
// Step 2: AI Analysis for enhanced metadata
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`,
result.notes?.substring(0, 2000) || 'PDF document content',
result.source.substring(0, 2000) || 'PDF document content',
'pdf'
);
@@ -153,17 +152,16 @@ export const paperExtractTool = tool({
body: JSON.stringify({
title: nodeTitle,
description: aiAnalysis?.nodeDescription,
notes: enhancedDescription,
source: result.source,
link: url,
dimensions: trimmedDimensions,
chunk: result.chunk || result.notes,
metadata: {
source: 'pdf',
hostname: new URL(url).hostname,
author: result.metadata?.author || result.metadata?.info?.Author,
pages: result.metadata?.pages,
file_size: result.metadata?.file_size,
content_length: (result.chunk || result.notes)?.length,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'python_pdfplumber',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: enhancedDescription,
@@ -197,7 +195,7 @@ export const paperExtractTool = tool({
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: (result.chunk || result.notes || '').length,
contentLength: result.source.length,
url: url,
dimensions: actualDimensions
}
src/tools/other/websiteExtract.ts
+8 -10
View File
@@ -149,7 +149,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
}
export const websiteExtractTool = tool({
description: 'Extract website content and metadata into a node with summary, tags, and raw chunk',
description: 'Extract website content and metadata into a node with summary, tags, and raw source text',
inputSchema: z.object({
url: z.string().describe('The website URL to add to knowledge base'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
@@ -166,14 +166,13 @@ export const websiteExtractTool = tool({
};
}
let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string };
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractWebsite(url);
result = {
success: true,
notes: extractionResult.content,
chunk: extractionResult.chunk,
source: extractionResult.chunk || extractionResult.content,
metadata: {
title: extractionResult.metadata.title,
author: extractionResult.metadata.author,
@@ -191,7 +190,7 @@ export const websiteExtractTool = tool({
};
}
if (!result.success || (!result.notes && !result.chunk)) {
if (!result.success || !result.source) {
return {
success: false,
error: result.error || 'Failed to extract website content',
@@ -206,7 +205,7 @@ export const websiteExtractTool = tool({
const contentType = inferWebsiteContentType(url);
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.notes?.substring(0, 2000) || 'Website content',
result.source.substring(0, 2000) || 'Website content',
contentType,
existingDimensions
);
@@ -231,17 +230,16 @@ export const websiteExtractTool = tool({
body: JSON.stringify({
title: nodeTitle,
description: aiAnalysis?.nodeDescription,
notes: enhancedDescription,
source: result.source,
link: url,
event_date: result.metadata?.published_date || result.metadata?.date || null,
dimensions: finalDimensions,
chunk: result.chunk || result.notes,
metadata: {
source: contentType,
hostname: new URL(url).hostname,
author: result.metadata?.author,
published_date: result.metadata?.published_date || result.metadata?.date,
content_length: (result.chunk || result.notes)?.length,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'python_beautifulsoup',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: enhancedDescription,
@@ -275,7 +273,7 @@ export const websiteExtractTool = tool({
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: (result.chunk || result.notes || '').length,
contentLength: result.source.length,
url: url,
dimensions: actualDimensions
}
src/tools/other/youtubeExtract.ts
+6 -9
View File
@@ -193,15 +193,14 @@ export const youtubeExtractTool = tool({
};
}
let result: { success: boolean; notes?: string; chunk?: string; metadata?: any; error?: string };
let result: { success: boolean; source?: string; metadata?: any; error?: string };
console.log('📝 Using TypeScript yt-dlp extractor');
try {
const extractionResult = await extractYouTube(url);
result = {
success: extractionResult.success,
notes: extractionResult.content,
chunk: extractionResult.chunk,
source: extractionResult.chunk || extractionResult.content,
metadata: {
video_title: extractionResult.metadata.video_title,
channel_name: extractionResult.metadata.channel_name,
@@ -222,7 +221,7 @@ export const youtubeExtractTool = tool({
};
}
if (!result.success || (!result.notes && !result.chunk)) {
if (!result.success || !result.source) {
return {
success: false,
error: result.error || 'Failed to extract YouTube content',
@@ -243,8 +242,7 @@ export const youtubeExtractTool = tool({
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.video_title || `YouTube Video ${url.split('/').pop()?.split('?')[0]}`;
const transcriptSummary = await summariseTranscript(nodeTitle, result.chunk || result.notes || '');
const nodeNotes = transcriptSummary || aiAnalysis?.enhancedDescription || `YouTube video by ${result.metadata?.channel_name || 'Unknown Channel'}`;
const transcriptSummary = await summariseTranscript(nodeTitle, result.source);
const suppliedDimensions = Array.isArray(dimensions) ? dimensions : [];
let trimmedDimensions = suppliedDimensions
@@ -262,10 +260,9 @@ export const youtubeExtractTool = tool({
body: JSON.stringify({
title: nodeTitle,
description: aiAnalysis?.nodeDescription,
notes: nodeNotes,
source: result.source,
link: url,
dimensions: finalDimensions,
chunk: result.chunk || result.notes,
metadata: {
source: 'youtube',
video_id: result.metadata?.video_id,
@@ -308,7 +305,7 @@ export const youtubeExtractTool = tool({
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: (result.chunk || result.notes || '').length,
contentLength: result.source.length,
url: url,
dimensions: actualDimensions
}