SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port holistic node refinement contract

Browse Source
This commit is contained in:
“BeeRad”
2026-04-11 21:37:52 +10:00
parent 35f9ecf89c
commit 3ae46245ec
119 changed files with 6596 additions and 10982 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tools/database/createDimension.ts
-64
View File
@@ -1,64 +0,0 @@
import { tool } from 'ai';
import { z } from 'zod';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
export const createDimensionTool = tool({
description: 'Create a new dimension only when the user explicitly instructs you to do so. Always provide a description explaining what belongs in this category.',
inputSchema: z.object({
name: z.string().describe('Dimension name'),
description: z.string().min(1).max(500).describe('Dimension description explaining what content belongs in this dimension (required, max 500 characters)')
}),
execute: async (params) => {
console.log('📁 CreateDimension tool called with params:', JSON.stringify(params, null, 2));
try {
const trimmedName = params.name.trim();
if (!trimmedName) {
return {
success: false,
error: 'Dimension name is required',
data: null
};
}
// Call POST /api/dimensions
const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
name: trimmedName,
description: params.description.trim()
})
});
if (!response.ok) {
let errorMessage = 'Failed to create dimension';
try {
const errorResult = await response.json();
errorMessage = errorResult.error || errorMessage;
} catch {
// If response is not JSON (e.g., HTML error page), use status text
errorMessage = `Failed to create dimension: ${response.status} ${response.statusText}`;
}
return {
success: false,
error: errorMessage,
data: null
};
}
const result = await response.json();
return {
success: true,
data: result.data,
message: `Created dimension "${trimmedName}"${params.description ? ' with description' : ''}`
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to create dimension',
data: null
};
}
}
});
src/tools/database/createEdge.ts
+2 -4
View File
@@ -119,14 +119,12 @@ export const createEdgeTool = tool({
const fromLabel = formatNodeForChat({
id: fromNode.id,
title: fromNode.title,
dimensions: fromNode.dimensions || []
title: fromNode.title
});
const toLabel = formatNodeForChat({
id: toNode.id,
title: toNode.title,
dimensions: toNode.dimensions || []
title: toNode.title
});
return {
src/tools/database/createNode.ts
+13 -11
View File
@@ -2,7 +2,6 @@ import { tool } from 'ai';
import { z } from 'zod';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { normalizeDimensions } from '@/services/database/quality';
function extractTextFromMessageContent(content: unknown): string {
if (typeof content === 'string') {
@@ -57,7 +56,7 @@ function inferSourceFromContext(params: { title: string; description?: string; s
}
export const createNodeTool = tool({
description: 'Create node. Set the primary context explicitly when it is clear; otherwise the server will infer the best-fit context automatically so the node is not left unscoped. Infer a clean title, dimensions, and natural description with best effort. When the node comes from the user\'s own idea, note, or dictated thought, preserve their actual wording in source with only minimal cleanup instead of flattening it into a summary. Do not block creation if the description is incomplete. If the description framing is materially inferred, create the node first and then invite one concise user correction pass.',
description: 'Create a node. Set context explicitly only when it is clear and useful; otherwise leave it blank. Focus on a clean title, a strong natural description, preserved source text, and the right metadata. When the node comes from the user\'s own idea, note, or dictated thought, preserve their actual wording in source with only minimal cleanup instead of flattening it into a summary. Do not block creation if the description is incomplete. If the description framing is materially inferred, create the node first and then invite one concise user correction pass.',
inputSchema: z.object({
title: z.string().describe('The title of the node'),
description: z.string().max(500).optional().describe('Optional natural description. If you have enough context, describe what this is, why it belongs in Brad\'s graph, and its current workflow status in normal prose. Do not use labels like WHAT:, WHY:, or STATUS:.'),
@@ -66,23 +65,18 @@ export const createNodeTool = tool({
event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'),
context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Use when the node clearly belongs to a known context.'),
context_name: z.string().optional().describe('Optional convenience context name. Resolved to a stable context_id before persistence.'),
dimensions: z
.array(z.string())
.max(5)
.optional()
.describe('Optional secondary dimension tags to apply to this node (0-5 items).'),
metadata: z.record(z.any()).optional().describe('Optional node metadata. Use canonical keys when known: type, state, captured_method, captured_by, and source_metadata. Source-specific facts belong inside source_metadata.')
}),
execute: async (params, context) => {
console.log('🎯 CreateNode tool called with params:', JSON.stringify(params, null, 2));
try {
const trimmedDimensions = normalizeDimensions(params.dimensions || [], 5);
const canonicalSource = inferSourceFromContext(params, context);
// Call the nodes API endpoint
const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ...params, source: canonicalSource, dimensions: trimmedDimensions })
body: JSON.stringify({ ...params, source: canonicalSource })
});
const result = await response.json();
@@ -95,10 +89,18 @@ export const createNodeTool = tool({
};
}
const formattedDisplay = formatNodeForChat({
id: result.data.id,
title: result.data.title,
});
return {
success: true,
data: result.data,
message: `Created: ${formatNodeForChat(result.data)}`
data: {
...result.data,
formatted_display: formattedDisplay,
},
message: `Created: ${formattedDisplay}`
};
} catch (error) {
return {
src/tools/database/deleteDimension.ts
-58
View File
@@ -1,58 +0,0 @@
import { tool } from 'ai';
import { z } from 'zod';
export const deleteDimensionTool = tool({
description: 'Delete a dimension and remove all node associations',
inputSchema: z.object({
name: z.string().describe('Dimension name to delete')
}),
execute: async (params) => {
console.log('🗑️ DeleteDimension tool called with params:', JSON.stringify(params, null, 2));
try {
const trimmedName = params.name.trim();
if (!trimmedName) {
return {
success: false,
error: 'Dimension name is required',
data: null
};
}
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/dimensions?name=${encodeURIComponent(trimmedName)}`, {
method: 'DELETE',
headers: { 'Content-Type': 'application/json' }
});
if (!response.ok) {
let errorMessage = 'Failed to delete dimension';
try {
const errorResult = await response.json();
errorMessage = errorResult.error || errorMessage;
} catch {
// If response is not JSON (e.g., HTML error page), use status text
errorMessage = `Failed to delete dimension: ${response.status} ${response.statusText}`;
}
return {
success: false,
error: errorMessage,
data: null
};
}
const result = await response.json();
return {
success: true,
data: result.data,
message: `Deleted dimension "${trimmedName}" and removed all node associations`
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to delete dimension',
data: null
};
}
}
});
src/tools/database/getDimension.ts
-91
View File
@@ -1,91 +0,0 @@
import { tool } from 'ai';
import { z } from 'zod';
import { DimensionService } from '@/services/database/dimensionService';
import { getSQLiteClient } from '@/services/database/sqlite-client';
export const getDimensionTool = tool({
description: 'Get dimension details: description and node count.',
inputSchema: z.object({
name: z.string().describe('The exact name of the dimension to retrieve')
}),
execute: async ({ name }) => {
console.log('📁 GetDimension tool called for:', name);
try {
const trimmedName = name.trim();
if (!trimmedName) {
return {
success: false,
error: 'Dimension name is required',
data: null
};
}
// Get dimension details from service
const dimension = await DimensionService.getDimensionByName(trimmedName);
// Get node count for this dimension
const sqlite = getSQLiteClient();
const countResult = sqlite.query(`
SELECT COUNT(*) AS count
FROM node_dimensions
WHERE dimension = ?
`, [trimmedName]);
const nodeCount = countResult.rows.length > 0
? Number((countResult.rows[0] as { count: number }).count)
: 0;
if (!dimension) {
// Dimension might exist in node_dimensions but not in dimensions table
if (nodeCount > 0) {
return {
success: true,
data: {
name: trimmedName,
description: null,
nodeCount,
exists: true,
hasMetadata: false
},
message: `Dimension "${trimmedName}" exists with ${nodeCount} nodes but has no metadata (description/priority not set).`
};
}
return {
success: false,
error: `Dimension "${trimmedName}" not found`,
data: null
};
}
const result = {
name: dimension.name,
description: dimension.description,
nodeCount,
updatedAt: dimension.updated_at,
exists: true,
hasMetadata: true
};
// Build descriptive message
const parts: string[] = [];
parts.push(`Dimension: ${result.name}`);
parts.push(`Nodes: ${result.nodeCount}`);
if (result.description) parts.push(`Description: ${result.description}`);
parts.push(`Last updated: ${result.updatedAt}`);
return {
success: true,
data: result,
message: parts.join('\n')
};
} catch (error) {
console.error('GetDimension tool error:', error);
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to get dimension',
data: null
};
}
}
});
src/tools/database/getNodesById.ts
+2 -1
View File
@@ -36,7 +36,8 @@ export const getNodesByIdTool = tool({
title: node.title,
link: node.link,
event_date: node.event_date ?? null,
dimensions: node.dimensions || [],
context_id: node.context_id ?? null,
context: node.context ?? null,
chunk_status: node.chunk_status || 'unknown',
created_at: node.created_at,
updated_at: node.updated_at,
src/tools/database/queryContexts.ts
+38 -24
View File
@@ -19,11 +19,11 @@ export const queryContextsTool = tool({
description: 'List and inspect contexts. Use this to discover available primary scopes before filtering nodes or assigning node context.',
inputSchema: z.object({
filters: z.object({
id: z.number().int().positive().optional(),
name: z.string().optional(),
search: z.string().optional(),
limit: z.number().min(1).max(100).default(50).optional(),
includeNodes: z.boolean().default(false).optional(),
id: z.number().int().positive().describe('Exact context ID lookup.').optional(),
name: z.string().describe('Exact context name lookup.').optional(),
search: z.string().describe('Case-insensitive search across context names and descriptions.').optional(),
limit: z.number().min(1).max(100).default(50).describe('Maximum number of contexts to return.').optional(),
includeNodes: z.boolean().default(false).describe('Include the node list for an exact single-context lookup.').optional(),
}).optional(),
}),
execute: async ({ filters = {} }: { filters?: QueryContextFilters }) => {
@@ -33,12 +33,15 @@ export const queryContextsTool = tool({
const normalizedSearch = filters.search?.trim().toLowerCase();
let contexts = await contextService.listContexts();
if (filters.id !== undefined) {
contexts = contexts.filter((context) => context.id === filters.id);
}
if (normalizedName) {
contexts = contexts.filter((context) => context.name.toLowerCase() === normalizedName.toLowerCase());
}
if (normalizedSearch) {
contexts = contexts.filter((context) =>
matchesSearch(context.name, normalizedSearch) ||
@@ -47,35 +50,46 @@ export const queryContextsTool = tool({
}
const limitedContexts = contexts.slice(0, limit);
const includeNodes = filters.includeNodes === true && limitedContexts.length === 1 && (filters.id !== undefined || Boolean(normalizedName));
const canIncludeNodes =
filters.includeNodes === true &&
limitedContexts.length === 1 &&
(filters.id !== undefined || Boolean(normalizedName));
const enriched = await Promise.all(limitedContexts.map(async (context) => {
if (!includeNodes) return context;
const nodes = await contextService.getNodesForContext(context.id);
return {
...context,
nodes: nodes.map((node) => ({
id: node.id,
title: node.title,
description: node.description ?? null,
dimensions: node.dimensions || [],
context_id: node.context_id ?? null,
updated_at: node.updated_at,
})),
};
}));
const contextsWithNodes = await Promise.all(
limitedContexts.map(async (context) => {
if (!canIncludeNodes) return context;
const nodes = await contextService.getNodesForContext(context.id);
return {
...context,
nodes: nodes.map((node) => ({
id: node.id,
title: node.title,
description: node.description ?? null,
context_id: node.context_id ?? null,
context: node.context ?? null,
updated_at: node.updated_at,
})),
};
})
);
const message = contextsWithNodes.length === 0
? 'No contexts found.'
: `Found ${contextsWithNodes.length} context${contextsWithNodes.length === 1 ? '' : 's'}:\n${contextsWithNodes.map((context) => `• ${context.name} (#${context.id}, ${context.count} nodes)`).join('\n')}`;
return {
success: true,
data: {
contexts: enriched,
count: enriched.length,
contexts: contextsWithNodes,
count: contextsWithNodes.length,
total_available: contexts.length,
filters_applied: filters,
},
message: enriched.length === 0 ? 'No contexts found.' : `Found ${enriched.length} context(s).`,
message,
};
} catch (error) {
console.error('QueryContexts tool error:', error);
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to query contexts',
src/tools/database/queryDimensionNodes.ts
-78
View File
@@ -1,78 +0,0 @@
import { tool } from 'ai';
import { z } from 'zod';
import { nodeService } from '@/services/database/nodes';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import type { Node } from '@/types/database';
export const queryDimensionNodesTool = tool({
description: 'Get nodes in a dimension, sorted by connection count.',
inputSchema: z.object({
dimension: z.string().describe('The dimension name to query nodes from'),
limit: z.number().optional().default(20).describe('Maximum number of nodes to return (default: 20)'),
offset: z.number().optional().default(0).describe('Number of nodes to skip for pagination'),
includeContent: z.boolean().optional().default(false).describe('Include truncated content preview (default: false)'),
}),
execute: async ({ dimension, limit = 20, offset = 0, includeContent = false }) => {
try {
// Query nodes with this dimension
const nodes = await nodeService.getNodes({
dimensions: [dimension],
limit,
offset,
sortBy: 'edges',
});
if (!nodes || nodes.length === 0) {
return {
success: true,
dimension,
nodes: [],
total: 0,
message: `No nodes found in dimension "${dimension}"`,
};
}
const formattedNodes = nodes.map((node: Node) => {
const formatted: Record<string, unknown> = {
id: node.id,
title: node.title,
label: formatNodeForChat({
id: node.id,
title: node.title,
dimensions: node.dimensions || [],
}),
edgeCount: node.edge_count || 0,
dimensions: node.dimensions || [],
created_at: node.created_at,
updated_at: node.updated_at,
event_date: node.event_date ?? null,
};
if (includeContent && node.source) {
const previewSource = node.source;
// Truncate to ~100 chars
formatted.sourcePreview = previewSource.length > 100
? previewSource.substring(0, 100) + '...'
: previewSource;
}
return formatted;
});
return {
success: true,
dimension,
nodes: formattedNodes,
total: nodes.length,
hasMore: nodes.length >= limit,
message: `Found ${nodes.length} nodes in dimension "${dimension}"`,
};
} catch (error) {
console.error('queryDimensionNodes error:', error);
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to query dimension nodes',
};
}
},
});
src/tools/database/queryDimensions.ts
-100
View File
@@ -1,100 +0,0 @@
import { tool } from 'ai';
import { z } from 'zod';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
export const queryDimensionsTool = tool({
description: 'List the existing canonical dimensions with node counts. Use this before assigning dimensions to nodes. Do not invent new dimensions without explicit user instruction.',
inputSchema: z.object({
filters: z.object({
search: z.string().describe('Search term to match against dimension names').optional(),
limit: z.number().min(1).max(100).default(50).describe('Maximum number of results to return')
}).optional()
}),
execute: async ({ filters = {} }) => {
console.log('📁 QueryDimensions tool called with filters:', JSON.stringify(filters, null, 2));
try {
const limit = filters.limit || 50;
const baseUrl = getInternalApiBaseUrl();
// Use existing API endpoint for dimension listing
const response = await fetch(`${baseUrl}/api/dimensions/popular`);
if (!response.ok) {
let errorMessage = 'Failed to query dimensions';
try {
const errorResult = await response.json();
errorMessage = errorResult.error || errorMessage;
} catch {
errorMessage = `Failed to query dimensions: ${response.status} ${response.statusText}`;
}
return {
success: false,
error: errorMessage,
data: { dimensions: [], count: 0 }
};
}
const result = await response.json();
if (!result.success || !result.data) {
return {
success: false,
error: 'Invalid response from dimensions API',
data: { dimensions: [], count: 0 }
};
}
// Apply filters
let dimensions = result.data as Array<{
dimension: string;
count: number;
description: string | null;
}>;
// Filter by search term
if (filters.search) {
const searchLower = filters.search.toLowerCase();
dimensions = dimensions.filter(d =>
d.dimension.toLowerCase().includes(searchLower)
);
}
// Apply limit
const limitedDimensions = dimensions.slice(0, limit);
// Format for display
const formattedDimensions = limitedDimensions.map(d => ({
name: d.dimension,
count: d.count,
description: d.description
}));
// Build message
const filterParts: string[] = [];
if (filters.search) filterParts.push(`matching "${filters.search}"`);
const filterDesc = filterParts.length > 0 ? ` (${filterParts.join(', ')})` : '';
const dimensionList = formattedDimensions
.map(d => `• ${d.name} (${d.count} nodes)${d.description ? ` - ${d.description}` : ''}`)
.join('\n');
return {
success: true,
data: {
dimensions: formattedDimensions,
count: formattedDimensions.length,
total_available: dimensions.length,
filters_applied: filters
},
message: `Found ${formattedDimensions.length} dimensions${filterDesc}:\n${dimensionList || 'No dimensions found'}`
};
} catch (error) {
console.error('QueryDimensions tool error:', error);
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to query dimensions',
data: { dimensions: [], count: 0 }
};
}
}
});
src/tools/database/queryEdge.ts
+2 -3
View File
@@ -62,8 +62,7 @@ export const queryEdgeTool = tool({
const formattedConnections = limitedConnections.map(connection => {
const formattedNode = formatNodeForChat({
id: connection.connected_node.id,
title: connection.connected_node.title,
dimensions: connection.connected_node.dimensions || []
title: connection.connected_node.title
});
const context = connection.edge.context as Record<string, unknown> | undefined;
@@ -85,7 +84,7 @@ export const queryEdgeTool = tool({
id: connection.connected_node.id,
title: connection.connected_node.title,
description: truncateText(connection.connected_node.description, 140),
dimensions: connection.connected_node.dimensions || [],
context: connection.connected_node.context ?? null,
formatted_display: formattedNode
}
};
src/tools/database/queryNodes.ts
+3 -12
View File
@@ -6,7 +6,6 @@ import type { Node } from '@/types/database';
import { scoreNodeSearchMatch } from '@/services/database/searchRanking';
type QueryNodeFilters = {
dimensions?: string[];
contextId?: number;
search?: string;
limit?: number;
@@ -17,10 +16,9 @@ type QueryNodeFilters = {
};
export const queryNodesTool = tool({
description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Do not use dimensions to constrain keyword search unless the user is explicitly asking about a known dimension.',
description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Use context filtering only when the user is explicitly asking about a known context.',
inputSchema: z.object({
filters: z.object({
dimensions: z.array(z.string()).describe('Filter by dimensions (e.g., ["research", "ai", "technology"]). Replaces old type/stage filtering.').optional(),
contextId: z.number().int().positive().describe('Optional primary context filter.').optional(),
search: z.string().describe('Search term to match against node title, description, or source').optional(),
limit: z.number().min(1).max(50).default(10).describe('Maximum number of results to return'),
@@ -54,7 +52,6 @@ export const queryNodesTool = tool({
const formatted = formatNodeForChat({
id: node.id,
title: node.title,
dimensions: node.dimensions || [],
});
return {
@@ -63,7 +60,6 @@ export const queryNodesTool = tool({
nodes: [{
id: node.id,
title: node.title,
dimensions: node.dimensions || [],
created_at: node.created_at,
updated_at: node.updated_at,
event_date: node.event_date ?? null,
@@ -83,7 +79,6 @@ export const queryNodesTool = tool({
const nodesPromise: Promise<Node[] | undefined> = nodeService.getNodes({
limit,
dimensions: queryFilters.dimensions,
contextId: queryFilters.contextId,
search: queryFilters.search,
searchMode: searchTerm ? 'hybrid' : 'standard',
@@ -97,9 +92,7 @@ export const queryNodesTool = tool({
return Array.isArray(nodes) ? nodes : [];
};
const effectiveFilters = searchTerm
? { ...filters, dimensions: undefined }
: { ...filters };
const effectiveFilters = { ...filters };
let safeNodes = await runQuery(effectiveFilters);
@@ -118,12 +111,10 @@ export const queryNodesTool = tool({
const formatted = formatNodeForChat({
id: node.id,
title: node.title,
dimensions: node.dimensions || []
});
return {
id: node.id,
title: node.title,
dimensions: node.dimensions || [],
created_at: node.created_at,
updated_at: node.updated_at,
event_date: node.event_date ?? null,
@@ -133,7 +124,7 @@ export const queryNodesTool = tool({
// Create message with formatted node labels only (no full node payload)
const formattedLabels = formattedNodes.map(node => node.formatted_display).join(', ');
const message = `Found ${safeNodes.length} nodes${effectiveFilters.dimensions ? ` with dimensions: ${effectiveFilters.dimensions.join(', ')}` : ''}${effectiveFilters.search ? ` matching: "${effectiveFilters.search}"` : ''}${formattedLabels ? `:\n${formattedLabels}` : ''}`;
const message = `Found ${safeNodes.length} nodes${effectiveFilters.search ? ` matching: "${effectiveFilters.search}"` : ''}${formattedLabels ? `:\n${formattedLabels}` : ''}`;
return {
success: true,
src/tools/database/updateDimension.ts
-75
View File
@@ -1,75 +0,0 @@
import { tool } from 'ai';
import { z } from 'zod';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
export const updateDimensionTool = tool({
description: 'Update a dimension name or description.',
inputSchema: z.object({
currentName: z.string().describe('Current dimension name'),
newName: z.string().optional().describe('New dimension name (if renaming)'),
description: z.string().max(500).optional().describe('New description (max 500 characters)')
}),
execute: async (params) => {
console.log('📝 UpdateDimension tool called with params:', JSON.stringify(params, null, 2));
try {
// Validate at least one update field
if (!params.newName && params.description === undefined) {
return {
success: false,
error: 'At least one update field (newName or description) must be provided',
data: null
};
}
// Handle rename + other updates
const body: any = {
currentName: params.currentName.trim(),
description: params.description?.trim() || ''
};
if (params.newName) {
body.newName = params.newName.trim();
}
const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
});
if (!response.ok) {
let errorMessage = 'Failed to update dimension';
try {
const errorResult = await response.json();
errorMessage = errorResult.error || errorMessage;
} catch {
// If response is not JSON (e.g., HTML error page), use status text
errorMessage = `Failed to update dimension: ${response.status} ${response.statusText}`;
}
return {
success: false,
error: errorMessage,
data: null
};
}
const result = await response.json();
const updates = [];
if (params.newName) updates.push(`renamed to "${params.newName}"`);
if (params.description !== undefined) updates.push('description updated');
return {
success: true,
data: result.data,
message: `Updated dimension "${params.currentName}": ${updates.join(', ')}`
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to update dimension',
data: null
};
}
}
});
src/tools/database/updateNode.ts
+4 -8
View File
@@ -1,7 +1,6 @@
import { tool } from 'ai';
import { z } from 'zod';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { normalizeDimensions } from '@/services/database/quality';
export const updateNodeTool = tool({
description: 'Update node fields. Use this to enrich or correct nodes without losing canonical source content. Context is preserved unless context_id is supplied explicitly. When fixing a user-authored idea node, source should preserve the user\'s original wording as fully as possible. Never block an update because the description is incomplete. If the new description framing is materially inferred, complete the update and then invite one concise user feedback pass.',
@@ -14,7 +13,6 @@ export const updateNodeTool = tool({
link: z.string().optional().describe('New link'),
event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'),
context_id: z.number().int().positive().nullable().optional().describe('Primary context ID. Omit to preserve the existing context. Use null only to clear it intentionally.'),
dimensions: z.array(z.string()).optional().describe('New secondary dimension tags - completely replaces existing dimensions'),
metadata: z.record(z.any()).optional().describe('Metadata patch. It now merges with existing metadata instead of replacing the full blob. Use canonical keys: type, state, captured_method, captured_by, source_metadata.')
}).describe('Object containing the fields to update. Derived analysis should be stored in a separate linked node, not appended to the source node.')
}),
@@ -28,10 +26,7 @@ export const updateNodeTool = tool({
};
}
if (Array.isArray(updates.dimensions)) {
updates.dimensions = normalizeDimensions(updates.dimensions, 5);
}
// Call the nodes API endpoint
const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes/${id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
@@ -51,16 +46,17 @@ export const updateNodeTool = tool({
return {
success: true,
data: result.node,
message: `Updated node ID ${id}${updates.dimensions ? ` with dimensions: ${updates.dimensions.join(', ')}` : ''}`
message: `Updated node ID ${id}`
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to update node',
data: null
};
};
}
}
});
// Legacy export for backwards compatibility
export const updateItemTool = updateNodeTool;
src/tools/infrastructure/groups.ts
+2 -5
View File
@@ -18,7 +18,7 @@ export const TOOL_GROUPS: Record<string, ToolGroup> = {
orchestration: {
id: 'orchestration',
name: 'Orchestration',
description: 'Web search and reasoning tools',
description: 'Workflows, web search, and reasoning (orchestrator only)',
icon: '●',
color: '#8b5cf6'
},
@@ -37,13 +37,10 @@ export const TOOL_GROUP_ASSIGNMENTS: Record<string, string> = {
queryNodes: 'core',
getNodesById: 'core',
queryEdge: 'core',
queryDimensions: 'core',
getDimension: 'core',
queryDimensionNodes: 'core',
queryContexts: 'core',
searchContentEmbeddings: 'core',
// Orchestration: Web search and reasoning
// Orchestration: Web search and reasoning (orchestrator only)
webSearch: 'orchestration',
think: 'orchestration',
src/tools/infrastructure/nodeFormatter.ts
-2
View File
@@ -7,7 +7,6 @@
export interface NodeData {
id: number;
title: string;
dimensions?: string[];
}
/**
@@ -45,7 +44,6 @@ export function parseNodeMarkers(text: string): Array<NodeData & { raw: string }
raw,
id: parseInt(id, 10),
title,
dimensions: []
});
}
src/tools/infrastructure/registry.ts
+7 -39
View File
@@ -1,20 +1,13 @@
import { getToolGroup, groupTools, getAllToolsByGroup } from './groups';
import { queryNodesTool } from '../database/queryNodes';
import { getNodesByIdTool } from '../database/getNodesById';
import { queryEdgeTool } from '../database/queryEdge';
import { queryContextsTool } from '../database/queryContexts';
import { createNodeTool } from '../database/createNode';
import { updateNodeTool } from '../database/updateNode';
import { deleteNodeTool } from '../database/deleteNode';
import { createEdgeTool } from '../database/createEdge';
import { queryEdgeTool } from '../database/queryEdge';
import { updateEdgeTool } from '../database/updateEdge';
import { createDimensionTool } from '../database/createDimension';
import { updateDimensionTool } from '../database/updateDimension';
// lockDimension and unlockDimension consolidated into updateDimension (use isPriority param)
import { deleteDimensionTool } from '../database/deleteDimension';
import { queryDimensionsTool } from '../database/queryDimensions';
import { getDimensionTool } from '../database/getDimension';
import { queryDimensionNodesTool } from '../database/queryDimensionNodes';
import { queryContextsTool } from '../database/queryContexts';
import { searchContentEmbeddingsTool } from '../other/searchContentEmbeddings';
import { webSearchTool } from '../other/webSearch';
import { thinkTool } from '../other/think';
@@ -24,34 +17,29 @@ import { paperExtractTool } from '../other/paperExtract';
import { sqliteQueryTool } from '../other/sqliteQuery';
import { logEvalToolCall } from '@/services/evals/evalsLogger';
// Core tools available to all agents (read-only graph operations)
// Read tools (graph queries)
const CORE_TOOLS: Record<string, any> = {
sqliteQuery: sqliteQueryTool,
queryNodes: queryNodesTool,
getNodesById: getNodesByIdTool,
queryEdge: queryEdgeTool,
queryDimensions: queryDimensionsTool,
getDimension: getDimensionTool,
queryDimensionNodes: queryDimensionNodesTool,
queryContexts: queryContextsTool,
searchContentEmbeddings: searchContentEmbeddingsTool,
};
// Utility tools
const ORCHESTRATION_TOOLS: Record<string, any> = {
webSearch: webSearchTool,
think: thinkTool,
};
// Execution tools for worker agents (includes write operations)
// Write tools (includes extraction)
const EXECUTION_TOOLS: Record<string, any> = {
createNode: createNodeTool,
updateNode: updateNodeTool,
deleteNode: deleteNodeTool,
createEdge: createEdgeTool,
updateEdge: updateEdgeTool,
createDimension: createDimensionTool,
updateDimension: updateDimensionTool,
deleteDimension: deleteDimensionTool,
youtubeExtract: youtubeExtractTool,
websiteExtract: websiteExtractTool,
paperExtract: paperExtractTool,
@@ -69,28 +57,9 @@ export const TOOLS: Record<string, any> = {
...EXECUTION_TOOLS,
};
const ORCHESTRATOR_TOOL_NAMES = Array.from(new Set([
...Object.keys(CORE_TOOLS),
'webSearch',
'think',
'createNode',
'updateNode',
'deleteNode',
'createEdge',
'updateEdge',
'createDimension',
'updateDimension',
'deleteDimension',
'youtubeExtract',
'websiteExtract',
'paperExtract',
]));
const ORCHESTRATOR_TOOL_NAMES = Object.keys(TOOLS);
const EXECUTOR_TOOL_NAMES = [
...Object.keys(CORE_TOOLS),
...Object.keys(ORCHESTRATION_TOOLS),
...Object.keys(EXECUTION_TOOLS),
];
const EXECUTOR_TOOL_NAMES = Object.keys(TOOLS);
// Note: PLANNER_TOOL_NAMES kept for backwards compatibility but workflows now use specific tool sets
const PLANNER_TOOL_NAMES = [
@@ -99,7 +68,6 @@ const PLANNER_TOOL_NAMES = [
'think',
'updateNode',
'createEdge',
'updateDimension',
];
/**
src/tools/other/paperExtract.ts
+87 -38
View File
@@ -8,15 +8,21 @@ import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string' ? candidate.trim().replace(/\s+/g, ' ') : '';
const normalizedCandidate = typeof candidate === 'string'
? candidate.trim().replace(/\s+/g, ' ')
: '';
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
return normalizedCandidate.slice(0, 500);
}
const lead = normalizedCandidate || fallbackLead.trim();
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
return `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`.slice(0, 500);
const joined = `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`;
return joined.slice(0, 500);
}
// AI-powered content analysis
async function analyzeContentWithAI(title: string, description: string, contentType: string) {
try {
const prompt = `Analyze this ${contentType} content and provide classification.
@@ -25,35 +31,57 @@ Title: "${title}"
Description: "${description}"
CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose.
2. Make clear what this literally is.
3. State the actual finding, method, or contribution.
4. Make clear why it belongs in the graph. If unclear, say so naturally.
5. Make the workflow status clear.
1. Write natural prose, not labels or a checklist.
2. Make clear what this literally is: "Paper by…", "Research from…", "Preprint introducing…"
3. Name the authors if known from the metadata.
4. State the actual finding, method, or contribution — not "a study on X" but what they actually found or built.
5. Make clear why it belongs in the graph. If that cannot be inferred, say so naturally.
6. Make the workflow status clear. If unknown, say naturally that it has not been reviewed yet.
7. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to". State things directly.
Respond with ONLY valid JSON:
Examples:
- Title: "Attention Is All You Need" / Authors: Vaswani et al.
GOOD: "Vaswani et al. introduce the Transformer architecture — replaces recurrence with self-attention for sequence modeling. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "This paper discusses a new architecture called the Transformer and explores its applications."
- Title: "Scaling Laws for Neural Language Models" / Authors: Kaplan et al.
GOOD: "Kaplan et al. show that LLM performance scales as a power law with compute, data, and parameters — and compute matters most. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "A study examining how neural language models scale with different factors."
Respond with ONLY valid JSON (no markdown, no code blocks):
{
"enhancedDescription": "A comprehensive summary.",
"nodeDescription": "<your natural description>",
"reasoning": "Brief explanation"
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your natural description following the rules above>",
"tags": ["relevant", "semantic", "tags"],
"reasoning": "Brief explanation of classification choices"
}`;
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
maxOutputTokens: 800
});
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
tags: Array.isArray(result.tags) ? result.tags : [],
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
console.warn('Paper analysis fallback (using default description):', error);
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('Paper analysis fallback (using default description):', message);
return {
enhancedDescription: description,
nodeDescription: undefined,
tags: [],
reasoning: 'Fallback description used'
};
}
@@ -63,20 +91,30 @@ export const paperExtractTool = tool({
description: 'Extract a PDF or research paper into a node with summary, metadata, and full-text source',
inputSchema: z.object({
url: z.string().describe('The PDF URL to add to inbox'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
title: z.string().optional().describe('Custom title (auto-generated if not provided)')
}),
execute: async ({ url, title, dimensions }) => {
execute: async ({ url, title }) => {
try {
// Validate PDF URL
if (!url.startsWith('http://') && !url.startsWith('https://')) {
return { success: false, error: 'Invalid URL format - must start with http:// or https://', data: null };
return {
success: false,
error: 'Invalid URL format - must start with http:// or https://',
data: null
};
}
// Check if URL likely points to a PDF
if (!url.toLowerCase().includes('.pdf') && !url.includes('arxiv.org')) {
return { success: false, error: 'URL does not appear to point to a PDF file', data: null };
return {
success: false,
error: 'URL does not appear to point to a PDF file',
data: null
};
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractPaper(url);
result = {
@@ -92,73 +130,84 @@ export const paperExtractTool = tool({
}
};
} catch (error: any) {
result = { success: false, error: error.message || 'TypeScript extraction failed' };
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
}
if (!result.success || !result.source) {
return { success: false, error: result.error || 'Failed to extract PDF content', data: null };
return {
success: false,
error: result.error || 'Failed to extract PDF content',
data: null
};
}
console.log('🎯 PDF extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`,
result.source.substring(0, 2000) || 'PDF document content',
'pdf'
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`;
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
`Research paper or PDF from ${new URL(url).hostname} titled ${nodeTitle}`
);
const trimmedDimensions = Array.isArray(dimensions) ? dimensions.slice(0, 5) : [];
const fallbackDescriptionLead = `PDF document titled "${nodeTitle}"`;
const finalDescription = ensureNodeDescription(aiAnalysis?.nodeDescription, fallbackDescriptionLead);
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: nodeDescription,
description: finalDescription,
source: result.source,
link: url,
dimensions: trimmedDimensions,
metadata: {
type: 'pdf',
state: 'not_processed',
captured_method: 'paper_extract',
captured_by: 'agent',
captured_by: 'human',
source_metadata: {
hostname: new URL(url).hostname,
author: result.metadata?.author || result.metadata?.info?.Author,
pages: result.metadata?.pages,
file_size: result.metadata?.file_size,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'typescript',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: aiAnalysis?.enhancedDescription,
refined_at: new Date().toISOString()
extraction_method: result.metadata?.extraction_method || 'python_pdfplumber',
refined_at: new Date().toISOString(),
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return { success: false, error: createResult.error || 'Failed to create node', data: null };
return {
success: false,
error: createResult.error || 'Failed to create node',
data: null
};
}
const actualDimensions: string[] = createResult.data?.dimensions || trimmedDimensions || [];
console.log('🎯 PaperExtract completed successfully');
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle })
: nodeTitle;
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
message: `Added ${formattedNode}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url,
dimensions: actualDimensions
url: url
}
};
} catch (error) {
src/tools/other/sqliteQuery.ts
+10 -2
View File
@@ -2,10 +2,18 @@ import { tool } from 'ai';
import { z } from 'zod';
import { exec } from 'child_process';
import { promisify } from 'util';
import { getDatabasePath } from '@/services/database/sqlite-runtime';
import path from 'path';
const execAsync = promisify(exec);
// Get database path (same logic as sqlite-client.ts)
function getDatabasePath(): string {
return process.env.SQLITE_DB_PATH || path.join(
process.env.HOME || '~',
'Library/Application Support/RA-H/db/rah.sqlite'
);
}
// Security: Only allow SELECT statements
function isReadOnlyQuery(sql: string): boolean {
const normalized = sql.trim().toLowerCase();
@@ -37,7 +45,7 @@ function isReadOnlyQuery(sql: string): boolean {
}
export const sqliteQueryTool = tool({
description: 'Execute read-only SQL queries (SELECT/WITH/PRAGMA). Tables: nodes, edges, dimensions, chunks. Use PRAGMA table_info(tablename) for schema.',
description: 'Execute read-only SQL queries (SELECT/WITH/PRAGMA). Tables: nodes, contexts, edges, chunks, logs, and migration snapshots. Use PRAGMA table_info(tablename) for schema.',
inputSchema: z.object({
sql: z.string().describe('The SQL query to execute. Must be a SELECT, WITH, or PRAGMA statement.'),
src/tools/other/websiteExtract.ts
+91 -94
View File
@@ -7,19 +7,19 @@ import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
interface ExistingDimension {
name: string;
description: string | null;
}
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string' ? candidate.trim().replace(/\s+/g, ' ') : '';
const normalizedCandidate = typeof candidate === 'string'
? candidate.trim().replace(/\s+/g, ' ')
: '';
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
return normalizedCandidate.slice(0, 500);
}
const lead = normalizedCandidate || fallbackLead.trim();
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
return `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`.slice(0, 500);
const joined = `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`;
return joined.slice(0, 500);
}
function inferWebsiteContentType(url: string): 'website' | 'tweet' {
@@ -33,47 +33,13 @@ function inferWebsiteContentType(url: string): 'website' | 'tweet' {
}
}
async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
// AI-powered content analysis
async function analyzeContentWithAI(
title: string,
description: string,
contentType: string
) {
try {
const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions/popular`);
if (!response.ok) return [];
const result = await response.json();
if (!Array.isArray(result.data)) return [];
return result.data
.map((dimension: { dimension?: unknown; description?: unknown }) => ({
name: typeof dimension.dimension === 'string' ? dimension.dimension.trim() : '',
description: typeof dimension.description === 'string' ? dimension.description.trim() : null
}))
.filter((dimension: ExistingDimension) => dimension.name.length > 0);
} catch (error) {
console.warn('Website dimension fetch fallback (no dimension context):', error);
return [];
}
}
function selectExistingDimensions(selected: unknown, existingDimensions: ExistingDimension[], max = 5): string[] {
if (!Array.isArray(selected) || existingDimensions.length === 0) return [];
const byLowerName = new Map(existingDimensions.map((dimension) => [dimension.name.toLowerCase(), dimension.name]));
const normalized: string[] = [];
const seen = new Set<string>();
for (const value of selected) {
if (typeof value !== 'string') continue;
const matched = byLowerName.get(value.trim().toLowerCase());
if (!matched) continue;
const key = matched.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
normalized.push(matched);
if (normalized.length >= max) break;
}
return normalized;
}
async function analyzeContentWithAI(title: string, description: string, contentType: string, existingDimensions: ExistingDimension[]) {
try {
const availableDimensionsBlock = existingDimensions.length > 0
? existingDimensions.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`).join('\n')
: '- No existing dimensions available. Return an empty dimensions array.';
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
@@ -81,55 +47,77 @@ Description: "${description}"
CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose, not labels or a checklist.
2. Make clear what this literally is using explicit entity words only.
2. Make clear what this literally is using explicit entity words like "Blog post by…", "Article from…", "Essay arguing…", "Tutorial on…", "Thread by…", "Tweet by…", "Post by…"
3. Name the author/site if known from the metadata.
4. State the actual claim or thesis.
5. Make clear why it belongs in the graph. If that remains unclear, say so naturally.
6. Make workflow status clear.
4. State the actual claim or thesis — don't paraphrase into vague abstractions.
5. Make clear why it belongs in the graph. If that cannot be inferred, say so naturally.
6. Make the workflow status clear. If unknown, say naturally that it has not been reviewed yet.
7. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to". State things directly.
Available dimensions:
${availableDimensionsBlock}
Examples:
- Title: "Software is eating the world — again" / Author: Andrej Karpathy
GOOD: "Karpathy's blog post arguing AI agents make software fluid — they can rip functionality from repos instead of taking dependencies. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "By Karpathy — discusses the importance of software becoming more fluid and malleable with agents."
Respond with ONLY valid JSON:
- Title: "The case for slowing down AI" / Site: The Atlantic
GOOD: "Atlantic article making the case that AI labs should voluntarily slow capability research until safety catches up. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "This article explores ideas about slowing down AI development and its implications."
Respond with ONLY valid JSON (no markdown, no code blocks):
{
"enhancedDescription": "A comprehensive summary.",
"nodeDescription": "<your natural description>",
"dimensions": ["existing-dimension-1"],
"reasoning": "Brief explanation"
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your natural description following the rules above>",
"reasoning": "Brief explanation of classification choices"
}`;
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
maxOutputTokens: 800
});
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
dimensions: selectExistingDimensions(result.dimensions, existingDimensions, 5),
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
console.warn('Website analysis fallback (using default description):', error);
return { enhancedDescription: description, nodeDescription: undefined, dimensions: [], reasoning: 'Fallback description used' };
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('Website analysis fallback (using default description):', message);
return {
enhancedDescription: description,
nodeDescription: undefined,
reasoning: 'Fallback description used'
};
}
}
export const websiteExtractTool = tool({
description: 'Extract website content and metadata into a node with summary, tags, and raw source text',
description: 'Extract website content and metadata into a node with summary and raw source text',
inputSchema: z.object({
url: z.string().describe('The website URL to add to knowledge base'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
title: z.string().optional().describe('Custom title (auto-generated if not provided)')
}),
execute: async ({ url, title, dimensions }) => {
execute: async ({ url, title }) => {
try {
// Validate URL format
if (!url.startsWith('http://') && !url.startsWith('https://')) {
return { success: false, error: 'Invalid URL format - must start with http:// or https://', data: null };
return {
success: false,
error: 'Invalid URL format - must start with http:// or https://',
data: null
};
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractWebsite(url);
result = {
@@ -146,76 +134,85 @@ export const websiteExtractTool = tool({
}
};
} catch (error: any) {
result = { success: false, error: error.message || 'TypeScript extraction failed' };
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
}
if (!result.success || !result.source) {
return { success: false, error: result.error || 'Failed to extract website content', data: null };
return {
success: false,
error: result.error || 'Failed to extract website content',
data: null
};
}
const existingDimensions = await fetchExistingDimensions();
console.log('🎯 Website extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const contentType = inferWebsiteContentType(url);
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.source.substring(0, 2000) || 'Website content',
inferWebsiteContentType(url),
existingDimensions
contentType
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.title || `Website: ${new URL(url).hostname}`;
const suppliedDimensions = Array.isArray(dimensions) ? dimensions.slice(0, 5) : [];
const finalDimensions = suppliedDimensions.length > 0 ? suppliedDimensions : (aiAnalysis?.dimensions || []).slice(0, 5);
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
`Website article from ${result.metadata?.site_name || new URL(url).hostname} titled ${nodeTitle}`
);
const fallbackDescriptionLead = `${contentType === 'tweet' ? 'Tweet' : 'Website article'} from ${result.metadata?.author || result.metadata?.site_name || new URL(url).hostname} titled "${nodeTitle}"`;
const finalDescription = ensureNodeDescription(aiAnalysis?.nodeDescription, fallbackDescriptionLead);
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: nodeDescription,
description: finalDescription,
source: result.source,
link: url,
dimensions: finalDimensions,
event_date: result.metadata?.published_date || result.metadata?.date || null,
metadata: {
type: inferWebsiteContentType(url),
type: contentType,
state: 'not_processed',
captured_method: 'website_extract',
captured_by: 'agent',
captured_by: 'human',
source_metadata: {
hostname: new URL(url).hostname,
author: result.metadata?.author,
site_name: result.metadata?.site_name,
date: result.metadata?.date,
og_image: result.metadata?.og_image,
extraction_method: result.metadata?.extraction_method,
ai_analysis: aiAnalysis?.reasoning,
refined_at: new Date().toISOString()
published_date: result.metadata?.published_date || result.metadata?.date,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'python_beautifulsoup',
refined_at: new Date().toISOString(),
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return { success: false, error: createResult.error || 'Failed to create node', data: null };
return {
success: false,
error: createResult.error || 'Failed to create node',
data: null
};
}
const actualDimensions: string[] = createResult.data?.dimensions || finalDimensions || [];
console.log('🎯 WebsiteExtract completed successfully');
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle })
: nodeTitle;
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
message: `Added ${formattedNode}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url,
dimensions: actualDimensions
url: url
}
};
} catch (error) {
src/tools/other/youtubeExtract.ts
+83 -104
View File
@@ -7,11 +7,6 @@ import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
interface ExistingDimension {
name: string;
description: string | null;
}
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string'
? candidate.trim().replace(/\s+/g, ' ')
@@ -27,53 +22,13 @@ function ensureNodeDescription(candidate: string | undefined, fallbackLead: stri
return joined.slice(0, 500);
}
async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
// AI-powered content analysis
async function analyzeContentWithAI(
title: string,
description: string,
contentType: string
) {
try {
const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions/popular`);
if (!response.ok) return [];
const result = await response.json();
if (!Array.isArray(result.data)) return [];
return result.data
.map((dimension: { dimension?: unknown; description?: unknown }) => ({
name: typeof dimension.dimension === 'string' ? dimension.dimension.trim() : '',
description: typeof dimension.description === 'string' ? dimension.description.trim() : null
}))
.filter((dimension: ExistingDimension) => dimension.name.length > 0);
} catch (error) {
console.warn('YouTube dimension fetch fallback (no dimension context):', error);
return [];
}
}
function selectExistingDimensions(selected: unknown, existingDimensions: ExistingDimension[], max = 5): string[] {
if (!Array.isArray(selected) || existingDimensions.length === 0) return [];
const byLowerName = new Map(existingDimensions.map((dimension) => [dimension.name.toLowerCase(), dimension.name]));
const normalized: string[] = [];
const seen = new Set<string>();
for (const value of selected) {
if (typeof value !== 'string') continue;
const matched = byLowerName.get(value.trim().toLowerCase());
if (!matched) continue;
const key = matched.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
normalized.push(matched);
if (normalized.length >= max) break;
}
return normalized;
}
async function analyzeContentWithAI(title: string, description: string, contentType: string, existingDimensions: ExistingDimension[]) {
try {
const availableDimensionsBlock = existingDimensions.length > 0
? existingDimensions.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`).join('\n')
: '- No existing dimensions available. Return an empty dimensions array.';
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
@@ -83,24 +38,25 @@ CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose, not labels or a checklist.
2. Make clear what this literally is: "Podcast episode where…", "Talk by…", "Interview with…", "Video essay on…"
3. Name people by their role: the channel/host is the creator, anyone in the title is likely the guest or subject.
4. State the actual claim or thesis from the title.
4. State the actual claim or thesis from the title — don't paraphrase into vague abstractions.
5. Make clear why it belongs in the graph. If that cannot be inferred, say so naturally.
6. Make the workflow status clear. If unknown, say naturally that it has not been reviewed yet.
7. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to".
7. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to". State things directly.
DIMENSION SELECTION:
You must select 0-3 dimensions from the list below.
Do NOT invent new dimension names.
Examples:
- Title: "Dario Amodei — We are near the end of the exponential" / Channel: Dwarkesh Patel
GOOD: "Dwarkesh Patel interview with Anthropic CEO Dario Amodei — argues we're nearing the end of exponential AI scaling. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "By Dario Amodei — discusses reaching the limits of exponential growth in AI, emphasizing the need for a critical perspective."
Available dimensions:
${availableDimensionsBlock}
- Title: "The spell of language models" / Channel: Andrej Karpathy
GOOD: "Karpathy talk on how LLMs work under the hood — tokenization, attention, and why they feel like magic but aren't. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "By Andrej Karpathy — explores the nature of language models and their capabilities."
Respond with ONLY valid JSON:
Respond with ONLY valid JSON (no markdown, no code blocks):
{
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars).",
"nodeDescription": "<your natural description>",
"dimensions": ["existing-dimension-1"],
"reasoning": "Brief explanation"
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your natural description following the rules above>",
"reasoning": "Brief explanation of classification choices"
}`;
const response = await generateText({
@@ -109,21 +65,24 @@ Respond with ONLY valid JSON:
maxOutputTokens: 800
});
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
dimensions: selectExistingDimensions(result.dimensions, existingDimensions, 5),
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
console.warn('YouTube analysis fallback (using default description):', error);
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('YouTube analysis fallback (using default description):', message);
return {
enhancedDescription: description,
nodeDescription: undefined,
dimensions: [],
reasoning: 'Fallback description used'
};
}
@@ -134,21 +93,29 @@ async function summariseTranscript(title: string, transcript: string): Promise<s
return null;
}
const excerpt = transcript.trim().length > 16000
? `${transcript.trim().slice(0, 8000)}\n[...]\n${transcript.trim().slice(-8000)}`
: transcript.trim();
// Limit transcript length to keep token costs manageable
const MAX_CHARS = 16000;
let excerpt = transcript.trim();
if (excerpt.length > MAX_CHARS) {
const head = excerpt.slice(0, MAX_CHARS / 2);
const tail = excerpt.slice(-MAX_CHARS / 2);
excerpt = `${head}\n[...]\n${tail}`;
}
try {
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt: `You are summarising a long-form recording for a knowledge graph entry. Title: "${title}".
const prompt = `You are summarising a long-form recording for a knowledge graph entry. Title: "${title}".
Using the transcript excerpt below, write a concise 3-4 sentence summary covering the main themes, notable claims, and outcomes. Keep the tone factual.
Using the transcript excerpt below, write a concise 3-4 sentence summary covering the main themes, notable claims, and outcomes. If specific terms, frameworks, or memorable lines appear, mention them. Keep the tone factual (no marketing language). If the excerpt appears truncated, note that the summary is based on the portion provided.
Transcript excerpt:
"""
${excerpt}
"""`,
"""
`;
try {
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
maxOutputTokens: 400
});
return response.text?.trim() || null;
@@ -162,18 +129,23 @@ export const youtubeExtractTool = tool({
description: 'Extract a YouTube transcript and metadata, create a node, and return summary details',
inputSchema: z.object({
url: z.string().describe('The YouTube video URL to add to knowledge base'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
title: z.string().optional().describe('Custom title (auto-generated if not provided)')
}),
execute: async ({ url, title, dimensions }) => {
execute: async ({ url, title }) => {
console.log('🎯 YouTubeExtract tool called with URL:', url);
try {
// Validate YouTube URL
if (!url.includes('youtube.com') && !url.includes('youtu.be')) {
return { success: false, error: 'Invalid YouTube URL format', data: null };
return {
success: false,
error: 'Invalid YouTube URL format',
data: null
};
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
console.log('📝 Using TypeScript yt-dlp extractor');
try {
const extractionResult = await extractYouTube(url);
result = {
@@ -193,46 +165,48 @@ export const youtubeExtractTool = tool({
error: extractionResult.error
};
} catch (error: any) {
result = { success: false, error: error.message || 'TypeScript extraction failed' };
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
}
if (!result.success || !result.source) {
return { success: false, error: result.error || 'Failed to extract YouTube content', data: null };
return {
success: false,
error: result.error || 'Failed to extract YouTube content',
data: null
};
}
const existingDimensions = await fetchExistingDimensions();
console.log('🎯 YouTube extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.video_title || 'YouTube Video',
`Video by ${result.metadata?.channel_name || 'Unknown Channel'}`,
'youtube',
existingDimensions
'youtube'
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.video_title || `YouTube Video ${url.split('/').pop()?.split('?')[0]}`;
const transcriptSummary = await summariseTranscript(nodeTitle, result.source);
const suppliedDimensions = Array.isArray(dimensions) ? dimensions : [];
const finalDimensions = suppliedDimensions.slice(0, 5).length > 0
? suppliedDimensions.slice(0, 5)
: (aiAnalysis?.dimensions || []).slice(0, 5);
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
transcriptSummary || `YouTube video by ${result.metadata?.channel_name || 'an unknown creator'} about ${nodeTitle}`
);
const fallbackDescriptionLead = `YouTube video from ${result.metadata?.channel_name || 'an unknown channel'} titled "${nodeTitle}"`;
const finalDescription = ensureNodeDescription(aiAnalysis?.nodeDescription, fallbackDescriptionLead);
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: nodeDescription,
description: finalDescription,
source: result.source,
link: url,
dimensions: finalDimensions,
metadata: {
type: 'youtube',
state: 'not_processed',
captured_method: 'youtube_extract',
captured_by: 'agent',
captured_by: 'human',
source_metadata: {
video_id: result.metadata?.video_id,
channel_name: result.metadata?.channel_name,
@@ -242,33 +216,38 @@ export const youtubeExtractTool = tool({
total_segments: result.metadata?.total_segments,
language: result.metadata?.language,
extraction_method: result.metadata?.extraction_method,
ai_analysis: aiAnalysis?.reasoning,
summary_origin: transcriptSummary ? 'transcript_summary' : 'metadata_description',
refined_at: new Date().toISOString()
transcript_summary: transcriptSummary,
refined_at: new Date().toISOString(),
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return { success: false, error: createResult.error || 'Failed to create item', data: null };
return {
success: false,
error: createResult.error || 'Failed to create item',
data: null
};
}
const actualDimensions: string[] = createResult.data?.dimensions || finalDimensions || [];
console.log('🎯 YouTubeExtract completed successfully');
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle })
: nodeTitle;
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
message: `Added ${formattedNode}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url,
dimensions: actualDimensions
url: url
}
};
} catch (error) {