SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port contexts layer and MCP parity

Browse Source
This commit is contained in:
“BeeRad”
2026-04-10 19:41:26 +10:00
parent a8c5506fb7
commit 35f9ecf89c
62 changed files with 4206 additions and 1224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tools/database/createNode.ts
+66 -32
View File
@@ -2,46 +2,91 @@ import { tool } from 'ai';
import { z } from 'zod';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality';
import { normalizeDimensions } from '@/services/database/quality';
function extractTextFromMessageContent(content: unknown): string {
if (typeof content === 'string') {
return content;
}
if (!Array.isArray(content)) {
return '';
}
return content
.map((part) => {
if (!part || typeof part !== 'object') return '';
const candidate = part as Record<string, unknown>;
if (typeof candidate.text === 'string') return candidate.text;
if (candidate.type === 'text' && typeof candidate.value === 'string') return candidate.value;
return '';
})
.filter(Boolean)
.join('\n');
}
function inferSourceFromContext(params: { title: string; description?: string; source?: string }, context: any): string | undefined {
if (typeof params.source === 'string' && params.source.trim()) {
return params.source.trim();
}
const messages = Array.isArray(context?.messages) ? context.messages : [];
const latestUserMessage = [...messages].reverse().find((message: any) => message?.role === 'user');
if (!latestUserMessage) {
return undefined;
}
const rawUserText = extractTextFromMessageContent(latestUserMessage.content).trim();
if (!rawUserText) {
return undefined;
}
if (/^https?:\/\//i.test(rawUserText)) {
return undefined;
}
const normalized = rawUserText.replace(/\r\n/g, '\n').trim();
const descriptionLength = typeof params.description === 'string' ? params.description.trim().length : 0;
const isSubstantialCapture = normalized.length >= Math.max(280, descriptionLength + 120) || normalized.includes('\n');
if (!isSubstantialCapture) {
return undefined;
}
return normalized;
}
export const createNodeTool = tool({
description: 'Create node. Description is REQUIRED and must be explicit about what the thing is (podcast, chat summary, idea, etc).',
description: 'Create node. Set the primary context explicitly when it is clear; otherwise the server will infer the best-fit context automatically so the node is not left unscoped. Infer a clean title, dimensions, and natural description with best effort. When the node comes from the user\'s own idea, note, or dictated thought, preserve their actual wording in source with only minimal cleanup instead of flattening it into a summary. Do not block creation if the description is incomplete. If the description framing is materially inferred, create the node first and then invite one concise user correction pass.',
inputSchema: z.object({
title: z.string().describe('The title of the node'),
description: z.string().max(280).describe('REQUIRED. Explicitly state WHAT this is (e.g. podcast episode, conversation summary, user insight) + WHY it matters for context grounding.'),
source: z.string().optional().describe('Raw content for embedding: transcript, article text, book passages, or user thoughts. If omitted, falls back to title + description.'),
description: z.string().max(500).optional().describe('Optional natural description. If you have enough context, describe what this is, why it belongs in Brad\'s graph, and its current workflow status in normal prose. Do not use labels like WHAT:, WHY:, or STATUS:.'),
source: z.string().optional().describe('Canonical source content for embedding. For external content, store the actual transcript/article/document text. For user-authored ideas or dictated notes, store the user\'s original wording as fully as possible with only minimal cleanup such as obvious whitespace or transcription artifacts. Do not replace raw user thinking with a thin summary.'),
link: z.string().optional().describe('A URL link to the source'),
event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'),
context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Use when the node clearly belongs to a known context.'),
context_name: z.string().optional().describe('Optional convenience context name. Resolved to a stable context_id before persistence.'),
dimensions: z
.array(z.string())
.max(5)
.optional()
.describe('Optional dimension tags to apply to this node (0-5 items).'),
metadata: z.record(z.any()).optional().describe('Additional metadata like source info, extraction details, etc.')
.describe('Optional secondary dimension tags to apply to this node (0-5 items).'),
metadata: z.record(z.any()).optional().describe('Optional node metadata. Use canonical keys when known: type, state, captured_method, captured_by, and source_metadata. Source-specific facts belong inside source_metadata.')
}),
execute: async (params) => {
execute: async (params, context) => {
console.log('🎯 CreateNode tool called with params:', JSON.stringify(params, null, 2));
try {
const descriptionError = validateExplicitDescription(params.description);
if (descriptionError) {
return {
success: false,
error: `${descriptionError} Do not retry with minor rephrasing in the same turn. Rewrite the description so it explicitly names the entity type, such as note, node, person, episode, article, project, test node, or skill, and states why it matters.`,
data: null
};
}
const trimmedDimensions = normalizeDimensions(params.dimensions || [], 5);
const canonicalSource = inferSourceFromContext(params, context);
// Call the nodes API endpoint
const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ...params, dimensions: trimmedDimensions })
body: JSON.stringify({ ...params, source: canonicalSource, dimensions: trimmedDimensions })
});
const result = await response.json();
if (!response.ok) {
return {
success: false,
@@ -50,20 +95,10 @@ export const createNodeTool = tool({
};
}
// Format the created node for chat display
const formattedDisplay = formatNodeForChat({
id: result.data.id,
title: result.data.title,
dimensions: result.data.dimensions || trimmedDimensions
});
return {
success: true,
data: {
...result.data,
formatted_display: formattedDisplay
},
message: `Created node ${formattedDisplay} with dimensions: ${result.data.dimensions ? result.data.dimensions.join(', ') : 'none'}`
data: result.data,
message: `Created: ${formatNodeForChat(result.data)}`
};
} catch (error) {
return {
@@ -75,5 +110,4 @@ export const createNodeTool = tool({
}
});
// Legacy export for backwards compatibility
export const createItemTool = createNodeTool;
src/tools/database/queryContexts.ts
+90
View File
@@ -0,0 +1,90 @@
import { tool } from 'ai';
import { z } from 'zod';
import { contextService } from '@/services/database/contextService';
type QueryContextFilters = {
id?: number;
name?: string;
search?: string;
limit?: number;
includeNodes?: boolean;
};
function matchesSearch(value: string | null | undefined, search: string): boolean {
if (!value) return false;
return value.toLowerCase().includes(search);
}
export const queryContextsTool = tool({
description: 'List and inspect contexts. Use this to discover available primary scopes before filtering nodes or assigning node context.',
inputSchema: z.object({
filters: z.object({
id: z.number().int().positive().optional(),
name: z.string().optional(),
search: z.string().optional(),
limit: z.number().min(1).max(100).default(50).optional(),
includeNodes: z.boolean().default(false).optional(),
}).optional(),
}),
execute: async ({ filters = {} }: { filters?: QueryContextFilters }) => {
try {
const limit = filters.limit || 50;
const normalizedName = filters.name?.trim();
const normalizedSearch = filters.search?.trim().toLowerCase();
let contexts = await contextService.listContexts();
if (filters.id !== undefined) {
contexts = contexts.filter((context) => context.id === filters.id);
}
if (normalizedName) {
contexts = contexts.filter((context) => context.name.toLowerCase() === normalizedName.toLowerCase());
}
if (normalizedSearch) {
contexts = contexts.filter((context) =>
matchesSearch(context.name, normalizedSearch) ||
matchesSearch(context.description, normalizedSearch)
);
}
const limitedContexts = contexts.slice(0, limit);
const includeNodes = filters.includeNodes === true && limitedContexts.length === 1 && (filters.id !== undefined || Boolean(normalizedName));
const enriched = await Promise.all(limitedContexts.map(async (context) => {
if (!includeNodes) return context;
const nodes = await contextService.getNodesForContext(context.id);
return {
...context,
nodes: nodes.map((node) => ({
id: node.id,
title: node.title,
description: node.description ?? null,
dimensions: node.dimensions || [],
context_id: node.context_id ?? null,
updated_at: node.updated_at,
})),
};
}));
return {
success: true,
data: {
contexts: enriched,
count: enriched.length,
total_available: contexts.length,
filters_applied: filters,
},
message: enriched.length === 0 ? 'No contexts found.' : `Found ${enriched.length} context(s).`,
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : 'Failed to query contexts',
data: {
contexts: [],
count: 0,
filters_applied: filters,
},
};
}
},
});
src/tools/database/queryNodes.ts
+3
View File
@@ -7,6 +7,7 @@ import { scoreNodeSearchMatch } from '@/services/database/searchRanking';
type QueryNodeFilters = {
dimensions?: string[];
contextId?: number;
search?: string;
limit?: number;
createdAfter?: string;
@@ -20,6 +21,7 @@ export const queryNodesTool = tool({
inputSchema: z.object({
filters: z.object({
dimensions: z.array(z.string()).describe('Filter by dimensions (e.g., ["research", "ai", "technology"]). Replaces old type/stage filtering.').optional(),
contextId: z.number().int().positive().describe('Optional primary context filter.').optional(),
search: z.string().describe('Search term to match against node title, description, or source').optional(),
limit: z.number().min(1).max(50).default(10).describe('Maximum number of results to return'),
createdAfter: z.string().optional().describe('ISO date (YYYY-MM-DD). Only return nodes created on or after this date.'),
@@ -82,6 +84,7 @@ export const queryNodesTool = tool({
const nodesPromise: Promise<Node[] | undefined> = nodeService.getNodes({
limit,
dimensions: queryFilters.dimensions,
contextId: queryFilters.contextId,
search: queryFilters.search,
searchMode: searchTerm ? 'hybrid' : 'standard',
createdAfter: queryFilters.createdAfter,
src/tools/database/updateNode.ts
+9 -26
View File
@@ -1,20 +1,21 @@
import { tool } from 'ai';
import { z } from 'zod';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality';
import { normalizeDimensions } from '@/services/database/quality';
export const updateNodeTool = tool({
description: 'Update node fields. Description is REQUIRED on every update and must explicitly state WHAT this is + WHY it matters.',
description: 'Update node fields. Use this to enrich or correct nodes without losing canonical source content. Context is preserved unless context_id is supplied explicitly. When fixing a user-authored idea node, source should preserve the user\'s original wording as fully as possible. Never block an update because the description is incomplete. If the new description framing is materially inferred, complete the update and then invite one concise user feedback pass.',
inputSchema: z.object({
id: z.number().describe('The ID of the node to update'),
updates: z.object({
title: z.string().optional().describe('New title'),
description: z.string().max(280).describe('REQUIRED on every update. Explicitly state WHAT this is + WHY it matters. No "discusses/explores".'),
source: z.string().optional().describe('Canonical source content for embedding. Use this only to set or correct the raw source text.'),
description: z.string().max(500).optional().describe('Optional natural description. Replace the whole field with one clean description when you are improving context. It should read like normal prose, not labels.'),
source: z.string().optional().describe('Canonical source content for embedding. Use this to set or correct the raw source text. For user-authored ideas or dictated notes, preserve the user\'s original wording with only minimal cleanup rather than compressing it into a summary.'),
link: z.string().optional().describe('New link'),
event_date: z.string().optional().describe('When the thing actually happened (ISO 8601). Not when it was added to the graph.'),
dimensions: z.array(z.string()).optional().describe('New dimension tags - completely replaces existing dimensions'),
metadata: z.record(z.any()).optional().describe('New metadata - completely replaces existing metadata')
context_id: z.number().int().positive().nullable().optional().describe('Primary context ID. Omit to preserve the existing context. Use null only to clear it intentionally.'),
dimensions: z.array(z.string()).optional().describe('New secondary dimension tags - completely replaces existing dimensions'),
metadata: z.record(z.any()).optional().describe('Metadata patch. It now merges with existing metadata instead of replacing the full blob. Use canonical keys: type, state, captured_method, captured_by, source_metadata.')
}).describe('Object containing the fields to update. Derived analysis should be stored in a separate linked node, not appended to the source node.')
}),
execute: async ({ id, updates }) => {
@@ -27,27 +28,10 @@ export const updateNodeTool = tool({
};
}
if (!updates.description) {
return {
success: false,
error: 'Every node update requires an explicit description (WHAT this is + WHY it matters).',
data: null
};
}
const descriptionError = validateExplicitDescription(updates.description);
if (descriptionError) {
return {
success: false,
error: descriptionError,
data: null
};
}
if (Array.isArray(updates.dimensions)) {
updates.dimensions = normalizeDimensions(updates.dimensions, 5);
}
// Call the nodes API endpoint
const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes/${id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
@@ -55,7 +39,7 @@ export const updateNodeTool = tool({
});
const result = await response.json();
if (!response.ok) {
return {
success: false,
@@ -74,10 +58,9 @@ export const updateNodeTool = tool({
success: false,
error: error instanceof Error ? error.message : 'Failed to update node',
data: null
};
};
}
}
});
// Legacy export for backwards compatibility
export const updateItemTool = updateNodeTool;
src/tools/infrastructure/groups.ts
+1
View File
@@ -40,6 +40,7 @@ export const TOOL_GROUP_ASSIGNMENTS: Record<string, string> = {
queryDimensions: 'core',
getDimension: 'core',
queryDimensionNodes: 'core',
queryContexts: 'core',
searchContentEmbeddings: 'core',
// Orchestration: Web search and reasoning
src/tools/infrastructure/registry.ts
+2
View File
@@ -14,6 +14,7 @@ import { deleteDimensionTool } from '../database/deleteDimension';
import { queryDimensionsTool } from '../database/queryDimensions';
import { getDimensionTool } from '../database/getDimension';
import { queryDimensionNodesTool } from '../database/queryDimensionNodes';
import { queryContextsTool } from '../database/queryContexts';
import { searchContentEmbeddingsTool } from '../other/searchContentEmbeddings';
import { webSearchTool } from '../other/webSearch';
import { thinkTool } from '../other/think';
@@ -32,6 +33,7 @@ const CORE_TOOLS: Record<string, any> = {
queryDimensions: queryDimensionsTool,
getDimension: getDimensionTool,
queryDimensionNodes: queryDimensionNodesTool,
queryContexts: queryContextsTool,
searchContentEmbeddings: searchContentEmbeddingsTool,
};
src/tools/other/paperExtract.ts
+56 -95
View File
@@ -3,9 +3,20 @@ import { z } from 'zod';
import { openai } from '@ai-sdk/openai';
import { generateText } from 'ai';
import { extractPaper } from '@/services/typescript/extractors/paper';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string' ? candidate.trim().replace(/\s+/g, ' ') : '';
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
return normalizedCandidate.slice(0, 500);
}
const lead = normalizedCandidate || fallbackLead.trim();
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
return `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`.slice(0, 500);
}
// AI-powered content analysis
async function analyzeContentWithAI(title: string, description: string, contentType: string) {
try {
const prompt = `Analyze this ${contentType} content and provide classification.
@@ -13,56 +24,36 @@ async function analyzeContentWithAI(title: string, description: string, contentT
Title: "${title}"
Description: "${description}"
CRITICAL — nodeDescription rules (max 280 chars):
1. Say WHAT this literally is: "Paper by…", "Research from…", "Preprint introducing…"
2. Name the authors if known from the metadata.
3. State the actual finding, method, or contribution — not "a study on X" but what they actually found or built.
4. End with why it matters — one concrete phrase about impact or implication.
5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly.
CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose.
2. Make clear what this literally is.
3. State the actual finding, method, or contribution.
4. Make clear why it belongs in the graph. If unclear, say so naturally.
5. Make the workflow status clear.
Examples:
- Title: "Attention Is All You Need" / Authors: Vaswani et al.
GOOD: "Vaswani et al. introduce the Transformer architecture — replaces recurrence with self-attention for sequence modeling. Foundation of every modern LLM."
BAD: "This paper discusses a new architecture called the Transformer and explores its applications."
- Title: "Scaling Laws for Neural Language Models" / Authors: Kaplan et al.
GOOD: "Kaplan et al. show that LLM performance scales as a power law with compute, data, and parameters — and compute matters most. The paper that launched the scaling era."
BAD: "A study examining how neural language models scale with different factors."
Respond with ONLY valid JSON (no markdown, no code blocks):
Respond with ONLY valid JSON:
{
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your 280-char description following the rules above>",
"tags": ["relevant", "semantic", "tags"],
"reasoning": "Brief explanation of classification choices"
"enhancedDescription": "A comprehensive summary.",
"nodeDescription": "<your natural description>",
"reasoning": "Brief explanation"
}`;
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
maxOutputTokens: 800
});
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined,
tags: Array.isArray(result.tags) ? result.tags : [],
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('Paper analysis fallback (using default description):', message);
console.warn('Paper analysis fallback (using default description):', error);
return {
enhancedDescription: description,
nodeDescription: undefined,
tags: [],
reasoning: 'Fallback description used'
};
}
@@ -73,30 +64,19 @@ export const paperExtractTool = tool({
inputSchema: z.object({
url: z.string().describe('The PDF URL to add to inbox'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node (locked dimensions first)')
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
}),
execute: async ({ url, title, dimensions }) => {
try {
// Validate PDF URL
if (!url.startsWith('http://') && !url.startsWith('https://')) {
return {
success: false,
error: 'Invalid URL format - must start with http:// or https://',
data: null
};
return { success: false, error: 'Invalid URL format - must start with http:// or https://', data: null };
}
// Check if URL likely points to a PDF
if (!url.toLowerCase().includes('.pdf') && !url.includes('arxiv.org')) {
return {
success: false,
error: 'URL does not appear to point to a PDF file',
data: null
};
return { success: false, error: 'URL does not appear to point to a PDF file', data: null };
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractPaper(url);
result = {
@@ -112,91 +92,72 @@ export const paperExtractTool = tool({
}
};
} catch (error: any) {
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
result = { success: false, error: error.message || 'TypeScript extraction failed' };
}
if (!result.success || !result.source) {
return {
success: false,
error: result.error || 'Failed to extract PDF content',
data: null
};
return { success: false, error: result.error || 'Failed to extract PDF content', data: null };
}
console.log('🎯 PDF extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`,
result.source.substring(0, 2000) || 'PDF document content',
result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`,
result.source.substring(0, 2000) || 'PDF document content',
'pdf'
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`;
const enhancedDescription = aiAnalysis?.enhancedDescription || `PDF document from ${new URL(url).hostname}`;
const suppliedDimensions = Array.isArray(dimensions) ? dimensions : [];
let trimmedDimensions = suppliedDimensions
.map(dim => (typeof dim === 'string' ? dim.trim() : ''))
.filter(Boolean);
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
`Research paper or PDF from ${new URL(url).hostname} titled ${nodeTitle}`
);
const trimmedDimensions = Array.isArray(dimensions) ? dimensions.slice(0, 5) : [];
trimmedDimensions = trimmedDimensions.slice(0, 5);
const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: aiAnalysis?.nodeDescription,
description: nodeDescription,
source: result.source,
link: url,
dimensions: trimmedDimensions,
metadata: {
source: 'pdf',
hostname: new URL(url).hostname,
author: result.metadata?.author || result.metadata?.info?.Author,
pages: result.metadata?.pages,
file_size: result.metadata?.file_size,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'python_pdfplumber',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: enhancedDescription,
refined_at: new Date().toISOString()
type: 'pdf',
state: 'not_processed',
captured_method: 'paper_extract',
captured_by: 'agent',
source_metadata: {
hostname: new URL(url).hostname,
author: result.metadata?.author || result.metadata?.info?.Author,
pages: result.metadata?.pages,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'typescript',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: aiAnalysis?.enhancedDescription,
refined_at: new Date().toISOString()
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return {
success: false,
error: createResult.error || 'Failed to create node',
data: null
};
return { success: false, error: createResult.error || 'Failed to create node', data: null };
}
console.log('🎯 PaperExtract completed successfully');
// Use actual assigned dimensions from API response (includes auto-assigned locked + keywords)
const actualDimensions: string[] = createResult.data?.dimensions || trimmedDimensions || [];
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
: nodeTitle;
const dimsDisplay = actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none';
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${dimsDisplay}`,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url: url,
url,
dimensions: actualDimensions
}
};
src/tools/other/websiteExtract.ts
+64 -124
View File
@@ -3,13 +3,25 @@ import { z } from 'zod';
import { openai } from '@ai-sdk/openai';
import { generateText } from 'ai';
import { extractWebsite } from '@/services/typescript/extractors/website';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
interface ExistingDimension {
name: string;
description: string | null;
}
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string' ? candidate.trim().replace(/\s+/g, ' ') : '';
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
return normalizedCandidate.slice(0, 500);
}
const lead = normalizedCandidate || fallbackLead.trim();
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
return `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`.slice(0, 500);
}
function inferWebsiteContentType(url: string): 'website' | 'tweet' {
try {
const hostname = new URL(url).hostname.toLowerCase();
@@ -23,12 +35,10 @@ function inferWebsiteContentType(url: string): 'website' | 'tweet' {
async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
try {
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/dimensions/popular`);
const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions/popular`);
if (!response.ok) return [];
const result = await response.json();
if (!Array.isArray(result.data)) return [];
return result.data
.map((dimension: { dimension?: unknown; description?: unknown }) => ({
name: typeof dimension.dimension === 'string' ? dimension.dimension.trim() : '',
@@ -41,17 +51,11 @@ async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
}
}
function selectExistingDimensions(
selected: unknown,
existingDimensions: ExistingDimension[],
max = 5
): string[] {
function selectExistingDimensions(selected: unknown, existingDimensions: ExistingDimension[], max = 5): string[] {
if (!Array.isArray(selected) || existingDimensions.length === 0) return [];
const byLowerName = new Map(existingDimensions.map((dimension) => [dimension.name.toLowerCase(), dimension.name]));
const normalized: string[] = [];
const seen = new Set<string>();
for (const value of selected) {
if (typeof value !== 'string') continue;
const matched = byLowerName.get(value.trim().toLowerCase());
@@ -62,89 +66,53 @@ function selectExistingDimensions(
normalized.push(matched);
if (normalized.length >= max) break;
}
return normalized;
}
// AI-powered content analysis
async function analyzeContentWithAI(
title: string,
description: string,
contentType: string,
existingDimensions: ExistingDimension[]
) {
async function analyzeContentWithAI(title: string, description: string, contentType: string, existingDimensions: ExistingDimension[]) {
try {
const availableDimensionsBlock = existingDimensions.length > 0
? existingDimensions
.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`)
.join('\n')
? existingDimensions.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`).join('\n')
: '- No existing dimensions available. Return an empty dimensions array.';
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
Description: "${description}"
CRITICAL — nodeDescription rules (max 280 chars):
1. Say WHAT this literally is using explicit entity words only: "Blog post by…", "Article from…", "Essay arguing…", "Tutorial on…", "Thread by…", "Tweet by…", "Post by…"
2. Name the author/site if known from the metadata.
3. State the actual claim or thesis — don't paraphrase into vague abstractions.
4. End with why it's interesting or important — one concrete phrase.
5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly.
DIMENSION SELECTION (critical):
You must select 0-3 dimensions from the list below.
Do NOT invent new dimension names.
Pick only dimensions that genuinely fit this content.
If nothing fits, return an empty array.
CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose, not labels or a checklist.
2. Make clear what this literally is using explicit entity words only.
3. Name the author/site if known from the metadata.
4. State the actual claim or thesis.
5. Make clear why it belongs in the graph. If that remains unclear, say so naturally.
6. Make workflow status clear.
Available dimensions:
${availableDimensionsBlock}
Examples:
- Title: "Software is eating the world — again" / Author: Andrej Karpathy
GOOD: "Karpathy's blog post arguing AI agents make software fluid — they can rip functionality from repos instead of taking dependencies. Signals the end of monolithic libraries."
BAD: "By Karpathy — discusses the importance of software becoming more fluid and malleable with agents."
- Title: "The case for slowing down AI" / Site: The Atlantic
GOOD: "Atlantic article making the case that AI labs should voluntarily slow capability research until safety catches up. Notable because it cites internal lab disagreements."
BAD: "This article explores ideas about slowing down AI development and its implications."
Respond with ONLY valid JSON (no markdown, no code blocks):
Respond with ONLY valid JSON:
{
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your 280-char description following the rules above>",
"dimensions": ["existing-dimension-1", "existing-dimension-2"],
"reasoning": "Brief explanation of classification choices"
"enhancedDescription": "A comprehensive summary.",
"nodeDescription": "<your natural description>",
"dimensions": ["existing-dimension-1"],
"reasoning": "Brief explanation"
}`;
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
maxOutputTokens: 800
});
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
dimensions: selectExistingDimensions(result.dimensions, existingDimensions, 5),
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('Website analysis fallback (using default description):', message);
return {
enhancedDescription: description,
nodeDescription: undefined,
dimensions: [],
reasoning: 'Fallback description used'
};
console.warn('Website analysis fallback (using default description):', error);
return { enhancedDescription: description, nodeDescription: undefined, dimensions: [], reasoning: 'Fallback description used' };
}
}
@@ -153,21 +121,15 @@ export const websiteExtractTool = tool({
inputSchema: z.object({
url: z.string().describe('The website URL to add to knowledge base'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node (locked dimensions first)')
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
}),
execute: async ({ url, title, dimensions }) => {
try {
// Validate URL format
if (!url.startsWith('http://') && !url.startsWith('https://')) {
return {
success: false,
error: 'Invalid URL format - must start with http:// or https://',
data: null
};
return { success: false, error: 'Invalid URL format - must start with http:// or https://', data: null };
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractWebsite(url);
result = {
@@ -184,97 +146,75 @@ export const websiteExtractTool = tool({
}
};
} catch (error: any) {
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
result = { success: false, error: error.message || 'TypeScript extraction failed' };
}
if (!result.success || !result.source) {
return {
success: false,
error: result.error || 'Failed to extract website content',
data: null
};
return { success: false, error: result.error || 'Failed to extract website content', data: null };
}
console.log('🎯 Website extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const existingDimensions = await fetchExistingDimensions();
const contentType = inferWebsiteContentType(url);
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.source.substring(0, 2000) || 'Website content',
contentType,
result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.source.substring(0, 2000) || 'Website content',
inferWebsiteContentType(url),
existingDimensions
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.title || `Website: ${new URL(url).hostname}`;
const enhancedDescription = aiAnalysis?.enhancedDescription || `Website content from ${new URL(url).hostname}`;
const suppliedDimensions = Array.isArray(dimensions) ? dimensions : [];
let trimmedDimensions = suppliedDimensions
.map(dim => (typeof dim === 'string' ? dim.trim() : ''))
.filter(Boolean);
const suppliedDimensions = Array.isArray(dimensions) ? dimensions.slice(0, 5) : [];
const finalDimensions = suppliedDimensions.length > 0 ? suppliedDimensions : (aiAnalysis?.dimensions || []).slice(0, 5);
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
`Website article from ${result.metadata?.site_name || new URL(url).hostname} titled ${nodeTitle}`
);
trimmedDimensions = trimmedDimensions.slice(0, 5);
const finalDimensions = trimmedDimensions.length > 0
? trimmedDimensions
: (aiAnalysis?.dimensions || []).slice(0, 5);
const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: aiAnalysis?.nodeDescription,
description: nodeDescription,
source: result.source,
link: url,
event_date: result.metadata?.published_date || result.metadata?.date || null,
dimensions: finalDimensions,
metadata: {
source: contentType,
hostname: new URL(url).hostname,
author: result.metadata?.author,
published_date: result.metadata?.published_date || result.metadata?.date,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'python_beautifulsoup',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: enhancedDescription,
refined_at: new Date().toISOString()
type: inferWebsiteContentType(url),
state: 'not_processed',
captured_method: 'website_extract',
captured_by: 'agent',
source_metadata: {
hostname: new URL(url).hostname,
author: result.metadata?.author,
site_name: result.metadata?.site_name,
date: result.metadata?.date,
og_image: result.metadata?.og_image,
extraction_method: result.metadata?.extraction_method,
ai_analysis: aiAnalysis?.reasoning,
refined_at: new Date().toISOString()
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return {
success: false,
error: createResult.error || 'Failed to create node',
data: null
};
return { success: false, error: createResult.error || 'Failed to create node', data: null };
}
console.log('🎯 WebsiteExtract completed successfully');
// Use actual assigned dimensions from API response (includes auto-assigned locked + keywords)
const actualDimensions: string[] = createResult.data?.dimensions || finalDimensions || [];
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
: nodeTitle;
const dimsDisplay = actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none';
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${dimsDisplay}`,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url: url,
url,
dimensions: actualDimensions
}
};
src/tools/other/youtubeExtract.ts
+85 -124
View File
@@ -3,16 +3,33 @@ import { z } from 'zod';
import { openai } from '@ai-sdk/openai';
import { generateText } from 'ai';
import { extractYouTube } from '@/services/typescript/extractors/youtube';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
interface ExistingDimension {
name: string;
description: string | null;
}
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string'
? candidate.trim().replace(/\s+/g, ' ')
: '';
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
return normalizedCandidate.slice(0, 500);
}
const lead = normalizedCandidate || fallbackLead.trim();
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
const joined = `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`;
return joined.slice(0, 500);
}
async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
try {
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/dimensions/popular`);
const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions/popular`);
if (!response.ok) return [];
const result = await response.json();
@@ -30,11 +47,7 @@ async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
}
}
function selectExistingDimensions(
selected: unknown,
existingDimensions: ExistingDimension[],
max = 5
): string[] {
function selectExistingDimensions(selected: unknown, existingDimensions: ExistingDimension[], max = 5): string[] {
if (!Array.isArray(selected) || existingDimensions.length === 0) return [];
const byLowerName = new Map(existingDimensions.map((dimension) => [dimension.name.toLowerCase(), dimension.name]));
@@ -55,55 +68,39 @@ function selectExistingDimensions(
return normalized;
}
// AI-powered content analysis
async function analyzeContentWithAI(
title: string,
description: string,
contentType: string,
existingDimensions: ExistingDimension[]
) {
async function analyzeContentWithAI(title: string, description: string, contentType: string, existingDimensions: ExistingDimension[]) {
try {
const availableDimensionsBlock = existingDimensions.length > 0
? existingDimensions
.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`)
.join('\n')
? existingDimensions.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`).join('\n')
: '- No existing dimensions available. Return an empty dimensions array.';
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
Description: "${description}"
CRITICAL — nodeDescription rules (max 280 chars):
1. Say WHAT this literally is: "Podcast episode where…", "Talk by…", "Interview with…", "Video essay on…"
2. Name people by their role: the channel/host is the creator, anyone in the title is likely the guest or subject.
3. State the actual claim or thesis from the title — don't paraphrase into vague abstractions.
4. End with why it's interesting or important — one concrete phrase.
5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly.
CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose, not labels or a checklist.
2. Make clear what this literally is: "Podcast episode where…", "Talk by…", "Interview with…", "Video essay on…"
3. Name people by their role: the channel/host is the creator, anyone in the title is likely the guest or subject.
4. State the actual claim or thesis from the title.
5. Make clear why it belongs in the graph. If that cannot be inferred, say so naturally.
6. Make the workflow status clear. If unknown, say naturally that it has not been reviewed yet.
7. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to".
DIMENSION SELECTION (critical):
DIMENSION SELECTION:
You must select 0-3 dimensions from the list below.
Do NOT invent new dimension names.
Pick only dimensions that genuinely fit this content.
If nothing fits, return an empty array.
Available dimensions:
${availableDimensionsBlock}
Examples:
- Title: "Dario Amodei — We are near the end of the exponential" / Channel: Dwarkesh Patel
GOOD: "Dwarkesh Patel interview with Anthropic CEO Dario Amodei — argues we're nearing the end of exponential AI scaling. Key signal for what the next phase of AI development looks like."
BAD: "By Dario Amodei — discusses reaching the limits of exponential growth in AI, emphasizing the need for a critical perspective."
- Title: "The spell of language models" / Channel: Andrej Karpathy
GOOD: "Karpathy talk on how LLMs work under the hood — tokenization, attention, and why they feel like magic but aren't. Essential mental model for anyone building with LLMs."
BAD: "By Andrej Karpathy — explores the nature of language models and their capabilities."
Respond with ONLY valid JSON (no markdown, no code blocks):
Respond with ONLY valid JSON:
{
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your 280-char description following the rules above>",
"dimensions": ["existing-dimension-1", "existing-dimension-2"],
"reasoning": "Brief explanation of classification choices"
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars).",
"nodeDescription": "<your natural description>",
"dimensions": ["existing-dimension-1"],
"reasoning": "Brief explanation"
}`;
const response = await generateText({
@@ -112,22 +109,17 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
maxOutputTokens: 800
});
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
dimensions: selectExistingDimensions(result.dimensions, existingDimensions, 5),
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('YouTube analysis fallback (using default description):', message);
console.warn('YouTube analysis fallback (using default description):', error);
return {
enhancedDescription: description,
nodeDescription: undefined,
@@ -142,29 +134,21 @@ async function summariseTranscript(title: string, transcript: string): Promise<s
return null;
}
// Limit transcript length to keep token costs manageable
const MAX_CHARS = 16000;
let excerpt = transcript.trim();
if (excerpt.length > MAX_CHARS) {
const head = excerpt.slice(0, MAX_CHARS / 2);
const tail = excerpt.slice(-MAX_CHARS / 2);
excerpt = `${head}\n[...]\n${tail}`;
}
const prompt = `You are summarising a long-form recording for a knowledge graph entry. Title: "${title}".
Using the transcript excerpt below, write a concise 3-4 sentence summary covering the main themes, notable claims, and outcomes. If specific terms, frameworks, or memorable lines appear, mention them. Keep the tone factual (no marketing language). If the excerpt appears truncated, note that the summary is based on the portion provided.
Transcript excerpt:
"""
${excerpt}
"""
`;
const excerpt = transcript.trim().length > 16000
? `${transcript.trim().slice(0, 8000)}\n[...]\n${transcript.trim().slice(-8000)}`
: transcript.trim();
try {
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
prompt: `You are summarising a long-form recording for a knowledge graph entry. Title: "${title}".
Using the transcript excerpt below, write a concise 3-4 sentence summary covering the main themes, notable claims, and outcomes. Keep the tone factual.
Transcript excerpt:
"""
${excerpt}
"""`,
maxOutputTokens: 400
});
return response.text?.trim() || null;
@@ -179,23 +163,17 @@ export const youtubeExtractTool = tool({
inputSchema: z.object({
url: z.string().describe('The YouTube video URL to add to knowledge base'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node (locked dimensions first)')
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
}),
execute: async ({ url, title, dimensions }) => {
console.log('🎯 YouTubeExtract tool called with URL:', url);
try {
// Validate YouTube URL
if (!url.includes('youtube.com') && !url.includes('youtu.be')) {
return {
success: false,
error: 'Invalid YouTube URL format',
data: null
};
return { success: false, error: 'Invalid YouTube URL format', data: null };
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
console.log('📝 Using TypeScript yt-dlp extractor');
try {
const extractionResult = await extractYouTube(url);
result = {
@@ -215,98 +193,81 @@ export const youtubeExtractTool = tool({
error: extractionResult.error
};
} catch (error: any) {
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
result = { success: false, error: error.message || 'TypeScript extraction failed' };
}
if (!result.success || !result.source) {
return {
success: false,
error: result.error || 'Failed to extract YouTube content',
data: null
};
return { success: false, error: result.error || 'Failed to extract YouTube content', data: null };
}
console.log('🎯 YouTube extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const existingDimensions = await fetchExistingDimensions();
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.video_title || 'YouTube Video',
`Video by ${result.metadata?.channel_name || 'Unknown Channel'}`,
result.metadata?.video_title || 'YouTube Video',
`Video by ${result.metadata?.channel_name || 'Unknown Channel'}`,
'youtube',
existingDimensions
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.video_title || `YouTube Video ${url.split('/').pop()?.split('?')[0]}`;
const transcriptSummary = await summariseTranscript(nodeTitle, result.source);
const suppliedDimensions = Array.isArray(dimensions) ? dimensions : [];
let trimmedDimensions = suppliedDimensions
.map(dim => (typeof dim === 'string' ? dim.trim() : ''))
.filter(Boolean);
trimmedDimensions = trimmedDimensions.slice(0, 5);
const finalDimensions = trimmedDimensions.length > 0
? trimmedDimensions
const finalDimensions = suppliedDimensions.slice(0, 5).length > 0
? suppliedDimensions.slice(0, 5)
: (aiAnalysis?.dimensions || []).slice(0, 5);
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
transcriptSummary || `YouTube video by ${result.metadata?.channel_name || 'an unknown creator'} about ${nodeTitle}`
);
const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: aiAnalysis?.nodeDescription,
description: nodeDescription,
source: result.source,
link: url,
dimensions: finalDimensions,
metadata: {
source: 'youtube',
video_id: result.metadata?.video_id,
channel_name: result.metadata?.channel_name,
channel_url: result.metadata?.channel_url,
thumbnail_url: result.metadata?.thumbnail_url,
transcript_length: result.metadata?.transcript_length,
total_segments: result.metadata?.total_segments,
language: result.metadata?.language,
extraction_method: result.metadata?.extraction_method,
ai_analysis: aiAnalysis?.reasoning,
summary_origin: transcriptSummary ? 'transcript_summary' : 'metadata_description',
refined_at: new Date().toISOString()
type: 'youtube',
state: 'not_processed',
captured_method: 'youtube_extract',
captured_by: 'agent',
source_metadata: {
video_id: result.metadata?.video_id,
channel_name: result.metadata?.channel_name,
channel_url: result.metadata?.channel_url,
thumbnail_url: result.metadata?.thumbnail_url,
transcript_length: result.metadata?.transcript_length,
total_segments: result.metadata?.total_segments,
language: result.metadata?.language,
extraction_method: result.metadata?.extraction_method,
ai_analysis: aiAnalysis?.reasoning,
summary_origin: transcriptSummary ? 'transcript_summary' : 'metadata_description',
refined_at: new Date().toISOString()
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return {
success: false,
error: createResult.error || 'Failed to create item',
data: null
};
return { success: false, error: createResult.error || 'Failed to create item', data: null };
}
console.log('🎯 YouTubeExtract completed successfully');
// Use actual assigned dimensions from API response (includes auto-assigned locked + keywords)
const actualDimensions: string[] = createResult.data?.dimensions || finalDimensions || [];
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
: nodeTitle;
const dimsDisplay = actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none';
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${dimsDisplay}`,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url: url,
url,
dimensions: actualDimensions
}
};