feat: port contexts layer and MCP parity
This commit is contained in:
“BeeRad”
@@ -3,9 +3,20 @@ import { z } from 'zod';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
import { extractPaper } from '@/services/typescript/extractors/paper';
|
||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
||||
import { validateExplicitDescription } from '@/services/database/quality';
|
||||
|
||||
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
|
||||
const normalizedCandidate = typeof candidate === 'string' ? candidate.trim().replace(/\s+/g, ' ') : '';
|
||||
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
|
||||
return normalizedCandidate.slice(0, 500);
|
||||
}
|
||||
const lead = normalizedCandidate || fallbackLead.trim();
|
||||
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
|
||||
return `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`.slice(0, 500);
|
||||
}
|
||||
|
||||
// AI-powered content analysis
|
||||
async function analyzeContentWithAI(title: string, description: string, contentType: string) {
|
||||
try {
|
||||
const prompt = `Analyze this ${contentType} content and provide classification.
|
||||
@@ -13,56 +24,36 @@ async function analyzeContentWithAI(title: string, description: string, contentT
|
||||
Title: "${title}"
|
||||
Description: "${description}"
|
||||
|
||||
CRITICAL — nodeDescription rules (max 280 chars):
|
||||
1. Say WHAT this literally is: "Paper by…", "Research from…", "Preprint introducing…"
|
||||
2. Name the authors if known from the metadata.
|
||||
3. State the actual finding, method, or contribution — not "a study on X" but what they actually found or built.
|
||||
4. End with why it matters — one concrete phrase about impact or implication.
|
||||
5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly.
|
||||
CRITICAL — nodeDescription rules (max 500 chars):
|
||||
1. Write natural prose.
|
||||
2. Make clear what this literally is.
|
||||
3. State the actual finding, method, or contribution.
|
||||
4. Make clear why it belongs in the graph. If unclear, say so naturally.
|
||||
5. Make the workflow status clear.
|
||||
|
||||
Examples:
|
||||
- Title: "Attention Is All You Need" / Authors: Vaswani et al.
|
||||
GOOD: "Vaswani et al. introduce the Transformer architecture — replaces recurrence with self-attention for sequence modeling. Foundation of every modern LLM."
|
||||
BAD: "This paper discusses a new architecture called the Transformer and explores its applications."
|
||||
|
||||
- Title: "Scaling Laws for Neural Language Models" / Authors: Kaplan et al.
|
||||
GOOD: "Kaplan et al. show that LLM performance scales as a power law with compute, data, and parameters — and compute matters most. The paper that launched the scaling era."
|
||||
BAD: "A study examining how neural language models scale with different factors."
|
||||
|
||||
Respond with ONLY valid JSON (no markdown, no code blocks):
|
||||
Respond with ONLY valid JSON:
|
||||
{
|
||||
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
|
||||
"nodeDescription": "<your 280-char description following the rules above>",
|
||||
"tags": ["relevant", "semantic", "tags"],
|
||||
"reasoning": "Brief explanation of classification choices"
|
||||
"enhancedDescription": "A comprehensive summary.",
|
||||
"nodeDescription": "<your natural description>",
|
||||
"reasoning": "Brief explanation"
|
||||
}`;
|
||||
|
||||
const response = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 800
|
||||
});
|
||||
|
||||
let content = response.text || '{}';
|
||||
|
||||
// Clean up the response - remove markdown code blocks if present
|
||||
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
|
||||
|
||||
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
|
||||
const result = JSON.parse(content);
|
||||
|
||||
return {
|
||||
enhancedDescription: result.enhancedDescription || description,
|
||||
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined,
|
||||
tags: Array.isArray(result.tags) ? result.tags : [],
|
||||
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
|
||||
reasoning: result.reasoning || 'AI analysis completed'
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'unknown error';
|
||||
console.warn('Paper analysis fallback (using default description):', message);
|
||||
console.warn('Paper analysis fallback (using default description):', error);
|
||||
return {
|
||||
enhancedDescription: description,
|
||||
nodeDescription: undefined,
|
||||
tags: [],
|
||||
reasoning: 'Fallback description used'
|
||||
};
|
||||
}
|
||||
@@ -73,30 +64,19 @@ export const paperExtractTool = tool({
|
||||
inputSchema: z.object({
|
||||
url: z.string().describe('The PDF URL to add to inbox'),
|
||||
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
|
||||
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node (locked dimensions first)')
|
||||
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
|
||||
}),
|
||||
execute: async ({ url, title, dimensions }) => {
|
||||
try {
|
||||
// Validate PDF URL
|
||||
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Invalid URL format - must start with http:// or https://',
|
||||
data: null
|
||||
};
|
||||
return { success: false, error: 'Invalid URL format - must start with http:// or https://', data: null };
|
||||
}
|
||||
|
||||
// Check if URL likely points to a PDF
|
||||
if (!url.toLowerCase().includes('.pdf') && !url.includes('arxiv.org')) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'URL does not appear to point to a PDF file',
|
||||
data: null
|
||||
};
|
||||
return { success: false, error: 'URL does not appear to point to a PDF file', data: null };
|
||||
}
|
||||
|
||||
let result: { success: boolean; source?: string; metadata?: any; error?: string };
|
||||
|
||||
try {
|
||||
const extractionResult = await extractPaper(url);
|
||||
result = {
|
||||
@@ -112,91 +92,72 @@ export const paperExtractTool = tool({
|
||||
}
|
||||
};
|
||||
} catch (error: any) {
|
||||
result = {
|
||||
success: false,
|
||||
error: error.message || 'TypeScript extraction failed'
|
||||
};
|
||||
result = { success: false, error: error.message || 'TypeScript extraction failed' };
|
||||
}
|
||||
|
||||
if (!result.success || !result.source) {
|
||||
return {
|
||||
success: false,
|
||||
error: result.error || 'Failed to extract PDF content',
|
||||
data: null
|
||||
};
|
||||
return { success: false, error: result.error || 'Failed to extract PDF content', data: null };
|
||||
}
|
||||
|
||||
console.log('🎯 PDF extraction successful, analyzing with AI...');
|
||||
|
||||
// Step 2: AI Analysis for enhanced metadata
|
||||
const aiAnalysis = await analyzeContentWithAI(
|
||||
result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`,
|
||||
result.source.substring(0, 2000) || 'PDF document content',
|
||||
result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`,
|
||||
result.source.substring(0, 2000) || 'PDF document content',
|
||||
'pdf'
|
||||
);
|
||||
|
||||
// Step 3: Create node with extracted content and AI analysis
|
||||
const nodeTitle = title || result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`;
|
||||
const enhancedDescription = aiAnalysis?.enhancedDescription || `PDF document from ${new URL(url).hostname}`;
|
||||
|
||||
const suppliedDimensions = Array.isArray(dimensions) ? dimensions : [];
|
||||
let trimmedDimensions = suppliedDimensions
|
||||
.map(dim => (typeof dim === 'string' ? dim.trim() : ''))
|
||||
.filter(Boolean);
|
||||
const nodeDescription = ensureNodeDescription(
|
||||
aiAnalysis?.nodeDescription,
|
||||
`Research paper or PDF from ${new URL(url).hostname} titled ${nodeTitle}`
|
||||
);
|
||||
const trimmedDimensions = Array.isArray(dimensions) ? dimensions.slice(0, 5) : [];
|
||||
|
||||
trimmedDimensions = trimmedDimensions.slice(0, 5);
|
||||
|
||||
const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
|
||||
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
title: nodeTitle,
|
||||
description: aiAnalysis?.nodeDescription,
|
||||
description: nodeDescription,
|
||||
source: result.source,
|
||||
link: url,
|
||||
dimensions: trimmedDimensions,
|
||||
metadata: {
|
||||
source: 'pdf',
|
||||
hostname: new URL(url).hostname,
|
||||
author: result.metadata?.author || result.metadata?.info?.Author,
|
||||
pages: result.metadata?.pages,
|
||||
file_size: result.metadata?.file_size,
|
||||
content_length: result.source.length,
|
||||
extraction_method: result.metadata?.extraction_method || 'python_pdfplumber',
|
||||
ai_analysis: aiAnalysis?.reasoning,
|
||||
enhanced_description: enhancedDescription,
|
||||
refined_at: new Date().toISOString()
|
||||
type: 'pdf',
|
||||
state: 'not_processed',
|
||||
captured_method: 'paper_extract',
|
||||
captured_by: 'agent',
|
||||
source_metadata: {
|
||||
hostname: new URL(url).hostname,
|
||||
author: result.metadata?.author || result.metadata?.info?.Author,
|
||||
pages: result.metadata?.pages,
|
||||
content_length: result.source.length,
|
||||
extraction_method: result.metadata?.extraction_method || 'typescript',
|
||||
ai_analysis: aiAnalysis?.reasoning,
|
||||
enhanced_description: aiAnalysis?.enhancedDescription,
|
||||
refined_at: new Date().toISOString()
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
const createResult = await createResponse.json();
|
||||
|
||||
if (!createResponse.ok) {
|
||||
return {
|
||||
success: false,
|
||||
error: createResult.error || 'Failed to create node',
|
||||
data: null
|
||||
};
|
||||
return { success: false, error: createResult.error || 'Failed to create node', data: null };
|
||||
}
|
||||
|
||||
console.log('🎯 PaperExtract completed successfully');
|
||||
|
||||
// Use actual assigned dimensions from API response (includes auto-assigned locked + keywords)
|
||||
const actualDimensions: string[] = createResult.data?.dimensions || trimmedDimensions || [];
|
||||
const formattedNode = createResult.data?.id
|
||||
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
|
||||
: nodeTitle;
|
||||
const dimsDisplay = actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none';
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Added ${formattedNode} with dimensions: ${dimsDisplay}`,
|
||||
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
|
||||
data: {
|
||||
nodeId: createResult.data?.id,
|
||||
title: nodeTitle,
|
||||
contentLength: result.source.length,
|
||||
url: url,
|
||||
url,
|
||||
dimensions: actualDimensions
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user