SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port holistic node refinement contract

Browse Source
This commit is contained in:
“BeeRad”
2026-04-11 21:37:52 +10:00
parent 35f9ecf89c
commit 3ae46245ec
119 changed files with 6596 additions and 10982 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tools/other/paperExtract.ts
+87 -38
View File
@@ -8,15 +8,21 @@ import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string' ? candidate.trim().replace(/\s+/g, ' ') : '';
const normalizedCandidate = typeof candidate === 'string'
? candidate.trim().replace(/\s+/g, ' ')
: '';
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
return normalizedCandidate.slice(0, 500);
}
const lead = normalizedCandidate || fallbackLead.trim();
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
return `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`.slice(0, 500);
const joined = `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`;
return joined.slice(0, 500);
}
// AI-powered content analysis
async function analyzeContentWithAI(title: string, description: string, contentType: string) {
try {
const prompt = `Analyze this ${contentType} content and provide classification.
@@ -25,35 +31,57 @@ Title: "${title}"
Description: "${description}"
CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose.
2. Make clear what this literally is.
3. State the actual finding, method, or contribution.
4. Make clear why it belongs in the graph. If unclear, say so naturally.
5. Make the workflow status clear.
1. Write natural prose, not labels or a checklist.
2. Make clear what this literally is: "Paper by…", "Research from…", "Preprint introducing…"
3. Name the authors if known from the metadata.
4. State the actual finding, method, or contribution — not "a study on X" but what they actually found or built.
5. Make clear why it belongs in the graph. If that cannot be inferred, say so naturally.
6. Make the workflow status clear. If unknown, say naturally that it has not been reviewed yet.
7. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to". State things directly.
Respond with ONLY valid JSON:
Examples:
- Title: "Attention Is All You Need" / Authors: Vaswani et al.
GOOD: "Vaswani et al. introduce the Transformer architecture — replaces recurrence with self-attention for sequence modeling. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "This paper discusses a new architecture called the Transformer and explores its applications."
- Title: "Scaling Laws for Neural Language Models" / Authors: Kaplan et al.
GOOD: "Kaplan et al. show that LLM performance scales as a power law with compute, data, and parameters — and compute matters most. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "A study examining how neural language models scale with different factors."
Respond with ONLY valid JSON (no markdown, no code blocks):
{
"enhancedDescription": "A comprehensive summary.",
"nodeDescription": "<your natural description>",
"reasoning": "Brief explanation"
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your natural description following the rules above>",
"tags": ["relevant", "semantic", "tags"],
"reasoning": "Brief explanation of classification choices"
}`;
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
maxOutputTokens: 800
});
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
tags: Array.isArray(result.tags) ? result.tags : [],
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
console.warn('Paper analysis fallback (using default description):', error);
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('Paper analysis fallback (using default description):', message);
return {
enhancedDescription: description,
nodeDescription: undefined,
tags: [],
reasoning: 'Fallback description used'
};
}
@@ -63,20 +91,30 @@ export const paperExtractTool = tool({
description: 'Extract a PDF or research paper into a node with summary, metadata, and full-text source',
inputSchema: z.object({
url: z.string().describe('The PDF URL to add to inbox'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
title: z.string().optional().describe('Custom title (auto-generated if not provided)')
}),
execute: async ({ url, title, dimensions }) => {
execute: async ({ url, title }) => {
try {
// Validate PDF URL
if (!url.startsWith('http://') && !url.startsWith('https://')) {
return { success: false, error: 'Invalid URL format - must start with http:// or https://', data: null };
return {
success: false,
error: 'Invalid URL format - must start with http:// or https://',
data: null
};
}
// Check if URL likely points to a PDF
if (!url.toLowerCase().includes('.pdf') && !url.includes('arxiv.org')) {
return { success: false, error: 'URL does not appear to point to a PDF file', data: null };
return {
success: false,
error: 'URL does not appear to point to a PDF file',
data: null
};
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractPaper(url);
result = {
@@ -92,73 +130,84 @@ export const paperExtractTool = tool({
}
};
} catch (error: any) {
result = { success: false, error: error.message || 'TypeScript extraction failed' };
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
}
if (!result.success || !result.source) {
return { success: false, error: result.error || 'Failed to extract PDF content', data: null };
return {
success: false,
error: result.error || 'Failed to extract PDF content',
data: null
};
}
console.log('🎯 PDF extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`,
result.source.substring(0, 2000) || 'PDF document content',
'pdf'
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.title || `PDF: ${new URL(url).pathname.split('/').pop()?.replace('.pdf', '')}`;
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
`Research paper or PDF from ${new URL(url).hostname} titled ${nodeTitle}`
);
const trimmedDimensions = Array.isArray(dimensions) ? dimensions.slice(0, 5) : [];
const fallbackDescriptionLead = `PDF document titled "${nodeTitle}"`;
const finalDescription = ensureNodeDescription(aiAnalysis?.nodeDescription, fallbackDescriptionLead);
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: nodeDescription,
description: finalDescription,
source: result.source,
link: url,
dimensions: trimmedDimensions,
metadata: {
type: 'pdf',
state: 'not_processed',
captured_method: 'paper_extract',
captured_by: 'agent',
captured_by: 'human',
source_metadata: {
hostname: new URL(url).hostname,
author: result.metadata?.author || result.metadata?.info?.Author,
pages: result.metadata?.pages,
file_size: result.metadata?.file_size,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'typescript',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: aiAnalysis?.enhancedDescription,
refined_at: new Date().toISOString()
extraction_method: result.metadata?.extraction_method || 'python_pdfplumber',
refined_at: new Date().toISOString(),
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return { success: false, error: createResult.error || 'Failed to create node', data: null };
return {
success: false,
error: createResult.error || 'Failed to create node',
data: null
};
}
const actualDimensions: string[] = createResult.data?.dimensions || trimmedDimensions || [];
console.log('🎯 PaperExtract completed successfully');
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle })
: nodeTitle;
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
message: `Added ${formattedNode}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url,
dimensions: actualDimensions
url: url
}
};
} catch (error) {