SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port holistic node refinement contract

Browse Source
This commit is contained in:
“BeeRad”
2026-04-11 21:37:52 +10:00
parent 35f9ecf89c
commit 3ae46245ec
119 changed files with 6596 additions and 10982 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tools/other/websiteExtract.ts
+91 -94
View File
@@ -7,19 +7,19 @@ import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
interface ExistingDimension {
name: string;
description: string | null;
}
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string' ? candidate.trim().replace(/\s+/g, ' ') : '';
const normalizedCandidate = typeof candidate === 'string'
? candidate.trim().replace(/\s+/g, ' ')
: '';
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
return normalizedCandidate.slice(0, 500);
}
const lead = normalizedCandidate || fallbackLead.trim();
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
return `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`.slice(0, 500);
const joined = `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`;
return joined.slice(0, 500);
}
function inferWebsiteContentType(url: string): 'website' | 'tweet' {
@@ -33,47 +33,13 @@ function inferWebsiteContentType(url: string): 'website' | 'tweet' {
}
}
async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
// AI-powered content analysis
async function analyzeContentWithAI(
title: string,
description: string,
contentType: string
) {
try {
const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions/popular`);
if (!response.ok) return [];
const result = await response.json();
if (!Array.isArray(result.data)) return [];
return result.data
.map((dimension: { dimension?: unknown; description?: unknown }) => ({
name: typeof dimension.dimension === 'string' ? dimension.dimension.trim() : '',
description: typeof dimension.description === 'string' ? dimension.description.trim() : null
}))
.filter((dimension: ExistingDimension) => dimension.name.length > 0);
} catch (error) {
console.warn('Website dimension fetch fallback (no dimension context):', error);
return [];
}
}
function selectExistingDimensions(selected: unknown, existingDimensions: ExistingDimension[], max = 5): string[] {
if (!Array.isArray(selected) || existingDimensions.length === 0) return [];
const byLowerName = new Map(existingDimensions.map((dimension) => [dimension.name.toLowerCase(), dimension.name]));
const normalized: string[] = [];
const seen = new Set<string>();
for (const value of selected) {
if (typeof value !== 'string') continue;
const matched = byLowerName.get(value.trim().toLowerCase());
if (!matched) continue;
const key = matched.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
normalized.push(matched);
if (normalized.length >= max) break;
}
return normalized;
}
async function analyzeContentWithAI(title: string, description: string, contentType: string, existingDimensions: ExistingDimension[]) {
try {
const availableDimensionsBlock = existingDimensions.length > 0
? existingDimensions.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`).join('\n')
: '- No existing dimensions available. Return an empty dimensions array.';
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
@@ -81,55 +47,77 @@ Description: "${description}"
CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose, not labels or a checklist.
2. Make clear what this literally is using explicit entity words only.
2. Make clear what this literally is using explicit entity words like "Blog post by…", "Article from…", "Essay arguing…", "Tutorial on…", "Thread by…", "Tweet by…", "Post by…"
3. Name the author/site if known from the metadata.
4. State the actual claim or thesis.
5. Make clear why it belongs in the graph. If that remains unclear, say so naturally.
6. Make workflow status clear.
4. State the actual claim or thesis — don't paraphrase into vague abstractions.
5. Make clear why it belongs in the graph. If that cannot be inferred, say so naturally.
6. Make the workflow status clear. If unknown, say naturally that it has not been reviewed yet.
7. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for", "insightful for understanding", "relevant to". State things directly.
Available dimensions:
${availableDimensionsBlock}
Examples:
- Title: "Software is eating the world — again" / Author: Andrej Karpathy
GOOD: "Karpathy's blog post arguing AI agents make software fluid — they can rip functionality from repos instead of taking dependencies. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "By Karpathy — discusses the importance of software becoming more fluid and malleable with agents."
Respond with ONLY valid JSON:
- Title: "The case for slowing down AI" / Site: The Atlantic
GOOD: "Atlantic article making the case that AI labs should voluntarily slow capability research until safety catches up. It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet."
BAD: "This article explores ideas about slowing down AI development and its implications."
Respond with ONLY valid JSON (no markdown, no code blocks):
{
"enhancedDescription": "A comprehensive summary.",
"nodeDescription": "<your natural description>",
"dimensions": ["existing-dimension-1"],
"reasoning": "Brief explanation"
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your natural description following the rules above>",
"reasoning": "Brief explanation of classification choices"
}`;
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
maxOutputTokens: 800
});
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
dimensions: selectExistingDimensions(result.dimensions, existingDimensions, 5),
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
console.warn('Website analysis fallback (using default description):', error);
return { enhancedDescription: description, nodeDescription: undefined, dimensions: [], reasoning: 'Fallback description used' };
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('Website analysis fallback (using default description):', message);
return {
enhancedDescription: description,
nodeDescription: undefined,
reasoning: 'Fallback description used'
};
}
}
export const websiteExtractTool = tool({
description: 'Extract website content and metadata into a node with summary, tags, and raw source text',
description: 'Extract website content and metadata into a node with summary and raw source text',
inputSchema: z.object({
url: z.string().describe('The website URL to add to knowledge base'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
title: z.string().optional().describe('Custom title (auto-generated if not provided)')
}),
execute: async ({ url, title, dimensions }) => {
execute: async ({ url, title }) => {
try {
// Validate URL format
if (!url.startsWith('http://') && !url.startsWith('https://')) {
return { success: false, error: 'Invalid URL format - must start with http:// or https://', data: null };
return {
success: false,
error: 'Invalid URL format - must start with http:// or https://',
data: null
};
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractWebsite(url);
result = {
@@ -146,76 +134,85 @@ export const websiteExtractTool = tool({
}
};
} catch (error: any) {
result = { success: false, error: error.message || 'TypeScript extraction failed' };
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
}
if (!result.success || !result.source) {
return { success: false, error: result.error || 'Failed to extract website content', data: null };
return {
success: false,
error: result.error || 'Failed to extract website content',
data: null
};
}
const existingDimensions = await fetchExistingDimensions();
console.log('🎯 Website extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const contentType = inferWebsiteContentType(url);
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.source.substring(0, 2000) || 'Website content',
inferWebsiteContentType(url),
existingDimensions
contentType
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.title || `Website: ${new URL(url).hostname}`;
const suppliedDimensions = Array.isArray(dimensions) ? dimensions.slice(0, 5) : [];
const finalDimensions = suppliedDimensions.length > 0 ? suppliedDimensions : (aiAnalysis?.dimensions || []).slice(0, 5);
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
`Website article from ${result.metadata?.site_name || new URL(url).hostname} titled ${nodeTitle}`
);
const fallbackDescriptionLead = `${contentType === 'tweet' ? 'Tweet' : 'Website article'} from ${result.metadata?.author || result.metadata?.site_name || new URL(url).hostname} titled "${nodeTitle}"`;
const finalDescription = ensureNodeDescription(aiAnalysis?.nodeDescription, fallbackDescriptionLead);
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: nodeDescription,
description: finalDescription,
source: result.source,
link: url,
dimensions: finalDimensions,
event_date: result.metadata?.published_date || result.metadata?.date || null,
metadata: {
type: inferWebsiteContentType(url),
type: contentType,
state: 'not_processed',
captured_method: 'website_extract',
captured_by: 'agent',
captured_by: 'human',
source_metadata: {
hostname: new URL(url).hostname,
author: result.metadata?.author,
site_name: result.metadata?.site_name,
date: result.metadata?.date,
og_image: result.metadata?.og_image,
extraction_method: result.metadata?.extraction_method,
ai_analysis: aiAnalysis?.reasoning,
refined_at: new Date().toISOString()
published_date: result.metadata?.published_date || result.metadata?.date,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'python_beautifulsoup',
refined_at: new Date().toISOString(),
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return { success: false, error: createResult.error || 'Failed to create node', data: null };
return {
success: false,
error: createResult.error || 'Failed to create node',
data: null
};
}
const actualDimensions: string[] = createResult.data?.dimensions || finalDimensions || [];
console.log('🎯 WebsiteExtract completed successfully');
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle })
: nodeTitle;
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
message: `Added ${formattedNode}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url,
dimensions: actualDimensions
url: url
}
};
} catch (error) {