SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port contexts layer and MCP parity

Browse Source
This commit is contained in:
“BeeRad”
2026-04-10 19:41:26 +10:00
parent a8c5506fb7
commit 35f9ecf89c
62 changed files with 4206 additions and 1224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tools/other/websiteExtract.ts
+64 -124
View File
@@ -3,13 +3,25 @@ import { z } from 'zod';
import { openai } from '@ai-sdk/openai';
import { generateText } from 'ai';
import { extractWebsite } from '@/services/typescript/extractors/website';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
interface ExistingDimension {
name: string;
description: string | null;
}
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string' ? candidate.trim().replace(/\s+/g, ' ') : '';
if (normalizedCandidate && !validateExplicitDescription(normalizedCandidate)) {
return normalizedCandidate.slice(0, 500);
}
const lead = normalizedCandidate || fallbackLead.trim();
const suffix = 'It was added via extraction and the exact reason it belongs in the graph is not yet inferred from the available context, and it has not been reviewed yet.';
return `${lead}${/[.!?]$/.test(lead) ? ' ' : '. '}${suffix}`.slice(0, 500);
}
function inferWebsiteContentType(url: string): 'website' | 'tweet' {
try {
const hostname = new URL(url).hostname.toLowerCase();
@@ -23,12 +35,10 @@ function inferWebsiteContentType(url: string): 'website' | 'tweet' {
async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
try {
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/dimensions/popular`);
const response = await fetch(`${getInternalApiBaseUrl()}/api/dimensions/popular`);
if (!response.ok) return [];
const result = await response.json();
if (!Array.isArray(result.data)) return [];
return result.data
.map((dimension: { dimension?: unknown; description?: unknown }) => ({
name: typeof dimension.dimension === 'string' ? dimension.dimension.trim() : '',
@@ -41,17 +51,11 @@ async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
}
}
function selectExistingDimensions(
selected: unknown,
existingDimensions: ExistingDimension[],
max = 5
): string[] {
function selectExistingDimensions(selected: unknown, existingDimensions: ExistingDimension[], max = 5): string[] {
if (!Array.isArray(selected) || existingDimensions.length === 0) return [];
const byLowerName = new Map(existingDimensions.map((dimension) => [dimension.name.toLowerCase(), dimension.name]));
const normalized: string[] = [];
const seen = new Set<string>();
for (const value of selected) {
if (typeof value !== 'string') continue;
const matched = byLowerName.get(value.trim().toLowerCase());
@@ -62,89 +66,53 @@ function selectExistingDimensions(
normalized.push(matched);
if (normalized.length >= max) break;
}
return normalized;
}
// AI-powered content analysis
async function analyzeContentWithAI(
title: string,
description: string,
contentType: string,
existingDimensions: ExistingDimension[]
) {
async function analyzeContentWithAI(title: string, description: string, contentType: string, existingDimensions: ExistingDimension[]) {
try {
const availableDimensionsBlock = existingDimensions.length > 0
? existingDimensions
.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`)
.join('\n')
? existingDimensions.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`).join('\n')
: '- No existing dimensions available. Return an empty dimensions array.';
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
Description: "${description}"
CRITICAL — nodeDescription rules (max 280 chars):
1. Say WHAT this literally is using explicit entity words only: "Blog post by…", "Article from…", "Essay arguing…", "Tutorial on…", "Thread by…", "Tweet by…", "Post by…"
2. Name the author/site if known from the metadata.
3. State the actual claim or thesis — don't paraphrase into vague abstractions.
4. End with why it's interesting or important — one concrete phrase.
5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly.
DIMENSION SELECTION (critical):
You must select 0-3 dimensions from the list below.
Do NOT invent new dimension names.
Pick only dimensions that genuinely fit this content.
If nothing fits, return an empty array.
CRITICAL — nodeDescription rules (max 500 chars):
1. Write natural prose, not labels or a checklist.
2. Make clear what this literally is using explicit entity words only.
3. Name the author/site if known from the metadata.
4. State the actual claim or thesis.
5. Make clear why it belongs in the graph. If that remains unclear, say so naturally.
6. Make workflow status clear.
Available dimensions:
${availableDimensionsBlock}
Examples:
- Title: "Software is eating the world — again" / Author: Andrej Karpathy
GOOD: "Karpathy's blog post arguing AI agents make software fluid — they can rip functionality from repos instead of taking dependencies. Signals the end of monolithic libraries."
BAD: "By Karpathy — discusses the importance of software becoming more fluid and malleable with agents."
- Title: "The case for slowing down AI" / Site: The Atlantic
GOOD: "Atlantic article making the case that AI labs should voluntarily slow capability research until safety catches up. Notable because it cites internal lab disagreements."
BAD: "This article explores ideas about slowing down AI development and its implications."
Respond with ONLY valid JSON (no markdown, no code blocks):
Respond with ONLY valid JSON:
{
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your 280-char description following the rules above>",
"dimensions": ["existing-dimension-1", "existing-dimension-2"],
"reasoning": "Brief explanation of classification choices"
"enhancedDescription": "A comprehensive summary.",
"nodeDescription": "<your natural description>",
"dimensions": ["existing-dimension-1"],
"reasoning": "Brief explanation"
}`;
const response = await generateText({
model: openai('gpt-4o-mini'),
prompt,
maxOutputTokens: 800
});
let content = response.text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const content = (response.text || '{}').replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
const result = JSON.parse(content);
return {
enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 500) : undefined,
dimensions: selectExistingDimensions(result.dimensions, existingDimensions, 5),
reasoning: result.reasoning || 'AI analysis completed'
};
} catch (error) {
const message = error instanceof Error ? error.message : 'unknown error';
console.warn('Website analysis fallback (using default description):', message);
return {
enhancedDescription: description,
nodeDescription: undefined,
dimensions: [],
reasoning: 'Fallback description used'
};
console.warn('Website analysis fallback (using default description):', error);
return { enhancedDescription: description, nodeDescription: undefined, dimensions: [], reasoning: 'Fallback description used' };
}
}
@@ -153,21 +121,15 @@ export const websiteExtractTool = tool({
inputSchema: z.object({
url: z.string().describe('The website URL to add to knowledge base'),
title: z.string().optional().describe('Custom title (auto-generated if not provided)'),
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node (locked dimensions first)')
dimensions: z.array(z.string()).min(1).max(5).optional().describe('Dimension tags to apply to the created node')
}),
execute: async ({ url, title, dimensions }) => {
try {
// Validate URL format
if (!url.startsWith('http://') && !url.startsWith('https://')) {
return {
success: false,
error: 'Invalid URL format - must start with http:// or https://',
data: null
};
return { success: false, error: 'Invalid URL format - must start with http:// or https://', data: null };
}
let result: { success: boolean; source?: string; metadata?: any; error?: string };
try {
const extractionResult = await extractWebsite(url);
result = {
@@ -184,97 +146,75 @@ export const websiteExtractTool = tool({
}
};
} catch (error: any) {
result = {
success: false,
error: error.message || 'TypeScript extraction failed'
};
result = { success: false, error: error.message || 'TypeScript extraction failed' };
}
if (!result.success || !result.source) {
return {
success: false,
error: result.error || 'Failed to extract website content',
data: null
};
return { success: false, error: result.error || 'Failed to extract website content', data: null };
}
console.log('🎯 Website extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata
const existingDimensions = await fetchExistingDimensions();
const contentType = inferWebsiteContentType(url);
const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.source.substring(0, 2000) || 'Website content',
contentType,
result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.source.substring(0, 2000) || 'Website content',
inferWebsiteContentType(url),
existingDimensions
);
// Step 3: Create node with extracted content and AI analysis
const nodeTitle = title || result.metadata?.title || `Website: ${new URL(url).hostname}`;
const enhancedDescription = aiAnalysis?.enhancedDescription || `Website content from ${new URL(url).hostname}`;
const suppliedDimensions = Array.isArray(dimensions) ? dimensions : [];
let trimmedDimensions = suppliedDimensions
.map(dim => (typeof dim === 'string' ? dim.trim() : ''))
.filter(Boolean);
const suppliedDimensions = Array.isArray(dimensions) ? dimensions.slice(0, 5) : [];
const finalDimensions = suppliedDimensions.length > 0 ? suppliedDimensions : (aiAnalysis?.dimensions || []).slice(0, 5);
const nodeDescription = ensureNodeDescription(
aiAnalysis?.nodeDescription,
`Website article from ${result.metadata?.site_name || new URL(url).hostname} titled ${nodeTitle}`
);
trimmedDimensions = trimmedDimensions.slice(0, 5);
const finalDimensions = trimmedDimensions.length > 0
? trimmedDimensions
: (aiAnalysis?.dimensions || []).slice(0, 5);
const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
const createResponse = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title: nodeTitle,
description: aiAnalysis?.nodeDescription,
description: nodeDescription,
source: result.source,
link: url,
event_date: result.metadata?.published_date || result.metadata?.date || null,
dimensions: finalDimensions,
metadata: {
source: contentType,
hostname: new URL(url).hostname,
author: result.metadata?.author,
published_date: result.metadata?.published_date || result.metadata?.date,
content_length: result.source.length,
extraction_method: result.metadata?.extraction_method || 'python_beautifulsoup',
ai_analysis: aiAnalysis?.reasoning,
enhanced_description: enhancedDescription,
refined_at: new Date().toISOString()
type: inferWebsiteContentType(url),
state: 'not_processed',
captured_method: 'website_extract',
captured_by: 'agent',
source_metadata: {
hostname: new URL(url).hostname,
author: result.metadata?.author,
site_name: result.metadata?.site_name,
date: result.metadata?.date,
og_image: result.metadata?.og_image,
extraction_method: result.metadata?.extraction_method,
ai_analysis: aiAnalysis?.reasoning,
refined_at: new Date().toISOString()
}
}
})
});
const createResult = await createResponse.json();
if (!createResponse.ok) {
return {
success: false,
error: createResult.error || 'Failed to create node',
data: null
};
return { success: false, error: createResult.error || 'Failed to create node', data: null };
}
console.log('🎯 WebsiteExtract completed successfully');
// Use actual assigned dimensions from API response (includes auto-assigned locked + keywords)
const actualDimensions: string[] = createResult.data?.dimensions || finalDimensions || [];
const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
: nodeTitle;
const dimsDisplay = actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none';
return {
success: true,
message: `Added ${formattedNode} with dimensions: ${dimsDisplay}`,
message: `Added ${formattedNode} with dimensions: ${actualDimensions.length > 0 ? actualDimensions.join(', ') : 'none'}`,
data: {
nodeId: createResult.data?.id,
title: nodeTitle,
contentLength: result.source.length,
url: url,
url,
dimensions: actualDimensions
}
};