sync: Idea Genealogy + Quick Capture Overhaul
From private repo: Idea Genealogy: - Required edge explanations across UI/tools/API/MCP - Inferred edge schema (category/type/confidence) - Added part_of relationship type - Connection UI chips (Made by / Part of / Came from / Related) Quick Capture Overhaul: - Jina.ai fallback for JS-rendered sites (Twitter, SPAs) - Simplified UI: removed mode buttons, auto-detect input type - Renamed to "Add Stuff", moved to top-right - Auto-edge creation from description entities - Increased content summary length (500→1500 chars) Description Generation Fix: - Simplified prompt to "what is this" - Added link/dimensions context - Better creator attribution Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
“BeeRad”
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
f7b8b2058c
commit
2cbc950fd4
@@ -0,0 +1,245 @@
|
||||
/**
|
||||
* Auto-Edge Creation Service
|
||||
*
|
||||
* After Quick Capture creates a node, this service:
|
||||
* 1. Extracts candidate entity strings from the node's description
|
||||
* 2. Looks up existing entity nodes by exact title match
|
||||
* 3. Creates edges with explanations for matches
|
||||
*
|
||||
* This is a "fast path" for obvious connections only - conservative by design.
|
||||
*/
|
||||
|
||||
import { nodeService, edgeService } from '@/services/database';
|
||||
import { Node } from '@/types/database';
|
||||
|
||||
// Entity-like dimensions where we expect to find referenceable entities
|
||||
const ENTITY_DIMENSIONS = ['people', 'companies', 'organizations', 'books', 'papers', 'articles', 'podcasts', 'creators'];
|
||||
|
||||
/**
|
||||
* Clean up a candidate entity string by removing common prefixes/suffixes.
|
||||
*/
|
||||
function cleanEntityCandidate(candidate: string): string {
|
||||
let cleaned = candidate.trim();
|
||||
|
||||
// Remove common author/attribution prefixes
|
||||
const prefixPatterns = [
|
||||
/^by\s+/i,
|
||||
/^author:\s*/i,
|
||||
/^written by\s+/i,
|
||||
/^from\s+/i,
|
||||
/^via\s+/i,
|
||||
/^featuring\s+/i,
|
||||
/^with\s+/i,
|
||||
/^hosted by\s+/i,
|
||||
];
|
||||
|
||||
for (const pattern of prefixPatterns) {
|
||||
cleaned = cleaned.replace(pattern, '');
|
||||
}
|
||||
|
||||
return cleaned.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract candidate entity strings from text using conservative heuristics.
|
||||
* Returns proper names, quoted titles, and recognized patterns.
|
||||
*/
|
||||
function extractCandidateEntities(text: string): string[] {
|
||||
if (!text || typeof text !== 'string') return [];
|
||||
|
||||
const candidates: Set<string> = new Set();
|
||||
|
||||
// Pattern 1: "By [Name]" pattern - common in article descriptions
|
||||
// Matches: "By Simon Willison", "by Sam Altman"
|
||||
const byPattern = /\b[Bb]y\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,2})\b/g;
|
||||
let match;
|
||||
while ((match = byPattern.exec(text)) !== null) {
|
||||
const name = match[1].trim();
|
||||
if (name.length >= 4 && !isGenericPhrase(name)) {
|
||||
candidates.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern 2: Proper name sequences (2-4 capitalized words)
|
||||
// Matches: "Sam Altman", "Dario Amodei", "Peter Thiel"
|
||||
const properNamePattern = /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3})\b/g;
|
||||
while ((match = properNamePattern.exec(text)) !== null) {
|
||||
const name = match[1].trim();
|
||||
// Clean the candidate (remove "By ", etc.)
|
||||
const cleaned = cleanEntityCandidate(name);
|
||||
if (cleaned.length >= 4 && !isGenericPhrase(cleaned)) {
|
||||
candidates.add(cleaned);
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern 3: Quoted titles (single or double quotes)
|
||||
// Matches: "Zero to One", 'The Lean Startup'
|
||||
const quotedPattern = /["']([^"']{3,60})["']/g;
|
||||
while ((match = quotedPattern.exec(text)) !== null) {
|
||||
const title = match[1].trim();
|
||||
if (title.length >= 3 && !isGenericPhrase(title)) {
|
||||
candidates.add(title);
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern 4: Known organization patterns
|
||||
// Matches: OpenAI, DeepMind, Y Combinator, Fly.io
|
||||
const orgPattern = /\b(OpenAI|DeepMind|Anthropic|Google|Microsoft|Meta|Apple|Amazon|Y Combinator|YC|Stripe|Coinbase|Fly\.io|Vercel|Cloudflare)\b/gi;
|
||||
while ((match = orgPattern.exec(text)) !== null) {
|
||||
candidates.add(match[1]);
|
||||
}
|
||||
|
||||
return Array.from(candidates);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a phrase is too generic to be a useful entity reference.
|
||||
*/
|
||||
function isGenericPhrase(phrase: string): boolean {
|
||||
const normalized = phrase.toLowerCase();
|
||||
|
||||
// Common stopwords and generic terms
|
||||
const genericTerms = [
|
||||
'the author', 'the article', 'the book', 'the podcast',
|
||||
'this article', 'this book', 'this podcast', 'this paper',
|
||||
'new research', 'recent study', 'key points', 'main ideas',
|
||||
'artificial intelligence', 'machine learning', 'deep learning',
|
||||
'first section', 'last section', 'next chapter',
|
||||
'united states', 'new york', 'san francisco', 'silicon valley'
|
||||
];
|
||||
|
||||
return genericTerms.some(term => normalized === term || normalized.startsWith(term + ' '));
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up existing nodes that match candidate entity strings.
|
||||
* Uses exact title matching (case-insensitive) with optional dimension filtering.
|
||||
*/
|
||||
async function findMatchingEntityNodes(candidates: string[]): Promise<Map<string, Node>> {
|
||||
const matches = new Map<string, Node>();
|
||||
|
||||
if (candidates.length === 0) return matches;
|
||||
|
||||
// Get all nodes (we'll filter in memory for exact title matches)
|
||||
// In a larger system, we'd use a more efficient query
|
||||
const allNodes = await nodeService.getNodes({ limit: 10000 });
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const normalizedCandidate = candidate.toLowerCase().trim();
|
||||
|
||||
// Find exact title match (case-insensitive)
|
||||
const matchingNode = allNodes.find(node => {
|
||||
const normalizedTitle = (node.title || '').toLowerCase().trim();
|
||||
if (normalizedTitle !== normalizedCandidate) return false;
|
||||
|
||||
// Optionally filter to entity-like dimensions for higher confidence
|
||||
// But don't require it - some entities might not have dimensions yet
|
||||
const nodeDimensions = node.dimensions || [];
|
||||
const hasEntityDimension = nodeDimensions.some(dim =>
|
||||
ENTITY_DIMENSIONS.includes(dim.toLowerCase())
|
||||
);
|
||||
|
||||
// Accept if it has an entity dimension OR if it's a short title (likely a named entity)
|
||||
return hasEntityDimension || node.title.length < 50;
|
||||
});
|
||||
|
||||
if (matchingNode) {
|
||||
matches.set(candidate, matchingNode);
|
||||
}
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create edges from a new node to matched entity nodes.
|
||||
* Each edge includes an explanation for auditability.
|
||||
*/
|
||||
async function createAutoEdges(
|
||||
newNodeId: number,
|
||||
matches: Map<string, Node>
|
||||
): Promise<number> {
|
||||
let edgesCreated = 0;
|
||||
|
||||
for (const [candidateText, entityNode] of matches) {
|
||||
// Skip self-references
|
||||
if (entityNode.id === newNodeId) continue;
|
||||
|
||||
// Check if edge already exists
|
||||
const exists = await edgeService.edgeExists(newNodeId, entityNode.id);
|
||||
if (exists) continue;
|
||||
|
||||
try {
|
||||
await edgeService.createEdge({
|
||||
from_node_id: newNodeId,
|
||||
to_node_id: entityNode.id,
|
||||
explanation: `Explicitly mentioned in description: "${candidateText}"`,
|
||||
created_via: 'quick_capture_auto',
|
||||
source: 'ai_similarity',
|
||||
skip_inference: false, // Let Idea Genealogy classify the relationship
|
||||
});
|
||||
edgesCreated++;
|
||||
console.log(`[autoEdge] Created edge: ${newNodeId} → ${entityNode.id} (${entityNode.title})`);
|
||||
} catch (error) {
|
||||
console.warn(`[autoEdge] Failed to create edge to ${entityNode.id}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return edgesCreated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entry point: Run auto-edge creation for a newly created node.
|
||||
* This is designed to be called fire-and-forget (non-blocking).
|
||||
*/
|
||||
export async function runAutoEdgeCreation(nodeId: number): Promise<void> {
|
||||
try {
|
||||
// Fetch the newly created node
|
||||
const node = await nodeService.getNodeById(nodeId);
|
||||
if (!node) {
|
||||
console.warn(`[autoEdge] Node ${nodeId} not found, skipping auto-edge creation`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Use description as the source of truth for entity extraction
|
||||
const description = node.description || '';
|
||||
if (!description || description.length < 10) {
|
||||
console.log(`[autoEdge] Node ${nodeId} has no/short description, skipping`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract candidate entities from description
|
||||
const candidates = extractCandidateEntities(description);
|
||||
if (candidates.length === 0) {
|
||||
console.log(`[autoEdge] No entity candidates found in node ${nodeId} description`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[autoEdge] Found ${candidates.length} candidates for node ${nodeId}:`, candidates);
|
||||
|
||||
// Find matching existing nodes
|
||||
const matches = await findMatchingEntityNodes(candidates);
|
||||
if (matches.size === 0) {
|
||||
console.log(`[autoEdge] No matching entity nodes found for node ${nodeId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Create edges
|
||||
const edgesCreated = await createAutoEdges(nodeId, matches);
|
||||
console.log(`[autoEdge] Created ${edgesCreated} auto-edges for node ${nodeId}`);
|
||||
} catch (error) {
|
||||
console.error(`[autoEdge] Error in auto-edge creation for node ${nodeId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule auto-edge creation to run asynchronously (fire-and-forget).
|
||||
* Use this from the nodes API to avoid blocking the response.
|
||||
*/
|
||||
export function scheduleAutoEdgeCreation(nodeId: number): void {
|
||||
setImmediate(() => {
|
||||
runAutoEdgeCreation(nodeId).catch(error => {
|
||||
console.error(`[autoEdge] Scheduled auto-edge creation failed for node ${nodeId}:`, error);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -48,6 +48,34 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
|
||||
}
|
||||
|
||||
function buildDescriptionPrompt(input: DescriptionInput): string {
|
||||
const normalizedSource = (input.metadata?.source || '').toLowerCase();
|
||||
const url = typeof input.link === 'string' ? input.link.trim() : '';
|
||||
|
||||
// Best-effort creator hint from structured metadata (when available),
|
||||
// but never assume a particular extraction source (YouTube vs paper vs website vs note).
|
||||
const creatorHint =
|
||||
input.metadata?.author?.trim() ||
|
||||
input.metadata?.channel_name?.trim() ||
|
||||
'';
|
||||
|
||||
// Best-effort publisher / container hint (less ideal than a true author, but better than nothing).
|
||||
const publisherHint = input.metadata?.site_name?.trim() || '';
|
||||
|
||||
const likelyExternal =
|
||||
Boolean(url) ||
|
||||
normalizedSource.includes('youtube') ||
|
||||
normalizedSource.includes('extract') ||
|
||||
normalizedSource.includes('paper') ||
|
||||
normalizedSource.includes('pdf') ||
|
||||
normalizedSource.includes('website');
|
||||
|
||||
const likelyUserAuthored =
|
||||
!likelyExternal &&
|
||||
(normalizedSource.includes('quick-add-note') ||
|
||||
normalizedSource.includes('quick-add-chat') ||
|
||||
normalizedSource.includes('note') ||
|
||||
normalizedSource.length === 0);
|
||||
|
||||
const lines: string[] = [`Title: ${input.title}`];
|
||||
|
||||
if (input.link) lines.push(`URL: ${input.link}`);
|
||||
@@ -55,11 +83,27 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
|
||||
if (input.metadata?.channel_name) lines.push(`Channel: ${input.metadata.channel_name}`);
|
||||
if (input.metadata?.author) lines.push(`Author: ${input.metadata.author}`);
|
||||
if (input.metadata?.site_name) lines.push(`Site: ${input.metadata.site_name}`);
|
||||
if (creatorHint) lines.push(`Creator hint: ${creatorHint}`);
|
||||
if (publisherHint) lines.push(`Publisher hint: ${publisherHint}`);
|
||||
lines.push(`Likely user-authored: ${likelyUserAuthored ? 'yes' : 'no'}`);
|
||||
|
||||
const contentPreview = input.content?.slice(0, 800) || '';
|
||||
if (contentPreview) lines.push(`Content: ${contentPreview}${input.content && input.content.length > 800 ? '...' : ''}`);
|
||||
|
||||
return `Your job is to do your best to answer 'what is this' - the most simple, high level contextual information of what this thing is. Users will be adding a variety of different nodes (ideas, books, podcasts, people, papers etc). Do your best to take the available information and infer what it is - high level. If unsure, that's fine just give your best guess and say you're unsure. Max 280 characters.
|
||||
return `Your job is to answer: "what is this?" in one short line. Max 280 characters.
|
||||
|
||||
GOAL: Include "who created it" when possible.
|
||||
|
||||
RULES (in priority order):
|
||||
1) ONLY use a creator if you have a creator hint (Author/Channel) or the content explicitly says "by <X>" / "hosted by <X>".
|
||||
- Do NOT treat prominent people in the title/transcript (e.g. a guest) as the creator.
|
||||
- If a creator hint is provided, prefer it.
|
||||
2) If you can identify a creator/author/channel/person/org from a creator hint, start with: "By <creator> — ..."
|
||||
3) If it's likely user-authored, start with: "Your <thing> — ..." (don't invent a creator name).
|
||||
4) If creator is unknown, do NOT guess; omit the byline.
|
||||
|
||||
Then, in the remainder, state what it is (video/paper/article/note/idea/etc) + what it's about (high-signal).
|
||||
If unsure, say so briefly.
|
||||
|
||||
${lines.join('\n')}`;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,120 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { Edge, EdgeData, NodeConnection, Node } from '@/types/database';
|
||||
import { Edge, EdgeContext, EdgeData, EdgeCreatedVia, NodeConnection, Node } from '@/types/database';
|
||||
import { eventBroadcaster } from '../events';
|
||||
import { nodeService } from './nodes';
|
||||
import { apiKeyService } from '../storage/apiKeys';
|
||||
import { generateText } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { z } from 'zod';
|
||||
|
||||
const inferredEdgeContextSchema = z.object({
|
||||
category: z.enum(['attribution', 'intellectual']),
|
||||
type: z.enum([
|
||||
'created_by',
|
||||
'features',
|
||||
'part_of',
|
||||
'source_of',
|
||||
'extends',
|
||||
'supports',
|
||||
'contradicts',
|
||||
'related_to'
|
||||
]),
|
||||
confidence: z.number().min(0).max(1),
|
||||
});
|
||||
|
||||
async function inferEdgeContext(params: {
|
||||
explanation: string;
|
||||
fromNode: Node;
|
||||
toNode: Node;
|
||||
}): Promise<Pick<EdgeContext, 'category' | 'type' | 'confidence'>> {
|
||||
const { explanation, fromNode, toNode } = params;
|
||||
|
||||
// Heuristic fast-paths for the 4 core UI chips.
|
||||
// This makes classification robust and reduces reliance on the model.
|
||||
const norm = explanation.trim().toLowerCase();
|
||||
const startsWithAny = (prefixes: string[]) => prefixes.some((p) => norm.startsWith(p));
|
||||
if (startsWithAny(['created by', 'made by', 'authored by', 'written by', 'founded by'])) {
|
||||
return { category: 'attribution', type: 'created_by', confidence: 1.0 };
|
||||
}
|
||||
if (startsWithAny(['part of', 'episode of', 'belongs to', 'in the series', 'in this series'])) {
|
||||
return { category: 'attribution', type: 'part_of', confidence: 1.0 };
|
||||
}
|
||||
if (startsWithAny(['features', 'mentions', 'hosted by', 'guest:', 'host:'])) {
|
||||
return { category: 'attribution', type: 'features', confidence: 0.95 };
|
||||
}
|
||||
if (startsWithAny(['came from', 'inspired by', 'derived from', 'from'])) {
|
||||
return { category: 'intellectual', type: 'source_of', confidence: 0.9 };
|
||||
}
|
||||
if (startsWithAny(['related to', 'related'])) {
|
||||
return { category: 'intellectual', type: 'related_to', confidence: 0.8 };
|
||||
}
|
||||
|
||||
// If no API key is configured, degrade gracefully.
|
||||
// We still enforce explanation, but fall back to "related_to" classification.
|
||||
const apiKey = apiKeyService.getOpenAiKey();
|
||||
if (!apiKey) {
|
||||
return { category: 'intellectual', type: 'related_to', confidence: 0.0 };
|
||||
}
|
||||
|
||||
const provider = createOpenAI({ apiKey });
|
||||
const prompt = [
|
||||
`Given this edge explanation: "${explanation}"`,
|
||||
``,
|
||||
`From node:`,
|
||||
`- Title: "${fromNode.title}"`,
|
||||
`- Description: ${fromNode.description || 'No description available'}`,
|
||||
`- Dimensions: ${fromNode.dimensions?.join(', ') || 'none'}`,
|
||||
``,
|
||||
`To node:`,
|
||||
`- Title: "${toNode.title}"`,
|
||||
`- Description: ${toNode.description || 'No description available'}`,
|
||||
`- Dimensions: ${toNode.dimensions?.join(', ') || 'none'}`,
|
||||
``,
|
||||
`Classify the relationship:`,
|
||||
`- category: "attribution" (factual: author, creator, host, guest) or "intellectual" (idea relationship)`,
|
||||
`- type: one of [created_by, features, part_of, source_of, extends, supports, contradicts, related_to]`,
|
||||
`- confidence: 0-1`,
|
||||
``,
|
||||
`IMPORTANT: Interpret the direction as "FROM node → TO node". Pick a type that reads correctly in that direction:`,
|
||||
`- created_by: FROM was created/founded/authored by TO`,
|
||||
`- features: FROM features TO (host/guest/subject appearing in FROM)`,
|
||||
`- part_of: FROM is part of TO (episode→podcast, chapter→book, note→project)`,
|
||||
`- source_of: FROM came from TO / was inspired by TO`,
|
||||
`- extends/supports/contradicts: FROM extends/supports/contradicts TO`,
|
||||
``,
|
||||
`Return JSON only: {"category": "...", "type": "...", "confidence": 0.X}`
|
||||
].join('\n');
|
||||
|
||||
try {
|
||||
const { text } = await generateText({
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt,
|
||||
temperature: 0.0,
|
||||
maxOutputTokens: 120,
|
||||
});
|
||||
|
||||
const parsedJson = (() => {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
// Sometimes models wrap JSON in prose; try to recover.
|
||||
const match = text.match(/\{[\s\S]*\}/);
|
||||
if (match) return JSON.parse(match[0]);
|
||||
throw new Error('AI did not return valid JSON');
|
||||
}
|
||||
})();
|
||||
|
||||
const parsed = inferredEdgeContextSchema.safeParse(parsedJson);
|
||||
if (!parsed.success) {
|
||||
return { category: 'intellectual', type: 'related_to', confidence: 0.2 };
|
||||
}
|
||||
|
||||
return parsed.data;
|
||||
} catch (error) {
|
||||
console.warn('[edges] inferEdgeContext failed; falling back to related_to', error);
|
||||
return { category: 'intellectual', type: 'related_to', confidence: 0.2 };
|
||||
}
|
||||
}
|
||||
|
||||
export class EdgeService {
|
||||
async getEdges(): Promise<Edge[]> {
|
||||
@@ -24,14 +138,41 @@ export class EdgeService {
|
||||
private async createEdgeSQLite(edgeData: EdgeData): Promise<Edge> {
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
|
||||
const explanation = (edgeData.explanation || '').trim();
|
||||
if (!explanation) {
|
||||
throw new Error('Edge explanation is required');
|
||||
}
|
||||
|
||||
const createdVia: EdgeCreatedVia = edgeData.created_via;
|
||||
|
||||
// Fetch nodes for inference context
|
||||
const [fromNode, toNode] = await Promise.all([
|
||||
nodeService.getNodeById(edgeData.from_node_id),
|
||||
nodeService.getNodeById(edgeData.to_node_id),
|
||||
]);
|
||||
|
||||
if (!fromNode) throw new Error(`Source node ${edgeData.from_node_id} not found`);
|
||||
if (!toNode) throw new Error(`Target node ${edgeData.to_node_id} not found`);
|
||||
|
||||
const inferred = edgeData.skip_inference
|
||||
? { category: 'intellectual' as const, type: 'related_to' as const, confidence: 0.0 }
|
||||
: await inferEdgeContext({ explanation, fromNode, toNode });
|
||||
|
||||
const context: EdgeContext = {
|
||||
...inferred,
|
||||
inferred_at: now,
|
||||
explanation,
|
||||
created_via: createdVia,
|
||||
};
|
||||
|
||||
const result = sqlite.prepare(`
|
||||
INSERT INTO edges (from_node_id, to_node_id, context, source, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
edgeData.from_node_id,
|
||||
edgeData.to_node_id,
|
||||
JSON.stringify(edgeData.context || {}),
|
||||
JSON.stringify(context),
|
||||
edgeData.source,
|
||||
now
|
||||
);
|
||||
@@ -67,6 +208,51 @@ export class EdgeService {
|
||||
const updateFields: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
// If explanation changes, re-infer classification and write full EdgeContext
|
||||
if (Object.prototype.hasOwnProperty.call(updates, 'context') && updates.context && typeof updates.context === 'object') {
|
||||
const incomingContext = updates.context as Partial<EdgeContext> & { explanation?: unknown };
|
||||
if (typeof incomingContext.explanation === 'string') {
|
||||
const explanation = incomingContext.explanation.trim();
|
||||
if (!explanation) {
|
||||
throw new Error('Edge explanation is required');
|
||||
}
|
||||
|
||||
const existingEdge = await this.getEdgeById(id);
|
||||
if (!existingEdge) {
|
||||
throw new Error(`Edge with ID ${id} not found`);
|
||||
}
|
||||
|
||||
const [fromNode, toNode] = await Promise.all([
|
||||
nodeService.getNodeById(existingEdge.from_node_id),
|
||||
nodeService.getNodeById(existingEdge.to_node_id),
|
||||
]);
|
||||
|
||||
if (!fromNode) throw new Error(`Source node ${existingEdge.from_node_id} not found`);
|
||||
if (!toNode) throw new Error(`Target node ${existingEdge.to_node_id} not found`);
|
||||
|
||||
const inferred = await inferEdgeContext({ explanation, fromNode, toNode });
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const existingContext = (existingEdge.context && typeof existingEdge.context === 'object')
|
||||
? (existingEdge.context as Partial<EdgeContext>)
|
||||
: undefined;
|
||||
|
||||
const created_via: EdgeCreatedVia =
|
||||
(incomingContext.created_via as EdgeCreatedVia) ||
|
||||
(existingContext?.created_via as EdgeCreatedVia) ||
|
||||
'ui';
|
||||
|
||||
updates.context = {
|
||||
...existingContext,
|
||||
...incomingContext,
|
||||
...inferred,
|
||||
inferred_at: now,
|
||||
explanation,
|
||||
created_via,
|
||||
} satisfies EdgeContext;
|
||||
}
|
||||
}
|
||||
|
||||
// Build dynamic update query
|
||||
Object.entries(updates).forEach(([key, value]) => {
|
||||
if (key !== 'id' && key !== 'created_at' && value !== undefined) {
|
||||
@@ -309,17 +495,23 @@ export class EdgeService {
|
||||
}
|
||||
|
||||
async createBidirectionalEdge(fromId: number, toId: number, options?: {
|
||||
context?: any;
|
||||
explanation?: string;
|
||||
created_via?: EdgeCreatedVia;
|
||||
source?: 'user' | 'ai_similarity' | 'helper_name';
|
||||
skip_inference?: boolean;
|
||||
}): Promise<Edge[]> {
|
||||
const edges: Edge[] = [];
|
||||
const explanation = (options?.explanation || 'Similarity-based connection').trim();
|
||||
const created_via: EdgeCreatedVia = options?.created_via || 'workflow';
|
||||
|
||||
// Create edge from A to B
|
||||
const forwardEdge = await this.createEdge({
|
||||
from_node_id: fromId,
|
||||
to_node_id: toId,
|
||||
context: options?.context,
|
||||
source: options?.source || 'ai_similarity'
|
||||
explanation,
|
||||
created_via,
|
||||
source: options?.source || 'ai_similarity',
|
||||
skip_inference: options?.skip_inference,
|
||||
});
|
||||
edges.push(forwardEdge);
|
||||
|
||||
@@ -327,8 +519,10 @@ export class EdgeService {
|
||||
const backwardEdge = await this.createEdge({
|
||||
from_node_id: toId,
|
||||
to_node_id: fromId,
|
||||
context: options?.context,
|
||||
source: options?.source || 'ai_similarity'
|
||||
explanation,
|
||||
created_via,
|
||||
source: options?.source || 'ai_similarity',
|
||||
skip_inference: options?.skip_inference,
|
||||
});
|
||||
edges.push(backwardEdge);
|
||||
|
||||
|
||||
@@ -111,13 +111,18 @@ function parseMetadata(metadata: unknown): Record<string, any> {
|
||||
if (!metadata) return {};
|
||||
if (typeof metadata === 'string') {
|
||||
try {
|
||||
return JSON.parse(metadata);
|
||||
const parsed = JSON.parse(metadata);
|
||||
if (parsed && typeof parsed === 'object') {
|
||||
return parsed as Record<string, any>;
|
||||
}
|
||||
return {};
|
||||
} catch (error) {
|
||||
console.warn('Failed to parse node metadata JSON:', error);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
if (typeof metadata === 'object') {
|
||||
if (metadata === null) return {};
|
||||
return metadata as Record<string, any>;
|
||||
}
|
||||
return {};
|
||||
|
||||
@@ -27,6 +27,46 @@ export class WebsiteExtractor {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
};
|
||||
|
||||
/**
|
||||
* Extract content using Jina.ai Reader API
|
||||
* Used as fallback when cheerio fails (JS-rendered sites, Twitter, SPAs)
|
||||
* Free tier: 20 requests/minute per IP (no API key needed)
|
||||
*/
|
||||
private async extractWithJina(url: string): Promise<ExtractionResult> {
|
||||
const jinaUrl = `https://r.jina.ai/${url}`;
|
||||
|
||||
const response = await fetch(jinaUrl, {
|
||||
headers: { 'User-Agent': 'RA-H/1.0' },
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Jina extraction failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const markdown = await response.text();
|
||||
|
||||
// Parse title from first markdown heading or first line
|
||||
const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/^(.+)$/m);
|
||||
const title = titleMatch?.[1]?.trim() || 'Extracted Content';
|
||||
|
||||
// Build metadata
|
||||
const metadata: WebsiteMetadata = {
|
||||
title,
|
||||
extraction_method: 'jina',
|
||||
};
|
||||
|
||||
// Format content consistently with cheerio output
|
||||
const content = this.formatContent(metadata, markdown);
|
||||
|
||||
return {
|
||||
content,
|
||||
chunk: markdown,
|
||||
metadata,
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean extracted content for better readability
|
||||
*/
|
||||
@@ -189,55 +229,69 @@ export class WebsiteExtractor {
|
||||
return sections.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract content using cheerio (static HTML parser)
|
||||
* Fast, free, no external calls - but fails on JS-rendered sites
|
||||
*/
|
||||
private async extractWithCheerio(url: string): Promise<ExtractionResult> {
|
||||
const response = await fetch(url, {
|
||||
headers: this.headers,
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const metadata = this.extractMetadata($);
|
||||
metadata.extraction_method = 'cheerio';
|
||||
const mainContent = this.extractMainContent($);
|
||||
|
||||
const content = this.formatContent(metadata, mainContent);
|
||||
|
||||
return {
|
||||
content,
|
||||
chunk: mainContent,
|
||||
metadata,
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Main extraction method
|
||||
* Strategy: Try cheerio first (fast, free), fall back to Jina for JS-rendered sites
|
||||
*/
|
||||
async extract(url: string): Promise<ExtractionResult> {
|
||||
// Validate URL
|
||||
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
||||
throw new Error('Invalid URL format - must start with http:// or https://');
|
||||
}
|
||||
|
||||
|
||||
// Try cheerio first (fast, no external call)
|
||||
try {
|
||||
// Fetch the webpage
|
||||
const response = await fetch(url, {
|
||||
headers: this.headers,
|
||||
signal: AbortSignal.timeout(30000), // 30 second timeout
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
const result = await this.extractWithCheerio(url);
|
||||
|
||||
// If content is substantial, cheerio worked
|
||||
if (result.chunk.length > 200) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
|
||||
// Parse HTML with cheerio
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
// Extract metadata and content
|
||||
const metadata = this.extractMetadata($);
|
||||
// Mark extraction method for downstream metadata
|
||||
metadata.extraction_method = 'typescript_cheerio';
|
||||
const mainContent = this.extractMainContent($);
|
||||
|
||||
// Format content for display
|
||||
const content = this.formatContent(metadata, mainContent);
|
||||
|
||||
// Chunk is the main content text
|
||||
const chunk = mainContent;
|
||||
|
||||
return {
|
||||
content,
|
||||
chunk,
|
||||
metadata,
|
||||
url,
|
||||
};
|
||||
|
||||
|
||||
console.log('[WebsiteExtractor] Cheerio extraction too short, trying Jina...');
|
||||
} catch (error: any) {
|
||||
if (error.name === 'AbortError') {
|
||||
throw new Error('Request timeout - website took too long to respond');
|
||||
}
|
||||
throw error;
|
||||
console.log('[WebsiteExtractor] Cheerio failed, trying Jina...', error.message);
|
||||
}
|
||||
|
||||
// Fallback to Jina for JS-rendered sites
|
||||
try {
|
||||
return await this.extractWithJina(url);
|
||||
} catch (jinaError: any) {
|
||||
throw new Error(`Extraction failed: ${jinaError.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user