merge: sync quick add ingestion updates

This commit is contained in:
“BeeRad”
2026-03-19 21:15:41 +11:00
14 changed files with 558 additions and 140 deletions
+2 -3
View File
@@ -57,9 +57,8 @@ export async function POST(request: NextRequest) {
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: JSON.stringify({
title, title,
type: 'pdf', // Give the description generator and UI a readable summary plus a short excerpt.
// Keep content readable in UI (not the full extracted text) notes: `Imported PDF document: ${file.name} (${extraction.metadata.pages} pages, ${Math.round(extraction.metadata.text_length / 1000)}k characters).\n\nPreview:\n${extraction.chunk.slice(0, 1200)}`,
content: `Imported PDF: ${file.name} (${extraction.metadata.pages} pages, ${Math.round(extraction.metadata.text_length / 1000)}k characters)`,
// Full extracted text goes in chunk for universal chunking/embedding // Full extracted text goes in chunk for universal chunking/embedding
chunk: extraction.chunk, chunk: extraction.chunk,
metadata: { metadata: {
+3 -4
View File
@@ -75,10 +75,9 @@ export async function PUT(
if (typeof body.description === 'string') { if (typeof body.description === 'string') {
const descriptionError = validateExplicitDescription(body.description); const descriptionError = validateExplicitDescription(body.description);
if (descriptionError) { if (descriptionError) {
return NextResponse.json({ console.warn(
success: false, `[DescriptionQuality] User-updated description failed validation for node ${nodeId}: ${descriptionError}`
error: descriptionError );
}, { status: 400 });
} }
} }
+11 -3
View File
@@ -93,6 +93,7 @@ export async function POST(request: NextRequest) {
body.title = sanitizeTitle(body.title); body.title = sanitizeTitle(body.title);
const rawNotes = typeof body.notes === 'string' ? body.notes : null; const rawNotes = typeof body.notes === 'string' ? body.notes : null;
const rawChunk = typeof body.chunk === 'string' ? body.chunk : null;
const eventDate = typeof body.event_date === 'string' ? body.event_date : null; const eventDate = typeof body.event_date === 'string' ? body.event_date : null;
// Process provided dimensions first (needed for description generation) // Process provided dimensions first (needed for description generation)
@@ -107,7 +108,7 @@ export async function POST(request: NextRequest) {
try { try {
nodeDescription = await generateDescription({ nodeDescription = await generateDescription({
title: body.title, title: body.title,
notes: rawNotes || undefined, notes: rawNotes || rawChunk?.slice(0, 2000) || undefined,
link: body.link || undefined, link: body.link || undefined,
metadata: body.metadata, metadata: body.metadata,
dimensions: trimmedProvidedDimensions dimensions: trimmedProvidedDimensions
@@ -122,16 +123,24 @@ export async function POST(request: NextRequest) {
nodeDescription = body.title.slice(0, 280); nodeDescription = body.title.slice(0, 280);
} }
const finalDescription = nodeDescription ?? body.title.slice(0, 280); let finalDescription = nodeDescription ?? body.title.slice(0, 280);
const descriptionError = validateExplicitDescription(finalDescription); const descriptionError = validateExplicitDescription(finalDescription);
if (descriptionError) { if (descriptionError) {
const isUserSuppliedDescription = typeof body.description === 'string' && body.description.trim().length > 0;
if (isUserSuppliedDescription) {
return NextResponse.json({ return NextResponse.json({
success: false, success: false,
error: descriptionError error: descriptionError
}, { status: 400 }); }, { status: 400 });
} }
console.warn(
`[DescriptionQuality] Auto-generated description failed validation for "${body.title}": ${descriptionError}. Falling back to title.`
);
finalDescription = body.title.slice(0, 280);
}
// Monitor description quality // Monitor description quality
if (WEAK_PATTERNS.test(finalDescription)) { if (WEAK_PATTERNS.test(finalDescription)) {
console.warn(`[DescriptionQuality] Weak description for node "${body.title}": "${finalDescription}"`); console.warn(`[DescriptionQuality] Weak description for node "${body.title}": "${finalDescription}"`);
@@ -139,7 +148,6 @@ export async function POST(request: NextRequest) {
// Use only provided dimensions (no auto-assignment) // Use only provided dimensions (no auto-assignment)
const finalDimensions = trimmedProvidedDimensions; const finalDimensions = trimmedProvidedDimensions;
const rawChunk = typeof body.chunk === 'string' ? body.chunk : null;
let chunkToStore = rawChunk; let chunkToStore = rawChunk;
let chunkStatus: Node['chunk_status']; let chunkStatus: Node['chunk_status'];
+3 -3
View File
@@ -14,18 +14,18 @@ export async function POST(request: NextRequest) {
} }
const normalizedMode: QuickAddMode | undefined = const normalizedMode: QuickAddMode | undefined =
mode === 'link' || mode === 'note' || mode === 'chat' ? mode : undefined; mode === 'link' || mode === 'text' ? mode : undefined;
const normalizedDescription: string | undefined = const normalizedDescription: string | undefined =
typeof description === 'string' && description.trim() ? description.trim() : undefined; typeof description === 'string' && description.trim() ? description.trim() : undefined;
const delegation = await enqueueQuickAdd({ const result = await enqueueQuickAdd({
rawInput: input.trim(), rawInput: input.trim(),
mode: normalizedMode, mode: normalizedMode,
description: normalizedDescription, description: normalizedDescription,
}); });
return NextResponse.json({ success: true, delegation }); return NextResponse.json({ success: true, result });
} catch (error) { } catch (error) {
console.error('[Quick Add API] Error:', error); console.error('[Quick Add API] Error:', error);
const message = error instanceof Error ? error.message : 'Unknown error'; const message = error instanceof Error ? error.message : 'Unknown error';
+5
View File
@@ -197,9 +197,14 @@ function validateExplicitDescription(description) {
return 'Description must be explicit and substantial (at least 24 characters).'; return 'Description must be explicit and substantial (at least 24 characters).';
} }
const weakPatterns = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i; const weakPatterns = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i;
const explicitEntityPatterns = /\b(article|artifact|book|brief|claim|company|concept|conversation|dataset|decision|dimension|document|episode|essay|event|guide|idea|insight|interview|lesson|link|node|note|paper|person|plan|placeholder|podcast|post|presentation|project|question|record|research|resource|skill|source|status|summary|talk|target|test node|thread|tool|transcript|tweet|update|video|website|workflow)\b/i;
const uncertaintyPatterns = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i;
if (weakPatterns.test(text)) { if (weakPatterns.test(text)) {
return 'Description is too vague. State exactly what this is and why it matters.'; return 'Description is too vague. State exactly what this is and why it matters.';
} }
if (!explicitEntityPatterns.test(text) && !uncertaintyPatterns.test(text)) {
return 'Description must explicitly identify what this thing is, or state uncertainty explicitly.';
}
return null; return null;
} }
@@ -70,6 +70,7 @@ function initDatabase() {
source TEXT, source TEXT,
created_at TEXT, created_at TEXT,
context TEXT, context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE, FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
); );
@@ -112,6 +113,18 @@ function initDatabase() {
db.pragma('cache_size = 5000'); db.pragma('cache_size = 5000');
db.pragma('busy_timeout = 5000'); db.pragma('busy_timeout = 5000');
const edgeCols = db.prepare('PRAGMA table_info(edges)').all().map(c => c.name);
if (!edgeCols.includes('explanation')) {
db.exec('ALTER TABLE edges ADD COLUMN explanation TEXT;');
try {
db.exec(`
UPDATE edges SET explanation = json_extract(context, '$.explanation')
WHERE explanation IS NULL AND json_extract(context, '$.explanation') IS NOT NULL;
`);
} catch {}
console.error('[RA-H] Migrated edges: added explanation column');
}
return db; return db;
} }
+157 -39
View File
@@ -1,11 +1,11 @@
"use client"; "use client";
import { useState, useCallback, useMemo } from 'react'; import { useState, useCallback, useMemo, useRef } from 'react';
import { createPortal } from 'react-dom'; import { createPortal } from 'react-dom';
interface QuickAddSubmitPayload { interface QuickAddSubmitPayload {
input: string; input: string;
mode: 'link' | 'note' | 'chat'; mode: 'link' | 'text';
description?: string; description?: string;
} }
@@ -15,34 +15,28 @@ interface QuickAddInputProps {
onClose?: () => void; onClose?: () => void;
} }
type DetectedType = 'youtube' | 'website' | 'pdf-url' | 'note' | 'chat'; type SourceType = 'link' | 'text' | 'upload';
function detectType(raw: string): DetectedType { function detectSourceType(raw: string): Exclude<SourceType, 'upload'> {
const trimmed = raw.trim(); const trimmed = raw.trim();
if (!trimmed) return 'note'; if (!trimmed) return 'text';
if (/youtu(\.be|be\.com)/i.test(trimmed)) return 'youtube'; const isSingleLine = !trimmed.includes('\n');
if (/\.pdf($|\?)/i.test(trimmed) || /arxiv\.org\//i.test(trimmed)) return 'pdf-url'; if (isSingleLine) {
if (/^https?:\/\//i.test(trimmed)) return 'website'; if (/^https?:\/\//i.test(trimmed)) return 'link';
const newlines = (trimmed.match(/\n/g)?.length ?? 0); }
if (newlines >= 3 && trimmed.length > 300) return 'chat'; return 'text';
if (/You said:|ChatGPT said:|Claude said:/i.test(trimmed)) return 'chat';
return 'note';
} }
const TYPE_LABELS: Record<DetectedType, string> = { const SOURCE_LABELS: Record<SourceType, string> = {
youtube: 'YouTube', link: 'Link',
website: 'Link', text: 'Text',
'pdf-url': 'PDF', upload: 'Upload',
note: 'Note',
chat: 'Transcript',
}; };
const TYPE_COLORS: Record<DetectedType, string> = { const SOURCE_COLORS: Record<SourceType, string> = {
youtube: '#ef4444', link: '#3b82f6',
website: '#3b82f6', text: '#22c55e',
'pdf-url': '#f59e0b', upload: '#f59e0b',
note: '#22c55e',
chat: '#a78bfa',
}; };
export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInputProps) { export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInputProps) {
@@ -52,6 +46,8 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
const [dragOver, setDragOver] = useState(false); const [dragOver, setDragOver] = useState(false);
const [uploadedFile, setUploadedFile] = useState<File | null>(null); const [uploadedFile, setUploadedFile] = useState<File | null>(null);
const [uploadError, setUploadError] = useState<string | null>(null); const [uploadError, setUploadError] = useState<string | null>(null);
const [sourceOverride, setSourceOverride] = useState<Exclude<SourceType, 'upload'> | null>(null);
const fileInputRef = useRef<HTMLInputElement | null>(null);
const isControlled = isOpen !== undefined; const isControlled = isOpen !== undefined;
const isExpanded = isControlled ? isOpen : isExpandedInternal; const isExpanded = isControlled ? isOpen : isExpandedInternal;
@@ -59,8 +55,8 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
? (value: boolean) => { if (!value && onClose) onClose(); } ? (value: boolean) => { if (!value && onClose) onClose(); }
: setIsExpandedInternal; : setIsExpandedInternal;
const detectedType = useMemo(() => detectType(input), [input]); const detectedSource = useMemo(() => detectSourceType(input), [input]);
const showTypePill = input.trim().length > 0 && !uploadedFile; const effectiveSource: SourceType = uploadedFile ? 'upload' : (sourceOverride ?? detectedSource);
const handleFileUpload = useCallback(async (file: File) => { const handleFileUpload = useCallback(async (file: File) => {
setIsPosting(true); setIsPosting(true);
@@ -78,6 +74,7 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
} }
setUploadedFile(null); setUploadedFile(null);
setInput(''); setInput('');
setSourceOverride(null);
setIsExpanded(false); setIsExpanded(false);
} catch (error) { } catch (error) {
console.error('[QuickAddInput] Upload error:', error); console.error('[QuickAddInput] Upload error:', error);
@@ -95,8 +92,12 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
if (!input.trim() || isPosting) return; if (!input.trim() || isPosting) return;
setIsPosting(true); setIsPosting(true);
try { try {
await onSubmit({ input: input.trim(), mode: 'link' }); await onSubmit({
input: input.trim(),
mode: effectiveSource === 'link' ? 'link' : 'text',
});
setInput(''); setInput('');
setSourceOverride(null);
setIsExpanded(false); setIsExpanded(false);
} catch (error) { } catch (error) {
console.error('[QuickAddInput] Submit error:', error); console.error('[QuickAddInput] Submit error:', error);
@@ -115,6 +116,7 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
setInput(''); setInput('');
setUploadedFile(null); setUploadedFile(null);
setUploadError(null); setUploadError(null);
setSourceOverride(null);
} }
}; };
@@ -147,6 +149,7 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
} }
setUploadedFile(file); setUploadedFile(file);
setInput(''); setInput('');
setSourceOverride(null);
}, []); }, []);
const hasContent = input.trim() || uploadedFile; const hasContent = input.trim() || uploadedFile;
@@ -156,6 +159,37 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
setInput(''); setInput('');
setUploadedFile(null); setUploadedFile(null);
setUploadError(null); setUploadError(null);
setSourceOverride(null);
};
const handleSourceSelect = (source: Exclude<SourceType, 'upload'>) => {
setSourceOverride(source);
setUploadedFile(null);
setUploadError(null);
};
const handleUploadClick = () => {
setUploadError(null);
fileInputRef.current?.click();
};
const handleFileInputChange = (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (!file) return;
if (file.type !== 'application/pdf' && !file.name.toLowerCase().endsWith('.pdf')) {
setUploadError('Only PDF files are supported');
event.target.value = '';
return;
}
if (file.size > 50 * 1024 * 1024) {
setUploadError(`File too large (${Math.round(file.size / 1024 / 1024)}MB). Max 50MB.`);
event.target.value = '';
return;
}
setUploadedFile(file);
setInput('');
setSourceOverride(null);
event.target.value = '';
}; };
if (!isExpanded) { if (!isExpanded) {
@@ -210,7 +244,14 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
onDrop={handleDrop} onDrop={handleDrop}
onClick={(e) => e.stopPropagation()} onClick={(e) => e.stopPropagation()}
> >
{/* File preview */} <input
ref={fileInputRef}
type="file"
accept=".pdf,application/pdf"
style={{ display: 'none' }}
onChange={handleFileInputChange}
/>
{uploadedFile && ( {uploadedFile && (
<div className="qa-file-row"> <div className="qa-file-row">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="#ef4444" strokeWidth="1.5"> <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="#ef4444" strokeWidth="1.5">
@@ -231,12 +272,10 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
</div> </div>
)} )}
{/* Error */}
{uploadError && ( {uploadError && (
<div className="qa-error">{uploadError}</div> <div className="qa-error">{uploadError}</div>
)} )}
{/* Input area */}
{!uploadedFile && ( {!uploadedFile && (
<div className={`qa-input-area ${dragOver ? 'dragging' : ''}`}> <div className={`qa-input-area ${dragOver ? 'dragging' : ''}`}>
{dragOver && ( {dragOver && (
@@ -250,9 +289,18 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
)} )}
<textarea <textarea
value={input} value={input}
onChange={(e) => setInput(e.target.value)} onChange={(e) => {
const nextValue = e.target.value;
setInput(nextValue);
if (nextValue.trim()) {
setUploadedFile(null);
}
if (!nextValue.trim()) {
setSourceOverride(null);
}
}}
onKeyDown={handleKeyDown} onKeyDown={handleKeyDown}
placeholder="Paste a URL, write a note, or drop a PDF..." placeholder="Paste a link, write/paste text, or upload a PDF..."
disabled={isPosting} disabled={isPosting}
autoFocus autoFocus
className="qa-textarea" className="qa-textarea"
@@ -261,18 +309,40 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
</div> </div>
)} )}
{/* Footer */}
<div className="qa-footer"> <div className="qa-footer">
<div className="qa-footer-left"> <div className="qa-footer-left">
{showTypePill && ( <div className="qa-source-row">
<button
type="button"
className={`qa-control-chip qa-source-chip ${effectiveSource === 'link' ? 'active' : ''}`}
onClick={() => handleSourceSelect('link')}
>
{SOURCE_LABELS.link}
</button>
<button
type="button"
className={`qa-control-chip qa-source-chip ${effectiveSource === 'text' ? 'active' : ''}`}
onClick={() => handleSourceSelect('text')}
>
{SOURCE_LABELS.text}
</button>
<button
type="button"
className={`qa-control-chip qa-source-chip ${effectiveSource === 'upload' ? 'active upload' : 'upload'}`}
onClick={handleUploadClick}
>
{SOURCE_LABELS.upload}
</button>
</div>
</div>
<div className="qa-footer-right">
<span className="qa-type-pill" style={{ <span className="qa-type-pill" style={{
color: TYPE_COLORS[detectedType], color: SOURCE_COLORS[effectiveSource],
borderColor: TYPE_COLORS[detectedType] + '30', borderColor: SOURCE_COLORS[effectiveSource] + '30',
background: TYPE_COLORS[detectedType] + '0a', background: SOURCE_COLORS[effectiveSource] + '0a',
}}> }}>
{TYPE_LABELS[detectedType]} {effectiveSource === 'upload' ? 'Process as Upload' : `Process as ${SOURCE_LABELS[effectiveSource]}`}
</span> </span>
)}
<span className="qa-hint"> <span className="qa-hint">
{uploadedFile {uploadedFile
? 'Ready to upload' ? 'Ready to upload'
@@ -413,13 +483,61 @@ export default function QuickAddInput({ onSubmit, isOpen, onClose }: QuickAddInp
border-top: 1px solid rgba(255, 255, 255, 0.04); border-top: 1px solid rgba(255, 255, 255, 0.04);
} }
.qa-source-row {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 8px;
}
.qa-control-chip {
border: 1px solid rgba(255, 255, 255, 0.08);
background: rgba(255, 255, 255, 0.03);
color: #8a8a8a;
border-radius: 10px;
padding: 6px 12px;
font-size: 11px;
font-weight: 600;
letter-spacing: 0.04em;
text-transform: uppercase;
cursor: pointer;
transition: all 0.15s ease;
}
.qa-control-chip:hover {
border-color: rgba(255, 255, 255, 0.16);
color: #cfcfcf;
}
.qa-source-chip.active {
color: #f5f5f5;
border-color: rgba(255, 255, 255, 0.2);
background: rgba(255, 255, 255, 0.08);
}
.qa-source-chip.upload.active {
border-color: rgba(245, 158, 11, 0.35);
background: rgba(245, 158, 11, 0.12);
color: #fbbf24;
}
.qa-footer-left { .qa-footer-left {
display: flex;
align-items: center;
gap: 12px;
min-width: 0;
}
.qa-footer-right {
display: flex; display: flex;
align-items: center; align-items: center;
gap: 10px; gap: 10px;
margin-left: auto;
} }
.qa-type-pill { .qa-type-pill {
display: inline-flex;
align-items: center;
font-size: 10px; font-size: 10px;
font-weight: 600; font-weight: 600;
letter-spacing: 0.04em; letter-spacing: 0.04em;
+7 -8
View File
@@ -570,7 +570,7 @@ export default function ThreePanelLayout() {
}, [slotA, slotB, handlePaneTypeClick]); }, [slotA, slotB, handlePaneTypeClick]);
// Handle Quick Add submit (used by global Add Stuff modal) // Handle Quick Add submit (used by global Add Stuff modal)
const handleQuickAddSubmit = useCallback(async ({ input, mode, description }: { input: string; mode: 'link' | 'note' | 'chat'; description?: string }) => { const handleQuickAddSubmit = useCallback(async ({ input, mode, description }: { input: string; mode: 'link' | 'text'; description?: string }) => {
try { try {
const response = await fetch('/api/quick-add', { const response = await fetch('/api/quick-add', {
method: 'POST', method: 'POST',
@@ -583,15 +583,14 @@ export default function ThreePanelLayout() {
throw new Error(data.error || 'Failed to submit Quick Add'); throw new Error(data.error || 'Failed to submit Quick Add');
} }
const result = await response.json(); const data = await response.json();
const delegation = result.delegation; const result = data.result as { id: string; inputType: string } | undefined;
// Add pending placeholder if (result?.id) {
if (delegation?.id && delegation?.inputType) {
setPendingNodes(prev => [{ setPendingNodes(prev => [{
id: delegation.id, id: result.id,
input: input.slice(0, 120), input: input.trim(),
inputType: delegation.inputType, inputType: result.inputType || 'note',
submittedAt: Date.now(), submittedAt: Date.now(),
status: 'processing' as const, status: 'processing' as const,
}, ...prev]); }, ...prev]);
+102 -21
View File
@@ -6,7 +6,7 @@ import { formatNodeForChat } from '@/tools/infrastructure/nodeFormatter';
import { summarizeTranscript } from './transcriptSummarizer'; import { summarizeTranscript } from './transcriptSummarizer';
import { eventBroadcaster } from '@/services/events'; import { eventBroadcaster } from '@/services/events';
export type QuickAddMode = 'link' | 'note' | 'chat'; export type QuickAddMode = 'link' | 'text';
export type QuickAddInputType = 'youtube' | 'website' | 'pdf' | 'note' | 'chat'; export type QuickAddInputType = 'youtube' | 'website' | 'pdf' | 'note' | 'chat';
@@ -16,6 +16,15 @@ export interface QuickAddInput {
description?: string; description?: string;
} }
export interface QuickAddResult {
id: string;
task: string;
inputType: QuickAddInputType;
status: 'queued' | 'completed' | 'failed';
summary?: string;
error?: string;
}
function isLikelyChatTranscript(raw: string): boolean { function isLikelyChatTranscript(raw: string): boolean {
const newlineCount = (raw.match(/\n/g)?.length ?? 0); const newlineCount = (raw.match(/\n/g)?.length ?? 0);
if (newlineCount >= 3 && raw.length > 300) return true; if (newlineCount >= 3 && raw.length > 300) return true;
@@ -25,13 +34,23 @@ function isLikelyChatTranscript(raw: string): boolean {
} }
export function detectInputType(raw: string, mode?: QuickAddMode): QuickAddInputType { export function detectInputType(raw: string, mode?: QuickAddMode): QuickAddInputType {
if (mode === 'chat') return 'chat';
if (mode === 'note') return 'note';
const input = raw.trim(); const input = raw.trim();
const isSingleLine = !input.includes('\n');
if (mode === 'text') {
return isLikelyChatTranscript(input) ? 'chat' : 'note';
}
if (isSingleLine) {
if (/youtu(\.be|be\.com)/i.test(input)) return 'youtube'; if (/youtu(\.be|be\.com)/i.test(input)) return 'youtube';
if (/\.pdf($|\?)/i.test(input) || /arxiv\.org\//i.test(input)) return 'pdf'; if (/\.pdf($|\?)/i.test(input) || /arxiv\.org\//i.test(input)) return 'pdf';
if (/^https?:\/\//i.test(input)) return 'website'; if (/^https?:\/\//i.test(input)) return 'website';
}
if (mode === 'link') {
return 'website';
}
if (!mode && isLikelyChatTranscript(input)) return 'chat'; if (!mode && isLikelyChatTranscript(input)) return 'chat';
return 'note'; return 'note';
} }
@@ -84,6 +103,30 @@ interface CreateNodeResponse {
error?: string; error?: string;
} }
function deriveFallbackLinkTitle(rawUrl: string): string {
const trimmed = rawUrl.trim();
if (!trimmed) return 'Saved link';
try {
const parsed = new URL(trimmed);
const hostname = parsed.hostname.replace(/^www\./, '');
const lastSegment = parsed.pathname
.split('/')
.filter(Boolean)
.pop()
?.replace(/[-_]+/g, ' ')
.trim();
if (lastSegment) {
return `${hostname}: ${lastSegment}`.slice(0, 160);
}
return hostname.slice(0, 160);
} catch {
return trimmed.slice(0, 160);
}
}
function buildStructuredSummary({ task, action, resultMessage, nodeReference }: SummaryParts): string { function buildStructuredSummary({ task, action, resultMessage, nodeReference }: SummaryParts): string {
const normalizedResult = resultMessage?.trim().length ? resultMessage.trim() : `${action} completed.`; const normalizedResult = resultMessage?.trim().length ? resultMessage.trim() : `${action} completed.`;
const normalizedNode = nodeReference || 'None'; const normalizedNode = nodeReference || 'None';
@@ -160,6 +203,8 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin
if (!execute) { if (!execute) {
throw new Error(`Tool ${toolName} does not have an execute function`); throw new Error(`Tool ${toolName} does not have an execute function`);
} }
try {
const rawResult = await execute({ url }, { toolCallId: 'quickadd-extract', messages: [] }); const rawResult = await execute({ url }, { toolCallId: 'quickadd-extract', messages: [] });
if (!isExtractionToolResult(rawResult)) { if (!isExtractionToolResult(rawResult)) {
@@ -186,6 +231,56 @@ async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: strin
resultMessage: summaryLine, resultMessage: summaryLine,
nodeReference, nodeReference,
}); });
} catch (error) {
const message = error instanceof Error ? error.message : `Failed to execute ${toolName}`;
const title = deriveFallbackLinkTitle(url);
const description =
`Link record for this source. RA-H could not correctly process the URL during ingestion because ${message}. Stored so the source is not lost and can be revisited later.`;
const notes = [
`Original URL: ${url}`,
`Ingestion failure: ${message}`,
`Attempted pipeline: ${type}`,
].join('\n');
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
title,
description,
notes,
link: url,
metadata: {
source: 'quick-add-link-fallback',
attempted_pipeline: type,
extraction_failed: true,
extraction_error: message,
refined_at: new Date().toISOString(),
},
}),
});
const rawFallbackResult = await response.json();
if (!isCreateNodeResponse(rawFallbackResult)) {
throw new Error(message);
}
if (!response.ok) {
throw new Error(rawFallbackResult.error || message);
}
const nodeId = rawFallbackResult.data?.id;
const nodeReference = nodeId ? formatNodeForChat({ id: nodeId, title }) : 'None';
const resultMessage = nodeId
? `Link ingestion failed, so RA-H saved a fallback node ${nodeReference}. Reason: ${message}`
: `Link ingestion failed, so RA-H saved a fallback node. Reason: ${message}`;
return buildStructuredSummary({
task,
action: `${toolName} (fallback)`,
resultMessage,
nodeReference,
});
}
} }
async function handleNoteQuickAdd(rawInput: string, task: string, userDescription?: string): Promise<string> { async function handleNoteQuickAdd(rawInput: string, task: string, userDescription?: string): Promise<string> {
@@ -204,7 +299,6 @@ async function handleNoteQuickAdd(rawInput: string, task: string, userDescriptio
}, },
}; };
// If user provided a description, use it instead of auto-generating
if (userDescription && userDescription.trim()) { if (userDescription && userDescription.trim()) {
nodePayload.description = userDescription.trim(); nodePayload.description = userDescription.trim();
} }
@@ -330,15 +424,6 @@ async function handleChatTranscriptQuickAdd(rawInput: string, task: string): Pro
}); });
} }
export interface QuickAddResult {
id: string;
task: string;
inputType: QuickAddInputType;
status: 'queued' | 'completed' | 'failed';
summary?: string;
error?: string;
}
export async function enqueueQuickAdd({ rawInput, mode, description }: QuickAddInput): Promise<QuickAddResult> { export async function enqueueQuickAdd({ rawInput, mode, description }: QuickAddInput): Promise<QuickAddResult> {
const inputType = detectInputType(rawInput, mode); const inputType = detectInputType(rawInput, mode);
const task = buildTaskPrompt(inputType, rawInput); const task = buildTaskPrompt(inputType, rawInput);
@@ -351,25 +436,21 @@ export async function enqueueQuickAdd({ rawInput, mode, description }: QuickAddI
status: 'queued', status: 'queued',
}; };
// Run async - fire and forget
setImmediate(async () => { setImmediate(async () => {
try { try {
let summary: string;
if (inputType === 'note') { if (inputType === 'note') {
summary = await handleNoteQuickAdd(rawInput, task, description); await handleNoteQuickAdd(rawInput, task, description);
} else if (inputType === 'chat') { } else if (inputType === 'chat') {
summary = await handleChatTranscriptQuickAdd(rawInput, task); await handleChatTranscriptQuickAdd(rawInput, task);
} else { } else {
summary = await handleExtractionQuickAdd(inputType as ExtractionQuickAddType, rawInput, task); await handleExtractionQuickAdd(inputType as ExtractionQuickAddType, rawInput, task);
} }
console.log(`[QuickAdd] Completed: ${task}`); console.log(`[QuickAdd] Completed: ${task}`);
// Broadcast completion so ThreePanelLayout can remove the pending placeholder
eventBroadcaster.broadcast({ eventBroadcaster.broadcast({
type: 'QUICK_ADD_COMPLETED', type: 'QUICK_ADD_COMPLETED',
data: { quickAddId: id, source: 'quick-add' } data: { quickAddId: id, source: 'quick-add' }
}); });
// Also broadcast NODE_CREATED to refresh the feed
eventBroadcaster.broadcast({ eventBroadcaster.broadcast({
type: 'NODE_CREATED', type: 'NODE_CREATED',
data: { node: { title: task }, source: 'quick-add' } data: { node: { title: task }, source: 'quick-add' }
+32 -6
View File
@@ -11,6 +11,9 @@ export interface DescriptionInput {
channel_name?: string; channel_name?: string;
author?: string; author?: string;
site_name?: string; site_name?: string;
original_filename?: string;
pages?: number;
text_length?: number;
}; };
dimensions?: string[]; dimensions?: string[];
} }
@@ -75,10 +78,7 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
temperature: 0.3, temperature: 0.3,
}); });
const description = response.text.trim(); const finalDescription = sanitizeDescription(response.text, input);
// Ensure within character limit
const finalDescription = description.slice(0, 280);
console.log(`[DescriptionService] Generated: "${finalDescription}"`); console.log(`[DescriptionService] Generated: "${finalDescription}"`);
@@ -126,6 +126,10 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
if (input.metadata?.channel_name) lines.push(`Channel: ${input.metadata.channel_name}`); if (input.metadata?.channel_name) lines.push(`Channel: ${input.metadata.channel_name}`);
if (input.metadata?.author) lines.push(`Author: ${input.metadata.author}`); if (input.metadata?.author) lines.push(`Author: ${input.metadata.author}`);
if (input.metadata?.site_name) lines.push(`Site: ${input.metadata.site_name}`); if (input.metadata?.site_name) lines.push(`Site: ${input.metadata.site_name}`);
if (input.metadata?.source) lines.push(`Source type: ${input.metadata.source}`);
if (input.metadata?.original_filename) lines.push(`Original filename: ${input.metadata.original_filename}`);
if (typeof input.metadata?.pages === 'number') lines.push(`Pages: ${input.metadata.pages}`);
if (typeof input.metadata?.text_length === 'number') lines.push(`Text length: ${input.metadata.text_length}`);
if (creatorHint) lines.push(`Creator hint: ${creatorHint}`); if (creatorHint) lines.push(`Creator hint: ${creatorHint}`);
if (publisherHint) lines.push(`Publisher hint: ${publisherHint}`); if (publisherHint) lines.push(`Publisher hint: ${publisherHint}`);
lines.push(`Likely user-authored: ${likelyUserAuthored ? 'yes' : 'no'}`); lines.push(`Likely user-authored: ${likelyUserAuthored ? 'yes' : 'no'}`);
@@ -138,15 +142,19 @@ function buildDescriptionPrompt(input: DescriptionInput): string {
Say WHAT this literally is and WHY it matters. Be concrete and specific like you're telling a friend what this thing is in one breath. Say WHAT this literally is and WHY it matters. Be concrete and specific like you're telling a friend what this thing is in one breath.
RULES: RULES:
1) Name the format: "Podcast episode where…", "Blog post arguing…", "Your note on…", "Research paper showing…", "Idea that…" 1) Name the format only if the context clearly supports it: "Podcast episode where…", "Blog post arguing…", "Personal note capturing…", "Research paper showing…", "Resume/CV for…", "Document likely containing…", "Idea that…"
2) Name people by role channel/host is the creator, title figures are guests/subjects. Use the Creator hint if available. 2) Name people by role channel/host is the creator, title figures are guests/subjects. Use the Creator hint if available.
3) State the actual claim, finding, or insight from the content not a vague summary of the topic. 3) State the actual claim, finding, or insight from the content not a vague summary of the topic.
4) End with why it's interesting or important one concrete phrase. 4) End with why it's interesting or important one concrete phrase.
5) ABSOLUTELY FORBIDDEN these words will be rejected: "discusses", "explores", "examines", "talks about", "is about", "delves into", "emphasizing the need for". State things directly instead. 5) ABSOLUTELY FORBIDDEN these words will be rejected: "discusses", "explores", "examines", "talks about", "is about", "delves into", "emphasizing the need for". State things directly instead.
6) Do NOT start with "Your note —" or "This note —". Use a concrete opener tied to the actual artifact.
7) If the artifact type is unclear, say so explicitly using words like "likely", "appears to be", or "unclear" rather than guessing a confident format.
GOOD: "Karpathy blog post — AI agents make software fluid, ripping functionality from repos instead of taking dependencies. Signals the end of monolithic libraries." GOOD: "Karpathy blog post — AI agents make software fluid, ripping functionality from repos instead of taking dependencies. Signals the end of monolithic libraries."
GOOD: "Dwarkesh Patel interview with Anthropic CEO Dario Amodei — argues we're nearing the end of exponential AI scaling. Key signal for what comes next." GOOD: "Dwarkesh Patel interview with Anthropic CEO Dario Amodei — argues we're nearing the end of exponential AI scaling. Key signal for what comes next."
GOOD: "Your note — morning optimism consistently reverses to evening pessimism. Not energy — the belief itself flips. Pattern worth tracking." GOOD: "Personal note capturing a recurring pattern: morning optimism reverses to evening pessimism. Indicates a belief-level swing worth tracking."
GOOD: "Resume/CV for Brad Morris outlining work in AI systems, context engineering, and RA-H. Useful as a compact record of background, projects, and expertise."
GOOD: "Document likely related to Brad Morris's work history and AI consulting, but the exact artifact type is unclear from the available context. Still useful as a reference profile."
BAD: "By Dario Amodei — discusses reaching the limits of exponential growth in AI, emphasizing the need for a critical perspective on future advancements." BAD: "By Dario Amodei — discusses reaching the limits of exponential growth in AI, emphasizing the need for a critical perspective on future advancements."
BAD: "This article explores ideas about how software is changing." BAD: "This article explores ideas about how software is changing."
@@ -155,6 +163,24 @@ Return ONLY the description text. Nothing else.
${lines.join('\n')}`; ${lines.join('\n')}`;
} }
function sanitizeDescription(rawText: string, input: DescriptionInput): string {
const singleLine = rawText
.trim()
.replace(/\s+/g, ' ')
.replace(/^["']|["']$/g, '');
if (!singleLine) {
return input.title.slice(0, 280);
}
const noGenericPrefix = singleLine.replace(
/^(your note|this note)\s*[—:-]\s*/i,
'Personal note capturing '
);
return noGenericPrefix.slice(0, 280);
}
export const descriptionService = { export const descriptionService = {
generateDescription generateDescription
}; };
+1 -1
View File
@@ -1,5 +1,5 @@
const WEAK_DESCRIPTION_PATTERNS = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i; const WEAK_DESCRIPTION_PATTERNS = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i;
const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|conversation|dataset|decision|dimension|document|episode|essay|guide|idea|insight|interview|node|note|paper|person|plan|placeholder|podcast|presentation|project|question|record|research|skill|source|summary|talk|target|test node|thread|tool|transcript|tweet|video|website|workflow)\b/i; const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|concept|conversation|dataset|decision|dimension|document|episode|essay|event|guide|idea|insight|interview|lesson|link|node|note|paper|person|plan|placeholder|podcast|post|presentation|project|question|record|research|resource|skill|source|status|summary|talk|target|test node|thread|tool|transcript|tweet|update|video|website|workflow)\b/i;
const UNCERTAINTY_PATTERNS = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i; const UNCERTAINTY_PATTERNS = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i;
const GENERIC_EDGE_PATTERNS = /^(related|related to|connected|connected to|association|associated with)$/i; const GENERIC_EDGE_PATTERNS = /^(related|related to|connected|connected to|association|associated with)$/i;
+9
View File
@@ -657,6 +657,15 @@ class SQLiteClient {
if (edgeCols.some(c => c.name === 'user_feedback')) { if (edgeCols.some(c => c.name === 'user_feedback')) {
try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch {} try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch {}
} }
if (!edgeCols.some(c => c.name === 'explanation')) {
this.db.exec('ALTER TABLE edges ADD COLUMN explanation TEXT;');
try {
this.db.exec(`
UPDATE edges SET explanation = json_extract(context, '$.explanation')
WHERE explanation IS NULL AND json_extract(context, '$.explanation') IS NOT NULL;
`);
} catch {}
}
// Recreate nodes_fts to index title + description + notes // Recreate nodes_fts to index title + description + notes
try { try {
+97 -10
View File
@@ -5,21 +5,101 @@ import { generateText } from 'ai';
import { extractWebsite } from '@/services/typescript/extractors/website'; import { extractWebsite } from '@/services/typescript/extractors/website';
import { formatNodeForChat } from '../infrastructure/nodeFormatter'; import { formatNodeForChat } from '../infrastructure/nodeFormatter';
// AI-powered content analysis interface ExistingDimension {
async function analyzeContentWithAI(title: string, description: string, contentType: string) { name: string;
description: string | null;
}
function inferWebsiteContentType(url: string): 'website' | 'tweet' {
try { try {
const hostname = new URL(url).hostname.toLowerCase();
return hostname === 'x.com' || hostname.endsWith('.x.com') || hostname === 'twitter.com' || hostname.endsWith('.twitter.com')
? 'tweet'
: 'website';
} catch {
return 'website';
}
}
async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
try {
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/dimensions/popular`);
if (!response.ok) return [];
const result = await response.json();
if (!Array.isArray(result.data)) return [];
return result.data
.map((dimension: { dimension?: unknown; description?: unknown }) => ({
name: typeof dimension.dimension === 'string' ? dimension.dimension.trim() : '',
description: typeof dimension.description === 'string' ? dimension.description.trim() : null
}))
.filter((dimension: ExistingDimension) => dimension.name.length > 0);
} catch (error) {
console.warn('Website dimension fetch fallback (no dimension context):', error);
return [];
}
}
function selectExistingDimensions(
selected: unknown,
existingDimensions: ExistingDimension[],
max = 5
): string[] {
if (!Array.isArray(selected) || existingDimensions.length === 0) return [];
const byLowerName = new Map(existingDimensions.map((dimension) => [dimension.name.toLowerCase(), dimension.name]));
const normalized: string[] = [];
const seen = new Set<string>();
for (const value of selected) {
if (typeof value !== 'string') continue;
const matched = byLowerName.get(value.trim().toLowerCase());
if (!matched) continue;
const key = matched.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
normalized.push(matched);
if (normalized.length >= max) break;
}
return normalized;
}
// AI-powered content analysis
async function analyzeContentWithAI(
title: string,
description: string,
contentType: string,
existingDimensions: ExistingDimension[]
) {
try {
const availableDimensionsBlock = existingDimensions.length > 0
? existingDimensions
.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`)
.join('\n')
: '- No existing dimensions available. Return an empty dimensions array.';
const prompt = `Analyze this ${contentType} content and provide classification. const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}" Title: "${title}"
Description: "${description}" Description: "${description}"
CRITICAL nodeDescription rules (max 280 chars): CRITICAL nodeDescription rules (max 280 chars):
1. Say WHAT this literally is: "Blog post by…", "Article from…", "Essay arguing…", "Tutorial on…", "Thread by…" 1. Say WHAT this literally is using explicit entity words only: "Blog post by…", "Article from…", "Essay arguing…", "Tutorial on…", "Thread by…", "Tweet by…", "Post by…"
2. Name the author/site if known from the metadata. 2. Name the author/site if known from the metadata.
3. State the actual claim or thesis don't paraphrase into vague abstractions. 3. State the actual claim or thesis don't paraphrase into vague abstractions.
4. End with why it's interesting or important one concrete phrase. 4. End with why it's interesting or important one concrete phrase.
5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly. 5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly.
DIMENSION SELECTION (critical):
You must select 0-3 dimensions from the list below.
Do NOT invent new dimension names.
Pick only dimensions that genuinely fit this content.
If nothing fits, return an empty array.
Available dimensions:
${availableDimensionsBlock}
Examples: Examples:
- Title: "Software is eating the world — again" / Author: Andrej Karpathy - Title: "Software is eating the world — again" / Author: Andrej Karpathy
GOOD: "Karpathy's blog post arguing AI agents make software fluid — they can rip functionality from repos instead of taking dependencies. Signals the end of monolithic libraries." GOOD: "Karpathy's blog post arguing AI agents make software fluid — they can rip functionality from repos instead of taking dependencies. Signals the end of monolithic libraries."
@@ -33,7 +113,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
{ {
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.", "enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your 280-char description following the rules above>", "nodeDescription": "<your 280-char description following the rules above>",
"tags": ["relevant", "semantic", "tags"], "dimensions": ["existing-dimension-1", "existing-dimension-2"],
"reasoning": "Brief explanation of classification choices" "reasoning": "Brief explanation of classification choices"
}`; }`;
@@ -53,7 +133,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
return { return {
enhancedDescription: result.enhancedDescription || description, enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined, nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined,
tags: Array.isArray(result.tags) ? result.tags : [], dimensions: selectExistingDimensions(result.dimensions, existingDimensions, 5),
reasoning: result.reasoning || 'AI analysis completed' reasoning: result.reasoning || 'AI analysis completed'
}; };
} catch (error) { } catch (error) {
@@ -62,7 +142,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
return { return {
enhancedDescription: description, enhancedDescription: description,
nodeDescription: undefined, nodeDescription: undefined,
tags: [], dimensions: [],
reasoning: 'Fallback description used' reasoning: 'Fallback description used'
}; };
} }
@@ -122,10 +202,13 @@ export const websiteExtractTool = tool({
console.log('🎯 Website extraction successful, analyzing with AI...'); console.log('🎯 Website extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata // Step 2: AI Analysis for enhanced metadata
const existingDimensions = await fetchExistingDimensions();
const contentType = inferWebsiteContentType(url);
const aiAnalysis = await analyzeContentWithAI( const aiAnalysis = await analyzeContentWithAI(
result.metadata?.title || `Website: ${new URL(url).hostname}`, result.metadata?.title || `Website: ${new URL(url).hostname}`,
result.notes?.substring(0, 2000) || 'Website content', result.notes?.substring(0, 2000) || 'Website content',
'website' contentType,
existingDimensions
); );
// Step 3: Create node with extracted content and AI analysis // Step 3: Create node with extracted content and AI analysis
@@ -138,6 +221,9 @@ export const websiteExtractTool = tool({
.filter(Boolean); .filter(Boolean);
trimmedDimensions = trimmedDimensions.slice(0, 5); trimmedDimensions = trimmedDimensions.slice(0, 5);
const finalDimensions = trimmedDimensions.length > 0
? trimmedDimensions
: (aiAnalysis?.dimensions || []).slice(0, 5);
const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, { const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
method: 'POST', method: 'POST',
@@ -147,10 +233,11 @@ export const websiteExtractTool = tool({
description: aiAnalysis?.nodeDescription, description: aiAnalysis?.nodeDescription,
notes: enhancedDescription, notes: enhancedDescription,
link: url, link: url,
dimensions: trimmedDimensions, event_date: result.metadata?.published_date || result.metadata?.date || null,
dimensions: finalDimensions,
chunk: result.chunk || result.notes, chunk: result.chunk || result.notes,
metadata: { metadata: {
source: 'website', source: contentType,
hostname: new URL(url).hostname, hostname: new URL(url).hostname,
author: result.metadata?.author, author: result.metadata?.author,
published_date: result.metadata?.published_date || result.metadata?.date, published_date: result.metadata?.published_date || result.metadata?.date,
@@ -176,7 +263,7 @@ export const websiteExtractTool = tool({
console.log('🎯 WebsiteExtract completed successfully'); console.log('🎯 WebsiteExtract completed successfully');
// Use actual assigned dimensions from API response (includes auto-assigned locked + keywords) // Use actual assigned dimensions from API response (includes auto-assigned locked + keywords)
const actualDimensions: string[] = createResult.data?.dimensions || trimmedDimensions || []; const actualDimensions: string[] = createResult.data?.dimensions || finalDimensions || [];
const formattedNode = createResult.data?.id const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions }) ? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
: nodeTitle; : nodeTitle;
+82 -8
View File
@@ -5,9 +5,69 @@ import { generateText } from 'ai';
import { extractYouTube } from '@/services/typescript/extractors/youtube'; import { extractYouTube } from '@/services/typescript/extractors/youtube';
import { formatNodeForChat } from '../infrastructure/nodeFormatter'; import { formatNodeForChat } from '../infrastructure/nodeFormatter';
// AI-powered content analysis interface ExistingDimension {
async function analyzeContentWithAI(title: string, description: string, contentType: string) { name: string;
description: string | null;
}
async function fetchExistingDimensions(): Promise<ExistingDimension[]> {
try { try {
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/dimensions/popular`);
if (!response.ok) return [];
const result = await response.json();
if (!Array.isArray(result.data)) return [];
return result.data
.map((dimension: { dimension?: unknown; description?: unknown }) => ({
name: typeof dimension.dimension === 'string' ? dimension.dimension.trim() : '',
description: typeof dimension.description === 'string' ? dimension.description.trim() : null
}))
.filter((dimension: ExistingDimension) => dimension.name.length > 0);
} catch (error) {
console.warn('YouTube dimension fetch fallback (no dimension context):', error);
return [];
}
}
function selectExistingDimensions(
selected: unknown,
existingDimensions: ExistingDimension[],
max = 5
): string[] {
if (!Array.isArray(selected) || existingDimensions.length === 0) return [];
const byLowerName = new Map(existingDimensions.map((dimension) => [dimension.name.toLowerCase(), dimension.name]));
const normalized: string[] = [];
const seen = new Set<string>();
for (const value of selected) {
if (typeof value !== 'string') continue;
const matched = byLowerName.get(value.trim().toLowerCase());
if (!matched) continue;
const key = matched.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
normalized.push(matched);
if (normalized.length >= max) break;
}
return normalized;
}
// AI-powered content analysis
async function analyzeContentWithAI(
title: string,
description: string,
contentType: string,
existingDimensions: ExistingDimension[]
) {
try {
const availableDimensionsBlock = existingDimensions.length > 0
? existingDimensions
.map((dimension) => `- ${dimension.name}${dimension.description ? `: ${dimension.description}` : ''}`)
.join('\n')
: '- No existing dimensions available. Return an empty dimensions array.';
const prompt = `Analyze this ${contentType} content and provide classification. const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}" Title: "${title}"
@@ -20,6 +80,15 @@ CRITICAL — nodeDescription rules (max 280 chars):
4. End with why it's interesting or important one concrete phrase. 4. End with why it's interesting or important one concrete phrase.
5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly. 5. ABSOLUTELY FORBIDDEN: "discusses", "explores", "examines", "talks about", "delves into", "emphasizing the need for". State things directly.
DIMENSION SELECTION (critical):
You must select 0-3 dimensions from the list below.
Do NOT invent new dimension names.
Pick only dimensions that genuinely fit this content.
If nothing fits, return an empty array.
Available dimensions:
${availableDimensionsBlock}
Examples: Examples:
- Title: "Dario Amodei — We are near the end of the exponential" / Channel: Dwarkesh Patel - Title: "Dario Amodei — We are near the end of the exponential" / Channel: Dwarkesh Patel
GOOD: "Dwarkesh Patel interview with Anthropic CEO Dario Amodei — argues we're nearing the end of exponential AI scaling. Key signal for what the next phase of AI development looks like." GOOD: "Dwarkesh Patel interview with Anthropic CEO Dario Amodei — argues we're nearing the end of exponential AI scaling. Key signal for what the next phase of AI development looks like."
@@ -33,7 +102,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
{ {
"enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.", "enhancedDescription": "A comprehensive summary (3-6 paragraphs, 800-1500 chars). Cover key points, arguments, takeaways.",
"nodeDescription": "<your 280-char description following the rules above>", "nodeDescription": "<your 280-char description following the rules above>",
"tags": ["relevant", "semantic", "tags"], "dimensions": ["existing-dimension-1", "existing-dimension-2"],
"reasoning": "Brief explanation of classification choices" "reasoning": "Brief explanation of classification choices"
}`; }`;
@@ -53,7 +122,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
return { return {
enhancedDescription: result.enhancedDescription || description, enhancedDescription: result.enhancedDescription || description,
nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined, nodeDescription: typeof result.nodeDescription === 'string' ? result.nodeDescription.slice(0, 280) : undefined,
tags: Array.isArray(result.tags) ? result.tags : [], dimensions: selectExistingDimensions(result.dimensions, existingDimensions, 5),
reasoning: result.reasoning || 'AI analysis completed' reasoning: result.reasoning || 'AI analysis completed'
}; };
} catch (error) { } catch (error) {
@@ -62,7 +131,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
return { return {
enhancedDescription: description, enhancedDescription: description,
nodeDescription: undefined, nodeDescription: undefined,
tags: [], dimensions: [],
reasoning: 'Fallback description used' reasoning: 'Fallback description used'
}; };
} }
@@ -164,10 +233,12 @@ export const youtubeExtractTool = tool({
console.log('🎯 YouTube extraction successful, analyzing with AI...'); console.log('🎯 YouTube extraction successful, analyzing with AI...');
// Step 2: AI Analysis for enhanced metadata // Step 2: AI Analysis for enhanced metadata
const existingDimensions = await fetchExistingDimensions();
const aiAnalysis = await analyzeContentWithAI( const aiAnalysis = await analyzeContentWithAI(
result.metadata?.video_title || 'YouTube Video', result.metadata?.video_title || 'YouTube Video',
`Video by ${result.metadata?.channel_name || 'Unknown Channel'}`, `Video by ${result.metadata?.channel_name || 'Unknown Channel'}`,
'youtube' 'youtube',
existingDimensions
); );
// Step 3: Create node with extracted content and AI analysis // Step 3: Create node with extracted content and AI analysis
@@ -181,6 +252,9 @@ export const youtubeExtractTool = tool({
.filter(Boolean); .filter(Boolean);
trimmedDimensions = trimmedDimensions.slice(0, 5); trimmedDimensions = trimmedDimensions.slice(0, 5);
const finalDimensions = trimmedDimensions.length > 0
? trimmedDimensions
: (aiAnalysis?.dimensions || []).slice(0, 5);
const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, { const createResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
method: 'POST', method: 'POST',
@@ -190,7 +264,7 @@ export const youtubeExtractTool = tool({
description: aiAnalysis?.nodeDescription, description: aiAnalysis?.nodeDescription,
notes: nodeNotes, notes: nodeNotes,
link: url, link: url,
dimensions: trimmedDimensions, dimensions: finalDimensions,
chunk: result.chunk || result.notes, chunk: result.chunk || result.notes,
metadata: { metadata: {
source: 'youtube', source: 'youtube',
@@ -222,7 +296,7 @@ export const youtubeExtractTool = tool({
console.log('🎯 YouTubeExtract completed successfully'); console.log('🎯 YouTubeExtract completed successfully');
// Use actual assigned dimensions from API response (includes auto-assigned locked + keywords) // Use actual assigned dimensions from API response (includes auto-assigned locked + keywords)
const actualDimensions: string[] = createResult.data?.dimensions || trimmedDimensions || []; const actualDimensions: string[] = createResult.data?.dimensions || finalDimensions || [];
const formattedNode = createResult.data?.id const formattedNode = createResult.data?.id
? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions }) ? formatNodeForChat({ id: createResult.data.id, title: nodeTitle, dimensions: actualDimensions })
: nodeTitle; : nodeTitle;