From 4f030c7d29c8968e062a47c6c50d5e9d0df9815a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CBeeRad=E2=80=9D?= Date: Tue, 13 Jan 2026 11:03:56 +1100 Subject: [PATCH] sync: ingestion improvements from private repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fixed dimension auto-assignment (locked dimensions now properly assigned) - Added description field as grounding context for AI - Updated embedding format: Title → Description → Content → Dimensions - Description-weighted search (5x boost) - Bug fixes: extraction tool dimension display, DimensionSearchModal closing - Removed quickLink tool (use Quick Link workflow instead) - Added regenerate-description API endpoint --- .../[id]/regenerate-description/route.ts | 58 +++++++ app/api/nodes/route.ts | 23 ++- app/api/quick-add/route.ts | 6 +- src/components/agents/AgentsPanel.tsx | 4 +- src/components/agents/QuickAddInput.tsx | 57 ++++++- src/components/focus/FocusPanel.tsx | 153 ++++++++++++++++- .../focus/dimensions/DimensionSearchModal.tsx | 157 +++++++++++++---- src/components/views/GridView.tsx | 8 +- src/components/views/ListView.tsx | 6 +- src/config/prompts/rah-easy.ts | 2 +- src/config/prompts/rah-main.ts | 2 +- src/services/agents/quickAdd.ts | 30 ++-- src/services/database/descriptionService.ts | 88 ++++++++++ src/services/database/dimensionService.ts | 36 ++-- src/services/database/nodes.ts | 20 ++- src/services/typescript/embed-nodes.ts | 10 +- src/services/typescript/sqlite-vec.ts | 6 +- src/tools/database/createDimension.ts | 6 +- src/tools/database/quickLink.ts | 159 ------------------ src/tools/infrastructure/groups.ts | 1 - src/tools/infrastructure/registry.ts | 4 - src/tools/other/paperExtract.ts | 8 +- src/tools/other/websiteExtract.ts | 8 +- src/tools/other/youtubeExtract.ts | 8 +- 24 files changed, 590 insertions(+), 270 deletions(-) create mode 100644 app/api/nodes/[id]/regenerate-description/route.ts create mode 100644 src/services/database/descriptionService.ts delete mode 100644 src/tools/database/quickLink.ts diff --git a/app/api/nodes/[id]/regenerate-description/route.ts b/app/api/nodes/[id]/regenerate-description/route.ts new file mode 100644 index 0000000..6c469bb --- /dev/null +++ b/app/api/nodes/[id]/regenerate-description/route.ts @@ -0,0 +1,58 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { nodeService } from '@/services/database'; +import { generateDescription } from '@/services/database/descriptionService'; + +export const runtime = 'nodejs'; + +export async function POST( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const nodeId = parseInt(id, 10); + + if (isNaN(nodeId)) { + return NextResponse.json({ + success: false, + error: 'Invalid node ID' + }, { status: 400 }); + } + + const node = await nodeService.getNodeById(nodeId); + + if (!node) { + return NextResponse.json({ + success: false, + error: 'Node not found' + }, { status: 404 }); + } + + // Generate new description using the description service + const newDescription = await generateDescription({ + title: node.title, + content: node.content || undefined, + metadata: node.metadata as { source?: string; channel_name?: string; author?: string; site_name?: string } | undefined, + type: (node.metadata as { type?: string } | null)?.type + }); + + // Update the node with the new description + const updatedNode = await nodeService.updateNode(nodeId, { + description: newDescription + }); + + return NextResponse.json({ + success: true, + node: updatedNode, + description: newDescription, + message: 'Description regenerated successfully' + }); + } catch (error) { + console.error('Error regenerating description:', error); + + return NextResponse.json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to regenerate description' + }, { status: 500 }); + } +} diff --git a/app/api/nodes/route.ts b/app/api/nodes/route.ts index 69c2ada..dc48487 100644 --- a/app/api/nodes/route.ts +++ b/app/api/nodes/route.ts @@ -4,6 +4,7 @@ import { Node, NodeFilters } from '@/types/database'; import { autoEmbedQueue } from '@/services/embedding/autoEmbedQueue'; import { hasSufficientContent } from '@/services/embedding/constants'; import { DimensionService } from '@/services/database/dimensionService'; +import { generateDescription } from '@/services/database/descriptionService'; export const runtime = 'nodejs'; @@ -60,17 +61,32 @@ export async function POST(request: NextRequest) { const rawContent = typeof body.content === 'string' ? body.content : null; + // Generate description BEFORE dimension assignment (used as primary context for matching) + let nodeDescription: string | undefined; + try { + nodeDescription = await generateDescription({ + title: body.title, + content: rawContent || undefined, + metadata: body.metadata, + type: body.type + }); + } catch (error) { + console.error('Error generating description:', error); + // Continue without description - dimension assignment will use content as fallback + } + const providedDimensions = Array.isArray(body.dimensions) ? body.dimensions : []; const trimmedProvidedDimensions = providedDimensions .map((dim: unknown) => typeof dim === 'string' ? dim.trim() : '') .filter(Boolean) - .slice(0, 5); + .slice(0, 8); // Auto-assign locked dimensions + keyword dimensions for all new nodes const { locked, keywords } = await DimensionService.assignDimensions({ title: body.title, content: rawContent || undefined, - link: body.link + link: body.link, + description: nodeDescription }); // Ensure keyword dimensions exist in the database (create if new) @@ -80,7 +96,7 @@ export async function POST(request: NextRequest) { // Combine provided, locked, and keyword dimensions, remove duplicates const finalDimensions = [...new Set([...trimmedProvidedDimensions, ...locked, ...keywords])] - .slice(0, 5); // Ensure we don't exceed 5 total dimensions + .slice(0, 8); // max 8 total const rawChunk = typeof body.chunk === 'string' ? body.chunk : null; let chunkToStore = rawChunk; let chunkStatus: Node['chunk_status']; @@ -94,6 +110,7 @@ export async function POST(request: NextRequest) { const node = await nodeService.createNode({ title: body.title, + description: nodeDescription, content: rawContent ?? undefined, link: body.link, dimensions: finalDimensions, diff --git a/app/api/quick-add/route.ts b/app/api/quick-add/route.ts index 910e92c..b5b78a4 100644 --- a/app/api/quick-add/route.ts +++ b/app/api/quick-add/route.ts @@ -4,7 +4,7 @@ import { enqueueQuickAdd, QuickAddMode } from '@/services/agents/quickAdd'; export async function POST(request: NextRequest) { try { const body = await request.json(); - const { input, mode } = body as { input?: unknown; mode?: unknown }; + const { input, mode, description } = body as { input?: unknown; mode?: unknown; description?: unknown }; if (typeof input !== 'string' || input.trim().length === 0) { return NextResponse.json( @@ -16,9 +16,13 @@ export async function POST(request: NextRequest) { const normalizedMode: QuickAddMode | undefined = mode === 'link' || mode === 'note' || mode === 'chat' ? mode : undefined; + const normalizedDescription: string | undefined = + typeof description === 'string' && description.trim() ? description.trim() : undefined; + const delegation = await enqueueQuickAdd({ rawInput: input.trim(), mode: normalizedMode, + description: normalizedDescription, }); return NextResponse.json({ success: true, delegation }); diff --git a/src/components/agents/AgentsPanel.tsx b/src/components/agents/AgentsPanel.tsx index a45c1a3..356d9a4 100644 --- a/src/components/agents/AgentsPanel.tsx +++ b/src/components/agents/AgentsPanel.tsx @@ -168,12 +168,12 @@ export default function AgentsPanel({ openTabsData, activeTabId, activeDimension const selectedDelegation = orderedDelegations.find(d => d.sessionId === activeAgentTab); - const handleQuickAddSubmit = async ({ input, mode }: { input: string; mode: 'link' | 'note' | 'chat' }) => { + const handleQuickAddSubmit = async ({ input, mode, description }: { input: string; mode: 'link' | 'note' | 'chat'; description?: string }) => { try { const response = await fetch('/api/quick-add', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ input, mode }) + body: JSON.stringify({ input, mode, description }) }); if (!response.ok) { diff --git a/src/components/agents/QuickAddInput.tsx b/src/components/agents/QuickAddInput.tsx index dcb4133..d54c170 100644 --- a/src/components/agents/QuickAddInput.tsx +++ b/src/components/agents/QuickAddInput.tsx @@ -9,6 +9,7 @@ type QuickAddIntent = 'link' | 'note' | 'chat'; interface QuickAddSubmitPayload { input: string; mode: QuickAddIntent; + description?: string; } interface QuickAddInputProps { @@ -60,6 +61,7 @@ const MODE_CONFIG: Array<{ export default function QuickAddInput({ activeDelegations, onSubmit }: QuickAddInputProps) { const [input, setInput] = useState(''); + const [description, setDescription] = useState(''); const [isPosting, setIsPosting] = useState(false); const [isExpanded, setIsExpanded] = useState(false); const [manualMode, setManualMode] = useState(null); @@ -108,8 +110,13 @@ export default function QuickAddInput({ activeDelegations, onSubmit }: QuickAddI setIsPosting(true); try { - await onSubmit({ input: input.trim(), mode: effectiveMode }); + await onSubmit({ + input: input.trim(), + mode: effectiveMode, + description: description.trim() || undefined + }); setInput(''); + setDescription(''); setManualMode(null); setAutoMode('link'); setIsExpanded(false); @@ -165,6 +172,7 @@ export default function QuickAddInput({ activeDelegations, onSubmit }: QuickAddI e.currentTarget.style.background = 'transparent'; }} > + CAPTURE + - CAPTURE ); } @@ -243,6 +250,7 @@ export default function QuickAddInput({ activeDelegations, onSubmit }: QuickAddI onClick={() => { setIsExpanded(false); setInput(''); + setDescription(''); }} style={{ marginLeft: 'auto', @@ -299,6 +307,51 @@ export default function QuickAddInput({ activeDelegations, onSubmit }: QuickAddI /> + {/* Description field - optional */} +
+