From d825e7a78399db42972d663838b5e4d5dbc39dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CBeeRad=E2=80=9D?= Date: Mon, 13 Apr 2026 20:50:13 +1000 Subject: [PATCH] feat: align MCP retrieval contract with ra-h - add external retrieval and confirmation-gated writeback tools across MCP surfaces - port direct-search-first retrieval support and supporting ranking/query updates - update MCP docs and skills to treat agent memory files as optional reinforcement Generated with Claude Code --- app/api/retrieval/query-context/route.ts | 35 ++ apps/mcp-server-standalone/README.md | 27 +- apps/mcp-server-standalone/index.js | 116 +++- .../services/edgeService.js | 26 +- .../services/nodeService.js | 11 + .../services/retrievalService.js | 595 ++++++++++++++++++ .../skills/db-operations.md | 38 +- apps/mcp-server/server.js | 140 ++++- apps/mcp-server/stdio-server.js | 140 ++++- docs/8_mcp.md | 15 +- src/config/skills/db-operations.md | 33 +- src/services/agents/toolResultUtils.ts | 20 + src/services/database/searchRanking.ts | 64 +- src/services/retrieval/queryContext.ts | 534 ++++++++++++++++ src/tools/database/queryNodes.ts | 37 +- src/tools/database/retrieveQueryContext.ts | 37 ++ src/tools/database/writeContext.ts | 72 +++ src/tools/infrastructure/groups.ts | 4 +- src/tools/infrastructure/registry.ts | 4 + 19 files changed, 1875 insertions(+), 73 deletions(-) create mode 100644 app/api/retrieval/query-context/route.ts create mode 100644 apps/mcp-server-standalone/services/retrievalService.js create mode 100644 src/services/retrieval/queryContext.ts create mode 100644 src/tools/database/retrieveQueryContext.ts create mode 100644 src/tools/database/writeContext.ts diff --git a/app/api/retrieval/query-context/route.ts b/app/api/retrieval/query-context/route.ts new file mode 100644 index 0000000..14b9d88 --- /dev/null +++ b/app/api/retrieval/query-context/route.ts @@ -0,0 +1,35 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { retrieveQueryContext } from '@/services/retrieval/queryContext'; + +export const runtime = 'nodejs'; + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const query = typeof body.query === 'string' ? body.query : ''; + + if (!query.trim()) { + return NextResponse.json({ + success: false, + error: 'Missing required field: query', + }, { status: 400 }); + } + + const result = await retrieveQueryContext({ + query, + focused_node_id: typeof body.focused_node_id === 'number' ? body.focused_node_id : null, + active_context_id: typeof body.active_context_id === 'number' ? body.active_context_id : null, + limit: typeof body.limit === 'number' ? body.limit : undefined, + }); + + return NextResponse.json({ + success: true, + data: result, + }); + } catch (error) { + return NextResponse.json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to retrieve query context', + }, { status: 500 }); + } +} diff --git a/apps/mcp-server-standalone/README.md b/apps/mcp-server-standalone/README.md index aa4ad3d..2d173a3 100644 --- a/apps/mcp-server-standalone/README.md +++ b/apps/mcp-server-standalone/README.md @@ -41,17 +41,35 @@ Restart Claude. Done. ## What to Expect Once connected, Claude will: -- **Call `getContext` first** to orient itself (stats, contexts, anchors/hubs, skills) +- **Use `queryNodes` for explicit node lookup** when the user is trying to find a specific existing thing +- **Use `retrieveQueryContext` when graph context is helpful** for a broader task, question, or request +- **Use `getContext` only for orientation** when high-level graph state would actually help - **Proactively capture knowledge** — when a new insight, decision, person, or reference surfaces, it proposes a specific node (title, description, optional context) so you can approve with minimal friction - **Read skills for complex tasks** — skills are editable and shared across internal + external agents - **Search before creating** to avoid duplicates +## Recommended Agent Memory Line + +If you use external agents through this MCP server, add one short instruction line to your agent memory file (`AGENTS.md`, `CLAUDE.md`, etc.): + +```md +Retrieve relevant context from RA-H before substantive work, and only suggest writing durable context back when it is clearly valuable and the user can confirm yes. +``` + +Keep the writeback prompt brief. A good pattern is: + +```md +Add "X" as a node? +``` + ## Available Tools | Tool | Description | |------|-------------| | `getContext` | Get graph overview — stats, contexts, recent activity | +| `retrieveQueryContext` | Pull relevant graph context for a broader current-turn task | | `createNode` | Create a new node | +| `writeContext` | Save one confirmed durable context node after explicit user approval | | `queryNodes` | Search nodes by keyword | | `queryContexts` | List or inspect contexts | | `getNodesById` | Load nodes by ID (includes chunk + metadata) | @@ -87,6 +105,13 @@ Rules: - use `captured_by = "human"` for direct user creation and user-requested agent capture - reserve `captured_by = "agent"` for autonomous/background creation only +## Writeback Rule + +- Do not ask to save every moderately useful point from the conversation. +- Only suggest a save when the context is unusually durable and valuable. +- Keep the ask terse and concrete, for example: `Add "X" as a node?` +- Never call `writeContext` unless the user has explicitly said yes. + ## Skills Skills are detailed instruction sets that teach agents how to work with your knowledge base. The default seeded skills are editable and shared by internal + external agents. diff --git a/apps/mcp-server-standalone/index.js b/apps/mcp-server-standalone/index.js index f4b21b6..a435304 100644 --- a/apps/mcp-server-standalone/index.js +++ b/apps/mcp-server-standalone/index.js @@ -34,6 +34,7 @@ const nodeService = require('./services/nodeService'); const edgeService = require('./services/edgeService'); const contextService = require('./services/contextService'); const skillService = require('./services/skillService'); +const retrievalService = require('./services/retrievalService'); // Server info const serverInfo = { @@ -59,14 +60,24 @@ function buildInstructions() { return `Today's date: ${now}. RA-H is the user's personal knowledge graph — local SQLite, fully on-device. ## Quick start -1. Call getContext for orientation (high-level graph state, optional contexts, anchors/hubs). -2. For simple tasks, tool descriptions have everything you need. -3. For complex tasks, call readSkill("db-operations"). +1. If the user is trying to find a specific existing node, call queryNodes first. +2. If graph context would help with a broader task, call retrieveQueryContext. +3. Call getContext only when orientation about the overall graph would actually help. +4. Do not keep re-running retrieval if you already have enough relevant graph context in play. +5. For simple tasks, tool descriptions have everything you need. +6. For complex tasks, call readSkill("db-operations"). + +## Context field rule +`context_id` is optional on writes. +Do not include `context_id` unless you already know a real existing context ID that clearly fits. +Omitting `context_id` is the normal default and does not block create or update operations. ## Knowledge capture -Proactively offer to save valuable information when insights, decisions, or references surface. -Propose: "I'd add this as: [title] — want me to?" -Always search before creating to avoid duplicates. +Only suggest saving context when it seems unusually durable and valuable. +Keep the ask brief: Add "X" as a node? +Do not pester. Do not keep re-asking if the user says no, ignores it, or moves on. +Never write via writeContext unless the user has explicitly confirmed yes. +Always search or retrieve before creating to avoid duplicates. ## Available skills ${skillIndex} @@ -82,7 +93,7 @@ const addNodeInputSchema = { source: z.string().max(50000).optional().describe('Canonical source content for embedding'), link: z.string().url().optional().describe('Source URL'), description: z.string().optional().describe('Strongly recommended. Write the description as natural prose, not labels or a checklist. It should make clear what the artifact is and any surrounding context available. RA-H will accept whatever description is provided and will not block the write.'), - context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID.'), + context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Usually omit this field entirely unless you already know a real matching context.'), context_name: z.string().optional().describe('Optional convenience context name.'), metadata: z.record(z.any()).optional().describe('Optional metadata. Prefer canonical keys: type, state, captured_method, captured_by, source_metadata.'), chunk: z.string().max(50000).optional().describe('Legacy alias for source text') @@ -98,6 +109,22 @@ const searchNodesInputSchema = { event_before: z.string().optional().describe('ISO date (YYYY-MM-DD). Only return nodes with event_date before this date.') }; +const retrieveQueryContextInputSchema = { + query: z.string().min(1).max(800).describe('The raw user query for this turn'), + focused_node_id: z.number().int().positive().nullable().optional().describe('Optional currently focused node ID'), + active_context_id: z.number().int().positive().nullable().optional().describe('Optional active context ID as a soft hint'), + limit: z.number().min(1).max(12).optional().describe('Maximum number of nodes to return') +}; + +const writeContextInputSchema = { + title: z.string().min(1).max(160).describe('Clear proposed node title'), + description: z.string().min(1).max(500).describe('Natural description of what this context is and why it matters'), + source: z.string().max(50000).optional().describe('Optional source or verbatim user wording to preserve'), + context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Usually omit this field entirely unless you already know a real matching context.'), + metadata: z.record(z.any()).optional().describe('Optional metadata patch'), + confirmed_by_user: z.boolean().describe('Must be true before the write is allowed') +}; + const getNodesInputSchema = { nodeIds: z.array(z.number().int().positive()).min(1).max(10).describe('Node IDs to load') }; @@ -110,7 +137,7 @@ const updateNodeInputSchema = { content: z.string().optional().describe('Legacy alias for source'), source: z.string().optional().describe('Canonical source content for embedding'), link: z.string().optional().describe('New link'), - context_id: z.number().int().positive().nullable().optional().describe('Primary context ID. Omit to preserve existing context; use null to clear it.'), + context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Omit this field to preserve existing context. Only use null when you intentionally want to clear context.'), metadata: z.record(z.any()).optional().describe('Metadata patch. It now merges with existing metadata. Prefer canonical keys: type, state, captured_method, captured_by, source_metadata.') }).describe('Fields to update') }; @@ -270,7 +297,7 @@ async function main() { 'getContext', { title: 'Get RA-H context', - description: 'Get knowledge graph overview: stats, contexts, hub nodes (secondary diagnostics), recent activity, and available skills. Call this first to orient yourself. For deeper operating policy, follow up with readSkill("db-operations").', + description: 'Get knowledge graph overview: stats, contexts, hub nodes (secondary diagnostics), recent activity, and available skills. Use this for orientation only, not as the default retrieval path for substantive requests. For deeper operating policy, follow up with readSkill("db-operations").', inputSchema: {} }, async () => { @@ -301,11 +328,37 @@ async function main() { // ========== NODE TOOLS ========== + registerToolWithAliases( + 'retrieveQueryContext', + { + title: 'Retrieve RA-H query context', + description: 'Given the raw user query plus optional focused node state, retrieve the most relevant graph context for the current turn. It starts with direct graph search and broadens only if useful. Use this when graph context could help answer or complete a broader task. For explicit node lookup, use queryNodes.', + inputSchema: retrieveQueryContextInputSchema + }, + async ({ query: rawQuery, focused_node_id, active_context_id, limit = 6 }) => { + const result = retrievalService.retrieveQueryContext({ + query: rawQuery, + focused_node_id: focused_node_id ?? null, + active_context_id: active_context_id ?? null, + limit, + }); + + const summary = result.shouldRetrieve + ? `Retrieved ${result.nodes.length} node(s) and ${result.chunks.length} chunk(s) for this turn.` + : result.reason; + + return { + content: [{ type: 'text', text: summary }], + structuredContent: result + }; + } + ); + registerToolWithAliases( 'createNode', { title: 'Add RA-H node', - description: 'Create a new node. Always search first (queryNodes) to avoid duplicates. Set context only when one obvious existing context clearly fits; otherwise leave it empty. Title: max 160 chars, clear and descriptive. Description is strongly recommended and should explicitly describe what the thing is and any surrounding context available, but the write will never be blocked over description quality. Use "link" ONLY for external content (URL, video, article) — omit for synthesis/ideas derived from existing nodes. "source" = verbatim or canonical content for embedding. Legacy "content" and "chunk" are mapped to source for compatibility.', + description: 'Create a new node. Always search first (queryNodes) to avoid duplicates. `context_id` is optional and should usually be omitted entirely unless one obvious existing context clearly fits. Title: max 160 chars, clear and descriptive. Description is strongly recommended and should explicitly describe what the thing is and any surrounding context available, but the write will never be blocked over description quality. Use "link" ONLY for external content (URL, video, article) — omit for synthesis/ideas derived from existing nodes. "source" = verbatim or canonical content for embedding. Legacy "content" and "chunk" are mapped to source for compatibility.', inputSchema: addNodeInputSchema }, async ({ title, content, source, link, description, context_id, context_name, metadata, chunk }) => { @@ -345,7 +398,7 @@ async function main() { 'queryNodes', { title: 'Search RA-H nodes', - description: 'Search nodes by keyword across title, description, and source fields. Multi-word queries find nodes containing all words (not exact phrases). Returns up to 25 results (default 10). Call before creating nodes to check for duplicates. Optionally filter by context. NOT for searching source documents (transcripts, articles) — use searchContentEmbeddings for that.', + description: 'Search nodes by keyword across title, description, and source fields using the same indexed search path as the app search UI. Use this for direct node lookup or duplicate checks. For full current-turn grounding of a substantive query, prefer retrieveQueryContext. NOT for searching source documents (transcripts, articles) — use searchContentEmbeddings for that.', inputSchema: searchNodesInputSchema }, async ({ query: searchQuery, limit = 10, contextId, created_after, created_before, event_after, event_before }) => { @@ -385,6 +438,43 @@ async function main() { } ); + registerToolWithAliases( + 'writeContext', + { + title: 'Write RA-H context node', + description: 'Write one atomic durable context node to the graph only after the user has explicitly approved the save. Use this sparingly for unusually valuable context. Never call it unless the user has clearly said yes.', + inputSchema: writeContextInputSchema + }, + async ({ title, description, source, context_id, metadata, confirmed_by_user }) => { + if (!confirmed_by_user) { + throw new Error('writeContext requires explicit user confirmation before writing to the graph.'); + } + + const node = nodeService.createNode({ + title: title.trim(), + description: description.trim(), + source: source?.trim(), + context_id: context_id ?? null, + metadata: { + captured_by: 'human', + captured_method: 'write_context', + ...(metadata || {}) + } + }); + + const summary = `Saved context as node #${node.id}: ${node.title}`; + return { + content: [{ type: 'text', text: summary }], + structuredContent: { + success: true, + nodeId: node.id, + title: node.title, + message: summary + } + }; + } + ); + registerToolWithAliases( 'getNodesById', { @@ -437,7 +527,7 @@ async function main() { 'updateNode', { title: 'Update RA-H node', - description: 'Update an existing node. Description updates should explicitly state what this thing is and any surrounding context available, but the write will never be blocked over description quality. Source content lives in "source". Legacy "content" is mapped to source for compatibility. Title, description, and link are overwritten. Call getNodesById first to verify current state before updating.', + description: 'Update an existing node. `context_id` is optional and should usually be omitted entirely unless you are intentionally setting or clearing a real context. Description updates should explicitly state what this thing is and any surrounding context available, but the write will never be blocked over description quality. Source content lives in "source". Legacy "content" is mapped to source for compatibility. Title, description, and link are overwritten. Call getNodesById first to verify current state before updating.', inputSchema: updateNodeInputSchema }, async ({ id, updates }) => { @@ -558,7 +648,7 @@ async function main() { } ); - // ========== CONTEXT TOOLS ========== + // ========== DIMENSION TOOLS ========== registerToolWithAliases( 'queryContexts', diff --git a/apps/mcp-server-standalone/services/edgeService.js b/apps/mcp-server-standalone/services/edgeService.js index 53f52fd..cefa7bf 100644 --- a/apps/mcp-server-standalone/services/edgeService.js +++ b/apps/mcp-server-standalone/services/edgeService.js @@ -151,7 +151,23 @@ function getNodeConnections(nodeId) { CASE WHEN e.from_node_id = ? THEN n_to.description ELSE n_from.description - END as connected_node_description + END as connected_node_description, + CASE + WHEN e.from_node_id = ? THEN n_to.link + ELSE n_from.link + END as connected_node_link, + CASE + WHEN e.from_node_id = ? THEN n_to.source + ELSE n_from.source + END as connected_node_source, + CASE + WHEN e.from_node_id = ? THEN n_to.updated_at + ELSE n_from.updated_at + END as connected_node_updated_at, + CASE + WHEN e.from_node_id = ? THEN n_to.metadata + ELSE n_from.metadata + END as connected_node_metadata FROM edges e LEFT JOIN nodes n_from ON e.from_node_id = n_from.id LEFT JOIN nodes n_to ON e.to_node_id = n_to.id @@ -159,7 +175,7 @@ function getNodeConnections(nodeId) { ORDER BY e.created_at DESC `; - const rows = query(sql, [nodeId, nodeId, nodeId, nodeId, nodeId]); + const rows = query(sql, [nodeId, nodeId, nodeId, nodeId, nodeId, nodeId, nodeId, nodeId, nodeId]); return rows.map(row => ({ edgeId: row.id, @@ -169,7 +185,11 @@ function getNodeConnections(nodeId) { connected_node: { id: row.connected_node_id, title: row.connected_node_title, - description: row.connected_node_description + description: row.connected_node_description, + link: row.connected_node_link, + source: row.connected_node_source, + updated_at: row.connected_node_updated_at, + metadata: parseContext(row.connected_node_metadata) } })); } diff --git a/apps/mcp-server-standalone/services/nodeService.js b/apps/mcp-server-standalone/services/nodeService.js index 74c1395..e8f2789 100644 --- a/apps/mcp-server-standalone/services/nodeService.js +++ b/apps/mcp-server-standalone/services/nodeService.js @@ -114,6 +114,16 @@ function getNodes(filters = {}) { return rows.map(mapNodeRow); } +/** + * Search nodes using the same filter object as getNodes. + */ +function searchNodes(filters = {}) { + if (typeof filters === 'string') { + return getNodes({ search: filters }); + } + return getNodes(filters); +} + /** * Get a single node by ID. */ @@ -316,6 +326,7 @@ function getContext() { module.exports = { getNodes, + searchNodes, getNodeById, createNode, updateNode, diff --git a/apps/mcp-server-standalone/services/retrievalService.js b/apps/mcp-server-standalone/services/retrievalService.js new file mode 100644 index 0000000..8c2bf60 --- /dev/null +++ b/apps/mcp-server-standalone/services/retrievalService.js @@ -0,0 +1,595 @@ +'use strict'; + +const { getDb, query } = require('./sqlite-client'); +const nodeService = require('./nodeService'); +const edgeService = require('./edgeService'); +const contextService = require('./contextService'); + +const LOW_SIGNAL_PATTERNS = [ + /^(yes|yeah|yep|no|nope|nah|ok|okay|cool|great|nice|thanks|thank you|sure|sounds good|go ahead|do it)[.!]?$/i, + /^(hi|hello|hey)[.!]?$/i, + /^(test|testing)[.!]?$/i, +]; + +const FOCUSED_SOURCE_PATTERN = /\b(this|focused|current)\s+(node|source|transcript|paper|article|video|document|note)\b|\b(inside|within|in)\s+(this|focused|current)\s+(node|source|transcript|paper|article|video|document|note)\b/i; +const SOURCE_DETAIL_PATTERN = /\b(quote|quotes|exact|specific|where|search|find|what did|what does|say|inside|within|transcript|paper|article|source|document|video)\b/i; +const USER_RECALL_PATTERN = /\b(what were (they|those)|what was (it|that)|what did i|what was my|what were my|do you remember|remind me)\b/i; +const FIRST_PERSON_PATTERN = /\b(i|my|me)\b/i; +const FIRST_PERSON_SHARE_PATTERN = /\b(i|my|me)\b.*\b(shared|mentioned|talked about|spoke about|said|posted)\b|\b(shared|mentioned|talked about|spoke about|said|posted)\b.*\b(i|my|me)\b/i; +const LOOKUP_PATTERN = /\b(find|look|search|show|get|pull)\b/i; +const EXPLICIT_HISTORY_QUERY_PATTERN = /\b(what have i said about|what did i say about)\b/i; +const DIRECT_NODE_TYPE_PATTERN = /\b(node|podcast|article|paper|video|note|idea|project|person|reflection|post|thread)\b/i; +const NOTE_HINT_PATTERN = /\b(idea|ideas|insight|insights|note|notes|thought|thoughts|reflection|reflections|realisation|realization|observation|observations|writing)\b/i; +const RECENT_REFERENCE_PATTERN = /\b(recent|recently|today|this morning|earlier|just)\b/i; +const NOTE_TERMS = new Set([ + 'idea', + 'insight', + 'note', + 'node', + 'thought', + 'reflection', + 'realisation', + 'realization', + 'observation', + 'writing', +]); +const HIGH_SIGNAL_STOP_WORDS = new Set([ + 'a', 'about', 'added', 'already', 'an', 'and', 'are', 'as', 'at', 'be', 'created', + 'db', 'did', 'do', 'does', 'earlier', 'find', 'for', 'from', 'get', 'had', + 'have', 'i', 'if', 'in', 'into', 'is', 'it', 'just', 'look', 'me', 'my', + 'being', 'said', 'going', 'having', 'node', 'of', 'on', 'pull', 'recent', + 'recently', 'search', 'shared', 'show', 'some', 'that', 'the', 'they', 'this', + 'those', 'to', 'today', 'user', 'versus', 'was', 'were', 'what', 'with', 'doing', +]); + +function normalizeWhitespace(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); +} + +function collapseCompactHyphenatedTerms(value) { + return String(value || '').replace(/\b([a-z0-9]{1,3})-([a-z0-9]{1,3})(?:-([a-z0-9]{1,3}))?\b/gi, (_match, a, b, c) => { + return `${a}${b}${c || ''}`; + }); +} + +function truncateText(value, maxLength = 180) { + const text = normalizeWhitespace(value); + if (!text) return ''; + if (text.length <= maxLength) return text; + return `${text.slice(0, maxLength - 1)}…`; +} + +function queryTermCount(queryText) { + return normalizeWhitespace(queryText).split(' ').filter(Boolean).length; +} + +function singularizeTerm(term) { + if (term.endsWith('ies') && term.length > 4) { + return `${term.slice(0, -3)}y`; + } + if (term.endsWith('s') && term.length > 4 && !term.endsWith('ss') && !term.endsWith('us')) { + return term.slice(0, -1); + } + return term; +} + +function normalizeRecallText(value) { + return collapseCompactHyphenatedTerms(normalizeWhitespace(value || '')) + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function normalizeSearchText(value) { + return normalizeRecallText(value); +} + +function extractHighSignalTerms(queryText) { + const rawTerms = collapseCompactHyphenatedTerms(normalizeWhitespace(queryText)) + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' ') + .split(/\s+/) + .map((term) => singularizeTerm(term.trim())) + .filter(Boolean); + + const seen = new Set(); + const result = []; + + for (const term of rawTerms) { + if (term.length < 3) continue; + if (HIGH_SIGNAL_STOP_WORDS.has(term)) continue; + if (seen.has(term)) continue; + seen.add(term); + result.push(term); + } + + return result; +} + +function extractRecallPhraseVariants(queryText) { + const normalized = collapseCompactHyphenatedTerms(normalizeWhitespace(queryText)) + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' '); + const variants = []; + const tokens = normalized.split(/\s+/).filter(Boolean); + + const pushVariant = (value) => { + const compact = normalizeWhitespace(value); + if (!compact || variants.includes(compact)) return; + variants.push(compact); + }; + + if (/\ball\s+in\b/.test(normalized)) { + pushVariant('all in'); + } + + if (/\bbackup(?:\s+plan)?\b/.test(normalized)) { + pushVariant('backup plan'); + pushVariant('backup'); + } + + for (let index = 0; index < tokens.length - 1; index += 1) { + const first = singularizeTerm(tokens[index]); + const second = singularizeTerm(tokens[index + 1]); + if (!first || !second) continue; + + const firstAllowed = first.length >= 4 && !HIGH_SIGNAL_STOP_WORDS.has(first); + const secondAllowed = second.length >= 4 && !HIGH_SIGNAL_STOP_WORDS.has(second); + if (firstAllowed && secondAllowed) { + pushVariant(`${first} ${second}`); + } + } + + return variants.slice(0, 6); +} + +function buildRecallSearchVariants(queryText) { + const terms = extractHighSignalTerms(queryText); + if (terms.length === 0) return []; + + const topicalTerms = terms.filter((term) => !NOTE_TERMS.has(term)); + const noteTerms = terms.filter((term) => NOTE_TERMS.has(term)); + const phraseVariants = extractRecallPhraseVariants(queryText); + const variants = []; + + const pushVariant = (value) => { + const normalized = normalizeWhitespace(value); + if (!normalized || variants.includes(normalized)) return; + variants.push(normalized); + }; + + phraseVariants.forEach(pushVariant); + + if (phraseVariants.includes('all in') && topicalTerms.includes('backup')) { + pushVariant('all in backup'); + } + + if (topicalTerms.length > 0 && noteTerms.length > 0) { + pushVariant(`${topicalTerms.join(' ')} ${noteTerms[0]}`); + pushVariant(`${topicalTerms[topicalTerms.length - 1]} ${noteTerms[0]}`); + } + if (topicalTerms.length >= 2) { + pushVariant(topicalTerms.slice(0, 2).join(' ')); + pushVariant(topicalTerms.slice(-2).join(' ')); + } + if (topicalTerms.length > 0) { + pushVariant(topicalTerms.join(' ')); + const lastTopicalTerm = topicalTerms[topicalTerms.length - 1]; + if (lastTopicalTerm.length >= 6) { + pushVariant(lastTopicalTerm); + } + } + if (terms.length > 1) { + pushVariant(terms.join(' ')); + } + + return variants.slice(0, 6); +} + +function buildDirectSearchVariants(queryText) { + const variants = [normalizeWhitespace(queryText), ...buildRecallSearchVariants(queryText)]; + return variants.filter((value, index) => value && variants.indexOf(value) === index).slice(0, 6); +} + +function isFocusedSourceRequest(queryText) { + return FOCUSED_SOURCE_PATTERN.test(queryText); +} + +function isLikelyUserNoteRecallQuery(queryText) { + const normalized = normalizeWhitespace(queryText); + if (!normalized) return false; + + const explicitRecall = USER_RECALL_PATTERN.test(normalized); + const firstPersonRecall = FIRST_PERSON_PATTERN.test(normalized) && /\bwhat\b/i.test(normalized); + const explicitLookup = LOOKUP_PATTERN.test(normalized) + && (FIRST_PERSON_PATTERN.test(normalized) || /\b(created|saved|wrote|added)\b/i.test(normalized)); + const firstPersonShareRecall = FIRST_PERSON_SHARE_PATTERN.test(normalized); + const noteHint = NOTE_HINT_PATTERN.test(normalized); + const recentHint = RECENT_REFERENCE_PATTERN.test(normalized); + + return (explicitRecall || firstPersonRecall || explicitLookup || firstPersonShareRecall) && (noteHint || recentHint); +} + +function isDirectNodeRetrievalQuery(queryText) { + const normalized = normalizeWhitespace(queryText); + if (!normalized) return false; + + const explicitHistoryQuery = EXPLICIT_HISTORY_QUERY_PATTERN.test(normalized); + const explicitLookup = LOOKUP_PATTERN.test(normalized) + && (FIRST_PERSON_PATTERN.test(normalized) || DIRECT_NODE_TYPE_PATTERN.test(normalized) || RECENT_REFERENCE_PATTERN.test(normalized)); + + return explicitHistoryQuery + || explicitLookup + || FIRST_PERSON_SHARE_PATTERN.test(normalized) + || isLikelyUserNoteRecallQuery(normalized); +} + +function shouldRetrieveForQuery(queryText) { + const trimmed = normalizeWhitespace(queryText); + if (!trimmed) return false; + if (LOW_SIGNAL_PATTERNS.some((pattern) => pattern.test(trimmed))) return false; + if (isFocusedSourceRequest(trimmed)) return true; + if (SOURCE_DETAIL_PATTERN.test(trimmed)) return true; + return trimmed.length >= 12 || queryTermCount(trimmed) >= 3; +} + +function countOccurrences(text, term) { + if (!text || !term) return 0; + const matches = text.match(new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'g')); + return matches ? matches.length : 0; +} + +function orderedTermMatches(text, terms) { + let position = 0; + for (const term of terms) { + const index = text.indexOf(term, position); + if (index === -1) return false; + position = index + term.length; + } + return terms.length > 0; +} + +function scoreNodeSearchMatch(node, queryText) { + const normalizedQuery = normalizeSearchText(queryText); + const normalizedTitle = normalizeSearchText(node.title || ''); + const normalizedDescription = normalizeSearchText(node.description || ''); + const normalizedSource = normalizeSearchText(node.source || ''); + const terms = extractHighSignalTerms(queryText); + + let score = 0; + + if (normalizedTitle === normalizedQuery) score += 2000; + if (normalizedTitle.startsWith(normalizedQuery)) score += 1200; + if (normalizedTitle.includes(normalizedQuery)) score += 700; + if (orderedTermMatches(normalizedTitle, terms)) score += 500; + if (terms.length > 0 && terms.every((term) => normalizedTitle.includes(term))) score += 350; + + if (normalizedDescription.includes(normalizedQuery)) score += 180; + if (orderedTermMatches(normalizedDescription, terms)) score += 120; + if (normalizedSource.includes(normalizedQuery)) score += 90; + + for (const term of terms) { + score += countOccurrences(normalizedTitle, term) * 40; + score += countOccurrences(normalizedDescription, term) * 8; + score += countOccurrences(normalizedSource, term) * 3; + } + + if (node.updated_at) { + score += new Date(node.updated_at).getTime() / 1e13; + } + + return score; +} + +function countHighSignalQueryTermMatches(node, queryText) { + const terms = extractHighSignalTerms(queryText); + if (terms.length === 0) return 0; + const combined = normalizeRecallText(`${node.title || ''} ${node.description || ''} ${node.source || ''}`); + return terms.filter((term) => combined.includes(term)).length; +} + +function isLikelyUserAuthoredNote(node) { + return !node.link && node.metadata && node.metadata.captured_by === 'human'; +} + +function scoreRecallMatch(node, queryText) { + const normalizedTitle = normalizeRecallText(node.title); + const normalizedDescription = normalizeRecallText(node.description); + const normalizedSource = normalizeRecallText(node.source); + const combined = `${normalizedTitle} ${normalizedDescription} ${normalizedSource}`.trim(); + const terms = extractHighSignalTerms(queryText); + const phraseVariants = extractRecallPhraseVariants(queryText); + + let score = 0; + + if (isLikelyUserAuthoredNote(node)) score += 800; + if (!node.link) score += 150; + + for (const phrase of phraseVariants) { + const normalizedPhrase = normalizeRecallText(phrase); + if (!normalizedPhrase) continue; + + if (normalizedTitle === normalizedPhrase) score += 2500; + if (normalizedTitle.includes(normalizedPhrase)) score += 1400; + if (normalizedDescription.includes(normalizedPhrase)) score += 700; + if (normalizedSource.includes(normalizedPhrase)) score += 900; + } + + const titleTermMatches = terms.filter((term) => normalizedTitle.includes(term)).length; + const totalTermMatches = terms.filter((term) => combined.includes(term)).length; + + score += titleTermMatches * 250; + score += totalTermMatches * 120; + + if (terms.length > 0 && titleTermMatches === terms.length) score += 1200; + if (terms.length > 0 && totalTermMatches === terms.length) score += 700; + + if (node.updated_at) { + score += new Date(node.updated_at).getTime() / 1e13; + } + + return score; +} + +function hasStrongRecallMatch(nodes, queryText) { + return nodes.some((node) => scoreRecallMatch(node, queryText) >= 1800); +} + +function addNodeWithReason(target, node, input) { + if (!node) return; + + const existing = target.get(node.id); + if (existing) { + if (input.kind === 'focused' && existing.kind !== 'focused') { + existing.kind = 'focused'; + } + if (!existing.reason.includes(input.reason)) { + existing.reason = `${existing.reason} ${input.reason}`.trim(); + } + if (input.seedNodeId && !existing.seed_node_id) { + existing.seed_node_id = input.seedNodeId; + } + if (typeof input.searchRank === 'number') { + existing.search_rank = typeof existing.search_rank === 'number' + ? Math.min(existing.search_rank, input.searchRank) + : input.searchRank; + } + return; + } + + target.set(node.id, { + id: node.id, + title: node.title, + description: node.description || null, + link: node.link || null, + updated_at: node.updated_at || '', + kind: input.kind, + reason: input.reason, + seed_node_id: input.seedNodeId, + search_rank: input.searchRank, + }); +} + +function rankRetrievedNodes(nodes) { + const kindWeight = { + focused: 4, + query_match: 3, + context_hint: 2, + neighbor: 1, + }; + + return [...nodes].sort((a, b) => { + const kindDiff = kindWeight[b.kind] - kindWeight[a.kind]; + if (kindDiff !== 0) return kindDiff; + const rankA = typeof a.search_rank === 'number' ? a.search_rank : Number.POSITIVE_INFINITY; + const rankB = typeof b.search_rank === 'number' ? b.search_rank : Number.POSITIVE_INFINITY; + if (rankA !== rankB) return rankA - rankB; + return String(b.updated_at || '').localeCompare(String(a.updated_at || '')); + }); +} + +function rankDirectQueryMatches(nodes, queryText, directNodeRetrieval) { + return [...nodes].sort((a, b) => { + const scoreDiff = directNodeRetrieval + ? scoreRecallMatch(b, queryText) - scoreRecallMatch(a, queryText) + : scoreNodeSearchMatch(b, queryText) - scoreNodeSearchMatch(a, queryText); + if (scoreDiff !== 0) return scoreDiff; + return String(b.updated_at || '').localeCompare(String(a.updated_at || '')); + }); +} + +function findDirectQueryMatches(queryText, limit) { + const variants = buildDirectSearchVariants(queryText); + const matches = []; + const seen = new Set(); + + for (const variant of variants) { + const rows = nodeService.searchNodes({ search: variant, limit: Math.max(limit, 8) }); + for (const node of rows) { + if (seen.has(node.id)) continue; + seen.add(node.id); + matches.push(node); + } + } + + return matches; +} + +function sanitizeFtsQuery(input) { + return String(input || '') + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' ') + .trim() + .split(/\s+/) + .filter((word) => word.length > 0 && !/^(AND|OR|NOT|NEAR)$/i.test(word)) + .join(' '); +} + +function searchChunks(queryText, nodeIds, limit) { + if (!nodeIds || nodeIds.length === 0) return []; + + const db = getDb(); + const ftsExists = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks_fts'").get(); + const ftsQuery = sanitizeFtsQuery(queryText); + + if (ftsExists && ftsQuery) { + try { + return query(` + SELECT c.id, c.node_id, c.chunk_idx, c.text, 0.85 as similarity + FROM chunks c + WHERE c.node_id IN (${nodeIds.map(() => '?').join(',')}) + AND c.id IN ( + SELECT rowid + FROM chunks_fts + WHERE chunks_fts MATCH ? + ) + ORDER BY c.chunk_idx ASC + LIMIT ? + `, [...nodeIds, ftsQuery, limit]); + } catch { + // Fall through to LIKE search. + } + } + + const terms = normalizeWhitespace(queryText).split(' ').filter((term) => term.length > 2); + if (terms.length === 0) return []; + + let sql = ` + SELECT c.id, c.node_id, c.chunk_idx, c.text, 0.75 as similarity + FROM chunks c + WHERE c.node_id IN (${nodeIds.map(() => '?').join(',')}) + `; + const params = [...nodeIds]; + + for (const term of terms) { + sql += ' AND LOWER(c.text) LIKE ?'; + params.push(`%${term.toLowerCase()}%`); + } + + sql += ' ORDER BY c.chunk_idx ASC LIMIT ?'; + params.push(limit); + return query(sql, params); +} + +function retrieveQueryContext(input = {}) { + const queryText = normalizeWhitespace(input.query || ''); + const focusedNodeId = input.focused_node_id ?? null; + const requestedActiveContextId = input.active_context_id ?? null; + const limit = Math.min(Math.max(input.limit || 6, 1), 12); + const shouldRetrieve = shouldRetrieveForQuery(queryText); + + if (!shouldRetrieve) { + return { + query: queryText, + shouldRetrieve: false, + mode: 'skip', + reason: 'Query is too lightweight or conversational to justify retrieval.', + focused_node_id: focusedNodeId, + active_context_id: requestedActiveContextId, + nodes: [], + chunks: [], + }; + } + + const activeContext = requestedActiveContextId ? contextService.getContextById(requestedActiveContextId) : null; + const activeContextId = activeContext ? activeContext.id : null; + const focusedRequest = isFocusedSourceRequest(queryText); + const directNodeRetrieval = isDirectNodeRetrievalQuery(queryText); + const nodesById = new Map(); + + const focusedNode = focusedNodeId ? nodeService.getNodeById(focusedNodeId) : null; + const focusedOverlap = focusedNode ? countHighSignalQueryTermMatches(focusedNode, queryText) : 0; + if (focusedNode && (focusedRequest || focusedOverlap >= 2)) { + addNodeWithReason(nodesById, focusedNode, { + kind: 'focused', + reason: focusedRequest + ? 'Focused node is the primary source for this request.' + : 'Focused node strongly overlaps the user query and should be considered first.', + }); + } + + const searchLimit = Math.max(limit * 2, 8); + const directQueryMatches = rankDirectQueryMatches( + findDirectQueryMatches(queryText, searchLimit), + queryText, + directNodeRetrieval + ); + const strongRecallMatch = directNodeRetrieval && hasStrongRecallMatch(directQueryMatches, queryText); + + directQueryMatches.forEach((node, index) => { + addNodeWithReason(nodesById, node, { + kind: 'query_match', + reason: directNodeRetrieval + ? 'Matched the query through direct graph search for a specific existing node.' + : 'Matched the query through direct graph search.', + searchRank: index, + }); + }); + + if (activeContextId && !strongRecallMatch) { + const contextMatches = queryText + ? nodeService.getNodes({ search: queryText, contextId: activeContextId, limit: Math.max(limit, 4) }) + : []; + contextMatches.forEach((node, index) => { + addNodeWithReason(nodesById, node, { + kind: 'context_hint', + reason: 'Also matched inside the active context.', + searchRank: directQueryMatches.length + index, + }); + }); + } + + if (!strongRecallMatch) { + const rankedSeedNodes = rankRetrievedNodes(Array.from(nodesById.values())).slice(0, Math.max(3, limit)); + rankedSeedNodes.slice(0, 3).forEach((seed) => { + const connections = edgeService.getNodeConnections(seed.id); + connections.slice(0, 2).forEach((connection) => { + addNodeWithReason(nodesById, connection.connected_node, { + kind: 'neighbor', + reason: `Connected to [NODE:${seed.id}:"${seed.title}"] via graph edges.`, + seedNodeId: seed.id, + }); + }); + }); + } + + const finalNodes = rankRetrievedNodes(Array.from(nodesById.values())).slice(0, limit); + const chunkScopeNodeIds = focusedRequest && focusedNodeId + ? [focusedNodeId] + : SOURCE_DETAIL_PATTERN.test(queryText) + ? finalNodes.slice(0, 3).map((node) => node.id) + : []; + + const chunks = searchChunks(queryText, chunkScopeNodeIds, Math.min(4, limit)).map((chunk) => { + const owner = finalNodes.find((node) => node.id === chunk.node_id) + || directQueryMatches.find((node) => node.id === chunk.node_id); + return { + id: chunk.id, + node_id: chunk.node_id, + node_title: owner ? owner.title : `Node ${chunk.node_id}`, + preview: truncateText(chunk.text, 220), + similarity: chunk.similarity, + }; + }); + + return { + query: queryText, + shouldRetrieve: true, + mode: focusedRequest ? 'focused' : 'query', + reason: focusedRequest + ? 'Focused-source request: use the focused node first, then broaden only if needed.' + : directNodeRetrieval + ? 'Direct node retrieval query: search the graph directly first and broaden only if needed.' + : 'Substantive query: search the graph directly, then pull additional supporting context if helpful.', + focused_node_id: focusedNodeId, + active_context_id: activeContextId, + nodes: finalNodes, + chunks, + }; +} + +module.exports = { + retrieveQueryContext, + shouldRetrieveForQuery, +}; diff --git a/apps/mcp-server-standalone/skills/db-operations.md b/apps/mcp-server-standalone/skills/db-operations.md index c4d3dc3..63ddef8 100644 --- a/apps/mcp-server-standalone/skills/db-operations.md +++ b/apps/mcp-server-standalone/skills/db-operations.md @@ -7,13 +7,15 @@ description: "Use for graph read, write, connect, classify, or traverse operatio ## Core Rules -1. Search before create to avoid duplicates. -2. Always try to include a natural description that clearly says what the thing is and any surrounding context available. But description quality is guidance only; RA-H should never block or rewrite a write because of description quality. -3. Use event dates when known (when it happened, not when saved). -4. Apply context only when it is an obvious match to one of the user's existing contexts and genuinely helpful. One node gets at most one context. If explicit context is missing on create, leave it empty instead of guessing. -5. Do not rely on dimensions. Node quality comes from title, description, source, metadata, and strong edges. -5. Create edges when relationships are meaningful; edge explanations should read as a sentence. -6. For user-authored ideas, notes, or dictated thoughts, preserve the user's wording in `source` as fully as possible with only minimal cleanup. +1. First decide whether the user is trying to find a specific existing node or whether they want graph context to support a broader answer. +2. If the user is trying to find a specific existing node, use `queryNodes` first. +3. If the user is asking a substantive question or request that would benefit from prior graph context, use `retrieveQueryContext` for current-turn grounding instead of relying on orientation alone. +4. Search before create to avoid duplicates. +5. Every create/update must include a natural description that clearly says what the thing is, why it matters here, and its current workflow status. +6. Use event dates when known (when it happened, not when saved). +7. Apply context only when it is an obvious match to one of the user's existing contexts and genuinely useful. One node gets at most one primary context, and leaving context blank is valid. +8. Create edges when relationships are meaningful; edge explanations should read as a sentence. +9. For user-authored ideas, notes, or dictated thoughts, preserve the user's wording in `source` as fully as possible with only minimal cleanup. ## Write Quality Contract @@ -22,7 +24,7 @@ description: "Use for graph read, write, connect, classify, or traverse operatio - `source`: full verbatim or canonical content of the node (transcript, article text, book passage, user's thoughts). This is what gets chunked and embedded for semantic search. - For idea capture from chat, the `source` should usually be the raw user thought, not a compressed assistant summary. - `link`: external source URL only. -- `context_id`: the node's primary context. Prefer setting it only when it is an obvious existing match. Leave it null rather than guessing. +- `context_id`: the node's primary context. This field is optional. Omit it entirely unless it is an obvious existing match. Do not add `context_id: null` defensively. - `metadata`: use the canonical node metadata contract when metadata is needed: - `type` - `state` (`processed` or `not_processed`) @@ -42,7 +44,7 @@ It must still make three things clear: 2. Why — why it is in the graph; what Brad is interested in; what it connects to 3. Status — where it sits in his workflow (queued, in progress, processed, unknown) -If the agent has graph context (active context, context anchor, context capsule, focused nodes), it should infer the why from that context and write it naturally. +If the agent has graph context (context capsule, focused nodes, recent connected nodes, or an explicit active context), it should infer the why from that context and write it naturally. Do not let the service auto-generate a weak context-free description when you already have enough signal. If the why genuinely cannot be inferred, say that naturally. Do not use labels like `WHAT:`, `WHY:`, or `STATUS:` and do not substitute vague filler like `insightful for understanding` or `relevant to Brad's work`. @@ -55,7 +57,7 @@ For user-authored idea capture, do not treat the inferred description as final i - why it belongs here - where it sits in their workflow -Keep it concise, but do not block the write over length or quality. +Max 500 characters. ## Metadata Semantics @@ -66,14 +68,20 @@ Keep it concise, but do not block the write over length or quality. ## Execution Pattern -1. Read context (search + relevant nodes + relevant edges). -2. Decide: create vs update vs connect. -3. Execute minimum required writes. -4. If the node is a user-authored idea and the contextual framing was inferred, offer one concise feedback pass after the write. -5. Verify result reflects user intent exactly. +1. Decide whether this is direct node retrieval or broader contextual grounding. +2. If the user is trying to find a specific existing node, call `queryNodes` first. +3. If the user is asking a broader question that would benefit from prior graph context, call `retrieveQueryContext`. +4. Decide: answer only vs create vs update vs connect. +5. If something seems unusually durable and valuable, you may suggest a save in one short line like `Add "X" as a node?` +6. Do not pester. If the user says no, ignores it, or moves on, do not keep asking. +7. Only call `writeContext` or another write tool after explicit user confirmation. +8. Execute minimum required writes. +9. If the node is a user-authored idea and the contextual framing was inferred, offer one concise feedback pass after the write. +10. Verify result reflects user intent exactly. ## Do Not - Create duplicate nodes when an update is correct. - Write vague descriptions ("discusses", "explores", "is about"). - Create weak or directionless edges. +- Ask to save every moderately useful point from the conversation. diff --git a/apps/mcp-server/server.js b/apps/mcp-server/server.js index 6b2110b..ee3a9ef 100644 --- a/apps/mcp-server/server.js +++ b/apps/mcp-server/server.js @@ -45,9 +45,14 @@ let logger = (message) => console.log(`[mcp] ${message}`); const instructions = [ 'RA-H is a personal knowledge graph — local-first, vendor-neutral.', 'Core concepts: contexts (optional soft scopes, max 10), nodes (knowledge units), and edges (connections with explanations).', - 'Always call rah_get_context first to orient yourself — it returns high-level graph state, contexts, hub nodes, stats, and available guides.', - 'Use contexts only when one obvious existing context is explicit and helpful. If unsure or if none exist, leave context empty. Do not expect automatic context assignment.', + 'If the user is trying to find a specific existing node, use rah_search_nodes first.', + 'If graph context would help with a broader task, use rah_retrieve_query_context.', + 'Use rah_get_context only when high-level graph orientation would actually help.', + 'Do not keep re-running retrieval if you already have enough relevant graph context in play.', + 'Use contexts only when one obvious existing context is explicitly helpful. If unsure or if none exist, leave context empty. Do not assume the server will infer a best-fit context.', 'Search before creating: use rah_search_nodes to check if content already exists.', + 'Only suggest saving context when it is unusually durable and valuable. Keep the ask brief, for example: Add "X" as a node?', + 'Never write via rah_write_context unless the user has explicitly confirmed yes.', 'Every edge needs an explanation: why does this connection exist?', 'All data stays local on this device; nothing leaves 127.0.0.1.', ].join(' '); @@ -73,7 +78,7 @@ const addNodeInputSchema = { source: z.string().max(50000).optional(), link: z.string().url().optional(), description: z.string().max(500).optional().describe('Description of the node. Write it as natural prose, not labels or a checklist. It must still make clear what the artifact is, why it is in the graph (infer from conversation context; ask the user if needed), and its current workflow status. Max 500 characters. If the reason is unclear, say that naturally instead of inventing it. Never use filler phrases like "insightful for understanding" or "relevant to the user\'s work".'), - context_id: z.number().int().positive().nullable().optional(), + context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Usually omit this field entirely unless you already know a real matching context.'), context_name: z.string().optional(), metadata: z.record(z.any()).optional().describe('Optional metadata. Prefer canonical keys: type, state, captured_method, captured_by, source_metadata.'), chunk: z.string().max(50000).optional() @@ -105,6 +110,55 @@ const searchNodesOutputSchema = { ) }; +const retrieveQueryContextInputSchema = { + query: z.string().min(1).max(800), + focused_node_id: z.number().int().positive().nullable().optional(), + active_context_id: z.number().int().positive().nullable().optional(), + limit: z.number().min(1).max(12).optional() +}; + +const retrieveQueryContextOutputSchema = { + query: z.string(), + shouldRetrieve: z.boolean(), + mode: z.enum(['skip', 'focused', 'query']), + reason: z.string(), + focused_node_id: z.number().nullable(), + active_context_id: z.number().nullable(), + nodes: z.array(z.object({ + id: z.number(), + title: z.string(), + description: z.string().nullable(), + link: z.string().nullable(), + updated_at: z.string(), + kind: z.enum(['focused', 'query_match', 'context_hint', 'neighbor']), + reason: z.string(), + seed_node_id: z.number().optional() + })), + chunks: z.array(z.object({ + id: z.number(), + node_id: z.number(), + node_title: z.string(), + preview: z.string(), + similarity: z.number() + })) +}; + +const writeContextInputSchema = { + title: z.string().min(1).max(160), + description: z.string().min(1).max(500), + source: z.string().max(50000).optional(), + context_id: z.number().int().positive().nullable().optional(), + metadata: z.record(z.any()).optional(), + confirmed_by_user: z.boolean() +}; + +const writeContextOutputSchema = { + success: z.boolean(), + nodeId: z.number(), + title: z.string(), + message: z.string() +}; + const queryContextsInputSchema = { contextId: z.number().int().positive().optional(), name: z.string().optional(), @@ -145,7 +199,7 @@ const updateNodeInputSchema = { content: z.string().optional().describe('Legacy alias for source. Mapped to source for backward compatibility.'), source: z.string().optional().describe('Canonical source text for embedding.'), link: z.string().optional().describe('New link'), - context_id: z.number().int().positive().nullable().optional().describe('Primary context ID. Omit to preserve existing context; use null to clear.'), + context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Omit this field to preserve existing context. Only use null when you intentionally want to clear context.'), metadata: z.record(z.any()).optional().describe('Metadata patch. This now merges with existing metadata. Prefer canonical keys: type, state, captured_method, captured_by, source_metadata.') }).describe('Fields to update') }; @@ -321,7 +375,7 @@ mcpServer.registerTool( 'rah_add_node', { title: 'Add RA-H node', - description: 'Create a new node in the local RA-H knowledge base. Set context only when one obvious existing context clearly fits; otherwise leave it empty.', + description: 'Create a new node in the local RA-H knowledge base. `context_id` is optional and should usually be omitted entirely unless one obvious existing context clearly fits.', inputSchema: addNodeInputSchema, outputSchema: addNodeOutputSchema }, @@ -359,7 +413,7 @@ mcpServer.registerTool( 'rah_search_nodes', { title: 'Search RA-H nodes', - description: 'Find existing RA-H entries that mention a topic before adding new ones.', + description: 'Find existing RA-H entries that mention a topic before adding new ones. For full current-turn grounding of a substantive request, prefer rah_retrieve_query_context.', inputSchema: searchNodesInputSchema, outputSchema: searchNodesOutputSchema }, @@ -398,11 +452,37 @@ mcpServer.registerTool( } ); +mcpServer.registerTool( + 'rah_retrieve_query_context', + { + title: 'Retrieve RA-H query context', + description: 'Given the raw user query plus optional focused node state, retrieve the most relevant graph context for the current turn. It starts with direct graph search and broadens only if useful. Use this when graph context could help answer or complete a broader task. For explicit node lookup, use rah_search_nodes.', + inputSchema: retrieveQueryContextInputSchema, + outputSchema: retrieveQueryContextOutputSchema + }, + async ({ query, focused_node_id, active_context_id, limit = 6 }) => { + const result = await callRaHApi('/api/retrieval/query-context', { + method: 'POST', + body: JSON.stringify({ + query, + focused_node_id: focused_node_id ?? null, + active_context_id: active_context_id ?? null, + limit + }) + }); + + return { + content: [{ type: 'text', text: result.data.shouldRetrieve ? `Retrieved ${result.data.nodes.length} node(s) and ${result.data.chunks.length} chunk(s) for this turn.` : result.data.reason }], + structuredContent: result.data + }; + } +); + mcpServer.registerTool( 'rah_query_contexts', { title: 'Query RA-H contexts', - description: 'List or inspect contexts, the soft organizational layer for the graph. Use this before assigning or filtering by context.', + description: 'List or inspect optional contexts. Use this only when a context is already obviously relevant or the user asks for it.', inputSchema: queryContextsInputSchema, outputSchema: queryContextsOutputSchema }, @@ -485,7 +565,7 @@ mcpServer.registerTool( 'rah_update_node', { title: 'Update RA-H node', - description: 'Update an existing node. Context remains optional and explicit.', + description: 'Update an existing node. `context_id` is optional and should usually be omitted entirely unless you are intentionally setting or clearing a real context.', inputSchema: updateNodeInputSchema, outputSchema: updateNodeOutputSchema }, @@ -782,7 +862,7 @@ mcpServer.registerTool( 'rah_get_context', { title: 'Get RA-H context', - description: 'Get orientation context: contexts, hub nodes, stats, and available guides. Call this first.', + description: 'Get orientation context: high-level graph state, optional contexts, hub nodes, stats, and available guides. Use this for orientation only, not as the default retrieval path for substantive requests.', inputSchema: {}, outputSchema: { stats: z.object({ nodeCount: z.number(), edgeCount: z.number(), contextCount: z.number().optional() }), @@ -818,6 +898,48 @@ mcpServer.registerTool( } ); +mcpServer.registerTool( + 'rah_write_context', + { + title: 'Write RA-H context node', + description: 'Write one atomic durable context node to the graph only after the user has explicitly approved the save. Use this sparingly for unusually valuable context. Never call it unless the user has clearly said yes.', + inputSchema: writeContextInputSchema, + outputSchema: writeContextOutputSchema + }, + async ({ title, description, source, context_id, metadata, confirmed_by_user }) => { + if (!confirmed_by_user) { + throw new Error('rah_write_context requires explicit user confirmation before writing to the graph.'); + } + + const result = await callRaHApi('/api/nodes', { + method: 'POST', + body: JSON.stringify({ + title: title.trim(), + description: description.trim(), + source: source?.trim() || undefined, + context_id: context_id ?? null, + metadata: { + captured_by: 'human', + captured_method: 'write_context', + ...(metadata || {}) + } + }) + }); + + const node = result.data; + const message = result.message || `Saved context as node #${node.id}: ${node.title}`; + return { + content: [{ type: 'text', text: message }], + structuredContent: { + success: true, + nodeId: node.id, + title: node.title, + message + } + }; + } +); + async function readRequestBody(req) { if (req.method !== 'POST') return undefined; try { diff --git a/apps/mcp-server/stdio-server.js b/apps/mcp-server/stdio-server.js index 9d7c60d..706820e 100644 --- a/apps/mcp-server/stdio-server.js +++ b/apps/mcp-server/stdio-server.js @@ -12,9 +12,14 @@ const packageJson = require('../../package.json'); const instructions = [ 'RA-H is a personal knowledge graph — local-first, vendor-neutral.', 'Core concepts: contexts (optional soft scopes, max 10), nodes (knowledge units), and edges (connections with explanations).', - 'Always call rah_get_context first to orient yourself — it returns high-level graph state, contexts, hub nodes, stats, and available guides.', - 'Use contexts only when one obvious existing context is explicit and helpful. If unsure or if none exist, leave context empty. Do not expect automatic context assignment.', + 'If the user is trying to find a specific existing node, use rah_search_nodes first.', + 'If graph context would help with a broader task, use rah_retrieve_query_context.', + 'Use rah_get_context only when high-level graph orientation would actually help.', + 'Do not keep re-running retrieval if you already have enough relevant graph context in play.', + 'Use contexts only when one obvious existing context is explicitly helpful. If unsure or if none exist, leave context empty.', 'Search before creating: use rah_search_nodes to check if content already exists.', + 'Only suggest saving context when it is unusually durable and valuable. Keep the ask brief, for example: Add "X" as a node?', + 'Never write via rah_write_context unless the user has explicitly confirmed yes.', 'Every edge needs an explanation: why does this connection exist?', 'All data stays local on this device; nothing leaves 127.0.0.1.', ].join(' '); @@ -39,7 +44,7 @@ const addNodeInputSchema = { source: z.string().max(50000).optional(), link: z.string().url().optional(), description: z.string().max(500).optional().describe('Description of the node. Write it as natural prose, not labels or a checklist. It must still make clear what the artifact is, why it is in the graph (infer from conversation context; ask the user if needed), and its current workflow status. Max 500 characters. If the reason is unclear, say that naturally instead of inventing it. Never use filler phrases like "insightful for understanding" or "relevant to the user\'s work".'), - context_id: z.number().int().positive().nullable().optional(), + context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Usually omit this field entirely unless you already know a real matching context.'), context_name: z.string().optional(), metadata: z.record(z.any()).optional().describe('Optional metadata. Prefer canonical keys: type, state, captured_method, captured_by, source_metadata.'), chunk: z.string().max(50000).optional() @@ -71,6 +76,55 @@ const searchNodesOutputSchema = { ) }; +const retrieveQueryContextInputSchema = { + query: z.string().min(1).max(800), + focused_node_id: z.number().int().positive().nullable().optional(), + active_context_id: z.number().int().positive().nullable().optional(), + limit: z.number().min(1).max(12).optional() +}; + +const retrieveQueryContextOutputSchema = { + query: z.string(), + shouldRetrieve: z.boolean(), + mode: z.enum(['skip', 'focused', 'query']), + reason: z.string(), + focused_node_id: z.number().nullable(), + active_context_id: z.number().nullable(), + nodes: z.array(z.object({ + id: z.number(), + title: z.string(), + description: z.string().nullable(), + link: z.string().nullable(), + updated_at: z.string(), + kind: z.enum(['focused', 'query_match', 'context_hint', 'neighbor']), + reason: z.string(), + seed_node_id: z.number().optional() + })), + chunks: z.array(z.object({ + id: z.number(), + node_id: z.number(), + node_title: z.string(), + preview: z.string(), + similarity: z.number() + })) +}; + +const writeContextInputSchema = { + title: z.string().min(1).max(160), + description: z.string().min(1).max(500), + source: z.string().max(50000).optional(), + context_id: z.number().int().positive().nullable().optional(), + metadata: z.record(z.any()).optional(), + confirmed_by_user: z.boolean() +}; + +const writeContextOutputSchema = { + success: z.boolean(), + nodeId: z.number(), + title: z.string(), + message: z.string() +}; + const queryContextsInputSchema = { contextId: z.number().int().positive().optional(), name: z.string().optional(), @@ -111,7 +165,7 @@ const updateNodeInputSchema = { content: z.string().optional().describe('Legacy alias for source. Mapped to source for backward compatibility.'), source: z.string().optional().describe('Canonical source text for embedding.'), link: z.string().optional().describe('New link'), - context_id: z.number().int().positive().nullable().optional().describe('Primary context ID. Omit to preserve existing context; use null to clear.'), + context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID. Omit this field to preserve existing context. Only use null when you intentionally want to clear context.'), metadata: z.record(z.any()).optional().describe('Metadata patch. This now merges with existing metadata. Prefer canonical keys: type, state, captured_method, captured_by, source_metadata.') }).describe('Fields to update') }; @@ -295,7 +349,7 @@ server.registerTool( 'rah_add_node', { title: 'Add RA-H node', - description: 'Create a new node in the local RA-H knowledge base. Set context only when one obvious existing context clearly fits; otherwise leave it empty.', + description: 'Create a new node in the local RA-H knowledge base. `context_id` is optional and should usually be omitted entirely unless one obvious existing context clearly fits.', inputSchema: addNodeInputSchema, outputSchema: addNodeOutputSchema }, @@ -333,7 +387,7 @@ server.registerTool( 'rah_search_nodes', { title: 'Search RA-H nodes', - description: 'Find existing RA-H entries that mention a topic before adding new ones.', + description: 'Find existing RA-H entries that mention a topic before adding new ones. For full current-turn grounding of a substantive request, prefer rah_retrieve_query_context.', inputSchema: searchNodesInputSchema, outputSchema: searchNodesOutputSchema }, @@ -373,11 +427,37 @@ server.registerTool( } ); +server.registerTool( + 'rah_retrieve_query_context', + { + title: 'Retrieve RA-H query context', + description: 'Given the raw user query plus optional focused node state, retrieve the most relevant graph context for the current turn. It starts with direct graph search and broadens only if useful. Use this when graph context could help answer or complete a broader task. For explicit node lookup, use rah_search_nodes.', + inputSchema: retrieveQueryContextInputSchema, + outputSchema: retrieveQueryContextOutputSchema + }, + async ({ query, focused_node_id, active_context_id, limit = 6 }) => { + const result = await callRaHApi('/api/retrieval/query-context', { + method: 'POST', + body: JSON.stringify({ + query, + focused_node_id: focused_node_id ?? null, + active_context_id: active_context_id ?? null, + limit + }) + }); + + return { + content: [{ type: 'text', text: result.data.shouldRetrieve ? `Retrieved ${result.data.nodes.length} node(s) and ${result.data.chunks.length} chunk(s) for this turn.` : result.data.reason }], + structuredContent: result.data + }; + } +); + server.registerTool( 'rah_query_contexts', { title: 'Query RA-H contexts', - description: 'List or inspect contexts, the soft organizational layer for the graph. Use this before assigning or filtering by context.', + description: 'List or inspect optional contexts. Use this only when a context is already obviously relevant or the user asks for it.', inputSchema: queryContextsInputSchema, outputSchema: queryContextsOutputSchema }, @@ -461,7 +541,7 @@ server.registerTool( 'rah_update_node', { title: 'Update RA-H node', - description: 'Update an existing node. Context remains optional and explicit.', + description: 'Update an existing node. `context_id` is optional and should usually be omitted entirely unless you are intentionally setting or clearing a real context.', inputSchema: updateNodeInputSchema, outputSchema: updateNodeOutputSchema }, @@ -780,7 +860,7 @@ server.registerTool( 'rah_get_context', { title: 'Get RA-H context', - description: 'Get orientation context: contexts, hub nodes, stats, and available guides. Call this first.', + description: 'Get orientation context: high-level graph state, optional contexts, hub nodes, stats, and available guides. Use this for orientation only, not as the default retrieval path for substantive requests.', inputSchema: {}, outputSchema: getContextOutputSchema }, @@ -837,6 +917,48 @@ server.registerTool( } ); +server.registerTool( + 'rah_write_context', + { + title: 'Write RA-H context node', + description: 'Write one atomic durable context node to the graph only after the user has explicitly approved the save. Use this sparingly for unusually valuable context. Never call it unless the user has clearly said yes.', + inputSchema: writeContextInputSchema, + outputSchema: writeContextOutputSchema + }, + async ({ title, description, source, context_id, metadata, confirmed_by_user }) => { + if (!confirmed_by_user) { + throw new Error('rah_write_context requires explicit user confirmation before writing to the graph.'); + } + + const result = await callRaHApi('/api/nodes', { + method: 'POST', + body: JSON.stringify({ + title: title.trim(), + description: description.trim(), + source: source?.trim() || undefined, + context_id: context_id ?? null, + metadata: { + captured_by: 'human', + captured_method: 'write_context', + ...(metadata || {}) + } + }) + }); + + const node = result.data; + const message = result.message || `Saved context as node #${node.id}: ${node.title}`; + return { + content: [{ type: 'text', text: message }], + structuredContent: { + success: true, + nodeId: node.id, + title: node.title, + message + } + }; + } +); + async function main() { const transport = new StdioServerTransport(); await server.connect(transport); diff --git a/docs/8_mcp.md b/docs/8_mcp.md index 04916fc..56939bd 100644 --- a/docs/8_mcp.md +++ b/docs/8_mcp.md @@ -4,14 +4,18 @@ RA-H exposes MCP tools for direct graph work against the local database or app A ## Core MCP Contract +- `queryNodes` is the primary tool for direct node retrieval when the user is trying to find a specific existing node. +- `retrieveQueryContext` is the primary retrieval entrypoint for substantive current-turn work when the agent needs graph context to support a broader answer. - `getContext` returns graph orientation: stats, contexts, hubs, and skills. -- `createNode` and `updateNode` accept optional `context_id` but do not require context. +- `createNode` and `updateNode` accept optional `context_id` but do not require context. Omitting `context_id` is the normal default. +- `writeContext` writes one confirmed durable context node and must never be called before explicit user approval. - `queryNodes` searches title, description, and source, with optional context filters. - `dimensions` are removed from the MCP contract. ## Main Tools Read: +- `retrieveQueryContext` - `getContext` - `queryNodes` - `queryContexts` @@ -23,6 +27,7 @@ Read: - `sqliteQuery` Write: +- `writeContext` - `createNode` - `updateNode` - `createEdge` @@ -33,6 +38,10 @@ Write: ## Tool Behavior - Always search before creating. -- Prefer explicit context assignment when the primary scope is clear. -- Do not expect automatic context assignment. +- If the user is trying to find a specific existing node, use `queryNodes` first. +- If the user is asking a broader question that would benefit from graph context, use `retrieveQueryContext`. +- Prefer explicit context assignment only when the primary scope is clear and a real context is already known. +- Do not send `context_id: null` unless the tool call is intentionally clearing an existing context. +- Do not assume the server will infer a best-fit context. - Judge graph quality by node quality and edges, not taxonomy completeness. +- Keep writeback prompts terse and selective. The goal is not to ask constantly whether every useful sentence should be saved. diff --git a/src/config/skills/db-operations.md b/src/config/skills/db-operations.md index 4f2642b..63ddef8 100644 --- a/src/config/skills/db-operations.md +++ b/src/config/skills/db-operations.md @@ -7,12 +7,15 @@ description: "Use for graph read, write, connect, classify, or traverse operatio ## Core Rules -1. Search before create to avoid duplicates. -2. Every create/update must include a natural description that clearly says what the thing is, why it matters here, and its current workflow status. -3. Use event dates when known (when it happened, not when saved). -4. Apply context only when it is an obvious match to one of the user's existing contexts and genuinely useful. One node gets at most one primary context, and leaving context blank is valid. -5. Create edges when relationships are meaningful; edge explanations should read as a sentence. -6. For user-authored ideas, notes, or dictated thoughts, preserve the user's wording in `source` as fully as possible with only minimal cleanup. +1. First decide whether the user is trying to find a specific existing node or whether they want graph context to support a broader answer. +2. If the user is trying to find a specific existing node, use `queryNodes` first. +3. If the user is asking a substantive question or request that would benefit from prior graph context, use `retrieveQueryContext` for current-turn grounding instead of relying on orientation alone. +4. Search before create to avoid duplicates. +5. Every create/update must include a natural description that clearly says what the thing is, why it matters here, and its current workflow status. +6. Use event dates when known (when it happened, not when saved). +7. Apply context only when it is an obvious match to one of the user's existing contexts and genuinely useful. One node gets at most one primary context, and leaving context blank is valid. +8. Create edges when relationships are meaningful; edge explanations should read as a sentence. +9. For user-authored ideas, notes, or dictated thoughts, preserve the user's wording in `source` as fully as possible with only minimal cleanup. ## Write Quality Contract @@ -21,7 +24,7 @@ description: "Use for graph read, write, connect, classify, or traverse operatio - `source`: full verbatim or canonical content of the node (transcript, article text, book passage, user's thoughts). This is what gets chunked and embedded for semantic search. - For idea capture from chat, the `source` should usually be the raw user thought, not a compressed assistant summary. - `link`: external source URL only. -- `context_id`: the node's primary context. Prefer setting it only when it is an obvious existing match. Leave it null rather than guessing. +- `context_id`: the node's primary context. This field is optional. Omit it entirely unless it is an obvious existing match. Do not add `context_id: null` defensively. - `metadata`: use the canonical node metadata contract when metadata is needed: - `type` - `state` (`processed` or `not_processed`) @@ -65,14 +68,20 @@ Max 500 characters. ## Execution Pattern -1. Read context (search + relevant nodes + relevant edges). -2. Decide: create vs update vs connect. -3. Execute minimum required writes. -4. If the node is a user-authored idea and the contextual framing was inferred, offer one concise feedback pass after the write. -5. Verify result reflects user intent exactly. +1. Decide whether this is direct node retrieval or broader contextual grounding. +2. If the user is trying to find a specific existing node, call `queryNodes` first. +3. If the user is asking a broader question that would benefit from prior graph context, call `retrieveQueryContext`. +4. Decide: answer only vs create vs update vs connect. +5. If something seems unusually durable and valuable, you may suggest a save in one short line like `Add "X" as a node?` +6. Do not pester. If the user says no, ignores it, or moves on, do not keep asking. +7. Only call `writeContext` or another write tool after explicit user confirmation. +8. Execute minimum required writes. +9. If the node is a user-authored idea and the contextual framing was inferred, offer one concise feedback pass after the write. +10. Verify result reflects user intent exactly. ## Do Not - Create duplicate nodes when an update is correct. - Write vague descriptions ("discusses", "explores", "is about"). - Create weak or directionless edges. +- Ask to save every moderately useful point from the conversation. diff --git a/src/services/agents/toolResultUtils.ts b/src/services/agents/toolResultUtils.ts index 383aa44..779ff53 100644 --- a/src/services/agents/toolResultUtils.ts +++ b/src/services/agents/toolResultUtils.ts @@ -78,6 +78,19 @@ export function summarizeToolExecution(toolName: string, args: any, result: any) return `Embedding search${query ? ` for "${truncateOrDefault(query, 60, query)}"` : ''}: no matches.`; } + if (toolName === 'retrieveQueryContext') { + const nodes = Array.isArray(result.data?.nodes) ? result.data.nodes : []; + const chunks = Array.isArray(result.data?.chunks) ? result.data.chunks : []; + if (nodes.length > 0) { + const labels = nodes + .slice(0, 3) + .map((node: any) => `[NODE:${node.id}:"${ensureString(node.title) || node.id}"]`) + .join(', '); + return `Retrieved ${nodes.length} node(s)${chunks.length > 0 ? ` and ${chunks.length} chunk(s)` : ''}: ${labels}`; + } + return ensureString(result.data?.reason) || 'No relevant graph context retrieved.'; + } + if (toolName === 'youtubeExtract') { const title = ensureString(result.data?.title) || ensureString(args?.title); const formatted = ensureString(result.data?.formatted_display); @@ -110,6 +123,13 @@ export function summarizeToolExecution(toolName: string, args: any, result: any) return 'No edges found.'; } + if (toolName === 'writeContext') { + const formatted = ensureString(result.data?.formatted_display); + if (formatted) { + return `Saved context as ${formatted}.`; + } + } + if (result.data?.formatted_display) { return ensureString(result.data.formatted_display) || fallback; } diff --git a/src/services/database/searchRanking.ts b/src/services/database/searchRanking.ts index f8f7d47..ea16817 100644 --- a/src/services/database/searchRanking.ts +++ b/src/services/database/searchRanking.ts @@ -1,15 +1,67 @@ import type { Node } from '@/types/database'; +const SEARCH_STOP_WORDS = new Set([ + 'a', 'about', 'added', 'already', 'an', 'and', 'are', 'as', 'at', 'be', 'by', + 'can', 'created', 'do', 'find', 'for', 'from', 'hello', 'i', 'in', 'into', 'is', + 'it', 'just', 'look', 'me', 'my', 'node', 'of', 'on', 'or', 'pull', 'recent', + 'recently', 'saved', 'shared', 'show', 'some', 'stuff', 'term', 'that', 'the', 'this', + 'to', 'versus', 'were', 'what', 'with', 'wrote', 'you', 'doing', 'going', 'having', +]); + +function collapseCompactHyphenatedTerms(value: string): string { + return value.replace(/\b([a-z0-9]{1,3})-([a-z0-9]{1,3})(?:-([a-z0-9]{1,3}))?\b/gi, (_match, a, b, c) => { + return `${a}${b}${c ?? ''}`; + }); +} + function normalizeSearchText(value: string): string { - return value + return collapseCompactHyphenatedTerms(value) .toLowerCase() .replace(/[^a-z0-9]+/g, ' ') .replace(/\s+/g, ' ') .trim(); } -function getQueryTerms(query: string): string[] { - return normalizeSearchText(query).split(' ').filter(term => term.length > 0); +function singularizeTerm(term: string): string { + if (term.endsWith('ies') && term.length > 4) { + return `${term.slice(0, -3)}y`; + } + if (term.endsWith('s') && term.length > 4 && !term.endsWith('ss') && !term.endsWith('us')) { + return term.slice(0, -1); + } + return term; +} + +export function getHighSignalSearchTerms(query: string): string[] { + const seen = new Set(); + const terms: string[] = []; + + for (const rawTerm of normalizeSearchText(query).split(' ')) { + const term = singularizeTerm(rawTerm.trim()); + if (!term || term.length < 3) continue; + if (SEARCH_STOP_WORDS.has(term)) continue; + if (seen.has(term)) continue; + seen.add(term); + terms.push(term); + } + + return terms; +} + +export function countHighSignalQueryTermMatches( + node: Pick, + query: string, +): number { + const terms = getHighSignalSearchTerms(query); + if (terms.length === 0) return 0; + + const haystack = [ + normalizeSearchText(node.title || ''), + normalizeSearchText(node.description || ''), + normalizeSearchText(node.source || ''), + ].join(' '); + + return terms.filter(term => haystack.includes(term)).length; } function countOccurrences(text: string, term: string): number { @@ -33,7 +85,7 @@ export function scoreNodeSearchMatch(node: Pick 0 && matchedTermCount === terms.length) score += 300; + for (const term of terms) { score += countOccurrences(normalizedTitle, term) * 40; score += countOccurrences(normalizedDescription, term) * 8; diff --git a/src/services/retrieval/queryContext.ts b/src/services/retrieval/queryContext.ts new file mode 100644 index 0000000..ccc433c --- /dev/null +++ b/src/services/retrieval/queryContext.ts @@ -0,0 +1,534 @@ +import { chunkService } from '@/services/database/chunks'; +import { contextService } from '@/services/database/contextService'; +import { edgeService } from '@/services/database/edges'; +import { nodeService } from '@/services/database/nodes'; +import { countHighSignalQueryTermMatches, scoreNodeSearchMatch } from '@/services/database/searchRanking'; +import type { Node } from '@/types/database'; + +export type RetrievedContextNodeKind = 'focused' | 'query_match' | 'context_hint' | 'neighbor'; + +export interface RetrievedContextNode { + id: number; + title: string; + description: string | null; + link: string | null; + updated_at: string; + kind: RetrievedContextNodeKind; + reason: string; + seed_node_id?: number; + search_rank?: number; +} + +export interface RetrievedContextChunk { + id: number; + node_id: number; + node_title: string; + preview: string; + similarity: number; +} + +export interface QueryContextResult { + query: string; + shouldRetrieve: boolean; + mode: 'skip' | 'focused' | 'query'; + reason: string; + focused_node_id: number | null; + active_context_id: number | null; + nodes: RetrievedContextNode[]; + chunks: RetrievedContextChunk[]; +} + +export interface RetrieveQueryContextInput { + query: string; + focused_node_id?: number | null; + active_context_id?: number | null; + limit?: number; +} + +const LOW_SIGNAL_PATTERNS = [ + /^(yes|yeah|yep|no|nope|nah|ok|okay|cool|great|nice|thanks|thank you|sure|sounds good|go ahead|do it)[.!]?$/i, + /^(hi|hello|hey)[.!]?$/i, + /^(test|testing)[.!]?$/i, +]; + +const FOCUSED_SOURCE_PATTERN = /\b(this|focused|current)\s+(node|source|transcript|paper|article|video|document|note)\b|\b(inside|within|in)\s+(this|focused|current)\s+(node|source|transcript|paper|article|video|document|note)\b/i; +const SOURCE_DETAIL_PATTERN = /\b(quote|quotes|exact|specific|where|search|find|what did|what does|say|inside|within|transcript|paper|article|source|document|video)\b/i; +const USER_RECALL_PATTERN = /\b(what were (they|those)|what was (it|that)|what did i|what was my|what were my|do you remember|remind me)\b/i; +const FIRST_PERSON_PATTERN = /\b(i|my|me)\b/i; +const FIRST_PERSON_SHARE_PATTERN = /\b(i|my|me)\b.*\b(shared|mentioned|talked about|spoke about|said|posted)\b|\b(shared|mentioned|talked about|spoke about|said|posted)\b.*\b(i|my|me)\b/i; +const LOOKUP_PATTERN = /\b(find|look|search|show|get|pull)\b/i; +const EXPLICIT_HISTORY_QUERY_PATTERN = /\b(what have i said about|what did i say about)\b/i; +const DIRECT_NODE_TYPE_PATTERN = /\b(node|podcast|article|paper|video|note|idea|project|person|reflection|post|thread)\b/i; +const NOTE_HINT_PATTERN = /\b(idea|ideas|insight|insights|note|notes|thought|thoughts|reflection|reflections|realisation|realization|observation|observations|writing)\b/i; +const RECENT_REFERENCE_PATTERN = /\b(recent|recently|today|this morning|earlier|just)\b/i; +const NOTE_TERMS = new Set([ + 'idea', + 'insight', + 'note', + 'node', + 'thought', + 'reflection', + 'realisation', + 'realization', + 'observation', + 'writing', +]); +const HIGH_SIGNAL_STOP_WORDS = new Set([ + 'a', 'about', 'added', 'already', 'an', 'and', 'are', 'as', 'at', 'be', 'created', + 'db', 'did', 'do', 'does', 'earlier', 'find', 'for', 'from', 'get', 'had', + 'have', 'i', 'if', 'in', 'into', 'is', 'it', 'just', 'look', 'me', 'my', + 'being', 'said', + 'going', 'having', + 'node', 'of', 'on', 'pull', 'recent', 'recently', 'search', 'shared', 'show', 'some', + 'that', 'the', 'they', 'this', 'those', 'to', 'today', 'user', 'versus', 'was', 'were', + 'what', 'with', 'doing', +]); + +function normalizeWhitespace(value: string): string { + return value.replace(/\s+/g, ' ').trim(); +} + +function collapseCompactHyphenatedTerms(value: string): string { + return value.replace(/\b([a-z0-9]{1,3})-([a-z0-9]{1,3})(?:-([a-z0-9]{1,3}))?\b/gi, (_match, a, b, c) => { + return `${a}${b}${c ?? ''}`; + }); +} + +function truncateText(value: string | null | undefined, maxLength = 180): string { + const text = normalizeWhitespace(value || ''); + if (!text) return ''; + if (text.length <= maxLength) return text; + return `${text.slice(0, maxLength - 1)}…`; +} + +function queryTermCount(query: string): number { + return normalizeWhitespace(query) + .split(' ') + .map((term) => term.trim()) + .filter(Boolean) + .length; +} + +function singularizeTerm(term: string): string { + if (term.endsWith('ies') && term.length > 4) { + return `${term.slice(0, -3)}y`; + } + if (term.endsWith('s') && term.length > 4 && !term.endsWith('ss') && !term.endsWith('us')) { + return term.slice(0, -1); + } + return term; +} + +function extractRecallPhraseVariants(query: string): string[] { + const normalized = collapseCompactHyphenatedTerms(normalizeWhitespace(query)) + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' '); + const variants: string[] = []; + const tokens = normalized.split(/\s+/).filter(Boolean); + + const pushVariant = (value: string) => { + const compact = normalizeWhitespace(value); + if (!compact || variants.includes(compact)) return; + variants.push(compact); + }; + + if (/\ball\s+in\b/.test(normalized)) { + pushVariant('all in'); + } + + if (/\bbackup(?:\s+plan)?\b/.test(normalized)) { + pushVariant('backup plan'); + pushVariant('backup'); + } + + for (let index = 0; index < tokens.length - 1; index += 1) { + const first = singularizeTerm(tokens[index]); + const second = singularizeTerm(tokens[index + 1]); + if (!first || !second) continue; + + const firstAllowed = first.length >= 4 && !HIGH_SIGNAL_STOP_WORDS.has(first); + const secondAllowed = second.length >= 4 && !HIGH_SIGNAL_STOP_WORDS.has(second); + if (firstAllowed && secondAllowed) { + pushVariant(`${first} ${second}`); + } + } + + return variants.slice(0, 6); +} + +function extractHighSignalTerms(query: string): string[] { + const rawTerms = collapseCompactHyphenatedTerms(normalizeWhitespace(query)) + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' ') + .split(/\s+/) + .map((term) => singularizeTerm(term.trim())) + .filter(Boolean); + + const seen = new Set(); + const result: string[] = []; + + for (const term of rawTerms) { + if (term.length < 3) continue; + if (HIGH_SIGNAL_STOP_WORDS.has(term)) continue; + if (seen.has(term)) continue; + seen.add(term); + result.push(term); + } + + return result; +} + +function isLikelyUserNoteRecallQuery(query: string): boolean { + const normalized = normalizeWhitespace(query); + if (!normalized) return false; + + const explicitRecall = USER_RECALL_PATTERN.test(normalized); + const firstPersonRecall = FIRST_PERSON_PATTERN.test(normalized) && /\bwhat\b/i.test(normalized); + const explicitLookup = LOOKUP_PATTERN.test(normalized) + && (FIRST_PERSON_PATTERN.test(normalized) || /\b(created|saved|wrote|added)\b/i.test(normalized)); + const firstPersonShareRecall = FIRST_PERSON_SHARE_PATTERN.test(normalized); + const noteHint = NOTE_HINT_PATTERN.test(normalized); + const recentHint = RECENT_REFERENCE_PATTERN.test(normalized); + + return (explicitRecall || firstPersonRecall || explicitLookup || firstPersonShareRecall) && (noteHint || recentHint); +} + +function isDirectNodeRetrievalQuery(query: string): boolean { + const normalized = normalizeWhitespace(query); + if (!normalized) return false; + + const explicitHistoryQuery = EXPLICIT_HISTORY_QUERY_PATTERN.test(normalized); + const explicitLookup = LOOKUP_PATTERN.test(normalized) + && (FIRST_PERSON_PATTERN.test(normalized) || DIRECT_NODE_TYPE_PATTERN.test(normalized) || RECENT_REFERENCE_PATTERN.test(normalized)); + + return explicitHistoryQuery + || explicitLookup + || FIRST_PERSON_SHARE_PATTERN.test(normalized) + || isLikelyUserNoteRecallQuery(normalized); +} + +function buildRecallSearchVariants(query: string): string[] { + const terms = extractHighSignalTerms(query); + if (terms.length === 0) return []; + + const topicalTerms = terms.filter((term) => !NOTE_TERMS.has(term)); + const noteTerms = terms.filter((term) => NOTE_TERMS.has(term)); + const phraseVariants = extractRecallPhraseVariants(query); + + const variants: string[] = []; + const pushVariant = (value: string) => { + const normalized = normalizeWhitespace(value); + if (!normalized || variants.includes(normalized)) return; + variants.push(normalized); + }; + + phraseVariants.forEach(pushVariant); + + if (phraseVariants.includes('all in') && topicalTerms.includes('backup')) { + pushVariant('all in backup'); + } + + if (topicalTerms.length > 0 && noteTerms.length > 0) { + pushVariant(`${topicalTerms.join(' ')} ${noteTerms[0]}`); + pushVariant(`${topicalTerms[topicalTerms.length - 1]} ${noteTerms[0]}`); + } + if (topicalTerms.length >= 2) { + pushVariant(topicalTerms.slice(0, 2).join(' ')); + pushVariant(topicalTerms.slice(-2).join(' ')); + } + if (topicalTerms.length > 0) { + pushVariant(topicalTerms.join(' ')); + const lastTopicalTerm = topicalTerms[topicalTerms.length - 1]; + if (lastTopicalTerm.length >= 6) { + pushVariant(lastTopicalTerm); + } + } + if (terms.length > 1) { + pushVariant(terms.join(' ')); + } + + return variants.slice(0, 6); +} + +function normalizeRecallText(value: string | null | undefined): string { + return collapseCompactHyphenatedTerms(normalizeWhitespace(value || '')) + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function scoreRecallMatch(node: Node, query: string): number { + const normalizedTitle = normalizeRecallText(node.title); + const normalizedDescription = normalizeRecallText(node.description); + const normalizedSource = normalizeRecallText(node.source); + const combined = `${normalizedTitle} ${normalizedDescription} ${normalizedSource}`.trim(); + const terms = extractHighSignalTerms(query); + const phraseVariants = extractRecallPhraseVariants(query); + + let score = 0; + + if (isLikelyUserAuthoredNote(node)) score += 800; + if (!node.link) score += 150; + + for (const phrase of phraseVariants) { + const normalizedPhrase = normalizeRecallText(phrase); + if (!normalizedPhrase) continue; + + if (normalizedTitle === normalizedPhrase) score += 2500; + if (normalizedTitle.includes(normalizedPhrase)) score += 1400; + if (normalizedDescription.includes(normalizedPhrase)) score += 700; + if (normalizedSource.includes(normalizedPhrase)) score += 900; + } + + const titleTermMatches = terms.filter((term) => normalizedTitle.includes(term)).length; + const totalTermMatches = terms.filter((term) => combined.includes(term)).length; + + score += titleTermMatches * 250; + score += totalTermMatches * 120; + + if (terms.length > 0 && titleTermMatches === terms.length) score += 1200; + if (terms.length > 0 && totalTermMatches === terms.length) score += 700; + + if (node.updated_at) { + score += new Date(node.updated_at).getTime() / 1e13; + } + + return score; +} + +function hasStrongRecallMatch(nodes: Node[], query: string): boolean { + return nodes.some((node) => scoreRecallMatch(node, query) >= 1800); +} + +function isLikelyUserAuthoredNote(node: Node): boolean { + return !node.link && node.metadata?.captured_by === 'human'; +} + +function buildDirectSearchVariants(query: string): string[] { + const variants = [normalizeWhitespace(query), ...buildRecallSearchVariants(query)]; + return variants.filter((value, index) => value && variants.indexOf(value) === index).slice(0, 6); +} + +async function findDirectQueryMatches(query: string, limit: number): Promise { + const variants = buildDirectSearchVariants(query); + const matches: Node[] = []; + const seen = new Set(); + + for (const variant of variants) { + const rows = await nodeService.searchNodes(variant, Math.max(limit, 8)); + + for (const node of rows) { + if (seen.has(node.id)) continue; + seen.add(node.id); + matches.push(node); + } + } + + return matches; +} + +function rankDirectQueryMatches(nodes: Node[], query: string, directNodeRetrieval: boolean): Node[] { + return [...nodes].sort((a, b) => { + const scoreDiff = directNodeRetrieval + ? scoreRecallMatch(b, query) - scoreRecallMatch(a, query) + : scoreNodeSearchMatch(b, query) - scoreNodeSearchMatch(a, query); + if (scoreDiff !== 0) return scoreDiff; + return (b.updated_at || '').localeCompare(a.updated_at || ''); + }); +} + +export function isFocusedSourceRequest(query: string): boolean { + return FOCUSED_SOURCE_PATTERN.test(query); +} + +export function shouldRetrieveForQuery(query: string): boolean { + const trimmed = normalizeWhitespace(query); + if (!trimmed) return false; + if (LOW_SIGNAL_PATTERNS.some((pattern) => pattern.test(trimmed))) return false; + + if (isFocusedSourceRequest(trimmed)) return true; + if (SOURCE_DETAIL_PATTERN.test(trimmed)) return true; + + return trimmed.length >= 12 || queryTermCount(trimmed) >= 3; +} + +function addNodeWithReason( + target: Map, + node: Node | null | undefined, + input: { kind: RetrievedContextNodeKind; reason: string; seedNodeId?: number; searchRank?: number }, +) { + if (!node) return; + + const existing = target.get(node.id); + if (existing) { + if (input.kind === 'focused' && existing.kind !== 'focused') { + existing.kind = 'focused'; + } + existing.reason = existing.reason.includes(input.reason) + ? existing.reason + : `${existing.reason} ${input.reason}`.trim(); + if (input.seedNodeId && !existing.seed_node_id) { + existing.seed_node_id = input.seedNodeId; + } + if (typeof input.searchRank === 'number') { + existing.search_rank = typeof existing.search_rank === 'number' + ? Math.min(existing.search_rank, input.searchRank) + : input.searchRank; + } + return; + } + + target.set(node.id, { + id: node.id, + title: node.title, + description: node.description ?? null, + link: node.link ?? null, + updated_at: node.updated_at, + kind: input.kind, + reason: input.reason, + seed_node_id: input.seedNodeId, + search_rank: input.searchRank, + }); +} + +function rankRetrievedNodes(nodes: RetrievedContextNode[]): RetrievedContextNode[] { + const kindWeight: Record = { + focused: 4, + query_match: 3, + context_hint: 2, + neighbor: 1, + }; + + return [...nodes].sort((a, b) => { + const kindDiff = kindWeight[b.kind] - kindWeight[a.kind]; + if (kindDiff !== 0) return kindDiff; + const rankA = typeof a.search_rank === 'number' ? a.search_rank : Number.POSITIVE_INFINITY; + const rankB = typeof b.search_rank === 'number' ? b.search_rank : Number.POSITIVE_INFINITY; + if (rankA !== rankB) return rankA - rankB; + return (b.updated_at || '').localeCompare(a.updated_at || ''); + }); +} + +export async function retrieveQueryContext(input: RetrieveQueryContextInput): Promise { + const query = normalizeWhitespace(input.query || ''); + const focusedNodeId = input.focused_node_id ?? null; + const requestedActiveContextId = input.active_context_id ?? null; + const limit = Math.min(Math.max(input.limit ?? 6, 1), 12); + const shouldRetrieve = shouldRetrieveForQuery(query); + + if (!shouldRetrieve) { + return { + query, + shouldRetrieve: false, + mode: 'skip', + reason: 'Query is too lightweight or conversational to justify retrieval.', + focused_node_id: focusedNodeId, + active_context_id: requestedActiveContextId, + nodes: [], + chunks: [], + }; + } + + const activeContextId = requestedActiveContextId + ? (await contextService.getContextById(requestedActiveContextId))?.id ?? null + : null; + + const focusedRequest = isFocusedSourceRequest(query); + const nodesById = new Map(); + + let focusedNode: Node | null = null; + if (focusedNodeId) { + focusedNode = await nodeService.getNodeById(focusedNodeId); + const focusedOverlap = focusedNode ? countHighSignalQueryTermMatches(focusedNode, query) : 0; + if (focusedNode && (focusedRequest || focusedOverlap >= 2)) { + addNodeWithReason(nodesById, focusedNode, { + kind: 'focused', + reason: focusedRequest + ? 'Focused node is the primary source for this request.' + : 'Focused node strongly overlaps the user query and should be considered first.', + }); + } + } + + const searchLimit = Math.max(limit * 2, 8); + const directNodeRetrieval = isDirectNodeRetrievalQuery(query); + const directQueryMatches = rankDirectQueryMatches( + await findDirectQueryMatches(query, searchLimit), + query, + directNodeRetrieval, + ); + const strongRecallMatch = directNodeRetrieval && hasStrongRecallMatch(directQueryMatches, query); + + directQueryMatches.forEach((node, index) => { + addNodeWithReason(nodesById, node, { + kind: 'query_match', + reason: directNodeRetrieval + ? 'Matched the query through direct graph search for a specific existing node.' + : 'Matched the query through direct graph search.', + searchRank: index, + }); + }); + + if (activeContextId && !strongRecallMatch) { + const contextMatches = query + ? await nodeService.getNodes({ search: query, contextId: activeContextId, limit: Math.max(limit, 4) }) + : []; + contextMatches.forEach((node, index) => { + addNodeWithReason(nodesById, node, { + kind: 'context_hint', + reason: 'Also matched inside the active context.', + searchRank: directQueryMatches.length + index, + }); + }); + } + + if (!strongRecallMatch) { + const rankedSeedNodes = rankRetrievedNodes(Array.from(nodesById.values())).slice(0, Math.max(3, limit)); + for (const seed of rankedSeedNodes.slice(0, 3)) { + const connections = await edgeService.getNodeConnections(seed.id); + connections.slice(0, 2).forEach((connection) => { + addNodeWithReason(nodesById, connection.connected_node, { + kind: 'neighbor', + reason: `Connected to [NODE:${seed.id}:"${seed.title}"] via graph edges.`, + seedNodeId: seed.id, + }); + }); + } + } + + const finalNodes = rankRetrievedNodes(Array.from(nodesById.values())).slice(0, limit); + const chunkScopeNodeIds = focusedRequest && focusedNodeId + ? [focusedNodeId] + : SOURCE_DETAIL_PATTERN.test(query) + ? finalNodes.slice(0, 3).map((node) => node.id) + : []; + + const chunks = chunkScopeNodeIds.length > 0 + ? await chunkService.textSearchFallback(query, Math.min(4, limit), chunkScopeNodeIds) + : []; + + return { + query, + shouldRetrieve: true, + mode: focusedRequest ? 'focused' : 'query', + reason: focusedRequest + ? 'Focused-source request: use the focused node first, then broaden only if needed.' + : directNodeRetrieval + ? 'Direct node retrieval query: search the graph directly first and broaden only if needed.' + : 'Substantive query: search the graph directly, then pull additional supporting context if helpful.', + focused_node_id: focusedNodeId, + active_context_id: activeContextId, + nodes: finalNodes, + chunks: chunks.map((chunk) => { + const owner = finalNodes.find((node) => node.id === chunk.node_id) || directQueryMatches.find((node) => node.id === chunk.node_id); + return { + id: chunk.id, + node_id: chunk.node_id, + node_title: owner?.title || `Node ${chunk.node_id}`, + preview: truncateText(chunk.text, 220), + similarity: chunk.similarity, + }; + }), + }; +} diff --git a/src/tools/database/queryNodes.ts b/src/tools/database/queryNodes.ts index 15a4626..599f1da 100644 --- a/src/tools/database/queryNodes.ts +++ b/src/tools/database/queryNodes.ts @@ -1,9 +1,10 @@ import { tool } from 'ai'; import { z } from 'zod'; +import { contextService } from '@/services/database'; import { nodeService } from '@/services/database/nodes'; import { formatNodeForChat } from '../infrastructure/nodeFormatter'; import type { Node } from '@/types/database'; -import { scoreNodeSearchMatch } from '@/services/database/searchRanking'; +import { countHighSignalQueryTermMatches, getHighSignalSearchTerms, scoreNodeSearchMatch } from '@/services/database/searchRanking'; type QueryNodeFilters = { contextId?: number; @@ -16,7 +17,7 @@ type QueryNodeFilters = { }; export const queryNodesTool = tool({ - description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Use context filtering only when the user is explicitly asking about a known context.', + description: 'Find specific existing nodes in the graph by searching title, description, and source. Use this first when the user is trying to locate a node they already created or a specific existing podcast, article, idea, person, project, or note. For broader current-turn grounding of a substantive question, use retrieveQueryContext instead. Leave contextId unset unless you know an actual context-table ID; never pass a hub node ID or arbitrary node ID as contextId.', inputSchema: z.object({ filters: z.object({ contextId: z.number().int().positive().describe('Optional primary context filter.').optional(), @@ -81,7 +82,8 @@ export const queryNodesTool = tool({ limit, contextId: queryFilters.contextId, search: queryFilters.search, - searchMode: searchTerm ? 'hybrid' : 'standard', + // Keep queryNodes literal-first. retrieveQueryContext is the broader semantic path. + searchMode: 'standard', createdAfter: queryFilters.createdAfter, createdBefore: queryFilters.createdBefore, eventAfter: queryFilters.eventAfter, @@ -93,9 +95,38 @@ export const queryNodesTool = tool({ }; const effectiveFilters = { ...filters }; + if (effectiveFilters.contextId !== undefined) { + const context = await contextService.getContextById(effectiveFilters.contextId); + if (!context) { + console.warn(`queryNodes received invalid contextId ${effectiveFilters.contextId}; ignoring context filter.`); + delete effectiveFilters.contextId; + } + } let safeNodes = await runQuery(effectiveFilters); + const hadExtraFilters = Boolean( + effectiveFilters.contextId !== undefined || + effectiveFilters.createdAfter || + effectiveFilters.createdBefore || + effectiveFilters.eventAfter || + effectiveFilters.eventBefore + ); + + const hasStrongAnchorMatch = (nodes: Node[]): boolean => { + if (!searchTerm || nodes.length === 0) return false; + const highSignalTerms = getHighSignalSearchTerms(searchTerm); + const requiredMatches = Math.min(2, highSignalTerms.length || 1); + return nodes.some(node => countHighSignalQueryTermMatches(node, searchTerm) >= requiredMatches); + }; + + // Match the nav search behavior when the model overconstrains a direct lookup. + // This prevents notes from disappearing behind synthetic date filters or weak filtered matches. + if (searchTerm && hadExtraFilters && (safeNodes.length === 0 || !hasStrongAnchorMatch(safeNodes))) { + console.warn(`queryNodes falling back to plain literal search for "${searchTerm}" after filtered lookup failed to return a strong anchor match.`); + safeNodes = await nodeService.searchNodes(searchTerm, limit); + } + if (searchTerm) { safeNodes = safeNodes .map(node => ({ node, score: scoreNodeSearchMatch(node, searchTerm) })) diff --git a/src/tools/database/retrieveQueryContext.ts b/src/tools/database/retrieveQueryContext.ts new file mode 100644 index 0000000..b1dc3c2 --- /dev/null +++ b/src/tools/database/retrieveQueryContext.ts @@ -0,0 +1,37 @@ +import { tool } from 'ai'; +import { z } from 'zod'; +import { retrieveQueryContext } from '@/services/retrieval/queryContext'; + +export const retrieveQueryContextTool = tool({ + description: 'Given a raw user query plus optional focused node state, retrieve the most relevant graph context to ground the current turn. Use this when the user is asking a substantive question or request that would benefit from one or more prior nodes, chunks, or neighbors. Do not use this as the first tool when the user is clearly trying to find a specific existing node; use queryNodes first for direct node retrieval.', + inputSchema: z.object({ + query: z.string().min(1).describe('The raw user query for this turn.'), + focused_node_id: z.number().int().positive().nullable().optional().describe('Optional currently focused node ID.'), + active_context_id: z.number().int().positive().nullable().optional().describe('Optional active context ID as a soft hint.'), + limit: z.number().int().min(1).max(12).optional().describe('Maximum number of nodes to return.'), + }), + execute: async ({ query, focused_node_id, active_context_id, limit }) => { + try { + const result = await retrieveQueryContext({ + query, + focused_node_id: focused_node_id ?? null, + active_context_id: active_context_id ?? null, + limit, + }); + + return { + success: true, + data: result, + message: result.shouldRetrieve + ? `Retrieved ${result.nodes.length} node(s) and ${result.chunks.length} supporting chunk(s) for the current turn.` + : result.reason, + }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to retrieve query context', + data: null, + }; + } + }, +}); diff --git a/src/tools/database/writeContext.ts b/src/tools/database/writeContext.ts new file mode 100644 index 0000000..8d12fde --- /dev/null +++ b/src/tools/database/writeContext.ts @@ -0,0 +1,72 @@ +import { tool } from 'ai'; +import { z } from 'zod'; +import { getInternalApiBaseUrl } from '@/services/runtime/apiBase'; +import { formatNodeForChat } from '../infrastructure/nodeFormatter'; + +export const writeContextTool = tool({ + description: 'Write one atomic durable context node to the graph only after the user has explicitly approved the save. Use this sparingly for unusually valuable context. Never call it unless the user has clearly said yes.', + inputSchema: z.object({ + title: z.string().min(1).max(160).describe('Clear proposed node title.'), + description: z.string().min(1).max(500).describe('Natural description of what this context is and why it matters.'), + source: z.string().optional().describe('Optional source or verbatim user wording to preserve.'), + context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID.'), + metadata: z.record(z.any()).optional().describe('Optional metadata patch.'), + confirmed_by_user: z.boolean().describe('Must be true. Reject the write otherwise.'), + }), + execute: async ({ title, description, source, context_id, metadata, confirmed_by_user }) => { + if (!confirmed_by_user) { + return { + success: false, + error: 'writeContext requires explicit user confirmation before writing to the graph.', + data: null, + }; + } + + try { + const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + title: title.trim(), + description: description.trim(), + source: source?.trim() || undefined, + context_id: context_id ?? null, + metadata: { + captured_by: 'human', + captured_method: 'write_context', + ...(metadata || {}), + }, + }), + }); + + const result = await response.json(); + if (!response.ok) { + return { + success: false, + error: result.error || 'Failed to write context node', + data: null, + }; + } + + const formattedDisplay = formatNodeForChat({ + id: result.data.id, + title: result.data.title, + }); + + return { + success: true, + data: { + ...result.data, + formatted_display: formattedDisplay, + }, + message: `Saved context as ${formattedDisplay}`, + }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to write context node', + data: null, + }; + } + }, +}); diff --git a/src/tools/infrastructure/groups.ts b/src/tools/infrastructure/groups.ts index 1700efc..5a730d0 100644 --- a/src/tools/infrastructure/groups.ts +++ b/src/tools/infrastructure/groups.ts @@ -35,6 +35,7 @@ export const TOOL_GROUPS: Record = { export const TOOL_GROUP_ASSIGNMENTS: Record = { // Core: Read-only graph operations (all agents) queryNodes: 'core', + retrieveQueryContext: 'core', getNodesById: 'core', queryEdge: 'core', queryContexts: 'core', @@ -46,6 +47,7 @@ export const TOOL_GROUP_ASSIGNMENTS: Record = { // Execution: Write operations and extraction (workers only) createNode: 'execution', + writeContext: 'execution', updateNode: 'execution', deleteNode: 'execution', createEdge: 'execution', @@ -53,7 +55,7 @@ export const TOOL_GROUP_ASSIGNMENTS: Record = { embedContent: 'execution', youtubeExtract: 'execution', websiteExtract: 'execution', - paperExtract: 'execution' + paperExtract: 'execution', }; /** diff --git a/src/tools/infrastructure/registry.ts b/src/tools/infrastructure/registry.ts index c2a77ef..f496cbb 100644 --- a/src/tools/infrastructure/registry.ts +++ b/src/tools/infrastructure/registry.ts @@ -1,10 +1,12 @@ import { getToolGroup, groupTools, getAllToolsByGroup } from './groups'; import { queryNodesTool } from '../database/queryNodes'; +import { retrieveQueryContextTool } from '../database/retrieveQueryContext'; import { getNodesByIdTool } from '../database/getNodesById'; import { queryEdgeTool } from '../database/queryEdge'; import { queryContextsTool } from '../database/queryContexts'; import { createNodeTool } from '../database/createNode'; import { updateNodeTool } from '../database/updateNode'; +import { writeContextTool } from '../database/writeContext'; import { deleteNodeTool } from '../database/deleteNode'; import { createEdgeTool } from '../database/createEdge'; import { updateEdgeTool } from '../database/updateEdge'; @@ -21,6 +23,7 @@ import { logEvalToolCall } from '@/services/evals/evalsLogger'; const CORE_TOOLS: Record = { sqliteQuery: sqliteQueryTool, queryNodes: queryNodesTool, + retrieveQueryContext: retrieveQueryContextTool, getNodesById: getNodesByIdTool, queryEdge: queryEdgeTool, queryContexts: queryContextsTool, @@ -36,6 +39,7 @@ const ORCHESTRATION_TOOLS: Record = { // Write tools (includes extraction) const EXECUTION_TOOLS: Record = { createNode: createNodeTool, + writeContext: writeContextTool, updateNode: updateNodeTool, deleteNode: deleteNodeTool, createEdge: createEdgeTool,