feat: align MCP retrieval contract with ra-h
- add external retrieval and confirmation-gated writeback tools across MCP surfaces - port direct-search-first retrieval support and supporting ranking/query updates - update MCP docs and skills to treat agent memory files as optional reinforcement Generated with Claude Code
This commit is contained in:
@@ -7,12 +7,15 @@ description: "Use for graph read, write, connect, classify, or traverse operatio
|
||||
|
||||
## Core Rules
|
||||
|
||||
1. Search before create to avoid duplicates.
|
||||
2. Every create/update must include a natural description that clearly says what the thing is, why it matters here, and its current workflow status.
|
||||
3. Use event dates when known (when it happened, not when saved).
|
||||
4. Apply context only when it is an obvious match to one of the user's existing contexts and genuinely useful. One node gets at most one primary context, and leaving context blank is valid.
|
||||
5. Create edges when relationships are meaningful; edge explanations should read as a sentence.
|
||||
6. For user-authored ideas, notes, or dictated thoughts, preserve the user's wording in `source` as fully as possible with only minimal cleanup.
|
||||
1. First decide whether the user is trying to find a specific existing node or whether they want graph context to support a broader answer.
|
||||
2. If the user is trying to find a specific existing node, use `queryNodes` first.
|
||||
3. If the user is asking a substantive question or request that would benefit from prior graph context, use `retrieveQueryContext` for current-turn grounding instead of relying on orientation alone.
|
||||
4. Search before create to avoid duplicates.
|
||||
5. Every create/update must include a natural description that clearly says what the thing is, why it matters here, and its current workflow status.
|
||||
6. Use event dates when known (when it happened, not when saved).
|
||||
7. Apply context only when it is an obvious match to one of the user's existing contexts and genuinely useful. One node gets at most one primary context, and leaving context blank is valid.
|
||||
8. Create edges when relationships are meaningful; edge explanations should read as a sentence.
|
||||
9. For user-authored ideas, notes, or dictated thoughts, preserve the user's wording in `source` as fully as possible with only minimal cleanup.
|
||||
|
||||
## Write Quality Contract
|
||||
|
||||
@@ -21,7 +24,7 @@ description: "Use for graph read, write, connect, classify, or traverse operatio
|
||||
- `source`: full verbatim or canonical content of the node (transcript, article text, book passage, user's thoughts). This is what gets chunked and embedded for semantic search.
|
||||
- For idea capture from chat, the `source` should usually be the raw user thought, not a compressed assistant summary.
|
||||
- `link`: external source URL only.
|
||||
- `context_id`: the node's primary context. Prefer setting it only when it is an obvious existing match. Leave it null rather than guessing.
|
||||
- `context_id`: the node's primary context. This field is optional. Omit it entirely unless it is an obvious existing match. Do not add `context_id: null` defensively.
|
||||
- `metadata`: use the canonical node metadata contract when metadata is needed:
|
||||
- `type`
|
||||
- `state` (`processed` or `not_processed`)
|
||||
@@ -65,14 +68,20 @@ Max 500 characters.
|
||||
|
||||
## Execution Pattern
|
||||
|
||||
1. Read context (search + relevant nodes + relevant edges).
|
||||
2. Decide: create vs update vs connect.
|
||||
3. Execute minimum required writes.
|
||||
4. If the node is a user-authored idea and the contextual framing was inferred, offer one concise feedback pass after the write.
|
||||
5. Verify result reflects user intent exactly.
|
||||
1. Decide whether this is direct node retrieval or broader contextual grounding.
|
||||
2. If the user is trying to find a specific existing node, call `queryNodes` first.
|
||||
3. If the user is asking a broader question that would benefit from prior graph context, call `retrieveQueryContext`.
|
||||
4. Decide: answer only vs create vs update vs connect.
|
||||
5. If something seems unusually durable and valuable, you may suggest a save in one short line like `Add "X" as a node?`
|
||||
6. Do not pester. If the user says no, ignores it, or moves on, do not keep asking.
|
||||
7. Only call `writeContext` or another write tool after explicit user confirmation.
|
||||
8. Execute minimum required writes.
|
||||
9. If the node is a user-authored idea and the contextual framing was inferred, offer one concise feedback pass after the write.
|
||||
10. Verify result reflects user intent exactly.
|
||||
|
||||
## Do Not
|
||||
|
||||
- Create duplicate nodes when an update is correct.
|
||||
- Write vague descriptions ("discusses", "explores", "is about").
|
||||
- Create weak or directionless edges.
|
||||
- Ask to save every moderately useful point from the conversation.
|
||||
|
||||
@@ -78,6 +78,19 @@ export function summarizeToolExecution(toolName: string, args: any, result: any)
|
||||
return `Embedding search${query ? ` for "${truncateOrDefault(query, 60, query)}"` : ''}: no matches.`;
|
||||
}
|
||||
|
||||
if (toolName === 'retrieveQueryContext') {
|
||||
const nodes = Array.isArray(result.data?.nodes) ? result.data.nodes : [];
|
||||
const chunks = Array.isArray(result.data?.chunks) ? result.data.chunks : [];
|
||||
if (nodes.length > 0) {
|
||||
const labels = nodes
|
||||
.slice(0, 3)
|
||||
.map((node: any) => `[NODE:${node.id}:"${ensureString(node.title) || node.id}"]`)
|
||||
.join(', ');
|
||||
return `Retrieved ${nodes.length} node(s)${chunks.length > 0 ? ` and ${chunks.length} chunk(s)` : ''}: ${labels}`;
|
||||
}
|
||||
return ensureString(result.data?.reason) || 'No relevant graph context retrieved.';
|
||||
}
|
||||
|
||||
if (toolName === 'youtubeExtract') {
|
||||
const title = ensureString(result.data?.title) || ensureString(args?.title);
|
||||
const formatted = ensureString(result.data?.formatted_display);
|
||||
@@ -110,6 +123,13 @@ export function summarizeToolExecution(toolName: string, args: any, result: any)
|
||||
return 'No edges found.';
|
||||
}
|
||||
|
||||
if (toolName === 'writeContext') {
|
||||
const formatted = ensureString(result.data?.formatted_display);
|
||||
if (formatted) {
|
||||
return `Saved context as ${formatted}.`;
|
||||
}
|
||||
}
|
||||
|
||||
if (result.data?.formatted_display) {
|
||||
return ensureString(result.data.formatted_display) || fallback;
|
||||
}
|
||||
|
||||
@@ -1,15 +1,67 @@
|
||||
import type { Node } from '@/types/database';
|
||||
|
||||
const SEARCH_STOP_WORDS = new Set([
|
||||
'a', 'about', 'added', 'already', 'an', 'and', 'are', 'as', 'at', 'be', 'by',
|
||||
'can', 'created', 'do', 'find', 'for', 'from', 'hello', 'i', 'in', 'into', 'is',
|
||||
'it', 'just', 'look', 'me', 'my', 'node', 'of', 'on', 'or', 'pull', 'recent',
|
||||
'recently', 'saved', 'shared', 'show', 'some', 'stuff', 'term', 'that', 'the', 'this',
|
||||
'to', 'versus', 'were', 'what', 'with', 'wrote', 'you', 'doing', 'going', 'having',
|
||||
]);
|
||||
|
||||
function collapseCompactHyphenatedTerms(value: string): string {
|
||||
return value.replace(/\b([a-z0-9]{1,3})-([a-z0-9]{1,3})(?:-([a-z0-9]{1,3}))?\b/gi, (_match, a, b, c) => {
|
||||
return `${a}${b}${c ?? ''}`;
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeSearchText(value: string): string {
|
||||
return value
|
||||
return collapseCompactHyphenatedTerms(value)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function getQueryTerms(query: string): string[] {
|
||||
return normalizeSearchText(query).split(' ').filter(term => term.length > 0);
|
||||
function singularizeTerm(term: string): string {
|
||||
if (term.endsWith('ies') && term.length > 4) {
|
||||
return `${term.slice(0, -3)}y`;
|
||||
}
|
||||
if (term.endsWith('s') && term.length > 4 && !term.endsWith('ss') && !term.endsWith('us')) {
|
||||
return term.slice(0, -1);
|
||||
}
|
||||
return term;
|
||||
}
|
||||
|
||||
export function getHighSignalSearchTerms(query: string): string[] {
|
||||
const seen = new Set<string>();
|
||||
const terms: string[] = [];
|
||||
|
||||
for (const rawTerm of normalizeSearchText(query).split(' ')) {
|
||||
const term = singularizeTerm(rawTerm.trim());
|
||||
if (!term || term.length < 3) continue;
|
||||
if (SEARCH_STOP_WORDS.has(term)) continue;
|
||||
if (seen.has(term)) continue;
|
||||
seen.add(term);
|
||||
terms.push(term);
|
||||
}
|
||||
|
||||
return terms;
|
||||
}
|
||||
|
||||
export function countHighSignalQueryTermMatches(
|
||||
node: Pick<Node, 'title' | 'description' | 'source'>,
|
||||
query: string,
|
||||
): number {
|
||||
const terms = getHighSignalSearchTerms(query);
|
||||
if (terms.length === 0) return 0;
|
||||
|
||||
const haystack = [
|
||||
normalizeSearchText(node.title || ''),
|
||||
normalizeSearchText(node.description || ''),
|
||||
normalizeSearchText(node.source || ''),
|
||||
].join(' ');
|
||||
|
||||
return terms.filter(term => haystack.includes(term)).length;
|
||||
}
|
||||
|
||||
function countOccurrences(text: string, term: string): number {
|
||||
@@ -33,7 +85,7 @@ export function scoreNodeSearchMatch(node: Pick<Node, 'title' | 'description' |
|
||||
const normalizedTitle = normalizeSearchText(node.title || '');
|
||||
const normalizedDescription = normalizeSearchText(node.description || '');
|
||||
const normalizedSource = normalizeSearchText(node.source || '');
|
||||
const terms = getQueryTerms(query);
|
||||
const terms = getHighSignalSearchTerms(query);
|
||||
|
||||
let score = 0;
|
||||
|
||||
@@ -47,6 +99,10 @@ export function scoreNodeSearchMatch(node: Pick<Node, 'title' | 'description' |
|
||||
if (orderedTermMatches(normalizedDescription, terms)) score += 120;
|
||||
if (normalizedSource.includes(normalizedQuery)) score += 90;
|
||||
|
||||
const matchedTermCount = countHighSignalQueryTermMatches(node, query);
|
||||
score += matchedTermCount * 120;
|
||||
if (terms.length > 0 && matchedTermCount === terms.length) score += 300;
|
||||
|
||||
for (const term of terms) {
|
||||
score += countOccurrences(normalizedTitle, term) * 40;
|
||||
score += countOccurrences(normalizedDescription, term) * 8;
|
||||
|
||||
@@ -0,0 +1,534 @@
|
||||
import { chunkService } from '@/services/database/chunks';
|
||||
import { contextService } from '@/services/database/contextService';
|
||||
import { edgeService } from '@/services/database/edges';
|
||||
import { nodeService } from '@/services/database/nodes';
|
||||
import { countHighSignalQueryTermMatches, scoreNodeSearchMatch } from '@/services/database/searchRanking';
|
||||
import type { Node } from '@/types/database';
|
||||
|
||||
export type RetrievedContextNodeKind = 'focused' | 'query_match' | 'context_hint' | 'neighbor';
|
||||
|
||||
export interface RetrievedContextNode {
|
||||
id: number;
|
||||
title: string;
|
||||
description: string | null;
|
||||
link: string | null;
|
||||
updated_at: string;
|
||||
kind: RetrievedContextNodeKind;
|
||||
reason: string;
|
||||
seed_node_id?: number;
|
||||
search_rank?: number;
|
||||
}
|
||||
|
||||
export interface RetrievedContextChunk {
|
||||
id: number;
|
||||
node_id: number;
|
||||
node_title: string;
|
||||
preview: string;
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
export interface QueryContextResult {
|
||||
query: string;
|
||||
shouldRetrieve: boolean;
|
||||
mode: 'skip' | 'focused' | 'query';
|
||||
reason: string;
|
||||
focused_node_id: number | null;
|
||||
active_context_id: number | null;
|
||||
nodes: RetrievedContextNode[];
|
||||
chunks: RetrievedContextChunk[];
|
||||
}
|
||||
|
||||
export interface RetrieveQueryContextInput {
|
||||
query: string;
|
||||
focused_node_id?: number | null;
|
||||
active_context_id?: number | null;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
const LOW_SIGNAL_PATTERNS = [
|
||||
/^(yes|yeah|yep|no|nope|nah|ok|okay|cool|great|nice|thanks|thank you|sure|sounds good|go ahead|do it)[.!]?$/i,
|
||||
/^(hi|hello|hey)[.!]?$/i,
|
||||
/^(test|testing)[.!]?$/i,
|
||||
];
|
||||
|
||||
const FOCUSED_SOURCE_PATTERN = /\b(this|focused|current)\s+(node|source|transcript|paper|article|video|document|note)\b|\b(inside|within|in)\s+(this|focused|current)\s+(node|source|transcript|paper|article|video|document|note)\b/i;
|
||||
const SOURCE_DETAIL_PATTERN = /\b(quote|quotes|exact|specific|where|search|find|what did|what does|say|inside|within|transcript|paper|article|source|document|video)\b/i;
|
||||
const USER_RECALL_PATTERN = /\b(what were (they|those)|what was (it|that)|what did i|what was my|what were my|do you remember|remind me)\b/i;
|
||||
const FIRST_PERSON_PATTERN = /\b(i|my|me)\b/i;
|
||||
const FIRST_PERSON_SHARE_PATTERN = /\b(i|my|me)\b.*\b(shared|mentioned|talked about|spoke about|said|posted)\b|\b(shared|mentioned|talked about|spoke about|said|posted)\b.*\b(i|my|me)\b/i;
|
||||
const LOOKUP_PATTERN = /\b(find|look|search|show|get|pull)\b/i;
|
||||
const EXPLICIT_HISTORY_QUERY_PATTERN = /\b(what have i said about|what did i say about)\b/i;
|
||||
const DIRECT_NODE_TYPE_PATTERN = /\b(node|podcast|article|paper|video|note|idea|project|person|reflection|post|thread)\b/i;
|
||||
const NOTE_HINT_PATTERN = /\b(idea|ideas|insight|insights|note|notes|thought|thoughts|reflection|reflections|realisation|realization|observation|observations|writing)\b/i;
|
||||
const RECENT_REFERENCE_PATTERN = /\b(recent|recently|today|this morning|earlier|just)\b/i;
|
||||
const NOTE_TERMS = new Set([
|
||||
'idea',
|
||||
'insight',
|
||||
'note',
|
||||
'node',
|
||||
'thought',
|
||||
'reflection',
|
||||
'realisation',
|
||||
'realization',
|
||||
'observation',
|
||||
'writing',
|
||||
]);
|
||||
const HIGH_SIGNAL_STOP_WORDS = new Set([
|
||||
'a', 'about', 'added', 'already', 'an', 'and', 'are', 'as', 'at', 'be', 'created',
|
||||
'db', 'did', 'do', 'does', 'earlier', 'find', 'for', 'from', 'get', 'had',
|
||||
'have', 'i', 'if', 'in', 'into', 'is', 'it', 'just', 'look', 'me', 'my',
|
||||
'being', 'said',
|
||||
'going', 'having',
|
||||
'node', 'of', 'on', 'pull', 'recent', 'recently', 'search', 'shared', 'show', 'some',
|
||||
'that', 'the', 'they', 'this', 'those', 'to', 'today', 'user', 'versus', 'was', 'were',
|
||||
'what', 'with', 'doing',
|
||||
]);
|
||||
|
||||
function normalizeWhitespace(value: string): string {
|
||||
return value.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function collapseCompactHyphenatedTerms(value: string): string {
|
||||
return value.replace(/\b([a-z0-9]{1,3})-([a-z0-9]{1,3})(?:-([a-z0-9]{1,3}))?\b/gi, (_match, a, b, c) => {
|
||||
return `${a}${b}${c ?? ''}`;
|
||||
});
|
||||
}
|
||||
|
||||
function truncateText(value: string | null | undefined, maxLength = 180): string {
|
||||
const text = normalizeWhitespace(value || '');
|
||||
if (!text) return '';
|
||||
if (text.length <= maxLength) return text;
|
||||
return `${text.slice(0, maxLength - 1)}…`;
|
||||
}
|
||||
|
||||
function queryTermCount(query: string): number {
|
||||
return normalizeWhitespace(query)
|
||||
.split(' ')
|
||||
.map((term) => term.trim())
|
||||
.filter(Boolean)
|
||||
.length;
|
||||
}
|
||||
|
||||
function singularizeTerm(term: string): string {
|
||||
if (term.endsWith('ies') && term.length > 4) {
|
||||
return `${term.slice(0, -3)}y`;
|
||||
}
|
||||
if (term.endsWith('s') && term.length > 4 && !term.endsWith('ss') && !term.endsWith('us')) {
|
||||
return term.slice(0, -1);
|
||||
}
|
||||
return term;
|
||||
}
|
||||
|
||||
function extractRecallPhraseVariants(query: string): string[] {
|
||||
const normalized = collapseCompactHyphenatedTerms(normalizeWhitespace(query))
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]+/g, ' ');
|
||||
const variants: string[] = [];
|
||||
const tokens = normalized.split(/\s+/).filter(Boolean);
|
||||
|
||||
const pushVariant = (value: string) => {
|
||||
const compact = normalizeWhitespace(value);
|
||||
if (!compact || variants.includes(compact)) return;
|
||||
variants.push(compact);
|
||||
};
|
||||
|
||||
if (/\ball\s+in\b/.test(normalized)) {
|
||||
pushVariant('all in');
|
||||
}
|
||||
|
||||
if (/\bbackup(?:\s+plan)?\b/.test(normalized)) {
|
||||
pushVariant('backup plan');
|
||||
pushVariant('backup');
|
||||
}
|
||||
|
||||
for (let index = 0; index < tokens.length - 1; index += 1) {
|
||||
const first = singularizeTerm(tokens[index]);
|
||||
const second = singularizeTerm(tokens[index + 1]);
|
||||
if (!first || !second) continue;
|
||||
|
||||
const firstAllowed = first.length >= 4 && !HIGH_SIGNAL_STOP_WORDS.has(first);
|
||||
const secondAllowed = second.length >= 4 && !HIGH_SIGNAL_STOP_WORDS.has(second);
|
||||
if (firstAllowed && secondAllowed) {
|
||||
pushVariant(`${first} ${second}`);
|
||||
}
|
||||
}
|
||||
|
||||
return variants.slice(0, 6);
|
||||
}
|
||||
|
||||
function extractHighSignalTerms(query: string): string[] {
|
||||
const rawTerms = collapseCompactHyphenatedTerms(normalizeWhitespace(query))
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]+/g, ' ')
|
||||
.split(/\s+/)
|
||||
.map((term) => singularizeTerm(term.trim()))
|
||||
.filter(Boolean);
|
||||
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
|
||||
for (const term of rawTerms) {
|
||||
if (term.length < 3) continue;
|
||||
if (HIGH_SIGNAL_STOP_WORDS.has(term)) continue;
|
||||
if (seen.has(term)) continue;
|
||||
seen.add(term);
|
||||
result.push(term);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function isLikelyUserNoteRecallQuery(query: string): boolean {
|
||||
const normalized = normalizeWhitespace(query);
|
||||
if (!normalized) return false;
|
||||
|
||||
const explicitRecall = USER_RECALL_PATTERN.test(normalized);
|
||||
const firstPersonRecall = FIRST_PERSON_PATTERN.test(normalized) && /\bwhat\b/i.test(normalized);
|
||||
const explicitLookup = LOOKUP_PATTERN.test(normalized)
|
||||
&& (FIRST_PERSON_PATTERN.test(normalized) || /\b(created|saved|wrote|added)\b/i.test(normalized));
|
||||
const firstPersonShareRecall = FIRST_PERSON_SHARE_PATTERN.test(normalized);
|
||||
const noteHint = NOTE_HINT_PATTERN.test(normalized);
|
||||
const recentHint = RECENT_REFERENCE_PATTERN.test(normalized);
|
||||
|
||||
return (explicitRecall || firstPersonRecall || explicitLookup || firstPersonShareRecall) && (noteHint || recentHint);
|
||||
}
|
||||
|
||||
function isDirectNodeRetrievalQuery(query: string): boolean {
|
||||
const normalized = normalizeWhitespace(query);
|
||||
if (!normalized) return false;
|
||||
|
||||
const explicitHistoryQuery = EXPLICIT_HISTORY_QUERY_PATTERN.test(normalized);
|
||||
const explicitLookup = LOOKUP_PATTERN.test(normalized)
|
||||
&& (FIRST_PERSON_PATTERN.test(normalized) || DIRECT_NODE_TYPE_PATTERN.test(normalized) || RECENT_REFERENCE_PATTERN.test(normalized));
|
||||
|
||||
return explicitHistoryQuery
|
||||
|| explicitLookup
|
||||
|| FIRST_PERSON_SHARE_PATTERN.test(normalized)
|
||||
|| isLikelyUserNoteRecallQuery(normalized);
|
||||
}
|
||||
|
||||
function buildRecallSearchVariants(query: string): string[] {
|
||||
const terms = extractHighSignalTerms(query);
|
||||
if (terms.length === 0) return [];
|
||||
|
||||
const topicalTerms = terms.filter((term) => !NOTE_TERMS.has(term));
|
||||
const noteTerms = terms.filter((term) => NOTE_TERMS.has(term));
|
||||
const phraseVariants = extractRecallPhraseVariants(query);
|
||||
|
||||
const variants: string[] = [];
|
||||
const pushVariant = (value: string) => {
|
||||
const normalized = normalizeWhitespace(value);
|
||||
if (!normalized || variants.includes(normalized)) return;
|
||||
variants.push(normalized);
|
||||
};
|
||||
|
||||
phraseVariants.forEach(pushVariant);
|
||||
|
||||
if (phraseVariants.includes('all in') && topicalTerms.includes('backup')) {
|
||||
pushVariant('all in backup');
|
||||
}
|
||||
|
||||
if (topicalTerms.length > 0 && noteTerms.length > 0) {
|
||||
pushVariant(`${topicalTerms.join(' ')} ${noteTerms[0]}`);
|
||||
pushVariant(`${topicalTerms[topicalTerms.length - 1]} ${noteTerms[0]}`);
|
||||
}
|
||||
if (topicalTerms.length >= 2) {
|
||||
pushVariant(topicalTerms.slice(0, 2).join(' '));
|
||||
pushVariant(topicalTerms.slice(-2).join(' '));
|
||||
}
|
||||
if (topicalTerms.length > 0) {
|
||||
pushVariant(topicalTerms.join(' '));
|
||||
const lastTopicalTerm = topicalTerms[topicalTerms.length - 1];
|
||||
if (lastTopicalTerm.length >= 6) {
|
||||
pushVariant(lastTopicalTerm);
|
||||
}
|
||||
}
|
||||
if (terms.length > 1) {
|
||||
pushVariant(terms.join(' '));
|
||||
}
|
||||
|
||||
return variants.slice(0, 6);
|
||||
}
|
||||
|
||||
function normalizeRecallText(value: string | null | undefined): string {
|
||||
return collapseCompactHyphenatedTerms(normalizeWhitespace(value || ''))
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function scoreRecallMatch(node: Node, query: string): number {
|
||||
const normalizedTitle = normalizeRecallText(node.title);
|
||||
const normalizedDescription = normalizeRecallText(node.description);
|
||||
const normalizedSource = normalizeRecallText(node.source);
|
||||
const combined = `${normalizedTitle} ${normalizedDescription} ${normalizedSource}`.trim();
|
||||
const terms = extractHighSignalTerms(query);
|
||||
const phraseVariants = extractRecallPhraseVariants(query);
|
||||
|
||||
let score = 0;
|
||||
|
||||
if (isLikelyUserAuthoredNote(node)) score += 800;
|
||||
if (!node.link) score += 150;
|
||||
|
||||
for (const phrase of phraseVariants) {
|
||||
const normalizedPhrase = normalizeRecallText(phrase);
|
||||
if (!normalizedPhrase) continue;
|
||||
|
||||
if (normalizedTitle === normalizedPhrase) score += 2500;
|
||||
if (normalizedTitle.includes(normalizedPhrase)) score += 1400;
|
||||
if (normalizedDescription.includes(normalizedPhrase)) score += 700;
|
||||
if (normalizedSource.includes(normalizedPhrase)) score += 900;
|
||||
}
|
||||
|
||||
const titleTermMatches = terms.filter((term) => normalizedTitle.includes(term)).length;
|
||||
const totalTermMatches = terms.filter((term) => combined.includes(term)).length;
|
||||
|
||||
score += titleTermMatches * 250;
|
||||
score += totalTermMatches * 120;
|
||||
|
||||
if (terms.length > 0 && titleTermMatches === terms.length) score += 1200;
|
||||
if (terms.length > 0 && totalTermMatches === terms.length) score += 700;
|
||||
|
||||
if (node.updated_at) {
|
||||
score += new Date(node.updated_at).getTime() / 1e13;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
function hasStrongRecallMatch(nodes: Node[], query: string): boolean {
|
||||
return nodes.some((node) => scoreRecallMatch(node, query) >= 1800);
|
||||
}
|
||||
|
||||
function isLikelyUserAuthoredNote(node: Node): boolean {
|
||||
return !node.link && node.metadata?.captured_by === 'human';
|
||||
}
|
||||
|
||||
function buildDirectSearchVariants(query: string): string[] {
|
||||
const variants = [normalizeWhitespace(query), ...buildRecallSearchVariants(query)];
|
||||
return variants.filter((value, index) => value && variants.indexOf(value) === index).slice(0, 6);
|
||||
}
|
||||
|
||||
async function findDirectQueryMatches(query: string, limit: number): Promise<Node[]> {
|
||||
const variants = buildDirectSearchVariants(query);
|
||||
const matches: Node[] = [];
|
||||
const seen = new Set<number>();
|
||||
|
||||
for (const variant of variants) {
|
||||
const rows = await nodeService.searchNodes(variant, Math.max(limit, 8));
|
||||
|
||||
for (const node of rows) {
|
||||
if (seen.has(node.id)) continue;
|
||||
seen.add(node.id);
|
||||
matches.push(node);
|
||||
}
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
function rankDirectQueryMatches(nodes: Node[], query: string, directNodeRetrieval: boolean): Node[] {
|
||||
return [...nodes].sort((a, b) => {
|
||||
const scoreDiff = directNodeRetrieval
|
||||
? scoreRecallMatch(b, query) - scoreRecallMatch(a, query)
|
||||
: scoreNodeSearchMatch(b, query) - scoreNodeSearchMatch(a, query);
|
||||
if (scoreDiff !== 0) return scoreDiff;
|
||||
return (b.updated_at || '').localeCompare(a.updated_at || '');
|
||||
});
|
||||
}
|
||||
|
||||
export function isFocusedSourceRequest(query: string): boolean {
|
||||
return FOCUSED_SOURCE_PATTERN.test(query);
|
||||
}
|
||||
|
||||
export function shouldRetrieveForQuery(query: string): boolean {
|
||||
const trimmed = normalizeWhitespace(query);
|
||||
if (!trimmed) return false;
|
||||
if (LOW_SIGNAL_PATTERNS.some((pattern) => pattern.test(trimmed))) return false;
|
||||
|
||||
if (isFocusedSourceRequest(trimmed)) return true;
|
||||
if (SOURCE_DETAIL_PATTERN.test(trimmed)) return true;
|
||||
|
||||
return trimmed.length >= 12 || queryTermCount(trimmed) >= 3;
|
||||
}
|
||||
|
||||
function addNodeWithReason(
|
||||
target: Map<number, RetrievedContextNode>,
|
||||
node: Node | null | undefined,
|
||||
input: { kind: RetrievedContextNodeKind; reason: string; seedNodeId?: number; searchRank?: number },
|
||||
) {
|
||||
if (!node) return;
|
||||
|
||||
const existing = target.get(node.id);
|
||||
if (existing) {
|
||||
if (input.kind === 'focused' && existing.kind !== 'focused') {
|
||||
existing.kind = 'focused';
|
||||
}
|
||||
existing.reason = existing.reason.includes(input.reason)
|
||||
? existing.reason
|
||||
: `${existing.reason} ${input.reason}`.trim();
|
||||
if (input.seedNodeId && !existing.seed_node_id) {
|
||||
existing.seed_node_id = input.seedNodeId;
|
||||
}
|
||||
if (typeof input.searchRank === 'number') {
|
||||
existing.search_rank = typeof existing.search_rank === 'number'
|
||||
? Math.min(existing.search_rank, input.searchRank)
|
||||
: input.searchRank;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
target.set(node.id, {
|
||||
id: node.id,
|
||||
title: node.title,
|
||||
description: node.description ?? null,
|
||||
link: node.link ?? null,
|
||||
updated_at: node.updated_at,
|
||||
kind: input.kind,
|
||||
reason: input.reason,
|
||||
seed_node_id: input.seedNodeId,
|
||||
search_rank: input.searchRank,
|
||||
});
|
||||
}
|
||||
|
||||
function rankRetrievedNodes(nodes: RetrievedContextNode[]): RetrievedContextNode[] {
|
||||
const kindWeight: Record<RetrievedContextNodeKind, number> = {
|
||||
focused: 4,
|
||||
query_match: 3,
|
||||
context_hint: 2,
|
||||
neighbor: 1,
|
||||
};
|
||||
|
||||
return [...nodes].sort((a, b) => {
|
||||
const kindDiff = kindWeight[b.kind] - kindWeight[a.kind];
|
||||
if (kindDiff !== 0) return kindDiff;
|
||||
const rankA = typeof a.search_rank === 'number' ? a.search_rank : Number.POSITIVE_INFINITY;
|
||||
const rankB = typeof b.search_rank === 'number' ? b.search_rank : Number.POSITIVE_INFINITY;
|
||||
if (rankA !== rankB) return rankA - rankB;
|
||||
return (b.updated_at || '').localeCompare(a.updated_at || '');
|
||||
});
|
||||
}
|
||||
|
||||
export async function retrieveQueryContext(input: RetrieveQueryContextInput): Promise<QueryContextResult> {
|
||||
const query = normalizeWhitespace(input.query || '');
|
||||
const focusedNodeId = input.focused_node_id ?? null;
|
||||
const requestedActiveContextId = input.active_context_id ?? null;
|
||||
const limit = Math.min(Math.max(input.limit ?? 6, 1), 12);
|
||||
const shouldRetrieve = shouldRetrieveForQuery(query);
|
||||
|
||||
if (!shouldRetrieve) {
|
||||
return {
|
||||
query,
|
||||
shouldRetrieve: false,
|
||||
mode: 'skip',
|
||||
reason: 'Query is too lightweight or conversational to justify retrieval.',
|
||||
focused_node_id: focusedNodeId,
|
||||
active_context_id: requestedActiveContextId,
|
||||
nodes: [],
|
||||
chunks: [],
|
||||
};
|
||||
}
|
||||
|
||||
const activeContextId = requestedActiveContextId
|
||||
? (await contextService.getContextById(requestedActiveContextId))?.id ?? null
|
||||
: null;
|
||||
|
||||
const focusedRequest = isFocusedSourceRequest(query);
|
||||
const nodesById = new Map<number, RetrievedContextNode>();
|
||||
|
||||
let focusedNode: Node | null = null;
|
||||
if (focusedNodeId) {
|
||||
focusedNode = await nodeService.getNodeById(focusedNodeId);
|
||||
const focusedOverlap = focusedNode ? countHighSignalQueryTermMatches(focusedNode, query) : 0;
|
||||
if (focusedNode && (focusedRequest || focusedOverlap >= 2)) {
|
||||
addNodeWithReason(nodesById, focusedNode, {
|
||||
kind: 'focused',
|
||||
reason: focusedRequest
|
||||
? 'Focused node is the primary source for this request.'
|
||||
: 'Focused node strongly overlaps the user query and should be considered first.',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const searchLimit = Math.max(limit * 2, 8);
|
||||
const directNodeRetrieval = isDirectNodeRetrievalQuery(query);
|
||||
const directQueryMatches = rankDirectQueryMatches(
|
||||
await findDirectQueryMatches(query, searchLimit),
|
||||
query,
|
||||
directNodeRetrieval,
|
||||
);
|
||||
const strongRecallMatch = directNodeRetrieval && hasStrongRecallMatch(directQueryMatches, query);
|
||||
|
||||
directQueryMatches.forEach((node, index) => {
|
||||
addNodeWithReason(nodesById, node, {
|
||||
kind: 'query_match',
|
||||
reason: directNodeRetrieval
|
||||
? 'Matched the query through direct graph search for a specific existing node.'
|
||||
: 'Matched the query through direct graph search.',
|
||||
searchRank: index,
|
||||
});
|
||||
});
|
||||
|
||||
if (activeContextId && !strongRecallMatch) {
|
||||
const contextMatches = query
|
||||
? await nodeService.getNodes({ search: query, contextId: activeContextId, limit: Math.max(limit, 4) })
|
||||
: [];
|
||||
contextMatches.forEach((node, index) => {
|
||||
addNodeWithReason(nodesById, node, {
|
||||
kind: 'context_hint',
|
||||
reason: 'Also matched inside the active context.',
|
||||
searchRank: directQueryMatches.length + index,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (!strongRecallMatch) {
|
||||
const rankedSeedNodes = rankRetrievedNodes(Array.from(nodesById.values())).slice(0, Math.max(3, limit));
|
||||
for (const seed of rankedSeedNodes.slice(0, 3)) {
|
||||
const connections = await edgeService.getNodeConnections(seed.id);
|
||||
connections.slice(0, 2).forEach((connection) => {
|
||||
addNodeWithReason(nodesById, connection.connected_node, {
|
||||
kind: 'neighbor',
|
||||
reason: `Connected to [NODE:${seed.id}:"${seed.title}"] via graph edges.`,
|
||||
seedNodeId: seed.id,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const finalNodes = rankRetrievedNodes(Array.from(nodesById.values())).slice(0, limit);
|
||||
const chunkScopeNodeIds = focusedRequest && focusedNodeId
|
||||
? [focusedNodeId]
|
||||
: SOURCE_DETAIL_PATTERN.test(query)
|
||||
? finalNodes.slice(0, 3).map((node) => node.id)
|
||||
: [];
|
||||
|
||||
const chunks = chunkScopeNodeIds.length > 0
|
||||
? await chunkService.textSearchFallback(query, Math.min(4, limit), chunkScopeNodeIds)
|
||||
: [];
|
||||
|
||||
return {
|
||||
query,
|
||||
shouldRetrieve: true,
|
||||
mode: focusedRequest ? 'focused' : 'query',
|
||||
reason: focusedRequest
|
||||
? 'Focused-source request: use the focused node first, then broaden only if needed.'
|
||||
: directNodeRetrieval
|
||||
? 'Direct node retrieval query: search the graph directly first and broaden only if needed.'
|
||||
: 'Substantive query: search the graph directly, then pull additional supporting context if helpful.',
|
||||
focused_node_id: focusedNodeId,
|
||||
active_context_id: activeContextId,
|
||||
nodes: finalNodes,
|
||||
chunks: chunks.map((chunk) => {
|
||||
const owner = finalNodes.find((node) => node.id === chunk.node_id) || directQueryMatches.find((node) => node.id === chunk.node_id);
|
||||
return {
|
||||
id: chunk.id,
|
||||
node_id: chunk.node_id,
|
||||
node_title: owner?.title || `Node ${chunk.node_id}`,
|
||||
preview: truncateText(chunk.text, 220),
|
||||
similarity: chunk.similarity,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { contextService } from '@/services/database';
|
||||
import { nodeService } from '@/services/database/nodes';
|
||||
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
||||
import type { Node } from '@/types/database';
|
||||
import { scoreNodeSearchMatch } from '@/services/database/searchRanking';
|
||||
import { countHighSignalQueryTermMatches, getHighSignalSearchTerms, scoreNodeSearchMatch } from '@/services/database/searchRanking';
|
||||
|
||||
type QueryNodeFilters = {
|
||||
contextId?: number;
|
||||
@@ -16,7 +17,7 @@ type QueryNodeFilters = {
|
||||
};
|
||||
|
||||
export const queryNodesTool = tool({
|
||||
description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Use context filtering only when the user is explicitly asking about a known context.',
|
||||
description: 'Find specific existing nodes in the graph by searching title, description, and source. Use this first when the user is trying to locate a node they already created or a specific existing podcast, article, idea, person, project, or note. For broader current-turn grounding of a substantive question, use retrieveQueryContext instead. Leave contextId unset unless you know an actual context-table ID; never pass a hub node ID or arbitrary node ID as contextId.',
|
||||
inputSchema: z.object({
|
||||
filters: z.object({
|
||||
contextId: z.number().int().positive().describe('Optional primary context filter.').optional(),
|
||||
@@ -81,7 +82,8 @@ export const queryNodesTool = tool({
|
||||
limit,
|
||||
contextId: queryFilters.contextId,
|
||||
search: queryFilters.search,
|
||||
searchMode: searchTerm ? 'hybrid' : 'standard',
|
||||
// Keep queryNodes literal-first. retrieveQueryContext is the broader semantic path.
|
||||
searchMode: 'standard',
|
||||
createdAfter: queryFilters.createdAfter,
|
||||
createdBefore: queryFilters.createdBefore,
|
||||
eventAfter: queryFilters.eventAfter,
|
||||
@@ -93,9 +95,38 @@ export const queryNodesTool = tool({
|
||||
};
|
||||
|
||||
const effectiveFilters = { ...filters };
|
||||
if (effectiveFilters.contextId !== undefined) {
|
||||
const context = await contextService.getContextById(effectiveFilters.contextId);
|
||||
if (!context) {
|
||||
console.warn(`queryNodes received invalid contextId ${effectiveFilters.contextId}; ignoring context filter.`);
|
||||
delete effectiveFilters.contextId;
|
||||
}
|
||||
}
|
||||
|
||||
let safeNodes = await runQuery(effectiveFilters);
|
||||
|
||||
const hadExtraFilters = Boolean(
|
||||
effectiveFilters.contextId !== undefined ||
|
||||
effectiveFilters.createdAfter ||
|
||||
effectiveFilters.createdBefore ||
|
||||
effectiveFilters.eventAfter ||
|
||||
effectiveFilters.eventBefore
|
||||
);
|
||||
|
||||
const hasStrongAnchorMatch = (nodes: Node[]): boolean => {
|
||||
if (!searchTerm || nodes.length === 0) return false;
|
||||
const highSignalTerms = getHighSignalSearchTerms(searchTerm);
|
||||
const requiredMatches = Math.min(2, highSignalTerms.length || 1);
|
||||
return nodes.some(node => countHighSignalQueryTermMatches(node, searchTerm) >= requiredMatches);
|
||||
};
|
||||
|
||||
// Match the nav search behavior when the model overconstrains a direct lookup.
|
||||
// This prevents notes from disappearing behind synthetic date filters or weak filtered matches.
|
||||
if (searchTerm && hadExtraFilters && (safeNodes.length === 0 || !hasStrongAnchorMatch(safeNodes))) {
|
||||
console.warn(`queryNodes falling back to plain literal search for "${searchTerm}" after filtered lookup failed to return a strong anchor match.`);
|
||||
safeNodes = await nodeService.searchNodes(searchTerm, limit);
|
||||
}
|
||||
|
||||
if (searchTerm) {
|
||||
safeNodes = safeNodes
|
||||
.map(node => ({ node, score: scoreNodeSearchMatch(node, searchTerm) }))
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { retrieveQueryContext } from '@/services/retrieval/queryContext';
|
||||
|
||||
export const retrieveQueryContextTool = tool({
|
||||
description: 'Given a raw user query plus optional focused node state, retrieve the most relevant graph context to ground the current turn. Use this when the user is asking a substantive question or request that would benefit from one or more prior nodes, chunks, or neighbors. Do not use this as the first tool when the user is clearly trying to find a specific existing node; use queryNodes first for direct node retrieval.',
|
||||
inputSchema: z.object({
|
||||
query: z.string().min(1).describe('The raw user query for this turn.'),
|
||||
focused_node_id: z.number().int().positive().nullable().optional().describe('Optional currently focused node ID.'),
|
||||
active_context_id: z.number().int().positive().nullable().optional().describe('Optional active context ID as a soft hint.'),
|
||||
limit: z.number().int().min(1).max(12).optional().describe('Maximum number of nodes to return.'),
|
||||
}),
|
||||
execute: async ({ query, focused_node_id, active_context_id, limit }) => {
|
||||
try {
|
||||
const result = await retrieveQueryContext({
|
||||
query,
|
||||
focused_node_id: focused_node_id ?? null,
|
||||
active_context_id: active_context_id ?? null,
|
||||
limit,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: result,
|
||||
message: result.shouldRetrieve
|
||||
? `Retrieved ${result.nodes.length} node(s) and ${result.chunks.length} supporting chunk(s) for the current turn.`
|
||||
: result.reason,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to retrieve query context',
|
||||
data: null,
|
||||
};
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
||||
|
||||
export const writeContextTool = tool({
|
||||
description: 'Write one atomic durable context node to the graph only after the user has explicitly approved the save. Use this sparingly for unusually valuable context. Never call it unless the user has clearly said yes.',
|
||||
inputSchema: z.object({
|
||||
title: z.string().min(1).max(160).describe('Clear proposed node title.'),
|
||||
description: z.string().min(1).max(500).describe('Natural description of what this context is and why it matters.'),
|
||||
source: z.string().optional().describe('Optional source or verbatim user wording to preserve.'),
|
||||
context_id: z.number().int().positive().nullable().optional().describe('Optional primary context ID.'),
|
||||
metadata: z.record(z.any()).optional().describe('Optional metadata patch.'),
|
||||
confirmed_by_user: z.boolean().describe('Must be true. Reject the write otherwise.'),
|
||||
}),
|
||||
execute: async ({ title, description, source, context_id, metadata, confirmed_by_user }) => {
|
||||
if (!confirmed_by_user) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'writeContext requires explicit user confirmation before writing to the graph.',
|
||||
data: null,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${getInternalApiBaseUrl()}/api/nodes`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
title: title.trim(),
|
||||
description: description.trim(),
|
||||
source: source?.trim() || undefined,
|
||||
context_id: context_id ?? null,
|
||||
metadata: {
|
||||
captured_by: 'human',
|
||||
captured_method: 'write_context',
|
||||
...(metadata || {}),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await response.json();
|
||||
if (!response.ok) {
|
||||
return {
|
||||
success: false,
|
||||
error: result.error || 'Failed to write context node',
|
||||
data: null,
|
||||
};
|
||||
}
|
||||
|
||||
const formattedDisplay = formatNodeForChat({
|
||||
id: result.data.id,
|
||||
title: result.data.title,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
...result.data,
|
||||
formatted_display: formattedDisplay,
|
||||
},
|
||||
message: `Saved context as ${formattedDisplay}`,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Failed to write context node',
|
||||
data: null,
|
||||
};
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -35,6 +35,7 @@ export const TOOL_GROUPS: Record<string, ToolGroup> = {
|
||||
export const TOOL_GROUP_ASSIGNMENTS: Record<string, string> = {
|
||||
// Core: Read-only graph operations (all agents)
|
||||
queryNodes: 'core',
|
||||
retrieveQueryContext: 'core',
|
||||
getNodesById: 'core',
|
||||
queryEdge: 'core',
|
||||
queryContexts: 'core',
|
||||
@@ -46,6 +47,7 @@ export const TOOL_GROUP_ASSIGNMENTS: Record<string, string> = {
|
||||
|
||||
// Execution: Write operations and extraction (workers only)
|
||||
createNode: 'execution',
|
||||
writeContext: 'execution',
|
||||
updateNode: 'execution',
|
||||
deleteNode: 'execution',
|
||||
createEdge: 'execution',
|
||||
@@ -53,7 +55,7 @@ export const TOOL_GROUP_ASSIGNMENTS: Record<string, string> = {
|
||||
embedContent: 'execution',
|
||||
youtubeExtract: 'execution',
|
||||
websiteExtract: 'execution',
|
||||
paperExtract: 'execution'
|
||||
paperExtract: 'execution',
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { getToolGroup, groupTools, getAllToolsByGroup } from './groups';
|
||||
import { queryNodesTool } from '../database/queryNodes';
|
||||
import { retrieveQueryContextTool } from '../database/retrieveQueryContext';
|
||||
import { getNodesByIdTool } from '../database/getNodesById';
|
||||
import { queryEdgeTool } from '../database/queryEdge';
|
||||
import { queryContextsTool } from '../database/queryContexts';
|
||||
import { createNodeTool } from '../database/createNode';
|
||||
import { updateNodeTool } from '../database/updateNode';
|
||||
import { writeContextTool } from '../database/writeContext';
|
||||
import { deleteNodeTool } from '../database/deleteNode';
|
||||
import { createEdgeTool } from '../database/createEdge';
|
||||
import { updateEdgeTool } from '../database/updateEdge';
|
||||
@@ -21,6 +23,7 @@ import { logEvalToolCall } from '@/services/evals/evalsLogger';
|
||||
const CORE_TOOLS: Record<string, any> = {
|
||||
sqliteQuery: sqliteQueryTool,
|
||||
queryNodes: queryNodesTool,
|
||||
retrieveQueryContext: retrieveQueryContextTool,
|
||||
getNodesById: getNodesByIdTool,
|
||||
queryEdge: queryEdgeTool,
|
||||
queryContexts: queryContextsTool,
|
||||
@@ -36,6 +39,7 @@ const ORCHESTRATION_TOOLS: Record<string, any> = {
|
||||
// Write tools (includes extraction)
|
||||
const EXECUTION_TOOLS: Record<string, any> = {
|
||||
createNode: createNodeTool,
|
||||
writeContext: writeContextTool,
|
||||
updateNode: updateNodeTool,
|
||||
deleteNode: deleteNodeTool,
|
||||
createEdge: createEdgeTool,
|
||||
|
||||
Reference in New Issue
Block a user