feat: port MCP retrieval and write hardening

- align os MCP/runtime/docs with ra-h contract - add safe direct node lookup and context-optional flows - gate edge and context writes behind confirmation
2026-04-14 20:57:07 +10:00
parent d825e7a783
commit 929423cb21
35 changed files with 1172 additions and 534 deletions
@@ -0,0 +1,223 @@
+'use strict';
+
+const nodeService = require('./nodeService');
+const contextService = require('./contextService');
+
+const SEARCH_STOP_WORDS = new Set([
+  'a', 'about', 'added', 'already', 'an', 'and', 'are', 'as', 'at', 'be', 'by',
+  'can', 'created', 'do', 'find', 'for', 'from', 'hello', 'i', 'in', 'into', 'is',
+  'it', 'just', 'look', 'me', 'my', 'node', 'of', 'on', 'or', 'pull', 'recent',
+  'recently', 'saved', 'shared', 'show', 'some', 'stuff', 'term', 'that', 'the', 'this',
+  'to', 'versus', 'were', 'what', 'with', 'wrote', 'you', 'doing', 'going', 'having',
+]);
+
+function normalizeSearchText(value) {
+  return String(value || '')
+    .toLowerCase()
+    .replace(/\b([a-z0-9]{1,3})-([a-z0-9]{1,3})(?:-([a-z0-9]{1,3}))?\b/gi, (_match, a, b, c) => `${a}${b}${c || ''}`)
+    .replace(/[^a-z0-9]+/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function singularizeTerm(term) {
+  if (term.endsWith('ies') && term.length > 4) return `${term.slice(0, -3)}y`;
+  if (term.endsWith('s') && term.length > 4 && !term.endsWith('ss') && !term.endsWith('us')) {
+    return term.slice(0, -1);
+  }
+  return term;
+}
+
+function getHighSignalSearchTerms(query) {
+  const seen = new Set();
+  const terms = [];
+
+  for (const rawTerm of normalizeSearchText(query).split(' ')) {
+    const term = singularizeTerm(rawTerm.trim());
+    if (!term || term.length < 3) continue;
+    if (SEARCH_STOP_WORDS.has(term)) continue;
+    if (seen.has(term)) continue;
+    seen.add(term);
+    terms.push(term);
+  }
+
+  return terms;
+}
+
+function countHighSignalQueryTermMatches(node, query) {
+  const terms = getHighSignalSearchTerms(query);
+  if (terms.length === 0) return 0;
+
+  const haystack = [
+    normalizeSearchText(node.title || ''),
+    normalizeSearchText(node.description || ''),
+    normalizeSearchText(node.source || ''),
+  ].join(' ');
+
+  return terms.filter(term => haystack.includes(term)).length;
+}
+
+function countOccurrences(text, term) {
+  if (!text || !term) return 0;
+  const matches = text.match(new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'g'));
+  return matches ? matches.length : 0;
+}
+
+function orderedTermMatches(text, terms) {
+  let position = 0;
+  for (const term of terms) {
+    const index = text.indexOf(term, position);
+    if (index === -1) return false;
+    position = index + term.length;
+  }
+  return terms.length > 0;
+}
+
+function scoreNodeSearchMatch(node, query) {
+  const normalizedQuery = normalizeSearchText(query);
+  const normalizedTitle = normalizeSearchText(node.title || '');
+  const normalizedDescription = normalizeSearchText(node.description || '');
+  const normalizedSource = normalizeSearchText(node.source || '');
+  const terms = getHighSignalSearchTerms(query);
+
+  let score = 0;
+
+  if (normalizedTitle === normalizedQuery) score += 2000;
+  if (normalizedTitle.startsWith(normalizedQuery)) score += 1200;
+  if (normalizedTitle.includes(normalizedQuery)) score += 700;
+  if (orderedTermMatches(normalizedTitle, terms)) score += 500;
+  if (terms.length > 0 && terms.every(term => normalizedTitle.includes(term))) score += 350;
+
+  if (normalizedDescription.includes(normalizedQuery)) score += 180;
+  if (orderedTermMatches(normalizedDescription, terms)) score += 120;
+  if (normalizedSource.includes(normalizedQuery)) score += 90;
+
+  const matchedTermCount = countHighSignalQueryTermMatches(node, query);
+  score += matchedTermCount * 120;
+  if (terms.length > 0 && matchedTermCount === terms.length) score += 300;
+
+  for (const term of terms) {
+    score += countOccurrences(normalizedTitle, term) * 40;
+    score += countOccurrences(normalizedDescription, term) * 8;
+    score += countOccurrences(normalizedSource, term) * 3;
+  }
+
+  if (node.updated_at) {
+    score += new Date(node.updated_at).getTime() / 1e13;
+  }
+
+  return score;
+}
+
+function normalizeContextName(value) {
+  if (typeof value !== 'string') return undefined;
+  const normalized = value.trim().replace(/\s+/g, ' ');
+  return normalized || undefined;
+}
+
+function resolveSearchContext({ context_name, contextId }) {
+  const normalizedName = normalizeContextName(context_name);
+  if (normalizedName) {
+    const context = contextService.getContextByName(normalizedName);
+    if (!context) {
+      console.warn(`directNodeLookupService received unknown context_name "${normalizedName}"; ignoring context filter.`);
+      return {};
+    }
+    return {
+      contextId: context.id,
+      context_name: context.name,
+    };
+  }
+
+  if (typeof contextId === 'number') {
+    const context = contextService.getContextById(contextId);
+    if (!context) {
+      console.warn(`directNodeLookupService received invalid legacy contextId ${contextId}; ignoring context filter.`);
+      return {};
+    }
+    return {
+      contextId: context.id,
+      context_name: context.name,
+    };
+  }
+
+  return {};
+}
+
+function hasStrongAnchorMatch(nodes, searchTerm) {
+  if (!searchTerm || nodes.length === 0) return false;
+  const highSignalTerms = getHighSignalSearchTerms(searchTerm);
+  const requiredMatches = Math.min(2, highSignalTerms.length || 1);
+  return nodes.some(node => countHighSignalQueryTermMatches(node, searchTerm) >= requiredMatches);
+}
+
+function directNodeLookup(input = {}) {
+  const limit = Math.min(Math.max(input.limit || 10, 1), 50);
+  const searchTerm = typeof input.search === 'string' ? input.search.trim() : '';
+
+  if (searchTerm && /^\d+$/.test(searchTerm)) {
+    const nodeId = Number(searchTerm);
+    const node = nodeService.getNodeById(nodeId);
+    return {
+      nodes: node ? [node] : [],
+      count: node ? 1 : 0,
+      filtersApplied: {
+        search: searchTerm,
+        limit,
+      },
+    };
+  }
+
+  const resolvedContext = resolveSearchContext(input);
+
+  let safeNodes = nodeService.getNodes({
+    search: searchTerm || undefined,
+    limit,
+    contextId: resolvedContext.contextId,
+    createdAfter: input.createdAfter,
+    createdBefore: input.createdBefore,
+    eventAfter: input.eventAfter,
+    eventBefore: input.eventBefore,
+  });
+
+  const hadExtraFilters = Boolean(
+    resolvedContext.contextId !== undefined ||
+    input.createdAfter ||
+    input.createdBefore ||
+    input.eventAfter ||
+    input.eventBefore
+  );
+
+  if (searchTerm && hadExtraFilters && (safeNodes.length === 0 || !hasStrongAnchorMatch(safeNodes, searchTerm))) {
+    console.warn(`directNodeLookupService falling back to plain literal search for "${searchTerm}" after filtered lookup missed a strong anchor match.`);
+    safeNodes = nodeService.searchNodes({ search: searchTerm, limit });
+  }
+
+  if (searchTerm) {
+    safeNodes = safeNodes
+      .map(node => ({ node, score: scoreNodeSearchMatch(node, searchTerm) }))
+      .sort((a, b) => b.score - a.score || String(b.node.updated_at || '').localeCompare(String(a.node.updated_at || '')))
+      .slice(0, limit)
+      .map(entry => entry.node);
+  } else {
+    safeNodes = safeNodes.slice(0, limit);
+  }
+
+  return {
+    nodes: safeNodes,
+    count: safeNodes.length,
+    filtersApplied: {
+      search: searchTerm || undefined,
+      limit,
+      context_name: resolvedContext.context_name,
+      createdAfter: input.createdAfter,
+      createdBefore: input.createdBefore,
+      eventAfter: input.eventAfter,
+      eventBefore: input.eventBefore,
+    },
+  };
+}
+
+module.exports = {
+  directNodeLookup,
+};
@@ -158,6 +158,69 @@ function sanitizeTitle(title) {
  return clean.slice(0, 160);
 }

+const CHUNK_SIZE = 1000;
+const CHUNK_OVERLAP = 200;
+
+function splitSourceIntoChunks(sourceText) {
+  const text = String(sourceText || '').trim();
+  if (!text) return [];
+
+  const chunks = [];
+  let start = 0;
+
+  while (start < text.length) {
+    const end = Math.min(start + CHUNK_SIZE, text.length);
+    const chunkText = text.slice(start, end).trim();
+
+    if (chunkText) {
+      chunks.push({
+        text: chunkText,
+        start_char: start,
+        end_char: end
+      });
+    }
+
+    if (end >= text.length) break;
+    start = Math.max(end - CHUNK_OVERLAP, start + 1);
+  }
+
+  return chunks;
+}
+
+function replaceNodeChunks(db, nodeId, title, sourceText) {
+  db.prepare('DELETE FROM chunks WHERE node_id = ?').run(nodeId);
+
+  const chunks = splitSourceIntoChunks(sourceText);
+  if (chunks.length === 0) {
+    return 0;
+  }
+
+  const now = new Date().toISOString();
+  const insertChunk = db.prepare(`
+    INSERT INTO chunks (node_id, chunk_idx, text, embedding_type, metadata, created_at)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+
+  chunks.forEach((chunk, index) => {
+    insertChunk.run(
+      nodeId,
+      index,
+      chunk.text,
+      'text-embedding-3-small',
+      JSON.stringify({
+        node_id: nodeId,
+        chunk_index: index,
+        start_char: chunk.start_char,
+        end_char: chunk.end_char,
+        title
+      }),
+      now
+    );
+  });
+
+  return chunks.length;
+}
+
 /**
 * Create a new node.
 */
@@ -180,11 +243,12 @@ function createNode(nodeData) {

  const sourceToStore = source ?? ([title, description].filter(Boolean).join('\n\n').trim() || null);
  const effectiveContextId = context_id ?? null;
+  const hasSource = !!normalizeString(sourceToStore);

  const nodeId = transaction(() => {
    const stmt = db.prepare(`
-      INSERT INTO nodes (title, description, source, link, event_date, metadata, context_id, created_at, updated_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      INSERT INTO nodes (title, description, source, link, event_date, metadata, chunk_status, context_id, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `);

    const result = stmt.run(
@@ -194,6 +258,7 @@ function createNode(nodeData) {
      link ?? null,
      event_date ?? null,
      JSON.stringify(canonicalMetadata),
+      hasSource ? 'chunked' : null,
      effectiveContextId ?? null,
      now,
      now
@@ -201,6 +266,10 @@ function createNode(nodeData) {

    const id = Number(result.lastInsertRowid);

+    if (hasSource) {
+      replaceNodeChunks(db, id, title, sourceToStore);
+    }
+
    return id;
  });

@@ -224,6 +293,9 @@ function updateNode(id, updates, options = {}) {
  const mergedMetadata = metadata !== undefined
    ? buildCanonicalMetadata({ existing: existing.metadata, metadata })
    : undefined;
+  const sourceWasProvided = Object.prototype.hasOwnProperty.call(updates, 'source');
+  const updatedTitle = title !== undefined ? title : existing.title;
+  const normalizedSource = sourceWasProvided ? normalizeString(source) : undefined;

  transaction(() => {
    const setFields = [];
@@ -249,6 +321,10 @@ function updateNode(id, updates, options = {}) {
      setFields.push('event_date = ?');
      params.push(event_date);
    }
+    if (sourceWasProvided) {
+      setFields.push('chunk_status = ?');
+      params.push(normalizedSource ? 'chunked' : null);
+    }
    if (Object.prototype.hasOwnProperty.call(updates, 'context_id')) {
      setFields.push('context_id = ?');
      params.push(updates.context_id ?? null);
@@ -268,6 +344,10 @@ function updateNode(id, updates, options = {}) {
      stmt.run(...params);
    }

+    if (sourceWasProvided) {
+      replaceNodeChunks(db, id, updatedTitle, normalizedSource || '');
+    }
+
  });

  return getNodeById(id);