fix: port search and dimension safety guardrails
- enforce canonical dimensions across app and MCP surfaces - prioritize title-first node search in UI and query tools - add regression tests for ranking and dimension validation
This commit is contained in:
@@ -88,7 +88,7 @@ export default function NodeSearchModal({
|
||||
|
||||
const timeoutId = setTimeout(async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/nodes/search?q=${encodeURIComponent(searchQuery)}&limit=10`);
|
||||
const response = await fetch(`/api/nodes/search?q=${encodeURIComponent(searchQuery)}&limit=20`);
|
||||
const result = await response.json();
|
||||
if (response.ok && result.success) {
|
||||
const nextSuggestions = (result.data as NodeSuggestion[])
|
||||
@@ -229,7 +229,9 @@ export default function NodeSearchModal({
|
||||
background: '#141414',
|
||||
border: '1px solid #2a2a2a',
|
||||
borderRadius: '12px',
|
||||
overflow: 'hidden',
|
||||
overflowX: 'hidden',
|
||||
overflowY: 'auto',
|
||||
maxHeight: 'min(60vh, 560px)',
|
||||
boxShadow: '0 24px 48px -12px rgba(0,0,0,0.6)',
|
||||
}}>
|
||||
{suggestions.map((suggestion, index) => (
|
||||
|
||||
@@ -104,7 +104,7 @@ export default function SearchModal({ isOpen, onClose, onNodeSelect, existingFil
|
||||
|
||||
const fetchSuggestions = async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/nodes/search?q=${encodeURIComponent(searchQuery)}&limit=10`);
|
||||
const response = await fetch(`/api/nodes/search?q=${encodeURIComponent(searchQuery)}&limit=20`);
|
||||
const result = await response.json();
|
||||
|
||||
if (result.success) {
|
||||
@@ -305,7 +305,9 @@ export default function SearchModal({ isOpen, onClose, onNodeSelect, existingFil
|
||||
background: var(--rah-bg-panel);
|
||||
border: 1px solid var(--rah-border-strong);
|
||||
border-radius: 16px;
|
||||
overflow: hidden;
|
||||
overflow-x: hidden;
|
||||
overflow-y: auto;
|
||||
max-height: min(60vh, 560px);
|
||||
box-shadow:
|
||||
0 0 0 1px rgba(255, 255, 255, 0.04),
|
||||
0 24px 48px -12px rgba(0, 0, 0, 0.6);
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { normalizeDimensionName } from './quality';
|
||||
|
||||
export function getUnknownDimensions(values: string[]): string[] {
|
||||
if (values.length === 0) return [];
|
||||
|
||||
const sqlite = getSQLiteClient();
|
||||
const placeholders = values.map(() => '?').join(', ');
|
||||
const result = sqlite.query<{ name: string }>(
|
||||
`SELECT name FROM dimensions WHERE name IN (${placeholders})`,
|
||||
values
|
||||
);
|
||||
|
||||
const existing = new Set(
|
||||
result.rows
|
||||
.map(row => (typeof row.name === 'string' ? normalizeDimensionName(row.name) : ''))
|
||||
.filter(Boolean)
|
||||
);
|
||||
|
||||
return values.filter(value => !existing.has(normalizeDimensionName(value)));
|
||||
}
|
||||
|
||||
export function formatUnknownDimensionsError(values: string[]): string {
|
||||
if (values.length === 1) {
|
||||
return `Unknown dimension: "${values[0]}". Create it first or use an existing dimension.`;
|
||||
}
|
||||
|
||||
return `Unknown dimensions: ${values.map(value => `"${value}"`).join(', ')}. Create them first or use existing dimensions.`;
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { getSQLiteClient } from './sqlite-client';
|
||||
import { Node, NodeFilters } from '@/types/database';
|
||||
import { eventBroadcaster } from '../events';
|
||||
import { EmbeddingService } from '@/services/embeddings';
|
||||
import { scoreNodeSearchMatch } from './searchRanking';
|
||||
|
||||
type NodeRow = Node & { dimensions_json: string };
|
||||
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
|
||||
@@ -496,17 +497,23 @@ export class NodeService {
|
||||
return [];
|
||||
}
|
||||
|
||||
const searchLimit = Math.max(limit + offset, Math.min(limit * 5, 100));
|
||||
let rankedRows = this.searchNodesFts(sqlite, search, filters, searchLimit);
|
||||
const searchLimit = Math.max(limit + offset, Math.min(limit * 20, 250));
|
||||
const candidateRows = new Map<number, NodeSearchRow>();
|
||||
|
||||
if (rankedRows.length === 0) {
|
||||
rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit);
|
||||
for (const row of this.searchNodesFts(sqlite, search, filters, searchLimit)) {
|
||||
candidateRows.set(row.id, row);
|
||||
}
|
||||
|
||||
if (rankedRows.length === 0) {
|
||||
rankedRows = this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit);
|
||||
for (const row of this.searchNodesLike(sqlite, search, filters, searchLimit)) {
|
||||
if (!candidateRows.has(row.id)) candidateRows.set(row.id, row);
|
||||
}
|
||||
|
||||
for (const row of this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit)) {
|
||||
if (!candidateRows.has(row.id)) candidateRows.set(row.id, row);
|
||||
}
|
||||
|
||||
let rankedRows = Array.from(candidateRows.values());
|
||||
|
||||
if ((filters.searchMode ?? 'standard') === 'hybrid') {
|
||||
const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
|
||||
if (vectorRows.length > 0) {
|
||||
@@ -514,6 +521,11 @@ export class NodeService {
|
||||
}
|
||||
}
|
||||
|
||||
rankedRows = rankedRows
|
||||
.map(row => ({ row, score: scoreNodeSearchMatch(row, search) }))
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.map(entry => entry.row);
|
||||
|
||||
return rankedRows
|
||||
.slice(offset, offset + limit)
|
||||
.map(row => this.mapNodeRow(row));
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { Node } from '@/types/database';
|
||||
|
||||
function normalizeSearchText(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function getQueryTerms(query: string): string[] {
|
||||
return normalizeSearchText(query).split(' ').filter(term => term.length > 0);
|
||||
}
|
||||
|
||||
function countOccurrences(text: string, term: string): number {
|
||||
if (!text || !term) return 0;
|
||||
const matches = text.match(new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'g'));
|
||||
return matches ? matches.length : 0;
|
||||
}
|
||||
|
||||
function orderedTermMatches(text: string, terms: string[]): boolean {
|
||||
let position = 0;
|
||||
for (const term of terms) {
|
||||
const index = text.indexOf(term, position);
|
||||
if (index === -1) return false;
|
||||
position = index + term.length;
|
||||
}
|
||||
return terms.length > 0;
|
||||
}
|
||||
|
||||
export function scoreNodeSearchMatch(node: Pick<Node, 'title' | 'description' | 'source' | 'updated_at'>, query: string): number {
|
||||
const normalizedQuery = normalizeSearchText(query);
|
||||
const normalizedTitle = normalizeSearchText(node.title || '');
|
||||
const normalizedDescription = normalizeSearchText(node.description || '');
|
||||
const normalizedSource = normalizeSearchText(node.source || '');
|
||||
const terms = getQueryTerms(query);
|
||||
|
||||
let score = 0;
|
||||
|
||||
if (normalizedTitle === normalizedQuery) score += 2000;
|
||||
if (normalizedTitle.startsWith(normalizedQuery)) score += 1200;
|
||||
if (normalizedTitle.includes(normalizedQuery)) score += 700;
|
||||
if (orderedTermMatches(normalizedTitle, terms)) score += 500;
|
||||
if (terms.length > 0 && terms.every(term => normalizedTitle.includes(term))) score += 350;
|
||||
|
||||
if (normalizedDescription.includes(normalizedQuery)) score += 180;
|
||||
if (orderedTermMatches(normalizedDescription, terms)) score += 120;
|
||||
if (normalizedSource.includes(normalizedQuery)) score += 90;
|
||||
|
||||
for (const term of terms) {
|
||||
score += countOccurrences(normalizedTitle, term) * 40;
|
||||
score += countOccurrences(normalizedDescription, term) * 8;
|
||||
score += countOccurrences(normalizedSource, term) * 3;
|
||||
}
|
||||
|
||||
if (node.updated_at) {
|
||||
score += new Date(node.updated_at).getTime() / 1e13;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import { z } from 'zod';
|
||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||
|
||||
export const createDimensionTool = tool({
|
||||
description: 'Create a new dimension. Always provide a description explaining what belongs in this category.',
|
||||
description: 'Create a new dimension only when the user explicitly instructs you to do so. Always provide a description explaining what belongs in this category.',
|
||||
inputSchema: z.object({
|
||||
name: z.string().describe('Dimension name'),
|
||||
description: z.string().min(1).max(500).describe('Dimension description explaining what content belongs in this dimension (required, max 500 characters)')
|
||||
|
||||
@@ -3,7 +3,7 @@ import { z } from 'zod';
|
||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||
|
||||
export const queryDimensionsTool = tool({
|
||||
description: 'List dimensions with node counts. Use this to inspect the user\'s organizational categories.',
|
||||
description: 'List the existing canonical dimensions with node counts. Use this before assigning dimensions to nodes. Do not invent new dimensions without explicit user instruction.',
|
||||
inputSchema: z.object({
|
||||
filters: z.object({
|
||||
search: z.string().describe('Search term to match against dimension names').optional(),
|
||||
|
||||
@@ -3,6 +3,7 @@ import { z } from 'zod';
|
||||
import { nodeService } from '@/services/database/nodes';
|
||||
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
||||
import type { Node } from '@/types/database';
|
||||
import { scoreNodeSearchMatch } from '@/services/database/searchRanking';
|
||||
|
||||
type QueryNodeFilters = {
|
||||
dimensions?: string[];
|
||||
@@ -14,55 +15,6 @@ type QueryNodeFilters = {
|
||||
eventBefore?: string;
|
||||
};
|
||||
|
||||
function extractSearchTerms(query: string): string[] {
|
||||
const stopWords = new Set(['a', 'an', 'and', 'for', 'from', 'in', 'of', 'on', 'or', 'recent', 'the', 'to', 'with']);
|
||||
|
||||
const rawTerms = query
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]+/g, ' ')
|
||||
.split(/\s+/)
|
||||
.map(term => term.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const terms = new Set<string>();
|
||||
for (const term of rawTerms) {
|
||||
if (!stopWords.has(term) && term.length >= 3) {
|
||||
terms.add(term);
|
||||
}
|
||||
const alphaParts = term.replace(/\d+/g, ' ').split(/\s+/).filter(Boolean);
|
||||
for (const part of alphaParts) {
|
||||
if (!stopWords.has(part) && part.length >= 3) {
|
||||
terms.add(part);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(terms).slice(0, 8);
|
||||
}
|
||||
|
||||
function scoreNodeForSearch(node: Node, searchTerm: string): number {
|
||||
const normalizedSearch = searchTerm.toLowerCase();
|
||||
const title = (node.title || '').toLowerCase();
|
||||
const description = (node.description || '').toLowerCase();
|
||||
const source = (node.source || '').toLowerCase();
|
||||
const terms = extractSearchTerms(searchTerm);
|
||||
|
||||
let score = 0;
|
||||
|
||||
if (title === normalizedSearch) score += 100;
|
||||
if (title.includes(normalizedSearch)) score += 40;
|
||||
if (description.includes(normalizedSearch)) score += 20;
|
||||
if (source.includes(normalizedSearch)) score += 10;
|
||||
|
||||
for (const term of terms) {
|
||||
if (title.includes(term)) score += 8;
|
||||
if (description.includes(term)) score += 3;
|
||||
if (source.includes(term)) score += 2;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
export const queryNodesTool = tool({
|
||||
description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Do not use dimensions to constrain keyword search unless the user is explicitly asking about a known dimension.',
|
||||
inputSchema: z.object({
|
||||
@@ -150,7 +102,7 @@ export const queryNodesTool = tool({
|
||||
|
||||
if (searchTerm) {
|
||||
safeNodes = safeNodes
|
||||
.map(node => ({ node, score: scoreNodeForSearch(node, searchTerm) }))
|
||||
.map(node => ({ node, score: scoreNodeSearchMatch(node, searchTerm) }))
|
||||
.sort((a, b) => b.score - a.score || b.node.updated_at.localeCompare(a.node.updated_at))
|
||||
.slice(0, limit)
|
||||
.map(entry => entry.node);
|
||||
|
||||
Reference in New Issue
Block a user