fix: port search and dimension safety guardrails
- enforce canonical dimensions across app and MCP surfaces - prioritize title-first node search in UI and query tools - add regression tests for ranking and dimension validation
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { normalizeDimensionName } from './quality';
|
||||
|
||||
export function getUnknownDimensions(values: string[]): string[] {
|
||||
if (values.length === 0) return [];
|
||||
|
||||
const sqlite = getSQLiteClient();
|
||||
const placeholders = values.map(() => '?').join(', ');
|
||||
const result = sqlite.query<{ name: string }>(
|
||||
`SELECT name FROM dimensions WHERE name IN (${placeholders})`,
|
||||
values
|
||||
);
|
||||
|
||||
const existing = new Set(
|
||||
result.rows
|
||||
.map(row => (typeof row.name === 'string' ? normalizeDimensionName(row.name) : ''))
|
||||
.filter(Boolean)
|
||||
);
|
||||
|
||||
return values.filter(value => !existing.has(normalizeDimensionName(value)));
|
||||
}
|
||||
|
||||
export function formatUnknownDimensionsError(values: string[]): string {
|
||||
if (values.length === 1) {
|
||||
return `Unknown dimension: "${values[0]}". Create it first or use an existing dimension.`;
|
||||
}
|
||||
|
||||
return `Unknown dimensions: ${values.map(value => `"${value}"`).join(', ')}. Create them first or use existing dimensions.`;
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { getSQLiteClient } from './sqlite-client';
|
||||
import { Node, NodeFilters } from '@/types/database';
|
||||
import { eventBroadcaster } from '../events';
|
||||
import { EmbeddingService } from '@/services/embeddings';
|
||||
import { scoreNodeSearchMatch } from './searchRanking';
|
||||
|
||||
type NodeRow = Node & { dimensions_json: string };
|
||||
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
|
||||
@@ -496,17 +497,23 @@ export class NodeService {
|
||||
return [];
|
||||
}
|
||||
|
||||
const searchLimit = Math.max(limit + offset, Math.min(limit * 5, 100));
|
||||
let rankedRows = this.searchNodesFts(sqlite, search, filters, searchLimit);
|
||||
const searchLimit = Math.max(limit + offset, Math.min(limit * 20, 250));
|
||||
const candidateRows = new Map<number, NodeSearchRow>();
|
||||
|
||||
if (rankedRows.length === 0) {
|
||||
rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit);
|
||||
for (const row of this.searchNodesFts(sqlite, search, filters, searchLimit)) {
|
||||
candidateRows.set(row.id, row);
|
||||
}
|
||||
|
||||
if (rankedRows.length === 0) {
|
||||
rankedRows = this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit);
|
||||
for (const row of this.searchNodesLike(sqlite, search, filters, searchLimit)) {
|
||||
if (!candidateRows.has(row.id)) candidateRows.set(row.id, row);
|
||||
}
|
||||
|
||||
for (const row of this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit)) {
|
||||
if (!candidateRows.has(row.id)) candidateRows.set(row.id, row);
|
||||
}
|
||||
|
||||
let rankedRows = Array.from(candidateRows.values());
|
||||
|
||||
if ((filters.searchMode ?? 'standard') === 'hybrid') {
|
||||
const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
|
||||
if (vectorRows.length > 0) {
|
||||
@@ -514,6 +521,11 @@ export class NodeService {
|
||||
}
|
||||
}
|
||||
|
||||
rankedRows = rankedRows
|
||||
.map(row => ({ row, score: scoreNodeSearchMatch(row, search) }))
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.map(entry => entry.row);
|
||||
|
||||
return rankedRows
|
||||
.slice(offset, offset + limit)
|
||||
.map(row => this.mapNodeRow(row));
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { Node } from '@/types/database';
|
||||
|
||||
function normalizeSearchText(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function getQueryTerms(query: string): string[] {
|
||||
return normalizeSearchText(query).split(' ').filter(term => term.length > 0);
|
||||
}
|
||||
|
||||
function countOccurrences(text: string, term: string): number {
|
||||
if (!text || !term) return 0;
|
||||
const matches = text.match(new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'g'));
|
||||
return matches ? matches.length : 0;
|
||||
}
|
||||
|
||||
function orderedTermMatches(text: string, terms: string[]): boolean {
|
||||
let position = 0;
|
||||
for (const term of terms) {
|
||||
const index = text.indexOf(term, position);
|
||||
if (index === -1) return false;
|
||||
position = index + term.length;
|
||||
}
|
||||
return terms.length > 0;
|
||||
}
|
||||
|
||||
export function scoreNodeSearchMatch(node: Pick<Node, 'title' | 'description' | 'source' | 'updated_at'>, query: string): number {
|
||||
const normalizedQuery = normalizeSearchText(query);
|
||||
const normalizedTitle = normalizeSearchText(node.title || '');
|
||||
const normalizedDescription = normalizeSearchText(node.description || '');
|
||||
const normalizedSource = normalizeSearchText(node.source || '');
|
||||
const terms = getQueryTerms(query);
|
||||
|
||||
let score = 0;
|
||||
|
||||
if (normalizedTitle === normalizedQuery) score += 2000;
|
||||
if (normalizedTitle.startsWith(normalizedQuery)) score += 1200;
|
||||
if (normalizedTitle.includes(normalizedQuery)) score += 700;
|
||||
if (orderedTermMatches(normalizedTitle, terms)) score += 500;
|
||||
if (terms.length > 0 && terms.every(term => normalizedTitle.includes(term))) score += 350;
|
||||
|
||||
if (normalizedDescription.includes(normalizedQuery)) score += 180;
|
||||
if (orderedTermMatches(normalizedDescription, terms)) score += 120;
|
||||
if (normalizedSource.includes(normalizedQuery)) score += 90;
|
||||
|
||||
for (const term of terms) {
|
||||
score += countOccurrences(normalizedTitle, term) * 40;
|
||||
score += countOccurrences(normalizedDescription, term) * 8;
|
||||
score += countOccurrences(normalizedSource, term) * 3;
|
||||
}
|
||||
|
||||
if (node.updated_at) {
|
||||
score += new Date(node.updated_at).getTime() / 1e13;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
Reference in New Issue
Block a user