fix: port search and dimension safety guardrails
- enforce canonical dimensions across app and MCP surfaces - prioritize title-first node search in UI and query tools - add regression tests for ranking and dimension validation
This commit is contained in:
@@ -26,20 +26,13 @@ async function getPopularDimensionsSQLite() {
|
|||||||
SELECT nd.dimension, COUNT(*) AS count
|
SELECT nd.dimension, COUNT(*) AS count
|
||||||
FROM node_dimensions nd
|
FROM node_dimensions nd
|
||||||
GROUP BY nd.dimension
|
GROUP BY nd.dimension
|
||||||
),
|
|
||||||
all_dimensions AS (
|
|
||||||
SELECT DISTINCT dimension AS name FROM node_dimensions
|
|
||||||
UNION
|
|
||||||
SELECT name FROM dimensions
|
|
||||||
)
|
)
|
||||||
SELECT ad.name AS dimension,
|
SELECT d.name AS dimension,
|
||||||
COALESCE(dc.count, 0) AS count,
|
COALESCE(dc.count, 0) AS count,
|
||||||
dim.description
|
d.description
|
||||||
FROM all_dimensions ad
|
FROM dimensions d
|
||||||
LEFT JOIN dimension_counts dc ON dc.dimension = ad.name
|
LEFT JOIN dimension_counts dc ON dc.dimension = d.name
|
||||||
LEFT JOIN dimensions dim ON dim.name = ad.name
|
ORDER BY LOWER(d.name) ASC
|
||||||
WHERE ad.name IS NOT NULL
|
|
||||||
ORDER BY LOWER(ad.name) ASC
|
|
||||||
`);
|
`);
|
||||||
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { nodeService } from '@/services/database';
|
|||||||
import { autoEmbedQueue } from '@/services/embedding/autoEmbedQueue';
|
import { autoEmbedQueue } from '@/services/embedding/autoEmbedQueue';
|
||||||
import { hasSufficientContent } from '@/services/embedding/constants';
|
import { hasSufficientContent } from '@/services/embedding/constants';
|
||||||
import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality';
|
import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality';
|
||||||
|
import { formatUnknownDimensionsError, getUnknownDimensions } from '@/services/database/dimensionValidation';
|
||||||
|
|
||||||
export const runtime = 'nodejs';
|
export const runtime = 'nodejs';
|
||||||
|
|
||||||
@@ -83,6 +84,13 @@ export async function PUT(
|
|||||||
|
|
||||||
if (Array.isArray(body.dimensions)) {
|
if (Array.isArray(body.dimensions)) {
|
||||||
updates.dimensions = normalizeDimensions(body.dimensions, 5);
|
updates.dimensions = normalizeDimensions(body.dimensions, 5);
|
||||||
|
const unknownDimensions = getUnknownDimensions(updates.dimensions as string[]);
|
||||||
|
if (unknownDimensions.length > 0) {
|
||||||
|
return NextResponse.json({
|
||||||
|
success: false,
|
||||||
|
error: formatUnknownDimensionsError(unknownDimensions)
|
||||||
|
}, { status: 400 });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
delete updates.notes;
|
delete updates.notes;
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { autoEmbedQueue } from '@/services/embedding/autoEmbedQueue';
|
|||||||
import { generateDescription } from '@/services/database/descriptionService';
|
import { generateDescription } from '@/services/database/descriptionService';
|
||||||
import { scheduleAutoEdgeCreation } from '@/services/agents/autoEdge';
|
import { scheduleAutoEdgeCreation } from '@/services/agents/autoEdge';
|
||||||
import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality';
|
import { normalizeDimensions, validateExplicitDescription } from '@/services/database/quality';
|
||||||
|
import { formatUnknownDimensionsError, getUnknownDimensions } from '@/services/database/dimensionValidation';
|
||||||
|
|
||||||
export const runtime = 'nodejs';
|
export const runtime = 'nodejs';
|
||||||
|
|
||||||
@@ -97,6 +98,13 @@ export async function POST(request: NextRequest) {
|
|||||||
|
|
||||||
// Process provided dimensions first (needed for description generation)
|
// Process provided dimensions first (needed for description generation)
|
||||||
const trimmedProvidedDimensions = normalizeDimensions(body.dimensions, 5);
|
const trimmedProvidedDimensions = normalizeDimensions(body.dimensions, 5);
|
||||||
|
const unknownDimensions = getUnknownDimensions(trimmedProvidedDimensions);
|
||||||
|
if (unknownDimensions.length > 0) {
|
||||||
|
return NextResponse.json({
|
||||||
|
success: false,
|
||||||
|
error: formatUnknownDimensionsError(unknownDimensions)
|
||||||
|
}, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
// Use provided description if present, otherwise auto-generate
|
// Use provided description if present, otherwise auto-generate
|
||||||
const isUserSuppliedDescription = typeof body.description === 'string' && body.description.trim().length > 0;
|
const isUserSuppliedDescription = typeof body.description === 'string' && body.description.trim().length > 0;
|
||||||
|
|||||||
@@ -66,6 +66,9 @@ function buildInstructions() {
|
|||||||
Proactively offer to save valuable information when insights, decisions, or references surface.
|
Proactively offer to save valuable information when insights, decisions, or references surface.
|
||||||
Propose: "I'd add this as: [title] in [dimensions] — want me to?"
|
Propose: "I'd add this as: [title] in [dimensions] — want me to?"
|
||||||
Always search before creating to avoid duplicates.
|
Always search before creating to avoid duplicates.
|
||||||
|
Use only existing dimensions returned by getContext or queryDimensions.
|
||||||
|
Do not invent new dimensions from node titles, concepts, or phrasing.
|
||||||
|
Only call createDimension when the user explicitly instructs you to create a new dimension.
|
||||||
|
|
||||||
## Available skills
|
## Available skills
|
||||||
${skillIndex}
|
${skillIndex}
|
||||||
@@ -81,7 +84,7 @@ const addNodeInputSchema = {
|
|||||||
source: z.string().max(50000).optional().describe('Full source text'),
|
source: z.string().max(50000).optional().describe('Full source text'),
|
||||||
link: z.string().url().optional().describe('Source URL'),
|
link: z.string().url().optional().describe('Source URL'),
|
||||||
description: z.string().min(24).max(280).describe('REQUIRED. One-sentence summary: WHAT this is (explicit, concrete) + WHY it matters. No weak verbs (discusses, explores, examines). Example: "Podcast — Lex Fridman interviews Sam Altman on AGI timelines. First public comments since board drama."'),
|
description: z.string().min(24).max(280).describe('REQUIRED. One-sentence summary: WHAT this is (explicit, concrete) + WHY it matters. No weak verbs (discusses, explores, examines). Example: "Podcast — Lex Fridman interviews Sam Altman on AGI timelines. First public comments since board drama."'),
|
||||||
dimensions: z.array(z.string()).min(1).max(5).describe('1-5 categories. Call queryDimensions first to use existing ones.'),
|
dimensions: z.array(z.string()).min(1).max(5).describe('1-5 existing categories. Call queryDimensions first to use existing ones. Do not invent new dimensions.'),
|
||||||
metadata: z.record(z.any()).optional().describe('Additional metadata'),
|
metadata: z.record(z.any()).optional().describe('Additional metadata'),
|
||||||
chunk: z.string().max(50000).optional().describe('Full source text')
|
chunk: z.string().max(50000).optional().describe('Full source text')
|
||||||
};
|
};
|
||||||
@@ -351,7 +354,7 @@ async function main() {
|
|||||||
'createNode',
|
'createNode',
|
||||||
{
|
{
|
||||||
title: 'Add RA-H node',
|
title: 'Add RA-H node',
|
||||||
description: 'Create a new node. Always search first (queryNodes) to avoid duplicates. Title: max 160 chars, clear and descriptive. Description is REQUIRED and must be explicit about what the thing is and why it matters for contextual grounding. Use "link" ONLY for external content (URL, video, article) — omit for synthesis/ideas derived from existing nodes. "source" = verbatim or canonical content for embedding. Legacy "content" and "chunk" are mapped to source for compatibility. Assign 1-5 dimensions — call queryDimensions first to use existing ones.',
|
description: 'Create a new node. Always search first (queryNodes) to avoid duplicates. Title: max 160 chars, clear and descriptive. Description is REQUIRED and must be explicit about what the thing is and why it matters for contextual grounding. Use "link" ONLY for external content (URL, video, article) — omit for synthesis/ideas derived from existing nodes. "source" = verbatim or canonical content for embedding. Legacy "content" and "chunk" are mapped to source for compatibility. Assign 1-5 existing dimensions — call queryDimensions first to use existing ones. Do not invent new dimensions.',
|
||||||
inputSchema: addNodeInputSchema
|
inputSchema: addNodeInputSchema
|
||||||
},
|
},
|
||||||
async ({ title, content, source, link, description, dimensions, metadata, chunk }) => {
|
async ({ title, content, source, link, description, dimensions, metadata, chunk }) => {
|
||||||
@@ -716,7 +719,7 @@ async function main() {
|
|||||||
'queryDimensions',
|
'queryDimensions',
|
||||||
{
|
{
|
||||||
title: 'List RA-H dimensions',
|
title: 'List RA-H dimensions',
|
||||||
description: 'Get all dimensions with node counts. Call before creating nodes (to assign existing dimensions) or before creating new dimensions (to avoid duplicates). Shows priority/locked status.',
|
description: 'Get all existing canonical dimensions with node counts. Call before creating nodes to assign existing dimensions. Only create a new dimension if the user explicitly instructs you to do so.',
|
||||||
inputSchema: listDimensionsInputSchema
|
inputSchema: listDimensionsInputSchema
|
||||||
},
|
},
|
||||||
async () => {
|
async () => {
|
||||||
@@ -736,7 +739,7 @@ async function main() {
|
|||||||
'createDimension',
|
'createDimension',
|
||||||
{
|
{
|
||||||
title: 'Create RA-H dimension',
|
title: 'Create RA-H dimension',
|
||||||
description: 'Create a new dimension/category. Use lowercase, singular form (e.g. "biology" not "Biology" or "biologies"). Set isPriority=true to lock it for automatic assignment to new nodes. Always include a description.',
|
description: 'Create a new dimension/category only when the user explicitly instructs you to do so. Use lowercase, singular form (e.g. "biology" not "Biology" or "biologies"). Set isPriority=true to lock it for automatic assignment to new nodes. Always include a description.',
|
||||||
inputSchema: createDimensionInputSchema
|
inputSchema: createDimensionInputSchema
|
||||||
},
|
},
|
||||||
async ({ name, description, isPriority }) => {
|
async ({ name, description, isPriority }) => {
|
||||||
|
|||||||
@@ -2,6 +2,34 @@
|
|||||||
|
|
||||||
const { query, transaction, getDb } = require('./sqlite-client');
|
const { query, transaction, getDb } = require('./sqlite-client');
|
||||||
|
|
||||||
|
function normalizeDimensionName(value) {
|
||||||
|
return String(value || '').trim().replace(/\s+/g, ' ');
|
||||||
|
}
|
||||||
|
|
||||||
|
function getUnknownDimensions(dimensions) {
|
||||||
|
if (!Array.isArray(dimensions) || dimensions.length === 0) return [];
|
||||||
|
|
||||||
|
const normalized = dimensions
|
||||||
|
.map(normalizeDimensionName)
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
if (normalized.length === 0) return [];
|
||||||
|
|
||||||
|
const placeholders = normalized.map(() => '?').join(', ');
|
||||||
|
const rows = query(`SELECT name FROM dimensions WHERE name IN (${placeholders})`, normalized);
|
||||||
|
const existing = new Set(rows.map(row => normalizeDimensionName(row.name)));
|
||||||
|
|
||||||
|
return normalized.filter(value => !existing.has(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatUnknownDimensionsError(values) {
|
||||||
|
if (values.length === 1) {
|
||||||
|
return `Unknown dimension: "${values[0]}". Create it first or use an existing dimension.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `Unknown dimensions: ${values.map(value => `"${value}"`).join(', ')}. Create them first or use existing dimensions.`;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get nodes with optional filtering.
|
* Get nodes with optional filtering.
|
||||||
*/
|
*/
|
||||||
@@ -116,6 +144,10 @@ function createNode(nodeData) {
|
|||||||
} = nodeData;
|
} = nodeData;
|
||||||
|
|
||||||
const title = sanitizeTitle(rawTitle);
|
const title = sanitizeTitle(rawTitle);
|
||||||
|
const unknownDimensions = getUnknownDimensions(dimensions);
|
||||||
|
if (unknownDimensions.length > 0) {
|
||||||
|
throw new Error(formatUnknownDimensionsError(unknownDimensions));
|
||||||
|
}
|
||||||
|
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const db = getDb();
|
const db = getDb();
|
||||||
@@ -174,6 +206,13 @@ function updateNode(id, updates, options = {}) {
|
|||||||
throw new Error(`Node with ID ${id} not found. Use rah_search_nodes to find nodes by keyword.`);
|
throw new Error(`Node with ID ${id} not found. Use rah_search_nodes to find nodes by keyword.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(dimensions)) {
|
||||||
|
const unknownDimensions = getUnknownDimensions(dimensions);
|
||||||
|
if (unknownDimensions.length > 0) {
|
||||||
|
throw new Error(formatUnknownDimensionsError(unknownDimensions));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
transaction(() => {
|
transaction(() => {
|
||||||
const setFields = [];
|
const setFields = [];
|
||||||
const params = [];
|
const params = [];
|
||||||
|
|||||||
@@ -46,6 +46,9 @@ const instructions = [
|
|||||||
'RA-H is a personal knowledge graph — local-first, vendor-neutral.',
|
'RA-H is a personal knowledge graph — local-first, vendor-neutral.',
|
||||||
'Core concepts: nodes (knowledge units), edges (connections with explanations), dimensions (categories).',
|
'Core concepts: nodes (knowledge units), edges (connections with explanations), dimensions (categories).',
|
||||||
'Always call rah_get_context first to orient yourself — it returns hub nodes, dimensions, stats, and available guides.',
|
'Always call rah_get_context first to orient yourself — it returns hub nodes, dimensions, stats, and available guides.',
|
||||||
|
'When assigning dimensions, use only existing dimensions returned by rah_get_context or rah_query_dimensions.',
|
||||||
|
'Do not invent new dimensions from node titles, concepts, or phrasing.',
|
||||||
|
'Only call rah_create_dimension when the user explicitly instructs you to create a new dimension.',
|
||||||
'Search before creating: use rah_search_nodes to check if content already exists.',
|
'Search before creating: use rah_search_nodes to check if content already exists.',
|
||||||
'Every edge needs an explanation: why does this connection exist?',
|
'Every edge needs an explanation: why does this connection exist?',
|
||||||
'All data stays local on this device; nothing leaves 127.0.0.1.',
|
'All data stays local on this device; nothing leaves 127.0.0.1.',
|
||||||
|
|||||||
@@ -13,6 +13,9 @@ const instructions = [
|
|||||||
'RA-H is a personal knowledge graph — local-first, vendor-neutral.',
|
'RA-H is a personal knowledge graph — local-first, vendor-neutral.',
|
||||||
'Core concepts: nodes (knowledge units), edges (connections with explanations), dimensions (categories).',
|
'Core concepts: nodes (knowledge units), edges (connections with explanations), dimensions (categories).',
|
||||||
'Always call rah_get_context first to orient yourself — it returns hub nodes, dimensions, stats, and available guides.',
|
'Always call rah_get_context first to orient yourself — it returns hub nodes, dimensions, stats, and available guides.',
|
||||||
|
'When assigning dimensions, use only existing dimensions returned by rah_get_context or rah_query_dimensions.',
|
||||||
|
'Do not invent new dimensions from node titles, concepts, or phrasing.',
|
||||||
|
'Only call rah_create_dimension when the user explicitly instructs you to create a new dimension.',
|
||||||
'Search before creating: use rah_search_nodes to check if content already exists.',
|
'Search before creating: use rah_search_nodes to check if content already exists.',
|
||||||
'Every edge needs an explanation: why does this connection exist?',
|
'Every edge needs an explanation: why does this connection exist?',
|
||||||
'All data stays local on this device; nothing leaves 127.0.0.1.',
|
'All data stays local on this device; nothing leaves 127.0.0.1.',
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ export default function NodeSearchModal({
|
|||||||
|
|
||||||
const timeoutId = setTimeout(async () => {
|
const timeoutId = setTimeout(async () => {
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`/api/nodes/search?q=${encodeURIComponent(searchQuery)}&limit=10`);
|
const response = await fetch(`/api/nodes/search?q=${encodeURIComponent(searchQuery)}&limit=20`);
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
if (response.ok && result.success) {
|
if (response.ok && result.success) {
|
||||||
const nextSuggestions = (result.data as NodeSuggestion[])
|
const nextSuggestions = (result.data as NodeSuggestion[])
|
||||||
@@ -229,7 +229,9 @@ export default function NodeSearchModal({
|
|||||||
background: '#141414',
|
background: '#141414',
|
||||||
border: '1px solid #2a2a2a',
|
border: '1px solid #2a2a2a',
|
||||||
borderRadius: '12px',
|
borderRadius: '12px',
|
||||||
overflow: 'hidden',
|
overflowX: 'hidden',
|
||||||
|
overflowY: 'auto',
|
||||||
|
maxHeight: 'min(60vh, 560px)',
|
||||||
boxShadow: '0 24px 48px -12px rgba(0,0,0,0.6)',
|
boxShadow: '0 24px 48px -12px rgba(0,0,0,0.6)',
|
||||||
}}>
|
}}>
|
||||||
{suggestions.map((suggestion, index) => (
|
{suggestions.map((suggestion, index) => (
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ export default function SearchModal({ isOpen, onClose, onNodeSelect, existingFil
|
|||||||
|
|
||||||
const fetchSuggestions = async () => {
|
const fetchSuggestions = async () => {
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`/api/nodes/search?q=${encodeURIComponent(searchQuery)}&limit=10`);
|
const response = await fetch(`/api/nodes/search?q=${encodeURIComponent(searchQuery)}&limit=20`);
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
|
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
@@ -305,7 +305,9 @@ export default function SearchModal({ isOpen, onClose, onNodeSelect, existingFil
|
|||||||
background: var(--rah-bg-panel);
|
background: var(--rah-bg-panel);
|
||||||
border: 1px solid var(--rah-border-strong);
|
border: 1px solid var(--rah-border-strong);
|
||||||
border-radius: 16px;
|
border-radius: 16px;
|
||||||
overflow: hidden;
|
overflow-x: hidden;
|
||||||
|
overflow-y: auto;
|
||||||
|
max-height: min(60vh, 560px);
|
||||||
box-shadow:
|
box-shadow:
|
||||||
0 0 0 1px rgba(255, 255, 255, 0.04),
|
0 0 0 1px rgba(255, 255, 255, 0.04),
|
||||||
0 24px 48px -12px rgba(0, 0, 0, 0.6);
|
0 24px 48px -12px rgba(0, 0, 0, 0.6);
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import { getSQLiteClient } from './sqlite-client';
|
||||||
|
import { normalizeDimensionName } from './quality';
|
||||||
|
|
||||||
|
export function getUnknownDimensions(values: string[]): string[] {
|
||||||
|
if (values.length === 0) return [];
|
||||||
|
|
||||||
|
const sqlite = getSQLiteClient();
|
||||||
|
const placeholders = values.map(() => '?').join(', ');
|
||||||
|
const result = sqlite.query<{ name: string }>(
|
||||||
|
`SELECT name FROM dimensions WHERE name IN (${placeholders})`,
|
||||||
|
values
|
||||||
|
);
|
||||||
|
|
||||||
|
const existing = new Set(
|
||||||
|
result.rows
|
||||||
|
.map(row => (typeof row.name === 'string' ? normalizeDimensionName(row.name) : ''))
|
||||||
|
.filter(Boolean)
|
||||||
|
);
|
||||||
|
|
||||||
|
return values.filter(value => !existing.has(normalizeDimensionName(value)));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatUnknownDimensionsError(values: string[]): string {
|
||||||
|
if (values.length === 1) {
|
||||||
|
return `Unknown dimension: "${values[0]}". Create it first or use an existing dimension.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `Unknown dimensions: ${values.map(value => `"${value}"`).join(', ')}. Create them first or use existing dimensions.`;
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ import { getSQLiteClient } from './sqlite-client';
|
|||||||
import { Node, NodeFilters } from '@/types/database';
|
import { Node, NodeFilters } from '@/types/database';
|
||||||
import { eventBroadcaster } from '../events';
|
import { eventBroadcaster } from '../events';
|
||||||
import { EmbeddingService } from '@/services/embeddings';
|
import { EmbeddingService } from '@/services/embeddings';
|
||||||
|
import { scoreNodeSearchMatch } from './searchRanking';
|
||||||
|
|
||||||
type NodeRow = Node & { dimensions_json: string };
|
type NodeRow = Node & { dimensions_json: string };
|
||||||
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
|
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
|
||||||
@@ -496,17 +497,23 @@ export class NodeService {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
const searchLimit = Math.max(limit + offset, Math.min(limit * 5, 100));
|
const searchLimit = Math.max(limit + offset, Math.min(limit * 20, 250));
|
||||||
let rankedRows = this.searchNodesFts(sqlite, search, filters, searchLimit);
|
const candidateRows = new Map<number, NodeSearchRow>();
|
||||||
|
|
||||||
if (rankedRows.length === 0) {
|
for (const row of this.searchNodesFts(sqlite, search, filters, searchLimit)) {
|
||||||
rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit);
|
candidateRows.set(row.id, row);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (rankedRows.length === 0) {
|
for (const row of this.searchNodesLike(sqlite, search, filters, searchLimit)) {
|
||||||
rankedRows = this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit);
|
if (!candidateRows.has(row.id)) candidateRows.set(row.id, row);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (const row of this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit)) {
|
||||||
|
if (!candidateRows.has(row.id)) candidateRows.set(row.id, row);
|
||||||
|
}
|
||||||
|
|
||||||
|
let rankedRows = Array.from(candidateRows.values());
|
||||||
|
|
||||||
if ((filters.searchMode ?? 'standard') === 'hybrid') {
|
if ((filters.searchMode ?? 'standard') === 'hybrid') {
|
||||||
const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
|
const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
|
||||||
if (vectorRows.length > 0) {
|
if (vectorRows.length > 0) {
|
||||||
@@ -514,6 +521,11 @@ export class NodeService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rankedRows = rankedRows
|
||||||
|
.map(row => ({ row, score: scoreNodeSearchMatch(row, search) }))
|
||||||
|
.sort((a, b) => b.score - a.score)
|
||||||
|
.map(entry => entry.row);
|
||||||
|
|
||||||
return rankedRows
|
return rankedRows
|
||||||
.slice(offset, offset + limit)
|
.slice(offset, offset + limit)
|
||||||
.map(row => this.mapNodeRow(row));
|
.map(row => this.mapNodeRow(row));
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
import type { Node } from '@/types/database';
|
||||||
|
|
||||||
|
function normalizeSearchText(value: string): string {
|
||||||
|
return value
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9]+/g, ' ')
|
||||||
|
.replace(/\s+/g, ' ')
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function getQueryTerms(query: string): string[] {
|
||||||
|
return normalizeSearchText(query).split(' ').filter(term => term.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function countOccurrences(text: string, term: string): number {
|
||||||
|
if (!text || !term) return 0;
|
||||||
|
const matches = text.match(new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'g'));
|
||||||
|
return matches ? matches.length : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function orderedTermMatches(text: string, terms: string[]): boolean {
|
||||||
|
let position = 0;
|
||||||
|
for (const term of terms) {
|
||||||
|
const index = text.indexOf(term, position);
|
||||||
|
if (index === -1) return false;
|
||||||
|
position = index + term.length;
|
||||||
|
}
|
||||||
|
return terms.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function scoreNodeSearchMatch(node: Pick<Node, 'title' | 'description' | 'source' | 'updated_at'>, query: string): number {
|
||||||
|
const normalizedQuery = normalizeSearchText(query);
|
||||||
|
const normalizedTitle = normalizeSearchText(node.title || '');
|
||||||
|
const normalizedDescription = normalizeSearchText(node.description || '');
|
||||||
|
const normalizedSource = normalizeSearchText(node.source || '');
|
||||||
|
const terms = getQueryTerms(query);
|
||||||
|
|
||||||
|
let score = 0;
|
||||||
|
|
||||||
|
if (normalizedTitle === normalizedQuery) score += 2000;
|
||||||
|
if (normalizedTitle.startsWith(normalizedQuery)) score += 1200;
|
||||||
|
if (normalizedTitle.includes(normalizedQuery)) score += 700;
|
||||||
|
if (orderedTermMatches(normalizedTitle, terms)) score += 500;
|
||||||
|
if (terms.length > 0 && terms.every(term => normalizedTitle.includes(term))) score += 350;
|
||||||
|
|
||||||
|
if (normalizedDescription.includes(normalizedQuery)) score += 180;
|
||||||
|
if (orderedTermMatches(normalizedDescription, terms)) score += 120;
|
||||||
|
if (normalizedSource.includes(normalizedQuery)) score += 90;
|
||||||
|
|
||||||
|
for (const term of terms) {
|
||||||
|
score += countOccurrences(normalizedTitle, term) * 40;
|
||||||
|
score += countOccurrences(normalizedDescription, term) * 8;
|
||||||
|
score += countOccurrences(normalizedSource, term) * 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (node.updated_at) {
|
||||||
|
score += new Date(node.updated_at).getTime() / 1e13;
|
||||||
|
}
|
||||||
|
|
||||||
|
return score;
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@ import { z } from 'zod';
|
|||||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||||
|
|
||||||
export const createDimensionTool = tool({
|
export const createDimensionTool = tool({
|
||||||
description: 'Create a new dimension. Always provide a description explaining what belongs in this category.',
|
description: 'Create a new dimension only when the user explicitly instructs you to do so. Always provide a description explaining what belongs in this category.',
|
||||||
inputSchema: z.object({
|
inputSchema: z.object({
|
||||||
name: z.string().describe('Dimension name'),
|
name: z.string().describe('Dimension name'),
|
||||||
description: z.string().min(1).max(500).describe('Dimension description explaining what content belongs in this dimension (required, max 500 characters)')
|
description: z.string().min(1).max(500).describe('Dimension description explaining what content belongs in this dimension (required, max 500 characters)')
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { z } from 'zod';
|
|||||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||||
|
|
||||||
export const queryDimensionsTool = tool({
|
export const queryDimensionsTool = tool({
|
||||||
description: 'List dimensions with node counts. Use this to inspect the user\'s organizational categories.',
|
description: 'List the existing canonical dimensions with node counts. Use this before assigning dimensions to nodes. Do not invent new dimensions without explicit user instruction.',
|
||||||
inputSchema: z.object({
|
inputSchema: z.object({
|
||||||
filters: z.object({
|
filters: z.object({
|
||||||
search: z.string().describe('Search term to match against dimension names').optional(),
|
search: z.string().describe('Search term to match against dimension names').optional(),
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { z } from 'zod';
|
|||||||
import { nodeService } from '@/services/database/nodes';
|
import { nodeService } from '@/services/database/nodes';
|
||||||
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
||||||
import type { Node } from '@/types/database';
|
import type { Node } from '@/types/database';
|
||||||
|
import { scoreNodeSearchMatch } from '@/services/database/searchRanking';
|
||||||
|
|
||||||
type QueryNodeFilters = {
|
type QueryNodeFilters = {
|
||||||
dimensions?: string[];
|
dimensions?: string[];
|
||||||
@@ -14,55 +15,6 @@ type QueryNodeFilters = {
|
|||||||
eventBefore?: string;
|
eventBefore?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
function extractSearchTerms(query: string): string[] {
|
|
||||||
const stopWords = new Set(['a', 'an', 'and', 'for', 'from', 'in', 'of', 'on', 'or', 'recent', 'the', 'to', 'with']);
|
|
||||||
|
|
||||||
const rawTerms = query
|
|
||||||
.toLowerCase()
|
|
||||||
.replace(/[^a-z0-9\s]+/g, ' ')
|
|
||||||
.split(/\s+/)
|
|
||||||
.map(term => term.trim())
|
|
||||||
.filter(Boolean);
|
|
||||||
|
|
||||||
const terms = new Set<string>();
|
|
||||||
for (const term of rawTerms) {
|
|
||||||
if (!stopWords.has(term) && term.length >= 3) {
|
|
||||||
terms.add(term);
|
|
||||||
}
|
|
||||||
const alphaParts = term.replace(/\d+/g, ' ').split(/\s+/).filter(Boolean);
|
|
||||||
for (const part of alphaParts) {
|
|
||||||
if (!stopWords.has(part) && part.length >= 3) {
|
|
||||||
terms.add(part);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Array.from(terms).slice(0, 8);
|
|
||||||
}
|
|
||||||
|
|
||||||
function scoreNodeForSearch(node: Node, searchTerm: string): number {
|
|
||||||
const normalizedSearch = searchTerm.toLowerCase();
|
|
||||||
const title = (node.title || '').toLowerCase();
|
|
||||||
const description = (node.description || '').toLowerCase();
|
|
||||||
const source = (node.source || '').toLowerCase();
|
|
||||||
const terms = extractSearchTerms(searchTerm);
|
|
||||||
|
|
||||||
let score = 0;
|
|
||||||
|
|
||||||
if (title === normalizedSearch) score += 100;
|
|
||||||
if (title.includes(normalizedSearch)) score += 40;
|
|
||||||
if (description.includes(normalizedSearch)) score += 20;
|
|
||||||
if (source.includes(normalizedSearch)) score += 10;
|
|
||||||
|
|
||||||
for (const term of terms) {
|
|
||||||
if (title.includes(term)) score += 8;
|
|
||||||
if (description.includes(term)) score += 3;
|
|
||||||
if (source.includes(term)) score += 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
return score;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const queryNodesTool = tool({
|
export const queryNodesTool = tool({
|
||||||
description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Do not use dimensions to constrain keyword search unless the user is explicitly asking about a known dimension.',
|
description: 'Search nodes across title, description, and source. For free-text lookups, search the graph broadly and prioritize title/description matches. Do not use dimensions to constrain keyword search unless the user is explicitly asking about a known dimension.',
|
||||||
inputSchema: z.object({
|
inputSchema: z.object({
|
||||||
@@ -150,7 +102,7 @@ export const queryNodesTool = tool({
|
|||||||
|
|
||||||
if (searchTerm) {
|
if (searchTerm) {
|
||||||
safeNodes = safeNodes
|
safeNodes = safeNodes
|
||||||
.map(node => ({ node, score: scoreNodeForSearch(node, searchTerm) }))
|
.map(node => ({ node, score: scoreNodeSearchMatch(node, searchTerm) }))
|
||||||
.sort((a, b) => b.score - a.score || b.node.updated_at.localeCompare(a.node.updated_at))
|
.sort((a, b) => b.score - a.score || b.node.updated_at.localeCompare(a.node.updated_at))
|
||||||
.slice(0, limit)
|
.slice(0, limit)
|
||||||
.map(entry => entry.node);
|
.map(entry => entry.node);
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('@/services/database/sqlite-client', () => ({
|
||||||
|
getSQLiteClient: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||||
|
import { formatUnknownDimensionsError, getUnknownDimensions } from '@/services/database/dimensionValidation';
|
||||||
|
|
||||||
|
describe('dimensionValidation', () => {
|
||||||
|
it('returns dimensions missing from the canonical dimensions table', () => {
|
||||||
|
vi.mocked(getSQLiteClient).mockReturnValue({
|
||||||
|
query: vi.fn().mockReturnValue({
|
||||||
|
rows: [{ name: 'ra-h' }, { name: 'ai' }],
|
||||||
|
}),
|
||||||
|
} as unknown as ReturnType<typeof getSQLiteClient>);
|
||||||
|
|
||||||
|
expect(getUnknownDimensions(['ra-h', 'Building RA-H — Personal Knowledge Graph', 'ai'])).toEqual([
|
||||||
|
'Building RA-H — Personal Knowledge Graph',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('formats a clear error for unknown dimensions', () => {
|
||||||
|
expect(formatUnknownDimensionsError(['Building RA-H — Personal Knowledge Graph']))
|
||||||
|
.toBe('Unknown dimension: "Building RA-H — Personal Knowledge Graph". Create it first or use an existing dimension.');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { scoreNodeSearchMatch } from '@/services/database/searchRanking';
|
||||||
|
|
||||||
|
describe('scoreNodeSearchMatch', () => {
|
||||||
|
it('strongly prefers the closest title match for hub-node queries', () => {
|
||||||
|
const query = 'building ra-h';
|
||||||
|
|
||||||
|
const hubScore = scoreNodeSearchMatch({
|
||||||
|
title: 'Building RA-H — Personal Knowledge Graph',
|
||||||
|
description: 'Brad project hub',
|
||||||
|
source: '',
|
||||||
|
updated_at: '2026-03-23T00:00:00.000Z',
|
||||||
|
}, query);
|
||||||
|
|
||||||
|
const broadScore = scoreNodeSearchMatch({
|
||||||
|
title: 'the ra-h project',
|
||||||
|
description: 'Foundational project document for RA-H',
|
||||||
|
source: '',
|
||||||
|
updated_at: '2026-03-23T00:00:00.000Z',
|
||||||
|
}, query);
|
||||||
|
|
||||||
|
expect(hubScore).toBeGreaterThan(broadScore);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
import { afterEach, vi } from 'vitest';
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user