SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: sync runtime search and schema quality updates from app repo

Browse Source 
- port retrieval, validation, and eval improvements relevant to os
- align prompts and dimensions with the flat single-agent model
- replace the old eval suite with the focused core scenarios

Generated with Codex
This commit is contained in:
“BeeRad”
2026-03-15 14:55:45 +11:00
parent 053c163e31
commit 4c75df101f
57 changed files with 1809 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/database/chunks.ts
+176 -4
View File
@@ -1,6 +1,75 @@
import { getSQLiteClient } from './sqlite-client';
import { Chunk, ChunkData } from '@/types/database';
type RankedChunk = Chunk & { similarity: number };
function sanitizeFtsQuery(input: string): string {
return input
.replace(/['"()*:^~{}[\]]/g, ' ')
.trim()
.split(/\s+/)
.filter(word => word.length > 0 && !/^(AND|OR|NOT|NEAR)$/i.test(word))
.join(' ');
}
function extractRelaxedSearchTerms(query: string): string[] {
const stopWords = new Set([
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'being', 'briefly', 'by', 'find',
'focused', 'for', 'from', 'in', 'inside', 'is', 'it', 'me', 'my', 'of', 'on',
'or', 'quote', 'search', 'solutions', 'specific', 'that', 'the', 'then', 'this',
'to', 'transcript', 'with'
]);
const rawTerms = query
.toLowerCase()
.replace(/[^a-z0-9\s]/g, ' ')
.split(/\s+/)
.map(term => term.trim())
.filter(term => term.length >= 4 && !stopWords.has(term));
const expanded = new Set<string>();
for (const term of rawTerms) {
expanded.add(term);
if (term.length >= 6) {
expanded.add(term.slice(0, 5));
expanded.add(term.slice(0, 6));
}
if (term.endsWith('ing') && term.length > 6) {
expanded.add(term.slice(0, -3));
}
if (term.endsWith('tion') && term.length > 7) {
expanded.add(term.slice(0, -4));
}
if (term.endsWith('ions') && term.length > 7) {
expanded.add(term.slice(0, -4));
}
}
return Array.from(expanded).slice(0, 12);
}
function reciprocalRankFuse<T extends { id: number }>(rankedLists: T[][], limit: number): T[] {
const scores = new Map<number, { score: number; item: T }>();
const k = 60;
rankedLists.forEach((list) => {
list.forEach((item, index) => {
const entry = scores.get(item.id);
const score = 1 / (k + index + 1);
if (entry) {
entry.score += score;
} else {
scores.set(item.id, { score, item });
}
});
});
return Array.from(scores.values())
.sort((a, b) => b.score - a.score)
.slice(0, limit)
.map(entry => entry.item);
}
export class ChunkService {
async getChunksByNodeId(nodeId: number): Promise<Chunk[]> {
const sqlite = getSQLiteClient();
@@ -157,7 +226,18 @@ export class ChunkService {
fallbackQuery?: string
): Promise<Array<Chunk & { similarity: number }>> {
try {
return await this.searchChunksSQLite(queryEmbedding, similarityThreshold, matchCount, nodeIds);
const vectorResults = await this.searchChunksSQLite(queryEmbedding, similarityThreshold, matchCount, nodeIds);
if (!fallbackQuery) {
return vectorResults;
}
const textResults = await this.textSearchFallback(fallbackQuery, matchCount, nodeIds);
if (textResults.length === 0) {
return vectorResults;
}
return reciprocalRankFuse<RankedChunk>([vectorResults, textResults], matchCount);
} catch (error) {
console.warn('Vector search failed, attempting text fallback:', error);
if (fallbackQuery) {
@@ -255,6 +335,12 @@ export class ChunkService {
): Promise<Array<Chunk & { similarity: number }>> {
const sqlite = getSQLiteClient();
const startTime = Date.now();
const ftsResults = this.ftsSearchChunks(sqlite, query, matchCount, nodeIds);
if (ftsResults.length > 0) {
const searchTime = Date.now() - startTime;
console.log(`📝 Text fallback (FTS): ${ftsResults.length} chunks found, time=${searchTime}ms`);
return ftsResults;
}
// Clean query for LIKE search
const cleanQuery = query.trim().toLowerCase();
@@ -285,12 +371,98 @@ export class ChunkService {
textQuery += ` ORDER BY LENGTH(text) ASC LIMIT ?`;
params.push(String(matchCount));
const result = sqlite.query<Chunk & { similarity: number }>(textQuery, params);
const result = sqlite.query<RankedChunk>(textQuery, params);
const searchTime = Date.now() - startTime;
console.log(`📝 Text fallback: ${result.rows.length} chunks found, time=${searchTime}ms`);
return result.rows;
if (result.rows.length > 0) {
return result.rows;
}
const relaxedTerms = extractRelaxedSearchTerms(query);
if (relaxedTerms.length === 0) {
return [];
}
const scoreClauses = relaxedTerms.map(() => 'CASE WHEN LOWER(text) LIKE ? THEN 1 ELSE 0 END');
const scoreParams = relaxedTerms.map(term => `%${term}%`);
const relaxedParams = [...scoreParams];
let relaxedQuery = `
SELECT *,
(${scoreClauses.join(' + ')}) * 0.7 AS similarity
FROM chunks
WHERE (${scoreClauses.join(' + ')}) > 0
`;
if (nodeIds && nodeIds.length > 0) {
relaxedQuery += ` AND node_id IN (${nodeIds.map(() => '?').join(',')})`;
relaxedParams.push(...nodeIds.map(String));
}
relaxedQuery += ' ORDER BY similarity DESC, chunk_idx ASC LIMIT ?';
relaxedParams.push(String(matchCount));
const relaxedResult = sqlite.query<RankedChunk>(
relaxedQuery,
[...scoreParams, ...relaxedParams]
);
const relaxedSearchTime = Date.now() - startTime;
console.log(`📝 Text fallback (relaxed): ${relaxedResult.rows.length} chunks found, time=${relaxedSearchTime}ms`);
return relaxedResult.rows;
}
private ftsSearchChunks(
sqlite: ReturnType<typeof getSQLiteClient>,
query: string,
matchCount: number,
nodeIds?: number[]
): RankedChunk[] {
const ftsExists = sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks_fts'"
).get();
if (!ftsExists) return [];
const ftsQuery = sanitizeFtsQuery(query);
if (!ftsQuery) return [];
try {
if (nodeIds && nodeIds.length > 0) {
const result = sqlite.query<RankedChunk>(`
SELECT c.*, 0.85 as similarity
FROM chunks c
WHERE c.node_id IN (${nodeIds.map(() => '?').join(',')})
AND c.id IN (
SELECT rowid
FROM chunks_fts
WHERE chunks_fts MATCH ?
)
ORDER BY c.chunk_idx ASC
LIMIT ?
`, [...nodeIds, ftsQuery, matchCount]);
return result.rows;
}
const result = sqlite.query<RankedChunk>(`
WITH fts_matches AS (
SELECT rowid, rank
FROM chunks_fts
WHERE chunks_fts MATCH ?
LIMIT ?
)
SELECT c.*, 0.85 as similarity
FROM fts_matches fm
JOIN chunks c ON c.id = fm.rowid
ORDER BY fm.rank
`, [ftsQuery, matchCount]);
return result.rows;
} catch (error) {
console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error);
return [];
}
}
async getChunkCount(): Promise<number> {
src/services/database/dimensionService.ts
+14 -79
View File
@@ -1,7 +1,4 @@
import { getSQLiteClient } from './sqlite-client';
import { openai as openaiProvider } from '@ai-sdk/openai';
import { generateText } from 'ai';
import { hasValidOpenAiKey } from '../storage/apiKeys';
export interface Dimension {
name: string;
@@ -18,85 +15,23 @@ export interface LockedDimension {
export class DimensionService {
/**
* Get all locked (priority) dimensions with their descriptions
* Legacy compatibility shim. Dimensions are now flat, so there is no locked subset.
*/
static async getLockedDimensions(): Promise<LockedDimension[]> {
const sqlite = getSQLiteClient();
const result = sqlite.query(`
WITH dimension_counts AS (
SELECT nd.dimension, COUNT(*) AS count
FROM node_dimensions nd
GROUP BY nd.dimension
)
SELECT
d.name,
d.description,
COALESCE(dc.count, 0) AS count
FROM dimensions d
LEFT JOIN dimension_counts dc ON dc.dimension = d.name
WHERE d.is_priority = 1
ORDER BY d.name ASC
`);
return result.rows.map((row: any) => ({
name: row.name,
description: row.description,
count: Number(row.count)
}));
return [];
}
/**
* Automatically assign locked dimensions + suggest keyword dimensions
* Returns { locked: string[], keywords: string[] }
*
* IMPORTANT: Returns empty result immediately if no valid API key is configured.
* This prevents slow node creation when OpenAI is unavailable.
* Automatic special-dimension assignment has been removed. Callers must provide dimensions explicitly.
*/
static async assignDimensions(nodeData: {
title: string;
notes?: string;
content?: string;
link?: string;
description?: string;
}): Promise<{ locked: string[]; keywords: string[] }> {
// Fast path: skip AI if no valid API key
if (!hasValidOpenAiKey()) {
console.log(`[DimensionAssignment] No valid OpenAI key, skipping for: "${nodeData.title}"`);
return { locked: [], keywords: [] };
}
try {
const lockedDimensions = await this.getLockedDimensions();
if (lockedDimensions.length === 0) {
console.log('[DimensionAssignment] No locked dimensions available');
return { locked: [], keywords: [] };
}
const prompt = this.buildAssignmentPrompt(nodeData, lockedDimensions);
console.log(`[DimensionAssignment] Processing: "${nodeData.title}"`);
const response = await generateText({
model: openaiProvider('gpt-4o-mini'),
prompt,
maxOutputTokens: 300, // Increased to accommodate more dimensions
temperature: 0.1,
});
console.log(`[DimensionAssignment] AI Response:\n${response.text}`);
const result = this.parseAssignmentResponse(response.text, lockedDimensions);
console.log(`[DimensionAssignment] Locked: ${result.locked.join(', ') || 'none'}`);
console.log(`[DimensionAssignment] Keywords: ${result.keywords.join(', ') || 'none'}`);
return result;
} catch (error) {
console.error('[DimensionAssignment] Error:', error);
return { locked: [], keywords: [] };
}
console.log(`[DimensionAssignment] Skipped for "${nodeData.title}" — flat dimensions require explicit assignment.`);
return { locked: [], keywords: [] };
}
/**
@@ -153,7 +88,7 @@ export class DimensionService {
}
/**
* Build AI prompt for dimension assignment (locked dimensions only)
* Legacy no-op prompt builder retained only for backward compatibility.
*/
private static buildAssignmentPrompt(
nodeData: { title: string; notes?: string; link?: string; description?: string },
@@ -162,13 +97,13 @@ export class DimensionService {
// Use description as primary context, content as fallback
let nodeContextSection: string;
if (nodeData.description) {
const contentPreview = nodeData.notes?.slice(0, 500) || '';
const notesPreview = nodeData.notes?.slice(0, 500) || '';
nodeContextSection = `DESCRIPTION: ${nodeData.description}
NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`;
NOTES PREVIEW: ${notesPreview}${nodeData.notes && nodeData.notes.length > 500 ? '...' : ''}`;
} else {
const contentPreview = nodeData.notes?.slice(0, 2000) || '';
nodeContextSection = `NOTES: ${contentPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`;
const notesPreview = nodeData.notes?.slice(0, 2000) || '';
nodeContextSection = `NOTES: ${notesPreview}${nodeData.notes && nodeData.notes.length > 2000 ? '...' : ''}`;
}
// Include ALL locked dimensions, using fallback text for those without descriptions
@@ -181,7 +116,7 @@ NOTES PREVIEW: ${contentPreview}${nodeData.notes && nodeData.notes.length > 500
})
.join('\n---\n');
return `You are categorizing a knowledge node into locked dimensions.
return `Dimensions are now flat categories with no locked subset.
=== NODE TO CATEGORIZE ===
Title: ${nodeData.title}
@@ -203,7 +138,7 @@ LOCKED:
}
/**
* Parse AI response and extract locked dimensions
* Legacy no-op parser retained only for backward compatibility.
*/
private static parseAssignmentResponse(
response: string,
@@ -251,4 +186,4 @@ LOCKED:
}
}
export const dimensionService = new DimensionService();
export const dimensionService = new DimensionService();
src/services/database/edges.ts
+63 -15
View File
@@ -2,9 +2,11 @@ import { getSQLiteClient } from './sqlite-client';
import { Edge, EdgeContext, EdgeData, EdgeCreatedVia, NodeConnection, Node } from '@/types/database';
import { eventBroadcaster } from '../events';
import { nodeService } from './nodes';
import { getOpenAiKey } from '../storage/apiKeys';
import { generateText } from 'ai';
import { createOpenAI } from '@ai-sdk/openai';
import { z } from 'zod';
import { validateEdgeExplanation } from './quality';
const inferredEdgeContextSchema = z.object({
type: z.enum(['created_by', 'part_of', 'source_of', 'related_to']),
@@ -53,7 +55,7 @@ async function inferEdgeContext(params: {
// If no API key is configured, degrade gracefully.
// We still enforce explanation, but fall back to "related_to" classification.
const apiKey = process.env.OPENAI_API_KEY;
const apiKey = getOpenAiKey();
if (!apiKey) {
return { type: 'related_to', confidence: 0.0, swap_direction: false };
}
@@ -118,11 +120,11 @@ async function autoInferEdge(params: {
}): Promise<{ explanation: string; type: EdgeContext['type']; confidence: number; swap_direction: boolean }> {
const { fromNode, toNode } = params;
const apiKey = process.env.OPENAI_API_KEY;
const apiKey = getOpenAiKey();
if (!apiKey) {
// Fallback without AI
return {
explanation: `Related to ${toNode.title}`,
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
type: 'related_to',
confidence: 0.0,
swap_direction: false,
@@ -181,8 +183,8 @@ async function autoInferEdge(params: {
const parsed = schema.safeParse(parsedJson);
if (!parsed.success) {
return {
explanation: `Related to ${toNode.title}`,
return {
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
type: 'related_to',
confidence: 0.2,
swap_direction: false,
@@ -193,7 +195,7 @@ async function autoInferEdge(params: {
} catch (error) {
console.warn('[edges] autoInferEdge failed; falling back', error);
return {
explanation: `Related to ${toNode.title}`,
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
type: 'related_to',
confidence: 0.2,
swap_direction: false,
@@ -205,13 +207,33 @@ export class EdgeService {
async getEdges(): Promise<Edge[]> {
const sqlite = getSQLiteClient();
const result = sqlite.query<Edge>('SELECT * FROM edges ORDER BY created_at DESC');
return result.rows;
return result.rows.map((row: any) => {
let context: any = row.context;
if (typeof context === 'string') {
try {
context = JSON.parse(context);
} catch {
// Keep raw context string if JSON parsing fails.
}
}
return { ...row, context };
});
}
async getEdgeById(id: number): Promise<Edge | null> {
const sqlite = getSQLiteClient();
const result = sqlite.query<Edge>('SELECT * FROM edges WHERE id = ?', [id]);
return result.rows[0] || null;
const row: any = result.rows[0];
if (!row) return null;
let context: any = row.context;
if (typeof context === 'string') {
try {
context = JSON.parse(context);
} catch {
// Keep raw context string if JSON parsing fails.
}
}
return { ...row, context };
}
async createEdge(edgeData: EdgeData): Promise<Edge> {
@@ -249,8 +271,12 @@ export class EdgeService {
};
} else if (edgeData.skip_inference) {
inferred = { type: 'related_to' as const, confidence: 0.0, swap_direction: false };
if (!explanation) explanation = `Related to ${toNode.title}`;
if (!explanation) explanation = `Connection to ${toNode.title}; exact relationship uncertain.`;
} else {
const explanationError = validateEdgeExplanation(explanation);
if (explanationError) {
throw new Error(explanationError);
}
inferred = await inferEdgeContext({ explanation, fromNode, toNode });
}
@@ -267,14 +293,15 @@ export class EdgeService {
};
const result = sqlite.prepare(`
INSERT INTO edges (from_node_id, to_node_id, context, source, created_at)
VALUES (?, ?, ?, ?, ?)
INSERT INTO edges (from_node_id, to_node_id, context, source, created_at, explanation)
VALUES (?, ?, ?, ?, ?, ?)
`).run(
finalFromId,
finalToId,
JSON.stringify(context),
edgeData.source,
now
now,
explanation
);
const edgeId = Number(result.lastInsertRowid);
@@ -316,6 +343,10 @@ export class EdgeService {
if (!explanation) {
throw new Error('Edge explanation is required');
}
const explanationError = validateEdgeExplanation(explanation);
if (explanationError) {
throw new Error(explanationError);
}
const existingEdge = await this.getEdgeById(id);
if (!existingEdge) {
@@ -342,8 +373,13 @@ export class EdgeService {
(existingContext?.created_via as EdgeCreatedVia) ||
'ui';
// Note: On update, we don't swap direction - the edge already exists with its direction.
// We only update the type and confidence based on the new explanation.
const nextFromId = inferred.swap_direction ? existingEdge.to_node_id : existingEdge.from_node_id;
const nextToId = inferred.swap_direction ? existingEdge.from_node_id : existingEdge.to_node_id;
if (inferred.swap_direction) {
updates.from_node_id = nextFromId;
updates.to_node_id = nextToId;
}
updates.context = {
...existingContext,
...incomingContext,
@@ -368,6 +404,18 @@ export class EdgeService {
}
});
if (Object.prototype.hasOwnProperty.call(updates, 'explanation')) {
const rawExplanation = (updates as any).explanation;
if (typeof rawExplanation === 'string') {
const explanationError = validateEdgeExplanation(rawExplanation);
if (explanationError) {
throw new Error(explanationError);
}
updateFields.push('explanation = ?');
params.push(rawExplanation.trim());
}
}
if (updateFields.length === 0) {
throw new Error('No valid fields to update');
}
@@ -429,7 +477,7 @@ export class EdgeService {
WHEN e.from_node_id = ? THEN n_to.title
ELSE n_from.title
END as connected_node_title,
CASE
CASE
WHEN e.from_node_id = ? THEN n_to.notes
ELSE n_from.notes
END as connected_node_notes,
src/services/database/nodes.ts
+311 -12
View File
@@ -1,6 +1,44 @@
import { getSQLiteClient } from './sqlite-client';
import { Node, NodeFilters } from '@/types/database';
import { eventBroadcaster } from '../events';
import { EmbeddingService } from '@/services/embeddings';
type NodeRow = Node & { dimensions_json: string };
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
function sanitizeFtsQuery(input: string): string {
return input
.replace(/['"()*:^~{}[\]]/g, ' ')
.trim()
.split(/\s+/)
.filter(word => word.length > 0 && !/^(AND|OR|NOT|NEAR)$/i.test(word))
.join(' ');
}
function reciprocalRankFuse<T extends { id: number }>(
rankedLists: T[][],
limit: number,
): T[] {
const scores = new Map<number, { score: number; item: T }>();
const k = 60;
rankedLists.forEach((list) => {
list.forEach((item, index) => {
const existing = scores.get(item.id);
const score = 1 / (k + index + 1);
if (existing) {
existing.score += score;
} else {
scores.set(item.id, { score, item });
}
});
});
return Array.from(scores.values())
.sort((a, b) => b.score - a.score)
.slice(0, limit)
.map(entry => entry.item);
}
export class NodeService {
async getNodes(filters: NodeFilters = {}): Promise<Node[]> {
@@ -10,6 +48,11 @@ export class NodeService {
async countNodes(filters: NodeFilters = {}): Promise<number> {
const { dimensions, search, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore } = filters;
if (search?.trim()) {
return this.countSearchNodesSQLite(filters);
}
const sqlite = getSQLiteClient();
let query = `SELECT COUNT(*) as total FROM nodes n WHERE 1=1`;
@@ -52,6 +95,11 @@ export class NodeService {
private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore } = filters;
if (search?.trim()) {
return this.searchNodesSQLite(filters);
}
const sqlite = getSQLiteClient();
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
@@ -135,6 +183,7 @@ export class NodeService {
} else if (sortBy === 'created') {
query += ' ORDER BY n.created_at DESC';
} else if (sortBy === 'event_date') {
// Nodes with event_date first (DESC), then by updated_at for nulls
query += ' ORDER BY n.event_date IS NULL, n.event_date DESC, n.updated_at DESC';
} else {
query += ' ORDER BY n.updated_at DESC';
@@ -150,14 +199,10 @@ export class NodeService {
params.push(offset);
}
const result = sqlite.query<Node & { dimensions_json: string }>(query, params);
const result = sqlite.query<NodeRow>(query, params);
// Parse dimensions_json and metadata back for compatibility
return result.rows.map(row => ({
...row,
dimensions: JSON.parse(row.dimensions_json || '[]'),
metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
}));
return result.rows.map(row => this.mapNodeRow(row));
}
async getNodeById(id: number): Promise<Node | null> {
@@ -177,16 +222,12 @@ export class NodeService {
FROM nodes n
WHERE n.id = ?
`;
const result = sqlite.query<Node & { dimensions_json: string }>(query, [id]);
const result = sqlite.query<NodeRow>(query, [id]);
if (result.rows.length === 0) return null;
const row = result.rows[0];
return {
...row,
dimensions: JSON.parse(row.dimensions_json || '[]'),
metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
};
return this.mapNodeRow(row);
}
async createNode(nodeData: Partial<Node>): Promise<Node> {
@@ -363,6 +404,264 @@ export class NodeService {
return this.getNodes({ search: searchTerm, limit });
}
private mapNodeRow(row: NodeRow): Node {
return {
...row,
dimensions: JSON.parse(row.dimensions_json || '[]'),
metadata: row.metadata ? (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) : null,
};
}
private buildNodeFilterClauses(filters: NodeFilters, alias = 'n'): { clauses: string[]; params: any[] } {
const {
dimensions,
dimensionsMatch = 'any',
createdAfter,
createdBefore,
eventAfter,
eventBefore,
} = filters;
const clauses: string[] = [];
const params: any[] = [];
if (dimensions && dimensions.length > 0) {
if (dimensionsMatch === 'all' && dimensions.length > 1) {
clauses.push(`(
SELECT COUNT(DISTINCT nd.dimension) FROM node_dimensions nd
WHERE nd.node_id = ${alias}.id
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
) = ?`);
params.push(...dimensions, dimensions.length);
} else {
clauses.push(`EXISTS (
SELECT 1 FROM node_dimensions nd
WHERE nd.node_id = ${alias}.id
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
)`);
params.push(...dimensions);
}
}
if (createdAfter) { clauses.push(`${alias}.created_at >= ?`); params.push(createdAfter); }
if (createdBefore) { clauses.push(`${alias}.created_at < ?`); params.push(createdBefore); }
if (eventAfter) { clauses.push(`${alias}.event_date >= ?`); params.push(eventAfter); }
if (eventBefore) { clauses.push(`${alias}.event_date < ?`); params.push(eventBefore); }
return { clauses, params };
}
private async searchNodesSQLite(filters: NodeFilters): Promise<Node[]> {
const sqlite = getSQLiteClient();
const search = filters.search?.trim();
const limit = Math.min(Math.max(filters.limit ?? 100, 1), 100);
const offset = Math.max(filters.offset ?? 0, 0);
if (!search) {
return [];
}
const searchLimit = Math.max(limit + offset, Math.min(limit * 5, 100));
let rankedRows = this.searchNodesFts(sqlite, search, filters, searchLimit);
if (rankedRows.length === 0) {
rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit);
}
if ((filters.searchMode ?? 'standard') === 'hybrid') {
const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
if (vectorRows.length > 0) {
rankedRows = reciprocalRankFuse<NodeSearchRow>([rankedRows, vectorRows], searchLimit);
}
}
return rankedRows
.slice(offset, offset + limit)
.map(row => this.mapNodeRow(row));
}
private countSearchNodesSQLite(filters: NodeFilters): number {
const sqlite = getSQLiteClient();
const search = filters.search?.trim();
if (!search) return 0;
const ftsQuery = sanitizeFtsQuery(search);
const ftsExists = sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
).get();
const { clauses, params } = this.buildNodeFilterClauses(filters);
if (ftsExists && ftsQuery) {
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const result = sqlite.query<{ total: number }>(`
WITH matched_nodes AS (
SELECT rowid
FROM nodes_fts
WHERE nodes_fts MATCH ?
)
SELECT COUNT(*) as total
FROM matched_nodes mn
JOIN nodes n ON n.id = mn.rowid
${whereClauses}
`, [ftsQuery, ...params]);
return Number(result.rows[0]?.total ?? 0);
}
const words = search.split(/\s+/).filter(Boolean);
let query = `SELECT COUNT(*) as total FROM nodes n WHERE 1=1`;
const queryParams = [...params];
if (clauses.length > 0) {
query += ` AND ${clauses.join(' AND ')}`;
}
for (const word of words) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
}
const result = sqlite.query<{ total: number }>(query, queryParams);
return Number(result.rows[0]?.total ?? 0);
}
private searchNodesFts(
sqlite: ReturnType<typeof getSQLiteClient>,
search: string,
filters: NodeFilters,
limit: number,
): NodeSearchRow[] {
const ftsQuery = sanitizeFtsQuery(search);
if (!ftsQuery) return [];
const ftsExists = sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
).get();
if (!ftsExists) return [];
const { clauses, params } = this.buildNodeFilterClauses(filters);
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
try {
const result = sqlite.query<NodeSearchRow>(`
WITH fts_matches AS (
SELECT rowid, rank
FROM nodes_fts
WHERE nodes_fts MATCH ?
LIMIT ?
)
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
fm.rank
FROM fts_matches fm
JOIN nodes n ON n.id = fm.rowid
${whereClauses}
ORDER BY fm.rank
LIMIT ?
`, [ftsQuery, Math.max(limit * 2, 50), ...params, limit]);
return result.rows;
} catch (error) {
console.warn('[NodeSearch] FTS search failed, falling back to LIKE:', error);
return [];
}
}
private searchNodesLike(
sqlite: ReturnType<typeof getSQLiteClient>,
search: string,
filters: NodeFilters,
limit: number,
): NodeSearchRow[] {
const words = search.split(/\s+/).filter(Boolean);
const { clauses, params } = this.buildNodeFilterClauses(filters);
let query = `
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
FROM nodes n
WHERE 1=1
`;
const queryParams = [...params];
if (clauses.length > 0) {
query += ` AND ${clauses.join(' AND ')}`;
}
for (const word of words) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
}
query += ` ORDER BY
CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 6 END,
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
n.updated_at DESC
LIMIT ?`;
queryParams.push(search, `${search}%`, `%${search}%`, `%${search}%`, `%${search}%`, limit);
const result = sqlite.query<NodeSearchRow>(query, queryParams);
return result.rows;
}
private async searchNodesVector(
sqlite: ReturnType<typeof getSQLiteClient>,
search: string,
filters: NodeFilters,
limit: number,
): Promise<NodeSearchRow[]> {
try {
const embedding = await EmbeddingService.generateQueryEmbedding(search);
if (!EmbeddingService.validateEmbedding(embedding)) {
return [];
}
const vecExists = sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='vec_nodes'"
).get();
if (!vecExists) return [];
const vectorString = `[${embedding.join(',')}]`;
const { clauses, params } = this.buildNodeFilterClauses(filters);
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const result = sqlite.query<NodeSearchRow>(`
WITH vector_matches AS (
SELECT node_id, distance
FROM vec_nodes
WHERE embedding MATCH ?
ORDER BY distance
LIMIT ?
)
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
(1.0 / (1.0 + vm.distance)) AS similarity
FROM vector_matches vm
JOIN nodes n ON n.id = vm.node_id
${whereClauses}
ORDER BY vm.distance
LIMIT ?
`, [vectorString, Math.max(limit * 2, 50), ...params, limit]);
return result.rows;
} catch (error) {
console.warn('[NodeSearch] Vector search unavailable, continuing without it:', error);
return [];
}
}
async getNodeCount(): Promise<number> {
const sqlite = getSQLiteClient();
const result = sqlite.query('SELECT COUNT(*) as count FROM nodes');
src/services/database/quality.ts
+64
View File
@@ -0,0 +1,64 @@
const WEAK_DESCRIPTION_PATTERNS = /\b(discusses|explores|examines|talks about|is about|delves into)\b/i;
const EXPLICIT_ENTITY_PATTERNS = /\b(article|artifact|book|brief|claim|company|conversation|dataset|decision|dimension|document|episode|essay|guide|idea|insight|interview|node|note|paper|person|plan|placeholder|podcast|presentation|project|question|record|research|skill|source|summary|talk|target|test node|thread|tool|transcript|tweet|video|website|workflow)\b/i;
const UNCERTAINTY_PATTERNS = /\b(likely|probably|possibly|appears to be|seems to be|unclear|uncertain)\b/i;
const GENERIC_EDGE_PATTERNS = /^(related|related to|connected|connected to|association|associated with)$/i;
export function normalizeDimensionName(value: string): string {
return value.trim().replace(/\s+/g, ' ');
}
export function normalizeDimensions(values: unknown, max = 5): string[] {
if (!Array.isArray(values)) return [];
const seen = new Set<string>();
const normalized: string[] = [];
for (const value of values) {
if (typeof value !== 'string') continue;
const trimmed = normalizeDimensionName(value);
if (!trimmed) continue;
const key = trimmed.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
normalized.push(trimmed);
if (normalized.length >= max) break;
}
return normalized;
}
export function validateExplicitDescription(description: string): string | null {
const text = description.trim();
if (text.length < 24) {
return 'Description must be explicit and substantial (at least 24 characters).';
}
if (WEAK_DESCRIPTION_PATTERNS.test(text)) {
return 'Description is too vague. State exactly what this is and why it matters.';
}
if (!EXPLICIT_ENTITY_PATTERNS.test(text) && !UNCERTAINTY_PATTERNS.test(text)) {
return 'Description must explicitly identify what this thing is, or state uncertainty explicitly.';
}
return null;
}
export function validateEdgeExplanation(explanation: string): string | null {
const text = explanation.trim();
if (text.length < 8) {
return 'Edge explanation must be explicit enough to describe the relationship.';
}
if (GENERIC_EDGE_PATTERNS.test(text)) {
return 'Edge explanation is too generic. State the actual relationship or explicitly note uncertainty.';
}
return null;
}
export function validateDimensionDescription(description: string): string | null {
const text = description.trim();
if (!text) {
return 'Dimension description is required.';
}
if (text.length > 500) {
return 'Description must be 500 characters or less.';
}
return null;
}
src/services/database/sqlite-client.ts
+60 -84
View File
@@ -19,12 +19,10 @@ class SQLiteClient {
private db: Database.Database;
private config: SQLiteConfig;
private readonly readOnly: boolean;
private readonly embeddingsDisabled: boolean;
private constructor() {
this.config = this.getSQLiteConfig();
this.readOnly = process.env.SQLITE_READONLY === 'true';
this.embeddingsDisabled = process.env.DISABLE_EMBEDDINGS === 'true';
// Initialize database connection
const dbDirectory = path.dirname(this.config.dbPath);
@@ -35,15 +33,13 @@ class SQLiteClient {
? new Database(this.config.dbPath, { readonly: true, fileMustExist: true })
: new Database(this.config.dbPath);
// Load sqlite-vec extension (skip entirely if embeddings are disabled)
if (!this.embeddingsDisabled) {
try {
this.db.loadExtension(this.config.vecExtensionPath);
console.log('SQLite vector extension loaded successfully');
} catch (error) {
// Do not fail hard — allow the app to run without vector features
console.error('Warning: Failed to load vector extension:', error);
}
// Load sqlite-vec extension
try {
this.db.loadExtension(this.config.vecExtensionPath);
console.log('SQLite vector extension loaded successfully');
} catch (error) {
// Do not fail hard — allow the app to run without vector features
console.error('Warning: Failed to load vector extension:', error);
}
// Configure SQLite settings
@@ -61,11 +57,9 @@ class SQLiteClient {
this.db.pragma('temp_store = memory');
this.db.pragma('busy_timeout = 5000');
// Ensure vector virtual tables are present and healthy (skip if disabled)
if (!this.embeddingsDisabled) {
this.ensureVectorTables();
this.healVectorTablesIfCorrupt();
}
// Ensure vector virtual tables are present and healthy
this.ensureVectorTables();
this.healVectorTablesIfCorrupt();
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
this.ensureLoggingAndMemorySchema();
@@ -140,9 +134,7 @@ class SQLiteClient {
} as DatabaseError;
}
// Proactively validate/repair vec vtables before any write transaction
if (!this.embeddingsDisabled) {
this.healVectorTablesIfCorrupt();
}
this.healVectorTablesIfCorrupt();
const txn = this.db.transaction(callback);
try {
return txn();
@@ -162,9 +154,6 @@ class SQLiteClient {
}
public async checkVectorExtension(): Promise<boolean> {
if (this.embeddingsDisabled) {
return false;
}
try {
const result = this.query('SELECT vec_version() as version');
return result.rows.length > 0;
@@ -266,7 +255,6 @@ class SQLiteClient {
}
};
ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;");
// type column removed in final schema pass
} catch (nodeErr) {
console.warn('Failed to ensure nodes columns:', nodeErr);
}
@@ -363,6 +351,17 @@ class SQLiteClient {
'cache_hit', COALESCE(json_extract(NEW.metadata, '$.cache_hit'), 0),
'model', COALESCE(json_extract(NEW.metadata, '$.model_used'), ''),
'tools_count', COALESCE(json_extract(NEW.metadata, '$.tool_calls_count'), 0),
'tools_used', COALESCE(json_extract(NEW.metadata, '$.tools_used'), json('[]')),
'latency_ms', COALESCE(json_extract(NEW.metadata, '$.latency_ms'), 0),
'prompt_build_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.promptBuildMs'), 0),
'tools_build_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.toolsBuildMs'), 0),
'model_resolve_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.modelResolveMs'), 0),
'message_assembly_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.messageAssemblyMs'), 0),
'stream_setup_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.streamSetupMs'), 0),
'tool_loop_ms', COALESCE(json_extract(NEW.metadata, '$.timing_breakdown.toolLoopMs'), 0),
'first_token_latency_ms', COALESCE(json_extract(NEW.metadata, '$.first_token_latency_ms'), 0),
'first_chunk_latency_ms', COALESCE(json_extract(NEW.metadata, '$.first_chunk_latency_ms'), 0),
'tool_timings', COALESCE(json_extract(NEW.metadata, '$.tool_timings'), json('[]')),
'trace_id', COALESCE(json_extract(NEW.metadata, '$.trace_id'), ''),
'voice_tts_chars', COALESCE(json_extract(NEW.metadata, '$.voice_tts_chars'), 0),
'voice_tts_cost_usd', COALESCE(json_extract(NEW.metadata, '$.voice_tts_cost_usd'), 0),
@@ -435,26 +434,18 @@ class SQLiteClient {
}
// Do not recreate memory_v; alias has been removed.
// 6) Drop orphaned chat_memory_state table (removed in final schema pass)
this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`);
// Agent delegation table for orchestrator/worker coordination
// 6) Clean up removed chat_memory_state table
try {
this.db.exec(`
CREATE TABLE IF NOT EXISTS agent_delegations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_id TEXT UNIQUE NOT NULL,
task TEXT NOT NULL,
context TEXT,
expected_outcome TEXT,
status TEXT NOT NULL DEFAULT 'queued',
summary TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
`);
this.db.exec(`DROP TABLE IF EXISTS chat_memory_state;`);
} catch (e) {
console.warn('Failed to ensure agent_delegations table:', e);
// Ignore if table doesn't exist
}
// Clean up removed agent_delegations table
try {
this.db.exec(`DROP TABLE IF EXISTS agent_delegations;`);
} catch (e) {
console.warn('Failed to drop agent_delegations table:', e);
}
// 8) Logs retention trigger (~10k most recent rows)
@@ -623,77 +614,62 @@ class SQLiteClient {
}
}
// 10) Final schema pass migrations (content→notes, event_date, icon, drop dead columns)
// 10) Final schema pass migrations (content→notes, event_date, dimensions.icon, drop dead columns)
try {
const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
const nodeColNames = nodeCols2.map((c: any) => c.name);
const nodeColNames = nodeCols2.map(c => c.name);
// Rename content → notes
// Rename content → notes (additive first)
if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) {
console.log('Renaming nodes.content → nodes.notes');
console.log('Migrating nodes.content → nodes.notes...');
this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;');
}
// Add event_date with backfill from metadata
// Add event_date
if (!nodeColNames.includes('event_date')) {
console.log('Adding nodes.event_date column');
this.db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;');
this.db.exec(`
UPDATE nodes SET event_date = json_extract(metadata, '$.published_date')
WHERE metadata IS NOT NULL
AND json_extract(metadata, '$.published_date') IS NOT NULL
AND json_extract(metadata, '$.published_date') != '';
`);
// Backfill from metadata.published_date where available
try {
this.db.exec(`
UPDATE nodes SET event_date = json_extract(metadata, '$.published_date')
WHERE event_date IS NULL AND json_extract(metadata, '$.published_date') IS NOT NULL;
`);
} catch {}
}
// Add dimensions.icon
const dimCols2 = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
if (!dimCols2.some((c: any) => c.name === 'icon')) {
console.log('Adding dimensions.icon column');
if (!dimCols2.some(c => c.name === 'icon')) {
this.db.exec('ALTER TABLE dimensions ADD COLUMN icon TEXT;');
}
// Drop dead columns (SQLite 3.35+)
// Drop dead columns (requires SQLite 3.35+)
// nodes.type
if (nodeColNames.includes('type')) {
console.log('Dropping nodes.type column');
try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_type;'); } catch {}
try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch (e) {
console.warn('Could not drop nodes.type (SQLite < 3.35?):', e);
}
try { this.db.exec('ALTER TABLE nodes DROP COLUMN type;'); } catch {}
}
// nodes.is_pinned
if (nodeColNames.includes('is_pinned')) {
console.log('Dropping nodes.is_pinned column');
try { this.db.exec('DROP INDEX IF EXISTS idx_nodes_pinned;'); } catch {}
try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch (e) {
console.warn('Could not drop nodes.is_pinned:', e);
}
try { this.db.exec('ALTER TABLE nodes DROP COLUMN is_pinned;'); } catch {}
}
// Drop edges.user_feedback
// edges.user_feedback
const edgeCols = this.db.prepare('PRAGMA table_info(edges)').all() as Array<{ name: string }>;
if (edgeCols.some((c: any) => c.name === 'user_feedback')) {
console.log('Dropping edges.user_feedback column');
try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch (e) {
console.warn('Could not drop edges.user_feedback:', e);
}
if (edgeCols.some(c => c.name === 'user_feedback')) {
try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch {}
}
// Rebuild FTS if it references 'content' instead of 'notes'
// Recreate nodes_fts to index title + description + notes
try {
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as any;
if (ftsCheck && ftsCheck.sql && ftsCheck.sql.includes('content')) {
console.log('Rebuilding nodes_fts to reference notes instead of content');
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined;
if (ftsCheck?.sql && (!ftsCheck.sql.includes('description') || ftsCheck.sql.includes('content'))) {
this.db.exec('DROP TABLE IF EXISTS nodes_fts;');
this.db.exec(`
CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content=nodes, content_rowid=id);
INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');
`);
this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content='nodes', content_rowid='id');");
// Rebuild FTS index
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
}
} catch (ftsErr) {
console.warn('FTS rebuild skipped:', ftsErr);
console.warn('Failed to rebuild nodes_fts:', ftsErr);
}
console.log('Final schema pass migrations complete');
} catch (schemaErr) {
console.warn('Final schema pass migration error:', schemaErr);
}