SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port source-first ingestion to os repo

Browse Source 
- make source the canonical field across os routes, tools, ui, and mcp
- align fresh-install schema, search, and embedding flows with source-first ingestion

Generated with Claude Code
This commit is contained in:
“BeeRad”
2026-03-20 12:36:26 +11:00
parent 41f8498d4a
commit 28e570696c
34 changed files with 588 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/database/edges.ts
+5 -11
View File
@@ -478,17 +478,13 @@ export class EdgeService {
ELSE n_from.title
END as connected_node_title,
CASE
WHEN e.from_node_id = ? THEN n_to.notes
ELSE n_from.notes
END as connected_node_notes,
CASE
WHEN e.from_node_id = ? THEN n_to.link
ELSE n_from.link
END as connected_node_link,
CASE
WHEN e.from_node_id = ? THEN n_to.chunk
ELSE n_from.chunk
END as connected_node_chunk,
WHEN e.from_node_id = ? THEN n_to.source
ELSE n_from.source
END as connected_node_source,
CASE
WHEN e.from_node_id = ? THEN n_to.metadata
ELSE n_from.metadata
@@ -547,11 +543,10 @@ export class EdgeService {
const connected_node: Node = {
id: row.connected_node_id,
title: row.connected_node_title,
notes: row.connected_node_notes,
link: row.connected_node_link,
dimensions: row.connected_node_dimensions,
embedding: undefined, // Not needed for display
chunk: row.connected_node_chunk,
source: row.connected_node_source,
metadata: row.connected_node_metadata,
created_at: row.connected_node_created_at,
updated_at: row.connected_node_updated_at
@@ -592,11 +587,10 @@ export class EdgeService {
const connected_node: Node = {
id: row.connected_node_id,
title: row.connected_node_title,
notes: row.connected_node_notes,
link: row.connected_node_link,
dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'),
embedding: undefined, // Not needed for display
chunk: row.connected_node_chunk,
source: row.connected_node_source,
metadata: typeof row.connected_node_metadata === 'string' ? JSON.parse(row.connected_node_metadata) : row.connected_node_metadata,
created_at: row.connected_node_created_at,
updated_at: row.connected_node_updated_at
src/services/database/nodes.ts
+119 -25
View File
@@ -16,6 +16,38 @@ function sanitizeFtsQuery(input: string): string {
.join(' ');
}
function extractRelaxedSearchTerms(query: string): string[] {
const stopWords = new Set([
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'can', 'do', 'find',
'for', 'from', 'hello', 'i', 'in', 'is', 'it', 'me', 'my', 'of', 'on',
'or', 'recent', 'stuff', 'term', 'that', 'the', 'this', 'to', 'with', 'you'
]);
const rawTerms = query
.toLowerCase()
.replace(/[^a-z0-9\s]+/g, ' ')
.split(/\s+/)
.map(term => term.trim())
.filter(Boolean);
const expanded = new Set<string>();
for (const term of rawTerms) {
if (!stopWords.has(term) && term.length >= 3) {
expanded.add(term);
}
const alphaParts = term.replace(/\d+/g, ' ').split(/\s+/).filter(Boolean);
for (const part of alphaParts) {
if (!stopWords.has(part) && part.length >= 3) {
expanded.add(part);
}
}
}
return Array.from(expanded).slice(0, 8);
}
function reciprocalRankFuse<T extends { id: number }>(
rankedLists: T[][],
limit: number,
@@ -48,7 +80,7 @@ export class NodeService {
async countNodes(filters: NodeFilters = {}): Promise<number> {
const { dimensions, search, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore } = filters;
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters;
if (search?.trim()) {
return this.countSearchNodesSQLite(filters);
@@ -78,7 +110,7 @@ export class NodeService {
}
if (search) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
}
@@ -86,6 +118,7 @@ export class NodeService {
if (createdBefore) { query += ` AND n.created_at < ?`; params.push(createdBefore); }
if (eventAfter) { query += ` AND n.event_date >= ?`; params.push(eventAfter); }
if (eventBefore) { query += ` AND n.event_date < ?`; params.push(eventBefore); }
if (chunkStatus) { query += ` AND n.chunk_status = ?`; params.push(chunkStatus); }
const result = sqlite.query<{ total: number }>(query, params);
return result.rows[0]?.total ?? 0;
@@ -95,7 +128,7 @@ export class NodeService {
private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore } = filters;
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters;
if (search?.trim()) {
return this.searchNodesSQLite(filters);
@@ -105,7 +138,7 @@ export class NodeService {
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
let query = `
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -137,9 +170,9 @@ export class NodeService {
}
}
// Text search in title, description, and notes (SQLite LIKE with COLLATE NOCASE)
// Text search in title, description, and source (SQLite LIKE with COLLATE NOCASE)
if (search) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
}
@@ -160,23 +193,27 @@ export class NodeService {
query += ` AND n.event_date < ?`;
params.push(eventBefore);
}
if (chunkStatus) {
query += ` AND n.chunk_status = ?`;
params.push(chunkStatus);
}
// Sorting logic
if (search) {
// For search queries, prioritize by relevance: exact title → starts with → contains in title → description → notes
// For search queries, prioritize by relevance: exact title → starts with → contains in title → description → source
query += ` ORDER BY
CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 6 END,
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
n.updated_at DESC`;
params.push(
search, // Exact match (case-insensitive)
`${search}%`, // Starts with search term
`%${search}%`, // Contains in title
`%${search}%`, // Contains in description
`%${search}%` // Contains in notes
`%${search}%` // Contains in source
);
} else if (sortBy === 'edges') {
// Sort by edge count (most connected first)
@@ -215,7 +252,7 @@ export class NodeService {
private async getNodeByIdSQLite(id: number): Promise<Node | null> {
const sqlite = getSQLiteClient();
const query = `
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -241,11 +278,10 @@ export class NodeService {
const {
title,
description,
notes,
source,
link,
event_date,
dimensions = [],
chunk,
chunk_status,
metadata = {}
} = nodeData;
@@ -255,16 +291,15 @@ export class NodeService {
const nodeId = sqlite.transaction(() => {
// Insert node using prepare/run for lastInsertRowid access
const nodeResult = sqlite.prepare(`
INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, chunk_status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO nodes (title, description, source, link, event_date, metadata, chunk_status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
title,
description ?? null,
notes ?? null,
source ?? null,
link ?? null,
event_date ?? null,
JSON.stringify(metadata),
chunk ?? null,
chunk_status ?? null,
now,
now
@@ -308,7 +343,7 @@ export class NodeService {
// PostgreSQL path removed in SQLite-only consolidation
private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates;
const { title, description, source, link, event_date, dimensions, metadata } = updates;
const now = new Date().toISOString();
const sqlite = getSQLiteClient();
@@ -327,10 +362,9 @@ export class NodeService {
if (title !== undefined) { setFields.push('title = ?'); params.push(title); }
if (description !== undefined) { setFields.push('description = ?'); params.push(description); }
if (notes !== undefined) { setFields.push('notes = ?'); params.push(notes); }
if (source !== undefined) { setFields.push('source = ?'); params.push(source); }
if (link !== undefined) { setFields.push('link = ?'); params.push(link); }
if (event_date !== undefined) { setFields.push('event_date = ?'); params.push(event_date); }
if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); }
if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) {
setFields.push('chunk_status = ?');
params.push(updates.chunk_status ?? null);
@@ -469,6 +503,10 @@ export class NodeService {
rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit);
}
if (rankedRows.length === 0) {
rankedRows = this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit);
}
if ((filters.searchMode ?? 'standard') === 'hybrid') {
const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
if (vectorRows.length > 0) {
@@ -518,7 +556,7 @@ export class NodeService {
}
for (const word of words) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
}
@@ -551,7 +589,7 @@ export class NodeService {
WHERE nodes_fts MATCH ?
LIMIT ?
)
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -580,7 +618,7 @@ export class NodeService {
const words = search.split(/\s+/).filter(Boolean);
const { clauses, params } = this.buildNodeFilterClauses(filters);
let query = `
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -595,7 +633,7 @@ export class NodeService {
}
for (const word of words) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
}
@@ -604,7 +642,7 @@ export class NodeService {
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
n.updated_at DESC
LIMIT ?`;
@@ -614,6 +652,62 @@ export class NodeService {
return result.rows;
}
private searchNodesLikeRelaxed(
sqlite: ReturnType<typeof getSQLiteClient>,
search: string,
filters: NodeFilters,
limit: number,
): NodeSearchRow[] {
const terms = extractRelaxedSearchTerms(search);
if (terms.length === 0) return [];
const { clauses, params } = this.buildNodeFilterClauses(filters);
let query = `
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
FROM nodes n
WHERE 1=1
`;
const queryParams = [...params];
if (clauses.length > 0) {
query += ` AND ${clauses.join(' AND ')}`;
}
const termClauses: string[] = [];
for (const term of terms) {
termClauses.push(`n.title LIKE ? COLLATE NOCASE`);
termClauses.push(`n.description LIKE ? COLLATE NOCASE`);
termClauses.push(`n.source LIKE ? COLLATE NOCASE`);
queryParams.push(`%${term}%`, `%${term}%`, `%${term}%`);
}
query += ` AND (${termClauses.join(' OR ')})`;
const scoreClauses: string[] = [];
const scoreParams: string[] = [];
for (const term of terms) {
scoreClauses.push(`CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 0 END`);
scoreClauses.push(`CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 2 ELSE 0 END`);
scoreClauses.push(`CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 1 ELSE 0 END`);
scoreParams.push(`%${term}%`, `%${term}%`, `%${term}%`);
}
query += ` ORDER BY
(${scoreClauses.join(' + ')}) DESC,
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 0 ELSE 1 END,
n.updated_at DESC
LIMIT ?`;
queryParams.push(...scoreParams, `%${search}%`, limit);
const result = sqlite.query<NodeSearchRow>(query, queryParams);
return result.rows;
}
private async searchNodesVector(
sqlite: ReturnType<typeof getSQLiteClient>,
search: string,
@@ -643,7 +737,7 @@ export class NodeService {
ORDER BY distance
LIMIT ?
)
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
src/services/database/sqlite-client.ts
+55 -6
View File
@@ -616,13 +616,60 @@ class SQLiteClient {
// 10) Final schema pass migrations (content→notes, event_date, dimensions.icon, drop dead columns)
try {
const nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
const nodeColNames = nodeCols2.map(c => c.name);
let nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
let nodeColNames = nodeCols2.map(c => c.name);
// Rename content → notes (additive first)
if (nodeColNames.includes('content') && !nodeColNames.includes('notes')) {
console.log('Migrating nodes.content → nodes.notes...');
this.db.exec('ALTER TABLE nodes RENAME COLUMN content TO notes;');
nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
nodeColNames = nodeCols2.map(c => c.name);
}
if (!nodeColNames.includes('source')) {
console.log('Adding nodes.source column...');
this.db.exec('ALTER TABLE nodes ADD COLUMN source TEXT;');
nodeCols2 = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
nodeColNames = nodeCols2.map(c => c.name);
}
if (nodeColNames.includes('source')) {
this.db.exec(`
UPDATE nodes
SET source = chunk
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND chunk IS NOT NULL
AND LENGTH(TRIM(chunk)) > 0;
`);
this.db.exec(`
UPDATE nodes
SET source = notes,
chunk_status = 'not_chunked'
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND notes IS NOT NULL
AND LENGTH(TRIM(notes)) > 0;
`);
this.db.exec(`
UPDATE nodes
SET source = title || CASE
WHEN description IS NOT NULL AND LENGTH(TRIM(description)) > 0
THEN char(10) || char(10) || description
ELSE ''
END,
chunk_status = 'not_chunked'
WHERE source IS NULL OR LENGTH(TRIM(source)) = 0;
`);
this.db.exec(`
UPDATE nodes
SET chunk_status = 'not_chunked'
WHERE source IS NOT NULL
AND LENGTH(TRIM(source)) > 0
AND (chunk_status IS NULL OR chunk_status != 'chunked');
`);
}
// Add event_date
@@ -657,8 +704,10 @@ class SQLiteClient {
if (edgeCols.some(c => c.name === 'user_feedback')) {
try { this.db.exec('ALTER TABLE edges DROP COLUMN user_feedback;'); } catch {}
}
// edges.explanation (top-level column added alongside context JSON)
if (!edgeCols.some(c => c.name === 'explanation')) {
this.db.exec('ALTER TABLE edges ADD COLUMN explanation TEXT;');
// Backfill from context JSON where available
try {
this.db.exec(`
UPDATE edges SET explanation = json_extract(context, '$.explanation')
@@ -667,13 +716,13 @@ class SQLiteClient {
} catch {}
}
// Recreate nodes_fts to index title + description + notes
// Recreate nodes_fts to index title + source + description
try {
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined;
if (ftsCheck?.sql && (!ftsCheck.sql.includes('description') || ftsCheck.sql.includes('content'))) {
const needsRebuild = !ftsCheck?.sql || !ftsCheck.sql.includes('source') || ftsCheck.sql.includes('notes') || ftsCheck.sql.includes('content');
if (needsRebuild) {
this.db.exec('DROP TABLE IF EXISTS nodes_fts;');
this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, description, notes, content='nodes', content_rowid='id');");
// Rebuild FTS index
this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');");
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
}
} catch (ftsErr) {