SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port source-first ingestion to os repo

Browse Source 
- make source the canonical field across os routes, tools, ui, and mcp
- align fresh-install schema, search, and embedding flows with source-first ingestion

Generated with Claude Code
This commit is contained in:
“BeeRad”
2026-03-20 12:36:26 +11:00
parent 41f8498d4a
commit 28e570696c
34 changed files with 588 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/database/nodes.ts
+119 -25
View File
@@ -16,6 +16,38 @@ function sanitizeFtsQuery(input: string): string {
.join(' ');
}
function extractRelaxedSearchTerms(query: string): string[] {
const stopWords = new Set([
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'can', 'do', 'find',
'for', 'from', 'hello', 'i', 'in', 'is', 'it', 'me', 'my', 'of', 'on',
'or', 'recent', 'stuff', 'term', 'that', 'the', 'this', 'to', 'with', 'you'
]);
const rawTerms = query
.toLowerCase()
.replace(/[^a-z0-9\s]+/g, ' ')
.split(/\s+/)
.map(term => term.trim())
.filter(Boolean);
const expanded = new Set<string>();
for (const term of rawTerms) {
if (!stopWords.has(term) && term.length >= 3) {
expanded.add(term);
}
const alphaParts = term.replace(/\d+/g, ' ').split(/\s+/).filter(Boolean);
for (const part of alphaParts) {
if (!stopWords.has(part) && part.length >= 3) {
expanded.add(part);
}
}
}
return Array.from(expanded).slice(0, 8);
}
function reciprocalRankFuse<T extends { id: number }>(
rankedLists: T[][],
limit: number,
@@ -48,7 +80,7 @@ export class NodeService {
async countNodes(filters: NodeFilters = {}): Promise<number> {
const { dimensions, search, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore } = filters;
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters;
if (search?.trim()) {
return this.countSearchNodesSQLite(filters);
@@ -78,7 +110,7 @@ export class NodeService {
}
if (search) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
}
@@ -86,6 +118,7 @@ export class NodeService {
if (createdBefore) { query += ` AND n.created_at < ?`; params.push(createdBefore); }
if (eventAfter) { query += ` AND n.event_date >= ?`; params.push(eventAfter); }
if (eventBefore) { query += ` AND n.event_date < ?`; params.push(eventBefore); }
if (chunkStatus) { query += ` AND n.chunk_status = ?`; params.push(chunkStatus); }
const result = sqlite.query<{ total: number }>(query, params);
return result.rows[0]?.total ?? 0;
@@ -95,7 +128,7 @@ export class NodeService {
private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
const { dimensions, search, limit = 100, offset = 0, sortBy, dimensionsMatch = 'any',
createdAfter, createdBefore, eventAfter, eventBefore } = filters;
createdAfter, createdBefore, eventAfter, eventBefore, chunkStatus } = filters;
if (search?.trim()) {
return this.searchNodesSQLite(filters);
@@ -105,7 +138,7 @@ export class NodeService {
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
let query = `
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -137,9 +170,9 @@ export class NodeService {
}
}
// Text search in title, description, and notes (SQLite LIKE with COLLATE NOCASE)
// Text search in title, description, and source (SQLite LIKE with COLLATE NOCASE)
if (search) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
}
@@ -160,23 +193,27 @@ export class NodeService {
query += ` AND n.event_date < ?`;
params.push(eventBefore);
}
if (chunkStatus) {
query += ` AND n.chunk_status = ?`;
params.push(chunkStatus);
}
// Sorting logic
if (search) {
// For search queries, prioritize by relevance: exact title → starts with → contains in title → description → notes
// For search queries, prioritize by relevance: exact title → starts with → contains in title → description → source
query += ` ORDER BY
CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 6 END,
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
n.updated_at DESC`;
params.push(
search, // Exact match (case-insensitive)
`${search}%`, // Starts with search term
`%${search}%`, // Contains in title
`%${search}%`, // Contains in description
`%${search}%` // Contains in notes
`%${search}%` // Contains in source
);
} else if (sortBy === 'edges') {
// Sort by edge count (most connected first)
@@ -215,7 +252,7 @@ export class NodeService {
private async getNodeByIdSQLite(id: number): Promise<Node | null> {
const sqlite = getSQLiteClient();
const query = `
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -241,11 +278,10 @@ export class NodeService {
const {
title,
description,
notes,
source,
link,
event_date,
dimensions = [],
chunk,
chunk_status,
metadata = {}
} = nodeData;
@@ -255,16 +291,15 @@ export class NodeService {
const nodeId = sqlite.transaction(() => {
// Insert node using prepare/run for lastInsertRowid access
const nodeResult = sqlite.prepare(`
INSERT INTO nodes (title, description, notes, link, event_date, metadata, chunk, chunk_status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO nodes (title, description, source, link, event_date, metadata, chunk_status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
title,
description ?? null,
notes ?? null,
source ?? null,
link ?? null,
event_date ?? null,
JSON.stringify(metadata),
chunk ?? null,
chunk_status ?? null,
now,
now
@@ -308,7 +343,7 @@ export class NodeService {
// PostgreSQL path removed in SQLite-only consolidation
private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
const { title, description, notes, link, event_date, dimensions, chunk, metadata } = updates;
const { title, description, source, link, event_date, dimensions, metadata } = updates;
const now = new Date().toISOString();
const sqlite = getSQLiteClient();
@@ -327,10 +362,9 @@ export class NodeService {
if (title !== undefined) { setFields.push('title = ?'); params.push(title); }
if (description !== undefined) { setFields.push('description = ?'); params.push(description); }
if (notes !== undefined) { setFields.push('notes = ?'); params.push(notes); }
if (source !== undefined) { setFields.push('source = ?'); params.push(source); }
if (link !== undefined) { setFields.push('link = ?'); params.push(link); }
if (event_date !== undefined) { setFields.push('event_date = ?'); params.push(event_date); }
if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); }
if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) {
setFields.push('chunk_status = ?');
params.push(updates.chunk_status ?? null);
@@ -469,6 +503,10 @@ export class NodeService {
rankedRows = this.searchNodesLike(sqlite, search, filters, searchLimit);
}
if (rankedRows.length === 0) {
rankedRows = this.searchNodesLikeRelaxed(sqlite, search, filters, searchLimit);
}
if ((filters.searchMode ?? 'standard') === 'hybrid') {
const vectorRows = await this.searchNodesVector(sqlite, search, filters, searchLimit);
if (vectorRows.length > 0) {
@@ -518,7 +556,7 @@ export class NodeService {
}
for (const word of words) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
}
@@ -551,7 +589,7 @@ export class NodeService {
WHERE nodes_fts MATCH ?
LIMIT ?
)
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -580,7 +618,7 @@ export class NodeService {
const words = search.split(/\s+/).filter(Boolean);
const { clauses, params } = this.buildNodeFilterClauses(filters);
let query = `
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
@@ -595,7 +633,7 @@ export class NodeService {
}
for (const word of words) {
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.notes LIKE ? COLLATE NOCASE)`;
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.description LIKE ? COLLATE NOCASE OR n.source LIKE ? COLLATE NOCASE)`;
queryParams.push(`%${word}%`, `%${word}%`, `%${word}%`);
}
@@ -604,7 +642,7 @@ export class NodeService {
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 6 END,
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 6 END,
CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 4 ELSE 6 END,
CASE WHEN n.notes LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 5 ELSE 6 END,
n.updated_at DESC
LIMIT ?`;
@@ -614,6 +652,62 @@ export class NodeService {
return result.rows;
}
private searchNodesLikeRelaxed(
sqlite: ReturnType<typeof getSQLiteClient>,
search: string,
filters: NodeFilters,
limit: number,
): NodeSearchRow[] {
const terms = extractRelaxedSearchTerms(search);
if (terms.length === 0) return [];
const { clauses, params } = this.buildNodeFilterClauses(filters);
let query = `
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
FROM nodes n
WHERE 1=1
`;
const queryParams = [...params];
if (clauses.length > 0) {
query += ` AND ${clauses.join(' AND ')}`;
}
const termClauses: string[] = [];
for (const term of terms) {
termClauses.push(`n.title LIKE ? COLLATE NOCASE`);
termClauses.push(`n.description LIKE ? COLLATE NOCASE`);
termClauses.push(`n.source LIKE ? COLLATE NOCASE`);
queryParams.push(`%${term}%`, `%${term}%`, `%${term}%`);
}
query += ` AND (${termClauses.join(' OR ')})`;
const scoreClauses: string[] = [];
const scoreParams: string[] = [];
for (const term of terms) {
scoreClauses.push(`CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 0 END`);
scoreClauses.push(`CASE WHEN n.description LIKE ? COLLATE NOCASE THEN 2 ELSE 0 END`);
scoreClauses.push(`CASE WHEN n.source LIKE ? COLLATE NOCASE THEN 1 ELSE 0 END`);
scoreParams.push(`%${term}%`, `%${term}%`, `%${term}%`);
}
query += ` ORDER BY
(${scoreClauses.join(' + ')}) DESC,
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 0 ELSE 1 END,
n.updated_at DESC
LIMIT ?`;
queryParams.push(...scoreParams, `%${search}%`, limit);
const result = sqlite.query<NodeSearchRow>(query, queryParams);
return result.rows;
}
private async searchNodesVector(
sqlite: ReturnType<typeof getSQLiteClient>,
search: string,
@@ -643,7 +737,7 @@ export class NodeService {
ORDER BY distance
LIMIT ?
)
SELECT n.id, n.title, n.description, n.notes, n.link, n.event_date, n.metadata, n.chunk,
SELECT n.id, n.title, n.description, n.source, n.notes, n.link, n.event_date, n.metadata, n.chunk,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)