SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add local AI and Qdrant vector backends

Browse Source
This commit is contained in:
“BeeRad”
2026-05-02 09:57:57 +10:00
parent 00f0afb8f4
commit 782ace9a34
37 changed files with 1440 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/agents/transcriptSummarizer.ts
+5 -6
View File
@@ -1,5 +1,4 @@
import { generateText } from 'ai';
import { createLocalOpenAIProvider } from '@/services/openai/localProvider';
import { generateUtilityText } from '@/services/llm/provider';
export interface TranscriptSummaryResult {
subject?: string;
@@ -48,14 +47,14 @@ export async function summarizeTranscript(transcript: string): Promise<Transcrip
: transcript;
try {
const provider = createLocalOpenAIProvider();
const response = await generateText({
model: provider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt: buildPrompt(limited),
maxOutputTokens: 600,
responseFormat: 'json',
task: 'transcript_summary',
});
let content = response.text || '';
let content = text || '';
content = content.replace(/```json/gi, '').replace(/```/g, '').trim();
const parsed = JSON.parse(content);
src/services/database/chunks.ts
+18 -53
View File
@@ -1,5 +1,7 @@
import { getSQLiteClient } from './sqlite-client';
import { Chunk, ChunkData } from '@/types/database';
import { getVectorBackendType } from '@/services/vectorBackend';
import { getVectorBackend } from '@/services/vectorBackend/factory';
type RankedChunk = Chunk & { similarity: number };
@@ -256,16 +258,11 @@ export class ChunkService {
matchCount = 5,
nodeIds?: number[]
): Promise<Array<Chunk & { similarity: number }>> {
const sqlite = getSQLiteClient();
const startTime = Date.now();
const vectorString = `[${queryEmbedding.join(',')}]`;
const vectorBackend = await getVectorBackend();
const vectorLimit = Math.max(matchCount * 10, 50);
// vec0 requires the knn constraint to live directly on the vec table query.
// A previous change pushed node-scoping into that WHERE clause in a way vec0 rejects,
// which made every node-scoped vector search throw and silently fall back to text.
if (nodeIds && nodeIds.length > 0) {
const sqlite = getSQLiteClient();
const chunkCountQuery = `SELECT COUNT(*) AS count FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
const chunkCountResult = sqlite.query<{ count: number }>(chunkCountQuery, nodeIds);
const chunkCount = Number(chunkCountResult.rows[0]?.count ?? 0);
@@ -276,62 +273,26 @@ export class ChunkService {
}
console.log(`🔍 Node-scoped search: ${chunkCount} chunks in nodes ${nodeIds.join(', ')}`);
let query = `
SELECT c.*, (1.0 / (1.0 + v.distance)) AS similarity
FROM (
SELECT chunk_id, distance
FROM vec_chunks
WHERE embedding MATCH ?
ORDER BY distance
LIMIT ?
) v
JOIN chunks c ON c.id = v.chunk_id
WHERE c.node_id IN (${nodeIds.map(() => '?').join(',')})
AND (1.0 / (1.0 + v.distance)) >= ?
ORDER BY similarity DESC
LIMIT ?
`;
const params = [vectorString, vectorLimit, ...nodeIds, similarityThreshold, matchCount];
const result = sqlite.query<Chunk & { similarity: number }>(query, params);
const rows = await vectorBackend.searchChunks(queryEmbedding, similarityThreshold, matchCount, nodeIds);
const searchTime = Date.now() - startTime;
console.log(`📊 Vector search (node-scoped): ${result.rows.length} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
if (result.rows.length > 0) {
console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
console.log(`📊 Vector search (${getVectorBackendType()}, node-scoped): ${rows.length} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
if (rows.length > 0) {
console.log(`🎯 Top result: chunk ${rows[0].id} (similarity: ${rows[0].similarity.toFixed(3)})`);
}
return result.rows;
return rows;
}
// Global search (no node filter)
const query = `
WITH vector_results AS (
SELECT chunk_id, distance
FROM vec_chunks
WHERE embedding MATCH ?
ORDER BY distance
LIMIT ?
)
SELECT c.*, (1.0 / (1.0 + vr.distance)) AS similarity
FROM vector_results vr
JOIN chunks c ON c.id = vr.chunk_id
WHERE (1.0 / (1.0 + vr.distance)) >= ?
ORDER BY similarity DESC
LIMIT ?
`;
const params = [vectorString, vectorLimit, similarityThreshold, matchCount];
const result = sqlite.query<Chunk & { similarity: number }>(query, params);
const rows = await vectorBackend.searchChunks(queryEmbedding, similarityThreshold, matchCount);
const searchTime = Date.now() - startTime;
console.log(`📊 Vector search (global): ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
if (result.rows.length > 0) {
console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
console.log(`📊 Vector search (${getVectorBackendType()}, global): ${rows.length} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
if (rows.length > 0) {
console.log(`🎯 Top result: chunk ${rows[0].id} (similarity: ${rows[0].similarity.toFixed(3)})`);
}
return result.rows;
return rows;
}
async textSearchFallback(
@@ -495,6 +456,10 @@ export class ChunkService {
}
async getChunksWithoutEmbeddings(): Promise<Chunk[]> {
if (getVectorBackendType() === 'qdrant') {
return [];
}
// In SQLite, chunk vectors live in vec_chunks; report chunks without corresponding vector rows
const sqlite = getSQLiteClient();
const result = sqlite.query<Chunk>(`
src/services/database/descriptionService.ts
+4 -12
View File
@@ -1,6 +1,4 @@
import { generateText } from 'ai';
import { createOpenAI } from '@ai-sdk/openai';
import { hasPreferredOpenAiKey, getPreferredOpenAiKey } from '../storage/openaiKeyServer';
import { generateUtilityText } from '@/services/llm/provider';
import type { CanonicalNodeMetadata } from '@/types/database';
export interface DescriptionInput {
@@ -23,25 +21,19 @@ export interface DescriptionInput {
* The result must cover what the artifact is, why it is in the graph, and workflow status.
*/
export async function generateDescription(input: DescriptionInput): Promise<string> {
if (!hasPreferredOpenAiKey()) {
console.log(`[DescriptionService] No valid OpenAI key, using fallback for: "${input.title}"`);
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
}
try {
const prompt = buildDescriptionPrompt(input);
console.log(`[DescriptionService] Generating description for: "${input.title}"`);
const provider = createOpenAI({ apiKey: getPreferredOpenAiKey() });
const response = await generateText({
model: provider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt,
maxOutputTokens: 100,
temperature: 0.3,
task: 'description',
});
const description = sanitizeDescription(response.text, input);
const description = sanitizeDescription(text, input);
console.log(`[DescriptionService] Generated: "${description}"`);
src/services/database/edges.ts
+7 -22
View File
@@ -2,11 +2,9 @@ import { getSQLiteClient } from './sqlite-client';
import { Edge, EdgeContext, EdgeData, EdgeCreatedVia, NodeConnection, Node } from '@/types/database';
import { eventBroadcaster } from '../events';
import { nodeService } from './nodes';
import { generateText } from 'ai';
import { createOpenAI } from '@ai-sdk/openai';
import { z } from 'zod';
import { validateEdgeExplanation } from './quality';
import { getPreferredOpenAiKey, hasPreferredOpenAiKey } from '../storage/openaiKeyServer';
import { generateUtilityText } from '@/services/llm/provider';
const inferredEdgeContextSchema = z.object({
type: z.enum(['created_by', 'part_of', 'source_of', 'related_to']),
@@ -75,16 +73,12 @@ async function inferEdgeContext(params: {
].join('\n');
try {
if (!hasPreferredOpenAiKey()) {
return { type: 'related_to', confidence: 0.2, swap_direction: false };
}
const provider = createOpenAI({ apiKey: getPreferredOpenAiKey() });
const { text } = await generateText({
model: provider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt,
temperature: 0.0,
maxOutputTokens: 120,
responseFormat: 'json',
task: 'edge_inference',
});
const parsedJson = (() => {
@@ -142,21 +136,12 @@ async function autoInferEdge(params: {
].join('\n');
try {
if (!hasPreferredOpenAiKey()) {
return {
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
type: 'related_to',
confidence: 0.2,
swap_direction: false,
};
}
const provider = createOpenAI({ apiKey: getPreferredOpenAiKey() });
const { text } = await generateText({
model: provider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt,
temperature: 0.0,
maxOutputTokens: 150,
responseFormat: 'json',
task: 'edge_inference',
});
const parsedJson = (() => {
src/services/database/nodes.ts
+22 -26
View File
@@ -4,6 +4,7 @@ import { eventBroadcaster } from '../events';
import { EmbeddingService } from '@/services/embeddings';
import { getHighSignalSearchTerms, scoreNodeSearchMatch } from './searchRanking';
import { buildCanonicalNodeMetadata, mergeNodeMetadata } from '@/services/nodes/metadata';
import { getVectorBackend } from '@/services/vectorBackend/factory';
type NodeRow = Node;
type NodeSearchRow = NodeRow & { rank?: number; similarity?: number };
@@ -345,6 +346,13 @@ export class NodeService {
private async deleteNodeSQLite(id: number): Promise<void> {
const sqlite = getSQLiteClient();
try {
const vectorBackend = await getVectorBackend();
await vectorBackend.deleteNode(id);
} catch (error) {
console.warn(`[NodeService] Could not delete vectors for node ${id}:`, error);
}
const result = sqlite.query('DELETE FROM nodes WHERE id = ?', [id]);
@@ -627,35 +635,27 @@ export class NodeService {
return [];
}
const vecExists = sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='vec_nodes'"
).get();
if (!vecExists) return [];
const vectorString = `[${embedding.join(',')}]`;
const { clauses, params } = this.buildNodeFilterClauses(filters);
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const extraClauses = clauses.length > 0 ? `AND ${clauses.join(' AND ')}` : '';
const vectorBackend = await getVectorBackend();
const matches = await vectorBackend.searchNodes(embedding, limit);
if (matches.length === 0) return [];
const matchIds = matches.map((match) => match.nodeId);
const scoreById = new Map(matches.map((match) => [match.nodeId, match.score]));
const result = sqlite.query<NodeSearchRow>(`
WITH vector_matches AS (
SELECT node_id, distance
FROM vec_nodes
WHERE embedding MATCH ?
ORDER BY distance
LIMIT ?
)
SELECT n.id, n.title, n.description, n.source, n.link, n.event_date, n.metadata,
n.chunk_status, n.embedding_updated_at, n.embedding_text,
n.created_at, n.updated_at,
(1.0 / (1.0 + vm.distance)) AS similarity
FROM vector_matches vm
JOIN nodes n ON n.id = vm.node_id
${whereClauses}
ORDER BY vm.distance
n.created_at, n.updated_at
FROM nodes n
WHERE n.id IN (${matchIds.map(() => '?').join(',')})
${extraClauses}
LIMIT ?
`, [vectorString, Math.max(limit * 2, 50), ...params, limit]);
`, [...matchIds, ...params, limit]);
return result.rows;
return result.rows
.map((row) => ({ ...row, similarity: scoreById.get(row.id) || 0 }))
.sort((a, b) => Number(b.similarity || 0) - Number(a.similarity || 0));
} catch (error) {
console.warn('[NodeSearch] Vector search unavailable, continuing without it:', error);
return [];
@@ -679,10 +679,6 @@ export class NodeService {
// PostgreSQL path removed in SQLite-only consolidation
private async bulkUpdateNodesSQLite(ids: number[], updates: Partial<Node>): Promise<Node[]> {
// For SQLite, use IN (SELECT value FROM json_each(?)) for safety
const sqlite = getSQLiteClient();
const idsJson = JSON.stringify(ids);
// For now, just update one by one - could optimize later
const updatedNodes: Node[] = [];
for (const id of ids) {
src/services/database/sqlite-client.ts
+119 -4
View File
@@ -3,6 +3,8 @@ import fs from 'fs';
import path from 'path';
import { DatabaseError } from '@/types/database';
import { getDatabasePath, getVecExtensionPath } from '@/services/database/sqlite-runtime';
import { getEmbeddingProviderInfo } from '@/services/embedding/provider';
import { getVectorBackendType } from '@/services/vectorBackend';
export interface SQLiteConfig {
dbPath: string;
@@ -43,6 +45,28 @@ export interface DatabaseIntegrityReport {
error?: string;
}
export interface EmbeddingProfileStatus {
active: {
profile: string;
model: string;
dimensions: number;
vector_backend: string;
};
stored: {
profile: string;
model: string;
dimensions: number;
vector_backend: string;
updated_at?: string;
} | null;
vectors: {
nodes: number;
chunks: number;
};
rebuild_required: boolean;
reason?: string;
}
class SQLiteClient {
private static instance: SQLiteClient;
private db: Database.Database;
@@ -258,13 +282,14 @@ class SQLiteClient {
public ensureVectorExtensions(): void {
try {
const dimensions = getEmbeddingProviderInfo().dimensions;
// Test for vec_nodes and vec_chunks; create them if missing
const hasVecNodes = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_nodes');
if (!hasVecNodes) {
this.db.exec(`
CREATE VIRTUAL TABLE vec_nodes USING vec0(
node_id INTEGER PRIMARY KEY,
embedding FLOAT[1536]
embedding FLOAT[${dimensions}]
);
`);
console.log('Created vec_nodes virtual table');
@@ -275,7 +300,7 @@ class SQLiteClient {
this.db.exec(`
CREATE VIRTUAL TABLE vec_chunks USING vec0(
chunk_id INTEGER PRIMARY KEY,
embedding FLOAT[1536]
embedding FLOAT[${dimensions}]
);
`);
console.log('Created vec_chunks virtual table');
@@ -353,6 +378,15 @@ class SQLiteClient {
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS embedding_profile_state (
id INTEGER PRIMARY KEY CHECK (id = 1),
profile TEXT NOT NULL,
model TEXT NOT NULL,
dimensions INTEGER NOT NULL,
vector_backend TEXT NOT NULL,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS chats (
id INTEGER PRIMARY KEY,
chat_type TEXT,
@@ -953,9 +987,10 @@ class SQLiteClient {
try {
this.db.exec(`DROP TABLE IF EXISTS ${table};`);
} catch {}
const dimensions = getEmbeddingProviderInfo().dimensions;
const ddl = table === 'vec_nodes'
? `CREATE VIRTUAL TABLE vec_nodes USING vec0(node_id INTEGER PRIMARY KEY, embedding FLOAT[1536]);`
: `CREATE VIRTUAL TABLE vec_chunks USING vec0(chunk_id INTEGER PRIMARY KEY, embedding FLOAT[1536]);`;
? `CREATE VIRTUAL TABLE vec_nodes USING vec0(node_id INTEGER PRIMARY KEY, embedding FLOAT[${dimensions}]);`
: `CREATE VIRTUAL TABLE vec_chunks USING vec0(chunk_id INTEGER PRIMARY KEY, embedding FLOAT[${dimensions}]);`;
try {
this.db.exec(ddl);
console.log(`Recreated ${table} virtual table`);
@@ -1229,6 +1264,86 @@ class SQLiteClient {
};
}
public getEmbeddingProfileStatus(): EmbeddingProfileStatus {
const embedding = getEmbeddingProviderInfo();
const active = {
profile: embedding.profile,
model: embedding.model,
dimensions: embedding.dimensions,
vector_backend: getVectorBackendType(),
};
const stored = this.db.prepare(`
SELECT profile, model, dimensions, vector_backend, updated_at
FROM embedding_profile_state
WHERE id = 1
`).get() as EmbeddingProfileStatus['stored'] | undefined;
const nodes = this.countVectorRows('vec_nodes');
const chunks = this.countVectorRows('vec_chunks');
const hasVectors = nodes > 0 || chunks > 0;
let rebuildRequired = false;
let reason: string | undefined;
if (!stored && hasVectors) {
rebuildRequired = true;
reason = 'Existing vectors do not have recorded provider/model/dimension metadata.';
} else if (stored) {
const mismatches = [
stored.profile !== active.profile ? `profile ${stored.profile} -> ${active.profile}` : '',
stored.model !== active.model ? `model ${stored.model} -> ${active.model}` : '',
Number(stored.dimensions) !== active.dimensions ? `dimensions ${stored.dimensions} -> ${active.dimensions}` : '',
stored.vector_backend !== active.vector_backend ? `backend ${stored.vector_backend} -> ${active.vector_backend}` : '',
].filter(Boolean);
if (mismatches.length > 0) {
rebuildRequired = hasVectors;
reason = `Embedding/vector profile changed (${mismatches.join(', ')}). Rebuild embeddings before semantic search.`;
}
}
return {
active,
stored: stored || null,
vectors: { nodes, chunks },
rebuild_required: rebuildRequired,
reason,
};
}
public markEmbeddingProfileCurrent(): void {
if (this.readOnly) return;
const embedding = getEmbeddingProviderInfo();
this.db.prepare(`
INSERT INTO embedding_profile_state (id, profile, model, dimensions, vector_backend, updated_at)
VALUES (1, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
profile = excluded.profile,
model = excluded.model,
dimensions = excluded.dimensions,
vector_backend = excluded.vector_backend,
updated_at = excluded.updated_at
`).run(
embedding.profile,
embedding.model,
embedding.dimensions,
getVectorBackendType(),
new Date().toISOString()
);
}
private countVectorRows(tableName: 'vec_nodes' | 'vec_chunks'): number {
try {
const exists = this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(tableName);
if (!exists) return 0;
const row = this.db.prepare(`SELECT COUNT(*) AS count FROM ${tableName}`).get() as { count?: number } | undefined;
return Number(row?.count ?? 0);
} catch {
return 0;
}
}
public getIntegrityReport(forceRefresh = false): DatabaseIntegrityReport {
if (!this.integrityReport || forceRefresh) {
this.integrityReport = this.inspectIntegrity();
src/services/embedding/provider.ts
+139
View File
@@ -0,0 +1,139 @@
import OpenAI from 'openai';
import { getPreferredOpenAiKey } from '@/services/storage/openaiKeyServer';
export type EmbeddingProfile = 'openai' | 'openai-compatible' | 'custom';
export interface EmbeddingProviderInfo {
profile: EmbeddingProfile;
model: string;
dimensions: number;
baseUrl?: string;
}
export interface EmbeddingHealth {
ok: boolean;
profile: EmbeddingProfile;
model: string;
dimensions: number;
detail?: string;
}
export interface EmbeddingProvider {
info(): EmbeddingProviderInfo;
generateEmbedding(text: string): Promise<number[]>;
healthCheck(): Promise<EmbeddingHealth>;
}
const DEFAULT_OPENAI_EMBEDDING_MODEL = 'text-embedding-3-small';
const DEFAULT_OPENAI_EMBEDDING_DIMENSIONS = 1536;
const DEFAULT_LOCAL_EMBEDDING_DIMENSIONS = 1024;
function normalizeProfile(raw: string | undefined): EmbeddingProfile {
if (raw === 'openai-compatible' || raw === 'custom') return raw;
return 'openai';
}
function parseDimensions(raw: string | undefined, fallback: number): number {
if (!raw) return fallback;
const parsed = Number(raw);
if (!Number.isInteger(parsed) || parsed <= 0) {
throw new Error(`Invalid EMBEDDING_DIMENSIONS="${raw}". Use a positive integer.`);
}
return parsed;
}
export function getEmbeddingProviderInfo(): EmbeddingProviderInfo {
const profile = normalizeProfile(process.env.EMBEDDING_PROFILE);
if (profile === 'openai') {
return {
profile,
model: process.env.EMBEDDING_MODEL || DEFAULT_OPENAI_EMBEDDING_MODEL,
dimensions: parseDimensions(process.env.EMBEDDING_DIMENSIONS, DEFAULT_OPENAI_EMBEDDING_DIMENSIONS),
};
}
const model = process.env.EMBEDDING_MODEL;
const baseUrl = process.env.EMBEDDING_BASE_URL || process.env.LLM_BASE_URL;
if (!model) {
throw new Error('EMBEDDING_MODEL is required when EMBEDDING_PROFILE=openai-compatible.');
}
if (!baseUrl) {
throw new Error('EMBEDDING_BASE_URL is required when EMBEDDING_PROFILE=openai-compatible.');
}
return {
profile,
model,
dimensions: parseDimensions(process.env.EMBEDDING_DIMENSIONS, DEFAULT_LOCAL_EMBEDDING_DIMENSIONS),
baseUrl,
};
}
function createOpenAiClient(info: EmbeddingProviderInfo): OpenAI {
if (info.profile === 'openai') {
const apiKey = getPreferredOpenAiKey();
if (!apiKey) {
throw new Error('OpenAI API key not configured. Add OPENAI_API_KEY to your .env.local file.');
}
return new OpenAI({ apiKey });
}
return new OpenAI({
apiKey: process.env.EMBEDDING_API_KEY || process.env.OPENAI_COMPATIBLE_API_KEY || 'local',
baseURL: info.baseUrl,
});
}
export function validateEmbeddingDimensions(embedding: number[], expectedDimensions = getEmbeddingProviderInfo().dimensions): boolean {
return Array.isArray(embedding) && embedding.length === expectedDimensions;
}
export function createEmbeddingProvider(): EmbeddingProvider {
const info = getEmbeddingProviderInfo();
return {
info: () => info,
async generateEmbedding(text: string): Promise<number[]> {
const client = createOpenAiClient(info);
const response = await client.embeddings.create({
model: info.model,
input: text.trim(),
encoding_format: 'float',
dimensions: info.dimensions,
});
const embedding = response.data?.[0]?.embedding;
if (!embedding) {
throw new Error(`No embedding returned from ${info.profile} provider.`);
}
if (!validateEmbeddingDimensions(embedding, info.dimensions)) {
throw new Error(
`Embedding dimension mismatch: expected ${info.dimensions}, got ${embedding.length}. ` +
'Run the local AI doctor and rebuild embeddings after changing providers.'
);
}
return embedding;
},
async healthCheck(): Promise<EmbeddingHealth> {
try {
const embedding = await this.generateEmbedding('RA-H embedding health check');
return {
ok: true,
profile: info.profile,
model: info.model,
dimensions: embedding.length,
detail: info.baseUrl ? `Connected to ${info.baseUrl}` : 'OpenAI embedding provider ready',
};
} catch (error) {
return {
ok: false,
profile: info.profile,
model: info.model,
dimensions: info.dimensions,
detail: error instanceof Error ? error.message : String(error),
};
}
},
};
}
src/services/embeddings.ts
+13 -26
View File
@@ -1,33 +1,16 @@
import OpenAI from 'openai';
import { getPreferredOpenAiKey } from './storage/openaiKeyServer';
function getOpenAiClient(): OpenAI {
const apiKey = getPreferredOpenAiKey();
if (!apiKey) {
throw new Error('OpenAI API key not configured. Add OPENAI_API_KEY to your .env.local file.');
}
return new OpenAI({ apiKey });
}
import {
createEmbeddingProvider,
getEmbeddingProviderInfo,
validateEmbeddingDimensions
} from '@/services/embedding/provider';
export class EmbeddingService {
/**
* Generate embedding for a search query using OpenAI's text-embedding-3-small model
* This matches the same model used in embed_universal.py for consistency
* Generate embedding for a search query using the active embedding profile.
*/
static async generateQueryEmbedding(query: string): Promise<number[]> {
try {
const openai = getOpenAiClient();
const response = await openai.embeddings.create({
model: "text-embedding-3-small",
input: query.trim(),
encoding_format: "float"
});
if (!response.data?.[0]?.embedding) {
throw new Error('No embedding returned from OpenAI API');
}
return response.data[0].embedding;
return await createEmbeddingProvider().generateEmbedding(query);
} catch (error) {
console.error('Failed to generate query embedding:', error);
throw new Error(`Embedding generation failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
@@ -35,9 +18,13 @@ export class EmbeddingService {
}
/**
* Validate embedding dimensions match expected size (1536 for text-embedding-3-small)
* Validate embedding dimensions match the active profile.
*/
static validateEmbedding(embedding: number[]): boolean {
return Array.isArray(embedding) && embedding.length === 1536;
return validateEmbeddingDimensions(embedding);
}
static getActiveEmbeddingInfo() {
return getEmbeddingProviderInfo();
}
}
src/services/llm/provider.ts
+127
View File
@@ -0,0 +1,127 @@
import { generateText } from 'ai';
import { createOpenAI } from '@ai-sdk/openai';
import { getPreferredOpenAiKey } from '@/services/storage/openaiKeyServer';
export type UtilityLlmProfile = 'openai' | 'openai-compatible' | 'custom';
export interface UtilityLlmRequest {
prompt: string;
maxOutputTokens?: number;
temperature?: number;
responseFormat?: 'text' | 'json';
task:
| 'description'
| 'edge_inference'
| 'extraction_analysis'
| 'transcript_summary'
| 'embedding_prep_analysis';
}
export interface UtilityLlmProviderInfo {
profile: UtilityLlmProfile;
model: string;
baseUrl?: string;
}
export interface UtilityLlmHealth {
ok: boolean;
profile: UtilityLlmProfile;
model: string;
detail?: string;
}
export interface UtilityLlmProvider {
info(): UtilityLlmProviderInfo;
generateText(input: UtilityLlmRequest): Promise<string>;
healthCheck(): Promise<UtilityLlmHealth>;
}
const DEFAULT_OPENAI_LLM_MODEL = 'gpt-4o-mini';
function normalizeProfile(raw: string | undefined): UtilityLlmProfile {
if (raw === 'openai-compatible' || raw === 'custom') return raw;
return 'openai';
}
export function getUtilityLlmProviderInfo(): UtilityLlmProviderInfo {
const profile = normalizeProfile(process.env.LLM_PROFILE);
if (profile === 'openai') {
return {
profile,
model: process.env.LLM_MODEL || DEFAULT_OPENAI_LLM_MODEL,
};
}
if (!process.env.LLM_MODEL) {
throw new Error('LLM_MODEL is required when LLM_PROFILE=openai-compatible.');
}
if (!process.env.LLM_BASE_URL) {
throw new Error('LLM_BASE_URL is required when LLM_PROFILE=openai-compatible.');
}
return {
profile,
model: process.env.LLM_MODEL,
baseUrl: process.env.LLM_BASE_URL,
};
}
function createProvider(info: UtilityLlmProviderInfo): ReturnType<typeof createOpenAI> {
if (info.profile === 'openai') {
const apiKey = getPreferredOpenAiKey();
if (!apiKey) {
throw new Error('OpenAI API key not configured. Add your key in Settings or .env.local.');
}
return createOpenAI({ apiKey });
}
return createOpenAI({
apiKey: process.env.LLM_API_KEY || process.env.OPENAI_COMPATIBLE_API_KEY || 'local',
baseURL: info.baseUrl,
});
}
export function createUtilityLlmProvider(): UtilityLlmProvider {
const info = getUtilityLlmProviderInfo();
return {
info: () => info,
async generateText(input: UtilityLlmRequest): Promise<string> {
const provider = createProvider(info);
const response = await generateText({
model: provider(info.model),
prompt: input.prompt,
maxOutputTokens: input.maxOutputTokens,
temperature: input.temperature,
});
return response.text;
},
async healthCheck(): Promise<UtilityLlmHealth> {
try {
await this.generateText({
prompt: 'Reply with exactly: ok',
maxOutputTokens: 8,
temperature: 0,
task: 'description',
});
return {
ok: true,
profile: info.profile,
model: info.model,
detail: info.baseUrl ? `Connected to ${info.baseUrl}` : 'OpenAI utility LLM ready',
};
} catch (error) {
return {
ok: false,
profile: info.profile,
model: info.model,
detail: error instanceof Error ? error.message : String(error),
};
}
},
};
}
export async function generateUtilityText(input: UtilityLlmRequest): Promise<string> {
return createUtilityLlmProvider().generateText(input);
}
src/services/typescript/embed-nodes.ts
+12 -37
View File
@@ -3,16 +3,15 @@
* Embeds node metadata (title, source, context, AI analysis) into nodes.embedding field
*/
import OpenAI from 'openai';
import { generateText } from 'ai';
import { createOpenAI } from '@ai-sdk/openai';
import { getPreferredOpenAiKey } from '@/services/storage/openaiKeyServer';
import {
createDatabaseConnection,
serializeFloat32Vector,
formatEmbeddingText,
batchProcess
} from './sqlite-vec';
import { createEmbeddingProvider } from '@/services/embedding/provider';
import { generateUtilityText } from '@/services/llm/provider';
import { getVectorBackend } from '@/services/vectorBackend/factory';
interface NodeRecord {
id: number;
@@ -31,20 +30,11 @@ interface EmbedNodeOptions {
}
export class NodeEmbedder {
private openaiClient: OpenAI;
private openaiProvider: ReturnType<typeof createOpenAI>;
private db: ReturnType<typeof createDatabaseConnection>;
private processedCount: number = 0;
private failedCount: number = 0;
constructor() {
const apiKey = getPreferredOpenAiKey();
if (!apiKey) {
throw new Error('OPENAI_API_KEY environment variable is not set');
}
this.openaiClient = new OpenAI({ apiKey });
this.openaiProvider = createOpenAI({ apiKey });
this.db = createDatabaseConnection();
}
@@ -57,14 +47,14 @@ export class NodeEmbedder {
Title: ${node.title}
Source: ${node.source || 'No source'}
Focus on the main concepts, key relationships, and practical implications.`;
Focus on the main concepts, key relationships, and practical implications.`;
try {
const { text } = await generateText({
model: this.openaiProvider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt,
maxOutputTokens: 150,
temperature: 0.3,
task: 'embedding_prep_analysis',
});
return text;
@@ -75,15 +65,10 @@ Focus on the main concepts, key relationships, and practical implications.`;
}
/**
* Generate embedding for text using OpenAI
* Generate embedding for text using the active embedding profile.
*/
private async generateEmbedding(text: string): Promise<number[]> {
const response = await this.openaiClient.embeddings.create({
model: 'text-embedding-3-small',
input: text,
});
return response.data[0].embedding;
return createEmbeddingProvider().generateEmbedding(text);
}
/**
@@ -132,20 +117,10 @@ Focus on the main concepts, key relationships, and practical implications.`;
// Update vec_nodes virtual table
try {
// Determine correct column name for primary key (node_id vs id)
// Use declared PK column from your DB schema (confirmed: node_id)
const pkCol = 'node_id';
// Delete existing entry if any
const deleteStmt = this.db.prepare(`DELETE FROM vec_nodes WHERE ${pkCol} = ?`);
deleteStmt.run(BigInt(node.id));
// Insert new entry (use bracketed string format compatible with sqlite-vec)
const vectorString = `[${embedding.join(',')}]`;
const insertStmt = this.db.prepare(`INSERT INTO vec_nodes (${pkCol}, embedding) VALUES (?, ?)`);
insertStmt.run(BigInt(node.id), vectorString);
const vectorBackend = await getVectorBackend();
await vectorBackend.upsertNode(node.id, embeddingText, embedding);
} catch (vecError) {
console.warn(`Could not update vec_nodes for node ${node.id}:`, vecError);
console.warn(`Could not update node vector backend for node ${node.id}:`, vecError);
// Continue - main embedding is still saved
}
@@ -212,7 +187,7 @@ Focus on the main concepts, key relationships, and practical implications.`;
async (node) => {
try {
await this.embedNode(node, forceReEmbed);
} catch (error) {
} catch {
// Error already logged in embedNode
}
},
src/services/typescript/embed-universal.ts
+17 -65
View File
@@ -3,13 +3,13 @@
* Takes a node_id, reads source content from nodes table, chunks it, and stores in chunks table
*/
import OpenAI from 'openai';
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
import { getPreferredOpenAiKey } from '@/services/storage/openaiKeyServer';
import {
createDatabaseConnection,
batchProcess
} from './sqlite-vec';
import { createEmbeddingProvider, getEmbeddingProviderInfo } from '@/services/embedding/provider';
import { getVectorBackend } from '@/services/vectorBackend/factory';
interface Node {
id: number;
@@ -18,34 +18,16 @@ interface Node {
chunk_status?: string | null;
}
interface ChunkData {
content: string;
metadata: {
node_id: number;
chunk_index: number;
start_char: number;
end_char: number;
};
}
interface EmbedUniversalOptions {
nodeId: number;
verbose?: boolean;
}
export class UniversalEmbedder {
private openaiClient: OpenAI;
private db: ReturnType<typeof createDatabaseConnection>;
private textSplitter: RecursiveCharacterTextSplitter;
private vecChunksInsertSQL: string | null = null;
constructor() {
const apiKey = getPreferredOpenAiKey();
if (!apiKey) {
throw new Error('OPENAI_API_KEY environment variable is not set');
}
this.openaiClient = new OpenAI({ apiKey });
this.db = createDatabaseConnection();
// Configure text splitter (same as old KMS system)
@@ -57,46 +39,23 @@ export class UniversalEmbedder {
}
/**
* Determine correct insert SQL for vec_chunks based on actual schema
*/
private resolveVecChunksInsertSQL(): string {
// Use declared PK column from your DB schema (confirmed: chunk_id)
if (!this.vecChunksInsertSQL) {
this.vecChunksInsertSQL = 'INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)';
}
return this.vecChunksInsertSQL;
}
/**
* Generate embedding for text using OpenAI
* Generate embedding for text using the active embedding profile.
*/
private async generateEmbedding(text: string): Promise<number[]> {
const response = await this.openaiClient.embeddings.create({
model: 'text-embedding-3-small',
input: text,
});
return response.data[0].embedding;
return createEmbeddingProvider().generateEmbedding(text);
}
/**
* Delete existing chunks for a node
*/
private deleteExistingChunks(nodeId: number): void {
// First, get all chunk IDs for this node
const chunkIds = this.db.prepare('SELECT id FROM chunks WHERE node_id = ?').all(nodeId) as Array<{ id: number }>;
// Delete from vec_chunks first, one by one to ensure they're removed
for (const chunk of chunkIds) {
try {
const deleteVecStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
deleteVecStmt.run(BigInt(chunk.id));
} catch (error) {
console.warn(`Could not delete vec_chunk ${chunk.id}:`, error);
}
private async deleteExistingChunks(nodeId: number): Promise<void> {
try {
const vectorBackend = await getVectorBackend();
await vectorBackend.deleteChunksByNode(nodeId);
} catch (error) {
console.warn(`Could not delete existing chunk vectors for node ${nodeId}:`, error);
}
// Then delete from chunks table
const deleteChunksStmt = this.db.prepare('DELETE FROM chunks WHERE node_id = ?');
deleteChunksStmt.run(nodeId);
}
@@ -108,7 +67,7 @@ export class UniversalEmbedder {
nodeId: number,
chunkContent: string,
chunkIndex: number,
metadata: any
metadata: Record<string, unknown>
): Promise<void> {
// Generate embedding
const embedding = await this.generateEmbedding(chunkContent);
@@ -124,7 +83,7 @@ export class UniversalEmbedder {
nodeId,
chunkIndex,
chunkContent,
'text-embedding-3-small',
getEmbeddingProviderInfo().model,
JSON.stringify(metadata),
now
);
@@ -132,17 +91,10 @@ export class UniversalEmbedder {
const chunkId = Number(result.lastInsertRowid);
try {
const vectorString = `[${embedding.join(',')}]`;
try {
const deleteStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
deleteStmt.run(BigInt(chunkId));
} catch {}
const sql = this.resolveVecChunksInsertSQL();
const vecInsertStmt = this.db.prepare(sql);
vecInsertStmt.run(BigInt(chunkId), vectorString);
const vectorBackend = await getVectorBackend();
await vectorBackend.upsertChunk(chunkId, nodeId, chunkIndex, chunkContent, embedding);
} catch (error) {
console.warn(`Could not insert into vec_chunks for chunk ${chunkId}:`, error);
console.warn(`Could not upsert vector for chunk ${chunkId}:`, error);
}
}
@@ -173,7 +125,7 @@ export class UniversalEmbedder {
console.log(`Processing node ${nodeId}: "${node.title}"`);
// Delete existing chunks
this.deleteExistingChunks(nodeId);
await this.deleteExistingChunks(nodeId);
// Split text into chunks
const chunks = await this.textSplitter.splitText(node.source);
@@ -274,7 +226,7 @@ export async function runCLI(args: string[]): Promise<void> {
const embedder = new UniversalEmbedder();
try {
const result = await embedder.processNode({ nodeId, verbose });
await embedder.processNode({ nodeId, verbose });
if (verbose) {
const stats = embedder.getStats();
src/services/vectorBackend/factory.ts
+20
View File
@@ -0,0 +1,20 @@
import { getVectorBackendType, type VectorBackend } from './index';
let instance: VectorBackend | null = null;
let instanceType: string | null = null;
export async function getVectorBackend(): Promise<VectorBackend> {
const type = getVectorBackendType();
if (instance && instanceType === type) return instance;
if (type === 'qdrant') {
const { QdrantBackend } = await import('./qdrant');
instance = new QdrantBackend();
} else {
const { SqliteVecBackend } = await import('./sqlite-vec-backend');
instance = new SqliteVecBackend();
}
instanceType = type;
return instance;
}
src/services/vectorBackend/index.ts
+38
View File
@@ -0,0 +1,38 @@
import type { Chunk } from '@/types/database';
export type VectorBackendType = 'sqlite-vec' | 'qdrant';
export interface RankedChunk extends Chunk {
similarity: number;
}
export interface RankedNode {
nodeId: number;
score: number;
}
export interface VectorBackendHealth {
ok: boolean;
backend: VectorBackendType;
detail?: string;
dimensions?: number;
}
export interface VectorBackend {
upsertChunk(chunkId: number, nodeId: number, chunkIndex: number, text: string, embedding: number[]): Promise<void>;
upsertNode(nodeId: number, text: string, embedding: number[]): Promise<void>;
searchChunks(
queryEmbedding: number[],
similarityThreshold: number,
limit: number,
nodeIds?: number[]
): Promise<RankedChunk[]>;
searchNodes(queryEmbedding: number[], limit: number): Promise<RankedNode[]>;
deleteChunksByNode(nodeId: number): Promise<void>;
deleteNode(nodeId: number): Promise<void>;
healthCheck(): Promise<VectorBackendHealth>;
}
export function getVectorBackendType(): VectorBackendType {
return process.env.VECTOR_BACKEND === 'qdrant' ? 'qdrant' : 'sqlite-vec';
}
src/services/vectorBackend/qdrant.ts
+234
View File
@@ -0,0 +1,234 @@
import { getEmbeddingProviderInfo } from '@/services/embedding/provider';
import type { RankedChunk, RankedNode, VectorBackend, VectorBackendHealth } from './index';
interface QdrantPoint {
id: number;
score?: number;
payload?: Record<string, unknown>;
}
function qdrantUrl(path: string): string {
const base = (process.env.QDRANT_URL || 'http://localhost:6333').replace(/\/+$/, '');
return `${base}${path}`;
}
function getApiKeyHeader(): Record<string, string> {
return process.env.QDRANT_API_KEY ? { 'api-key': process.env.QDRANT_API_KEY } : {};
}
async function qdrantFetch<T>(path: string, init?: RequestInit): Promise<T> {
const response = await fetch(qdrantUrl(path), {
...init,
headers: {
'Content-Type': 'application/json',
...getApiKeyHeader(),
...(init?.headers || {}),
},
});
if (!response.ok) {
const detail = await response.text().catch(() => response.statusText);
throw new Error(`Qdrant ${response.status} ${response.statusText}: ${detail}`);
}
return response.json() as Promise<T>;
}
function payloadNumber(payload: Record<string, unknown> | undefined, key: string, fallback: number): number {
const value = payload?.[key];
return typeof value === 'number' ? value : fallback;
}
function payloadString(payload: Record<string, unknown> | undefined, key: string): string {
const value = payload?.[key];
return typeof value === 'string' ? value : '';
}
export class QdrantBackend implements VectorBackend {
private readonly chunksCollection = process.env.QDRANT_CHUNKS_COLLECTION || 'rah_chunks';
private readonly nodesCollection = process.env.QDRANT_NODES_COLLECTION || 'rah_nodes';
private readonly dimensions = getEmbeddingProviderInfo().dimensions;
private ensured = new Set<string>();
private async ensureCollection(name: string): Promise<void> {
if (this.ensured.has(name)) return;
const existing = await fetch(qdrantUrl(`/collections/${encodeURIComponent(name)}`), {
headers: getApiKeyHeader(),
});
if (existing.status === 404) {
await qdrantFetch(`/collections/${encodeURIComponent(name)}`, {
method: 'PUT',
body: JSON.stringify({
vectors: {
size: this.dimensions,
distance: 'Cosine',
},
}),
});
this.ensured.add(name);
return;
}
if (!existing.ok) {
const detail = await existing.text().catch(() => existing.statusText);
throw new Error(`Qdrant ${existing.status} ${existing.statusText}: ${detail}`);
}
const data = await existing.json() as {
result?: { config?: { params?: { vectors?: { size?: number } | Record<string, { size?: number }> } } };
};
const vectors = data.result?.config?.params?.vectors;
const size = typeof vectors?.size === 'number'
? vectors.size
: vectors && typeof vectors === 'object'
? Object.values(vectors).find((value) => typeof value?.size === 'number')?.size
: undefined;
if (typeof size === 'number' && size !== this.dimensions) {
throw new Error(
`Qdrant collection ${name} has ${size} dimensions, but active embedding profile requires ${this.dimensions}. ` +
'Run the embedding rebuild script after changing embedding providers.'
);
}
this.ensured.add(name);
}
async upsertChunk(chunkId: number, nodeId: number, chunkIndex: number, text: string, embedding: number[]): Promise<void> {
await this.ensureCollection(this.chunksCollection);
await qdrantFetch(`/collections/${encodeURIComponent(this.chunksCollection)}/points?wait=true`, {
method: 'PUT',
body: JSON.stringify({
points: [{
id: chunkId,
vector: embedding,
payload: { chunkId, nodeId, chunkIndex, text },
}],
}),
});
}
async upsertNode(nodeId: number, text: string, embedding: number[]): Promise<void> {
await this.ensureCollection(this.nodesCollection);
await qdrantFetch(`/collections/${encodeURIComponent(this.nodesCollection)}/points?wait=true`, {
method: 'PUT',
body: JSON.stringify({
points: [{
id: nodeId,
vector: embedding,
payload: { nodeId, text },
}],
}),
});
}
async searchChunks(
queryEmbedding: number[],
similarityThreshold: number,
limit: number,
nodeIds?: number[]
): Promise<RankedChunk[]> {
await this.ensureCollection(this.chunksCollection);
const filter = nodeIds && nodeIds.length > 0
? {
must: [{
key: 'nodeId',
match: { any: nodeIds },
}],
}
: undefined;
const response = await qdrantFetch<{ result: QdrantPoint[] }>(
`/collections/${encodeURIComponent(this.chunksCollection)}/points/search`,
{
method: 'POST',
body: JSON.stringify({
vector: queryEmbedding,
limit,
score_threshold: similarityThreshold,
with_payload: true,
filter,
}),
}
);
return response.result.map((point) => {
const payload = point.payload;
return {
id: payloadNumber(payload, 'chunkId', Number(point.id)),
node_id: payloadNumber(payload, 'nodeId', 0),
chunk_idx: payloadNumber(payload, 'chunkIndex', 0),
text: payloadString(payload, 'text'),
embedding_type: getEmbeddingProviderInfo().model,
created_at: '',
similarity: Number(point.score ?? 0),
};
});
}
async searchNodes(queryEmbedding: number[], limit: number): Promise<RankedNode[]> {
await this.ensureCollection(this.nodesCollection);
const response = await qdrantFetch<{ result: QdrantPoint[] }>(
`/collections/${encodeURIComponent(this.nodesCollection)}/points/search`,
{
method: 'POST',
body: JSON.stringify({
vector: queryEmbedding,
limit,
with_payload: true,
}),
}
);
return response.result.map((point) => ({
nodeId: payloadNumber(point.payload, 'nodeId', Number(point.id)),
score: Number(point.score ?? 0),
}));
}
async deleteChunksByNode(nodeId: number): Promise<void> {
await this.ensureCollection(this.chunksCollection);
await qdrantFetch(`/collections/${encodeURIComponent(this.chunksCollection)}/points/delete?wait=true`, {
method: 'POST',
body: JSON.stringify({
filter: {
must: [{ key: 'nodeId', match: { value: nodeId } }],
},
}),
});
}
async deleteNode(nodeId: number): Promise<void> {
await this.deleteChunksByNode(nodeId);
await this.ensureCollection(this.nodesCollection);
await qdrantFetch(`/collections/${encodeURIComponent(this.nodesCollection)}/points/delete?wait=true`, {
method: 'POST',
body: JSON.stringify({
points: [nodeId],
}),
});
}
async healthCheck(): Promise<VectorBackendHealth> {
try {
await qdrantFetch('/collections', { method: 'GET' });
await this.ensureCollection(this.chunksCollection);
await this.ensureCollection(this.nodesCollection);
return {
ok: true,
backend: 'qdrant',
dimensions: this.dimensions,
detail: `Qdrant reachable at ${process.env.QDRANT_URL || 'http://localhost:6333'}`,
};
} catch (error) {
return {
ok: false,
backend: 'qdrant',
dimensions: this.dimensions,
detail: error instanceof Error ? error.message : String(error),
};
}
}
}
src/services/vectorBackend/sqlite-vec-backend.ts
+109
View File
@@ -0,0 +1,109 @@
import { getSQLiteClient } from '@/services/database/sqlite-client';
import { getEmbeddingProviderInfo } from '@/services/embedding/provider';
import type { Chunk } from '@/types/database';
import type { RankedChunk, RankedNode, VectorBackend, VectorBackendHealth } from './index';
function vectorLiteral(embedding: number[]): string {
return `[${embedding.join(',')}]`;
}
export class SqliteVecBackend implements VectorBackend {
async upsertChunk(chunkId: number, _nodeId: number, _chunkIndex: number, _text: string, embedding: number[]): Promise<void> {
const sqlite = getSQLiteClient();
sqlite.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?').run(BigInt(chunkId));
sqlite.prepare('INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)').run(BigInt(chunkId), vectorLiteral(embedding));
}
async upsertNode(nodeId: number, _text: string, embedding: number[]): Promise<void> {
const sqlite = getSQLiteClient();
sqlite.prepare('DELETE FROM vec_nodes WHERE node_id = ?').run(BigInt(nodeId));
sqlite.prepare('INSERT OR REPLACE INTO vec_nodes (node_id, embedding) VALUES (?, ?)').run(BigInt(nodeId), vectorLiteral(embedding));
}
async searchChunks(
queryEmbedding: number[],
similarityThreshold: number,
limit: number,
nodeIds?: number[]
): Promise<RankedChunk[]> {
const sqlite = getSQLiteClient();
const vectorLimit = Math.max(limit * 10, 50);
if (nodeIds && nodeIds.length > 0) {
const result = sqlite.query<RankedChunk>(`
SELECT c.*, (1.0 / (1.0 + v.distance)) AS similarity
FROM (
SELECT chunk_id, distance
FROM vec_chunks
WHERE embedding MATCH ?
ORDER BY distance
LIMIT ?
) v
JOIN chunks c ON c.id = v.chunk_id
WHERE c.node_id IN (${nodeIds.map(() => '?').join(',')})
AND (1.0 / (1.0 + v.distance)) >= ?
ORDER BY similarity DESC
LIMIT ?
`, [vectorLiteral(queryEmbedding), vectorLimit, ...nodeIds, similarityThreshold, limit]);
return result.rows;
}
const result = sqlite.query<RankedChunk>(`
WITH vector_results AS (
SELECT chunk_id, distance
FROM vec_chunks
WHERE embedding MATCH ?
ORDER BY distance
LIMIT ?
)
SELECT c.*, (1.0 / (1.0 + vr.distance)) AS similarity
FROM vector_results vr
JOIN chunks c ON c.id = vr.chunk_id
WHERE (1.0 / (1.0 + vr.distance)) >= ?
ORDER BY similarity DESC
LIMIT ?
`, [vectorLiteral(queryEmbedding), vectorLimit, similarityThreshold, limit]);
return result.rows;
}
async searchNodes(queryEmbedding: number[], limit: number): Promise<RankedNode[]> {
const sqlite = getSQLiteClient();
const result = sqlite.query<{ node_id: number; distance: number }>(`
SELECT node_id, distance
FROM vec_nodes
WHERE embedding MATCH ?
ORDER BY distance
LIMIT ?
`, [vectorLiteral(queryEmbedding), Math.max(limit * 2, 50)]);
return result.rows.map((row) => ({
nodeId: Number(row.node_id),
score: 1.0 / (1.0 + Number(row.distance)),
}));
}
async deleteChunksByNode(nodeId: number): Promise<void> {
const sqlite = getSQLiteClient();
const chunks = sqlite.query<Pick<Chunk, 'id'>>('SELECT id FROM chunks WHERE node_id = ?', [nodeId]).rows;
for (const chunk of chunks) {
sqlite.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?').run(BigInt(chunk.id));
}
}
async deleteNode(nodeId: number): Promise<void> {
const sqlite = getSQLiteClient();
await this.deleteChunksByNode(nodeId);
sqlite.prepare('DELETE FROM vec_nodes WHERE node_id = ?').run(BigInt(nodeId));
}
async healthCheck(): Promise<VectorBackendHealth> {
const sqlite = getSQLiteClient();
const ok = await sqlite.checkVectorExtension();
return {
ok,
backend: 'sqlite-vec',
dimensions: getEmbeddingProviderInfo().dimensions,
detail: ok ? 'sqlite-vec extension loaded' : 'sqlite-vec extension unavailable',
};
}
}
src/tools/other/paperExtract.ts
+6 -7
View File
@@ -1,11 +1,10 @@
import { tool } from 'ai';
import { z } from 'zod';
import { generateText } from 'ai';
import { extractPaper } from '@/services/typescript/extractors/paper';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
import { createLocalOpenAIProvider } from '@/services/openai/localProvider';
import { generateUtilityText } from '@/services/llm/provider';
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string'
@@ -25,7 +24,6 @@ function ensureNodeDescription(candidate: string | undefined, fallbackLead: stri
// AI-powered content analysis
async function analyzeContentWithAI(title: string, description: string, contentType: string) {
try {
const provider = createLocalOpenAIProvider();
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
@@ -57,13 +55,14 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
"reasoning": "Brief explanation of classification choices"
}`;
const response = await generateText({
model: provider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt,
maxOutputTokens: 800
maxOutputTokens: 800,
responseFormat: 'json',
task: 'extraction_analysis',
});
let content = response.text || '{}';
let content = text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
src/tools/other/websiteExtract.ts
+6 -7
View File
@@ -1,11 +1,10 @@
import { tool } from 'ai';
import { z } from 'zod';
import { generateText } from 'ai';
import { extractWebsite } from '@/services/typescript/extractors/website';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
import { createLocalOpenAIProvider } from '@/services/openai/localProvider';
import { generateUtilityText } from '@/services/llm/provider';
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string'
@@ -40,7 +39,6 @@ async function analyzeContentWithAI(
contentType: string
) {
try {
const provider = createLocalOpenAIProvider();
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
@@ -71,13 +69,14 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
"reasoning": "Brief explanation of classification choices"
}`;
const response = await generateText({
model: provider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt,
maxOutputTokens: 800
maxOutputTokens: 800,
responseFormat: 'json',
task: 'extraction_analysis',
});
let content = response.text || '{}';
let content = text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
src/tools/other/youtubeExtract.ts
+10 -12
View File
@@ -1,11 +1,10 @@
import { tool } from 'ai';
import { z } from 'zod';
import { generateText } from 'ai';
import { extractYouTube } from '@/services/typescript/extractors/youtube';
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
import { validateExplicitDescription } from '@/services/database/quality';
import { createLocalOpenAIProvider } from '@/services/openai/localProvider';
import { generateUtilityText } from '@/services/llm/provider';
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
const normalizedCandidate = typeof candidate === 'string'
@@ -29,7 +28,6 @@ async function analyzeContentWithAI(
contentType: string
) {
try {
const provider = createLocalOpenAIProvider();
const prompt = `Analyze this ${contentType} content and provide classification.
Title: "${title}"
@@ -60,13 +58,14 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
"reasoning": "Brief explanation of classification choices"
}`;
const response = await generateText({
model: provider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt,
maxOutputTokens: 800
maxOutputTokens: 800,
responseFormat: 'json',
task: 'extraction_analysis',
});
let content = response.text || '{}';
let content = text || '{}';
// Clean up the response - remove markdown code blocks if present
content = content.replace(/```json\s*/g, '').replace(/```\s*/g, '').trim();
@@ -114,13 +113,12 @@ ${excerpt}
`;
try {
const provider = createLocalOpenAIProvider();
const response = await generateText({
model: provider('gpt-4o-mini'),
const text = await generateUtilityText({
prompt,
maxOutputTokens: 400
maxOutputTokens: 400,
task: 'extraction_analysis',
});
return response.text?.trim() || null;
return text?.trim() || null;
} catch (error) {
console.warn('Transcript summarisation failed, falling back to AI analysis description:', error);
return null;