Initial commit: RA-H Open Source Edition
Local-first knowledge management system with BYO API keys. Features: - 3-panel UI (Nodes | Focus | Helpers) - SQLite + sqlite-vec for vector search - Agent system (Easy/Hard mode orchestrators) - Content extraction (YouTube, PDF, web) - Integrate workflow for connection discovery - Dimension system with auto-assignment Tech stack: - Next.js 15 + TypeScript + Tailwind CSS - Anthropic (Claude) + OpenAI (GPT) via Vercel AI SDK Setup: npm install && npm rebuild better-sqlite3 scripts/dev/bootstrap-local.sh npm run dev MIT License
This commit is contained in:
“BeeRad”
@@ -0,0 +1,331 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { Chunk, ChunkData } from '@/types/database';
|
||||
|
||||
export class ChunkService {
|
||||
async getChunksByNodeId(nodeId: number): Promise<Chunk[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Chunk>('SELECT * FROM chunks WHERE node_id = ? ORDER BY chunk_idx ASC', [nodeId]);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async getChunkById(id: number): Promise<Chunk | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Chunk>('SELECT * FROM chunks WHERE id = ?', [id]);
|
||||
return result.rows[0] || null;
|
||||
}
|
||||
|
||||
async createChunk(chunkData: ChunkData): Promise<Chunk> {
|
||||
return this.createChunkSQLite(chunkData);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async createChunkSQLite(chunkData: ChunkData): Promise<Chunk> {
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.prepare(`
|
||||
INSERT INTO chunks (node_id, chunk_idx, text, embedding, embedding_type, metadata, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
chunkData.node_id,
|
||||
chunkData.chunk_idx || null,
|
||||
chunkData.text,
|
||||
chunkData.embedding || null,
|
||||
chunkData.embedding_type,
|
||||
chunkData.metadata ? JSON.stringify(chunkData.metadata) : null,
|
||||
now
|
||||
);
|
||||
|
||||
const chunkId = Number(result.lastInsertRowid);
|
||||
const createdChunk = await this.getChunkById(chunkId);
|
||||
|
||||
if (!createdChunk) {
|
||||
throw new Error('Failed to create chunk');
|
||||
}
|
||||
|
||||
return createdChunk;
|
||||
}
|
||||
|
||||
async createChunks(chunksData: ChunkData[]): Promise<Chunk[]> {
|
||||
if (chunksData.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.createChunksSQLite(chunksData);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async createChunksSQLite(chunksData: ChunkData[]): Promise<Chunk[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const now = new Date().toISOString();
|
||||
const createdChunks: Chunk[] = [];
|
||||
|
||||
// Use transaction for bulk insert
|
||||
sqlite.transaction(() => {
|
||||
const stmt = sqlite.prepare(`
|
||||
INSERT INTO chunks (node_id, chunk_idx, text, embedding, embedding_type, metadata, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
for (const chunk of chunksData) {
|
||||
stmt.run(
|
||||
chunk.node_id,
|
||||
chunk.chunk_idx || null,
|
||||
chunk.text,
|
||||
chunk.embedding || null,
|
||||
chunk.embedding_type,
|
||||
chunk.metadata ? JSON.stringify(chunk.metadata) : null,
|
||||
now
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Get all created chunks by node_id (since we know they were just created)
|
||||
const nodeIds = [...new Set(chunksData.map(c => c.node_id))];
|
||||
for (const nodeId of nodeIds) {
|
||||
const chunks = await this.getChunksByNodeId(nodeId);
|
||||
createdChunks.push(...chunks.filter(c => c.created_at === now));
|
||||
}
|
||||
|
||||
return createdChunks;
|
||||
}
|
||||
|
||||
async updateChunk(id: number, updates: Partial<Chunk>): Promise<Chunk> {
|
||||
return this.updateChunkSQLite(id, updates);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async updateChunkSQLite(id: number, updates: Partial<Chunk>): Promise<Chunk> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const updateFields: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
// Build dynamic update query
|
||||
Object.entries(updates).forEach(([key, value]) => {
|
||||
if (key !== 'id' && key !== 'created_at' && value !== undefined) {
|
||||
updateFields.push(`${key} = ?`);
|
||||
if (key === 'metadata') {
|
||||
params.push(typeof value === 'object' ? JSON.stringify(value) : value);
|
||||
} else {
|
||||
params.push(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (updateFields.length === 0) {
|
||||
throw new Error('No valid fields to update');
|
||||
}
|
||||
|
||||
params.push(id); // Add ID for WHERE clause
|
||||
|
||||
const query = `UPDATE chunks SET ${updateFields.join(', ')} WHERE id = ?`;
|
||||
const result = sqlite.query(query, params);
|
||||
|
||||
if (result.changes === 0) {
|
||||
throw new Error(`Chunk with ID ${id} not found`);
|
||||
}
|
||||
|
||||
const updatedChunk = await this.getChunkById(id);
|
||||
if (!updatedChunk) {
|
||||
throw new Error(`Failed to retrieve updated chunk with ID ${id}`);
|
||||
}
|
||||
|
||||
return updatedChunk;
|
||||
}
|
||||
|
||||
async deleteChunk(id: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('DELETE FROM chunks WHERE id = ?', [id]);
|
||||
if ((result.changes || 0) === 0) {
|
||||
throw new Error(`Chunk with ID ${id} not found`);
|
||||
}
|
||||
}
|
||||
|
||||
async deleteChunksByNodeId(nodeId: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
sqlite.query('DELETE FROM chunks WHERE node_id = ?', [nodeId]);
|
||||
}
|
||||
|
||||
async searchChunks(
|
||||
queryEmbedding: number[],
|
||||
similarityThreshold = 0.5,
|
||||
matchCount = 5,
|
||||
nodeIds?: number[],
|
||||
fallbackQuery?: string
|
||||
): Promise<Array<Chunk & { similarity: number }>> {
|
||||
try {
|
||||
return await this.searchChunksSQLite(queryEmbedding, similarityThreshold, matchCount, nodeIds);
|
||||
} catch (error) {
|
||||
console.warn('Vector search failed, attempting text fallback:', error);
|
||||
if (fallbackQuery) {
|
||||
return await this.textSearchFallback(fallbackQuery, matchCount, nodeIds);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async searchChunksSQLite(
|
||||
queryEmbedding: number[],
|
||||
similarityThreshold = 0.5,
|
||||
matchCount = 5,
|
||||
nodeIds?: number[]
|
||||
): Promise<Array<Chunk & { similarity: number }>> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
// Step 1: Determine vector search limit - more conservative approach
|
||||
let vectorLimit = 50; // Start smaller for efficiency
|
||||
|
||||
if (nodeIds && nodeIds.length > 0) {
|
||||
// When searching specific nodes, get exact count and use reasonable multiplier
|
||||
const candidateCountQuery = `SELECT COUNT(*) as count FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
|
||||
const candidateResult = sqlite.query<{count: number}>(candidateCountQuery, nodeIds);
|
||||
const candidateCount = Number(candidateResult.rows[0].count);
|
||||
|
||||
// Use 2x the candidate count but cap at reasonable limits
|
||||
vectorLimit = Math.min(Math.max(candidateCount * 2, 50), 1000);
|
||||
console.log(`🔍 Node-scoped search: ${candidateCount} candidates, using vector limit ${vectorLimit}`);
|
||||
} else {
|
||||
// For global search, use adaptive limit based on expected results
|
||||
vectorLimit = Math.max(matchCount * 10, 50);
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
// Step 2: Use CTE-based query to avoid UNION ALL explosion
|
||||
const vectorString = `[${queryEmbedding.join(',')}]`;
|
||||
let query = `
|
||||
WITH vector_results AS (
|
||||
SELECT chunk_id, distance
|
||||
FROM vec_chunks
|
||||
WHERE embedding MATCH ?
|
||||
ORDER BY distance
|
||||
LIMIT ?
|
||||
)
|
||||
SELECT c.*, (1.0 / (1.0 + vr.distance)) AS similarity
|
||||
FROM vector_results vr
|
||||
JOIN chunks c ON c.id = vr.chunk_id
|
||||
`;
|
||||
|
||||
const params: any[] = [vectorString, vectorLimit];
|
||||
const conditions = [];
|
||||
|
||||
// Node ID filter if provided
|
||||
if (nodeIds && nodeIds.length > 0) {
|
||||
conditions.push(`c.node_id IN (${nodeIds.map(() => '?').join(',')})`);
|
||||
params.push(...nodeIds);
|
||||
}
|
||||
|
||||
// Similarity threshold filter
|
||||
conditions.push(`(1.0 / (1.0 + vr.distance)) >= ?`);
|
||||
params.push(similarityThreshold);
|
||||
|
||||
if (conditions.length > 0) {
|
||||
query += ` WHERE ${conditions.join(' AND ')}`;
|
||||
}
|
||||
|
||||
query += ` ORDER BY similarity DESC LIMIT ?`;
|
||||
params.push(matchCount);
|
||||
|
||||
const result = sqlite.query<Chunk & { similarity: number }>(query, params);
|
||||
const searchTime = Date.now() - startTime;
|
||||
|
||||
console.log(`📊 Vector search: ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
|
||||
if (result.rows.length > 0) {
|
||||
console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
|
||||
}
|
||||
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async textSearchFallback(
|
||||
query: string,
|
||||
matchCount = 5,
|
||||
nodeIds?: number[]
|
||||
): Promise<Array<Chunk & { similarity: number }>> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const startTime = Date.now();
|
||||
|
||||
// Clean query for LIKE search
|
||||
const cleanQuery = query.trim().toLowerCase();
|
||||
const searchTerms = cleanQuery.split(/\s+/).filter(term => term.length > 2);
|
||||
|
||||
if (searchTerms.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Build LIKE conditions for each term
|
||||
const likeConditions = searchTerms.map(() => 'LOWER(text) LIKE ?').join(' AND ');
|
||||
const likeParams = searchTerms.map(term => `%${term}%`);
|
||||
|
||||
let textQuery = `
|
||||
SELECT *, 0.8 as similarity
|
||||
FROM chunks
|
||||
WHERE ${likeConditions}
|
||||
`;
|
||||
|
||||
const params = [...likeParams];
|
||||
|
||||
// Add node filter if provided
|
||||
if (nodeIds && nodeIds.length > 0) {
|
||||
textQuery += ` AND node_id IN (${nodeIds.map(() => '?').join(',')})`;
|
||||
params.push(...nodeIds.map(String));
|
||||
}
|
||||
|
||||
textQuery += ` ORDER BY LENGTH(text) ASC LIMIT ?`;
|
||||
params.push(String(matchCount));
|
||||
|
||||
const result = sqlite.query<Chunk & { similarity: number }>(textQuery, params);
|
||||
const searchTime = Date.now() - startTime;
|
||||
|
||||
console.log(`📝 Text fallback: ${result.rows.length} chunks found, time=${searchTime}ms`);
|
||||
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async getChunkCount(): Promise<number> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT COUNT(*) as count FROM chunks');
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async getChunkCountByNodeId(nodeId: number): Promise<number> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT COUNT(*) as count FROM chunks WHERE node_id = ?', [nodeId]);
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async getNodesWithChunks(): Promise<Array<{ node_id: number; chunk_count: number }>> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query(`
|
||||
SELECT node_id, COUNT(*) as chunk_count
|
||||
FROM chunks
|
||||
GROUP BY node_id
|
||||
ORDER BY chunk_count DESC
|
||||
`);
|
||||
return result.rows.map((row: any) => ({
|
||||
node_id: Number(row.node_id),
|
||||
chunk_count: Number(row.chunk_count)
|
||||
}));
|
||||
}
|
||||
|
||||
async getChunksWithoutEmbeddings(): Promise<Chunk[]> {
|
||||
// In SQLite, chunk vectors live in vec_chunks; report chunks without corresponding vector rows
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Chunk>(`
|
||||
SELECT c.*
|
||||
FROM chunks c
|
||||
LEFT JOIN vec_chunks v ON v.chunk_id = c.id
|
||||
WHERE v.chunk_id IS NULL
|
||||
ORDER BY c.created_at ASC
|
||||
`);
|
||||
return result.rows;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const chunkService = new ChunkService();
|
||||
@@ -0,0 +1,202 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { openai as openaiProvider } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
|
||||
export interface Dimension {
|
||||
name: string;
|
||||
description: string | null;
|
||||
is_priority: boolean;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface LockedDimension {
|
||||
name: string;
|
||||
description: string | null;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export class DimensionService {
|
||||
/**
|
||||
* Get all locked (priority) dimensions with their descriptions
|
||||
*/
|
||||
static async getLockedDimensions(): Promise<LockedDimension[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.query(`
|
||||
WITH dimension_counts AS (
|
||||
SELECT nd.dimension, COUNT(*) AS count
|
||||
FROM node_dimensions nd
|
||||
GROUP BY nd.dimension
|
||||
)
|
||||
SELECT
|
||||
d.name,
|
||||
d.description,
|
||||
COALESCE(dc.count, 0) AS count
|
||||
FROM dimensions d
|
||||
LEFT JOIN dimension_counts dc ON dc.dimension = d.name
|
||||
WHERE d.is_priority = 1
|
||||
ORDER BY d.name ASC
|
||||
`);
|
||||
|
||||
return result.rows.map((row: any) => ({
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
count: Number(row.count)
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatically assign locked dimensions to a node based on its content
|
||||
*/
|
||||
static async assignLockedDimensions(nodeData: {
|
||||
title: string;
|
||||
content?: string;
|
||||
link?: string;
|
||||
}): Promise<string[]> {
|
||||
try {
|
||||
const lockedDimensions = await this.getLockedDimensions();
|
||||
|
||||
if (lockedDimensions.length === 0) {
|
||||
console.log('No locked dimensions available for assignment');
|
||||
return [];
|
||||
}
|
||||
|
||||
const prompt = this.buildAssignmentPrompt(nodeData, lockedDimensions);
|
||||
|
||||
const response = await generateText({
|
||||
model: openaiProvider('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 100,
|
||||
temperature: 0.1, // Low temperature for consistent results
|
||||
});
|
||||
|
||||
const assignedDimensions = this.parseAssignmentResponse(response.text, lockedDimensions);
|
||||
|
||||
console.log(`Assigned dimensions for "${nodeData.title}": ${assignedDimensions.join(', ')}`);
|
||||
return assignedDimensions;
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error assigning dimensions:', error);
|
||||
return []; // Graceful fallback
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update dimension description
|
||||
*/
|
||||
static async updateDimensionDescription(name: string, description: string): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
sqlite.query(`
|
||||
INSERT INTO dimensions(name, description, is_priority, updated_at)
|
||||
VALUES (?, ?, 0, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
description = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`, [name, description, description]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get dimension by name with description
|
||||
*/
|
||||
static async getDimensionByName(name: string): Promise<Dimension | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.query(`
|
||||
SELECT name, description, is_priority, updated_at
|
||||
FROM dimensions
|
||||
WHERE name = ?
|
||||
`, [name]);
|
||||
|
||||
if (result.rows.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const row = result.rows[0] as any;
|
||||
return {
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
is_priority: Boolean(row.is_priority),
|
||||
updated_at: row.updated_at
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build AI prompt for dimension assignment
|
||||
*/
|
||||
private static buildAssignmentPrompt(
|
||||
nodeData: { title: string; content?: string; link?: string },
|
||||
lockedDimensions: LockedDimension[]
|
||||
): string {
|
||||
const contentPreview = nodeData.content?.slice(0, 500) || '';
|
||||
const dimensionsList = lockedDimensions
|
||||
.map(d => `- ${d.name}: ${d.description || `Infer purpose from name: ${d.name}`}`)
|
||||
.join('\n');
|
||||
|
||||
return `Analyze this node content and assign appropriate dimensions from the available locked dimensions.
|
||||
|
||||
Node:
|
||||
Title: ${nodeData.title}
|
||||
Content: ${contentPreview}${nodeData.content && nodeData.content.length > 500 ? '...' : ''}
|
||||
URL: ${nodeData.link || 'none'}
|
||||
|
||||
Available Locked Dimensions:
|
||||
${dimensionsList}
|
||||
|
||||
Return only the dimension names that best match this content, maximum 3 dimensions.
|
||||
Respond with just the dimension names, one per line.
|
||||
If no dimensions are appropriate, respond with "none".
|
||||
|
||||
Examples of good responses:
|
||||
Work
|
||||
Learning
|
||||
|
||||
or
|
||||
|
||||
Research
|
||||
Ideas
|
||||
Tech
|
||||
|
||||
or
|
||||
|
||||
none`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse AI response and validate dimension names
|
||||
*/
|
||||
private static parseAssignmentResponse(
|
||||
response: string,
|
||||
availableDimensions: LockedDimension[]
|
||||
): string[] {
|
||||
const lines = response.trim().toLowerCase().split('\n');
|
||||
const availableNames = availableDimensions.map(d => d.name.toLowerCase());
|
||||
const validDimensions: string[] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
const dimensionName = line.trim();
|
||||
|
||||
if (dimensionName === 'none') {
|
||||
break;
|
||||
}
|
||||
|
||||
// Find matching dimension (case-insensitive)
|
||||
const matchedDimension = availableDimensions.find(
|
||||
d => d.name.toLowerCase() === dimensionName
|
||||
);
|
||||
|
||||
if (matchedDimension && !validDimensions.includes(matchedDimension.name)) {
|
||||
validDimensions.push(matchedDimension.name);
|
||||
|
||||
// Limit to 3 dimensions max
|
||||
if (validDimensions.length >= 3) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return validDimensions;
|
||||
}
|
||||
}
|
||||
|
||||
export const dimensionService = new DimensionService();
|
||||
@@ -0,0 +1,340 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { Edge, EdgeData, NodeConnection, Node } from '@/types/database';
|
||||
import { eventBroadcaster } from '../events';
|
||||
|
||||
export class EdgeService {
|
||||
async getEdges(): Promise<Edge[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Edge>('SELECT * FROM edges ORDER BY created_at DESC');
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async getEdgeById(id: number): Promise<Edge | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Edge>('SELECT * FROM edges WHERE id = ?', [id]);
|
||||
return result.rows[0] || null;
|
||||
}
|
||||
|
||||
async createEdge(edgeData: EdgeData): Promise<Edge> {
|
||||
return this.createEdgeSQLite(edgeData);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async createEdgeSQLite(edgeData: EdgeData): Promise<Edge> {
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.prepare(`
|
||||
INSERT INTO edges (from_node_id, to_node_id, context, source, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
edgeData.from_node_id,
|
||||
edgeData.to_node_id,
|
||||
JSON.stringify(edgeData.context || {}),
|
||||
edgeData.source,
|
||||
now
|
||||
);
|
||||
|
||||
const edgeId = Number(result.lastInsertRowid);
|
||||
const newEdge = await this.getEdgeById(edgeId);
|
||||
|
||||
if (!newEdge) {
|
||||
throw new Error('Failed to create edge');
|
||||
}
|
||||
|
||||
// Broadcast edge creation event
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'EDGE_CREATED',
|
||||
data: {
|
||||
fromNodeId: newEdge.from_node_id,
|
||||
toNodeId: newEdge.to_node_id,
|
||||
edge: newEdge
|
||||
}
|
||||
});
|
||||
|
||||
return newEdge;
|
||||
}
|
||||
|
||||
async updateEdge(id: number, updates: Partial<Edge>): Promise<Edge> {
|
||||
return this.updateEdgeSQLite(id, updates);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async updateEdgeSQLite(id: number, updates: Partial<Edge>): Promise<Edge> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const updateFields: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
// Build dynamic update query
|
||||
Object.entries(updates).forEach(([key, value]) => {
|
||||
if (key !== 'id' && key !== 'created_at' && value !== undefined) {
|
||||
updateFields.push(`${key} = ?`);
|
||||
if (key === 'context') {
|
||||
params.push(typeof value === 'object' ? JSON.stringify(value) : value);
|
||||
} else {
|
||||
params.push(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (updateFields.length === 0) {
|
||||
throw new Error('No valid fields to update');
|
||||
}
|
||||
|
||||
params.push(id); // Add ID for WHERE clause
|
||||
|
||||
const query = `UPDATE edges SET ${updateFields.join(', ')} WHERE id = ?`;
|
||||
const result = sqlite.query(query, params);
|
||||
|
||||
if (result.changes === 0) {
|
||||
throw new Error(`Edge with ID ${id} not found`);
|
||||
}
|
||||
|
||||
const updatedEdge = await this.getEdgeById(id);
|
||||
if (!updatedEdge) {
|
||||
throw new Error(`Failed to retrieve updated edge with ID ${id}`);
|
||||
}
|
||||
|
||||
return updatedEdge;
|
||||
}
|
||||
|
||||
async deleteEdge(id: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('DELETE FROM edges WHERE id = ?', [id]);
|
||||
if ((result.changes || 0) === 0) {
|
||||
throw new Error(`Edge with ID ${id} not found`);
|
||||
}
|
||||
// Broadcast edge deletion event
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'EDGE_DELETED',
|
||||
data: { edgeId: id }
|
||||
});
|
||||
}
|
||||
|
||||
async deleteEdgesByNodeId(nodeId: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
sqlite.query(
|
||||
'DELETE FROM edges WHERE from_node_id = ? OR to_node_id = ?',
|
||||
[nodeId, nodeId]
|
||||
);
|
||||
}
|
||||
|
||||
async getNodeConnections(nodeId: number): Promise<NodeConnection[]> {
|
||||
return this.getNodeConnectionsSQLite(nodeId);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async getNodeConnectionsSQLite(nodeId: number): Promise<NodeConnection[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query(`
|
||||
SELECT
|
||||
e.*,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.id
|
||||
ELSE n_from.id
|
||||
END as connected_node_id,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.title
|
||||
ELSE n_from.title
|
||||
END as connected_node_title,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.content
|
||||
ELSE n_from.content
|
||||
END as connected_node_content,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.link
|
||||
ELSE n_from.link
|
||||
END as connected_node_link,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.chunk
|
||||
ELSE n_from.chunk
|
||||
END as connected_node_chunk,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.metadata
|
||||
ELSE n_from.metadata
|
||||
END as connected_node_metadata,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.created_at
|
||||
ELSE n_from.created_at
|
||||
END as connected_node_created_at,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.updated_at
|
||||
ELSE n_from.updated_at
|
||||
END as connected_node_updated_at,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN (
|
||||
SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n_to.id
|
||||
)
|
||||
ELSE (
|
||||
SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n_from.id
|
||||
)
|
||||
END as connected_node_dimensions_json
|
||||
FROM edges e
|
||||
LEFT JOIN nodes n_from ON e.from_node_id = n_from.id
|
||||
LEFT JOIN nodes n_to ON e.to_node_id = n_to.id
|
||||
WHERE e.from_node_id = ? OR e.to_node_id = ?
|
||||
ORDER BY e.created_at DESC
|
||||
`, [
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId
|
||||
]);
|
||||
|
||||
return this.mapNodeConnectionsSQLite(result.rows);
|
||||
}
|
||||
|
||||
private mapNodeConnections(rows: any[]): NodeConnection[] {
|
||||
return rows.map(row => {
|
||||
const edge: Edge = {
|
||||
id: row.id,
|
||||
from_node_id: row.from_node_id,
|
||||
to_node_id: row.to_node_id,
|
||||
context: row.context,
|
||||
source: row.source,
|
||||
created_at: row.created_at
|
||||
};
|
||||
|
||||
const connected_node: Node = {
|
||||
id: row.connected_node_id,
|
||||
title: row.connected_node_title,
|
||||
content: row.connected_node_content,
|
||||
link: row.connected_node_link,
|
||||
dimensions: row.connected_node_dimensions,
|
||||
embedding: undefined, // Not needed for display
|
||||
chunk: row.connected_node_chunk,
|
||||
metadata: row.connected_node_metadata,
|
||||
created_at: row.connected_node_created_at,
|
||||
updated_at: row.connected_node_updated_at
|
||||
};
|
||||
|
||||
return {
|
||||
id: edge.id,
|
||||
connected_node,
|
||||
edge
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private mapNodeConnectionsSQLite(rows: any[]): NodeConnection[] {
|
||||
return rows.map(row => {
|
||||
let context: any = row.context;
|
||||
if (typeof row.context === 'string') {
|
||||
const trimmed = row.context.trim();
|
||||
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
||||
try {
|
||||
context = JSON.parse(trimmed);
|
||||
} catch (error) {
|
||||
console.warn('[edges] Failed to parse JSON context for edge', row.id, error);
|
||||
context = row.context;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const edge: Edge = {
|
||||
id: row.id,
|
||||
from_node_id: row.from_node_id,
|
||||
to_node_id: row.to_node_id,
|
||||
context,
|
||||
source: row.source,
|
||||
created_at: row.created_at
|
||||
};
|
||||
|
||||
const connected_node: Node = {
|
||||
id: row.connected_node_id,
|
||||
title: row.connected_node_title,
|
||||
content: row.connected_node_content,
|
||||
link: row.connected_node_link,
|
||||
dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'),
|
||||
embedding: undefined, // Not needed for display
|
||||
chunk: row.connected_node_chunk,
|
||||
metadata: typeof row.connected_node_metadata === 'string' ? JSON.parse(row.connected_node_metadata) : row.connected_node_metadata,
|
||||
created_at: row.connected_node_created_at,
|
||||
updated_at: row.connected_node_updated_at
|
||||
};
|
||||
|
||||
return {
|
||||
id: edge.id,
|
||||
connected_node,
|
||||
edge
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async edgeExists(fromId: number, toId: number): Promise<boolean> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT 1 FROM edges WHERE from_node_id = ? AND to_node_id = ?', [fromId, toId]);
|
||||
return result.rows.length > 0;
|
||||
}
|
||||
|
||||
async getEdgeCount(): Promise<number> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT COUNT(*) as count FROM edges');
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
|
||||
async getMostConnectedNodes(limit = 10): Promise<Array<{ node_id: number; connection_count: number }>> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query(`
|
||||
SELECT
|
||||
node_id,
|
||||
COUNT(*) as connection_count
|
||||
FROM (
|
||||
SELECT from_node_id as node_id FROM edges
|
||||
UNION ALL
|
||||
SELECT to_node_id as node_id FROM edges
|
||||
) combined
|
||||
GROUP BY node_id
|
||||
ORDER BY connection_count DESC
|
||||
LIMIT ?
|
||||
`, [limit]);
|
||||
|
||||
return result.rows.map((row: any) => ({
|
||||
node_id: Number(row.node_id),
|
||||
connection_count: Number(row.connection_count)
|
||||
}));
|
||||
}
|
||||
|
||||
async createBidirectionalEdge(fromId: number, toId: number, options?: {
|
||||
context?: any;
|
||||
source?: 'user' | 'ai_similarity' | 'helper_name';
|
||||
}): Promise<Edge[]> {
|
||||
const edges: Edge[] = [];
|
||||
|
||||
// Create edge from A to B
|
||||
const forwardEdge = await this.createEdge({
|
||||
from_node_id: fromId,
|
||||
to_node_id: toId,
|
||||
context: options?.context,
|
||||
source: options?.source || 'ai_similarity'
|
||||
});
|
||||
edges.push(forwardEdge);
|
||||
|
||||
// Create edge from B to A
|
||||
const backwardEdge = await this.createEdge({
|
||||
from_node_id: toId,
|
||||
to_node_id: fromId,
|
||||
context: options?.context,
|
||||
source: options?.source || 'ai_similarity'
|
||||
});
|
||||
edges.push(backwardEdge);
|
||||
|
||||
return edges;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const edgeService = new EdgeService();
|
||||
@@ -0,0 +1,70 @@
|
||||
// Service instances
|
||||
export { nodeService, NodeService } from './nodes';
|
||||
export { chunkService, ChunkService } from './chunks';
|
||||
export { edgeService, EdgeService } from './edges';
|
||||
export { dimensionService, DimensionService } from './dimensionService';
|
||||
// export { HelperService } from './helpers'; // Removed - migrated to JSON-based service
|
||||
|
||||
// Types
|
||||
export * from '@/types/database';
|
||||
|
||||
// Health check utility
|
||||
export async function checkDatabaseHealth(): Promise<{
|
||||
connected: boolean;
|
||||
vectorExtension: boolean;
|
||||
tablesExist: boolean;
|
||||
error?: string;
|
||||
}> {
|
||||
try {
|
||||
return checkSQLiteDatabaseHealth();
|
||||
} catch (error) {
|
||||
return {
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function checkSQLiteDatabaseHealth(): Promise<{
|
||||
connected: boolean;
|
||||
vectorExtension: boolean;
|
||||
tablesExist: boolean;
|
||||
error?: string;
|
||||
}> {
|
||||
try {
|
||||
const { getSQLiteClient } = await import('./sqlite-client');
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const connected = await sqlite.testConnection();
|
||||
if (!connected) {
|
||||
return {
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
error: 'SQLite connection failed'
|
||||
};
|
||||
}
|
||||
|
||||
const vectorExtension = await sqlite.checkVectorExtension();
|
||||
|
||||
// Check if main tables exist
|
||||
const tables = await sqlite.checkTables();
|
||||
const requiredTables = ['nodes', 'chunks', 'edges'];
|
||||
const tablesExist = requiredTables.every(table => tables.includes(table));
|
||||
|
||||
return {
|
||||
connected,
|
||||
vectorExtension,
|
||||
tablesExist
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { Node, NodeFilters } from '@/types/database';
|
||||
import { eventBroadcaster } from '../events';
|
||||
|
||||
export class NodeService {
|
||||
async getNodes(filters: NodeFilters = {}): Promise<Node[]> {
|
||||
return this.getNodesSQLite(filters);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
|
||||
const { dimensions, search, limit = 100, offset = 0, sortBy } = filters;
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
|
||||
let query = `
|
||||
SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
(SELECT COUNT(*) FROM edges WHERE from_node_id = n.id OR to_node_id = n.id) as edge_count
|
||||
FROM nodes n
|
||||
WHERE 1=1
|
||||
`;
|
||||
const params: any[] = [];
|
||||
|
||||
// Filter by dimensions (SQLite JOIN with node_dimensions)
|
||||
if (dimensions && dimensions.length > 0) {
|
||||
query += ` AND EXISTS (
|
||||
SELECT 1 FROM node_dimensions nd
|
||||
WHERE nd.node_id = n.id
|
||||
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
|
||||
)`;
|
||||
params.push(...dimensions);
|
||||
}
|
||||
|
||||
// Text search in title and content (SQLite LIKE with COLLATE NOCASE)
|
||||
if (search) {
|
||||
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.content LIKE ? COLLATE NOCASE)`;
|
||||
params.push(`%${search}%`, `%${search}%`);
|
||||
}
|
||||
|
||||
// Sorting logic
|
||||
if (search) {
|
||||
// For search queries, prioritize by relevance: exact title → starts with → contains in title → content
|
||||
query += ` ORDER BY
|
||||
CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 5 END,
|
||||
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 5 END,
|
||||
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 5 END,
|
||||
CASE WHEN n.content LIKE ? COLLATE NOCASE THEN 4 ELSE 5 END,
|
||||
n.updated_at DESC`;
|
||||
params.push(
|
||||
search, // Exact match (case-insensitive)
|
||||
`${search}%`, // Starts with search term
|
||||
`%${search}%`, // Contains in title
|
||||
`%${search}%` // Contains in content
|
||||
);
|
||||
} else if (sortBy === 'edges') {
|
||||
// Sort by edge count (most connected first)
|
||||
query += ' ORDER BY edge_count DESC, n.updated_at DESC';
|
||||
} else {
|
||||
query += ' ORDER BY n.updated_at DESC';
|
||||
}
|
||||
|
||||
if (limit) {
|
||||
query += ` LIMIT ?`;
|
||||
params.push(limit);
|
||||
}
|
||||
|
||||
if (offset > 0) {
|
||||
query += ` OFFSET ?`;
|
||||
params.push(offset);
|
||||
}
|
||||
|
||||
const result = sqlite.query<Node & { dimensions_json: string }>(query, params);
|
||||
|
||||
// Parse dimensions_json back to array for compatibility
|
||||
return result.rows.map(row => ({
|
||||
...row,
|
||||
dimensions: JSON.parse(row.dimensions_json || '[]')
|
||||
}));
|
||||
}
|
||||
|
||||
async getNodeById(id: number): Promise<Node | null> {
|
||||
return this.getNodeByIdSQLite(id);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async getNodeByIdSQLite(id: number): Promise<Node | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
FROM nodes n
|
||||
WHERE n.id = ?
|
||||
`;
|
||||
const result = sqlite.query<Node & { dimensions_json: string }>(query, [id]);
|
||||
|
||||
if (result.rows.length === 0) return null;
|
||||
|
||||
const row = result.rows[0];
|
||||
return {
|
||||
...row,
|
||||
dimensions: JSON.parse(row.dimensions_json || '[]')
|
||||
};
|
||||
}
|
||||
|
||||
async createNode(nodeData: Partial<Node>): Promise<Node> {
|
||||
return this.createNodeSQLite(nodeData);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async createNodeSQLite(nodeData: Partial<Node>): Promise<Node> {
|
||||
const {
|
||||
title,
|
||||
description,
|
||||
content,
|
||||
link,
|
||||
type,
|
||||
dimensions = [],
|
||||
chunk,
|
||||
chunk_status,
|
||||
metadata = {}
|
||||
} = nodeData;
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const nodeId = sqlite.transaction(() => {
|
||||
// Insert node using prepare/run for lastInsertRowid access
|
||||
const nodeResult = sqlite.prepare(`
|
||||
INSERT INTO nodes (title, description, content, link, type, metadata, chunk, chunk_status, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
title,
|
||||
description ?? null,
|
||||
content ?? null,
|
||||
link ?? null,
|
||||
type ?? null,
|
||||
JSON.stringify(metadata),
|
||||
chunk ?? null,
|
||||
chunk_status ?? null,
|
||||
now,
|
||||
now
|
||||
);
|
||||
|
||||
const id = Number(nodeResult.lastInsertRowid);
|
||||
|
||||
// Insert dimensions separately with INSERT OR IGNORE for safety
|
||||
if (dimensions.length > 0) {
|
||||
const stmt = sqlite.prepare(
|
||||
"INSERT OR IGNORE INTO node_dimensions (node_id, dimension) VALUES (?, ?)"
|
||||
);
|
||||
for (const dimension of dimensions) {
|
||||
stmt.run(id, dimension);
|
||||
}
|
||||
}
|
||||
|
||||
return id; // Returns number directly
|
||||
});
|
||||
|
||||
// Get the created node with dimensions (outside transaction)
|
||||
const createdNode = await this.getNodeByIdSQLite(nodeId);
|
||||
if (!createdNode) {
|
||||
throw new Error('Failed to create node');
|
||||
}
|
||||
|
||||
// Broadcast node creation event
|
||||
console.log('🚀 Broadcasting NODE_CREATED event for:', createdNode.title);
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'NODE_CREATED',
|
||||
data: { node: createdNode }
|
||||
});
|
||||
|
||||
return createdNode;
|
||||
}
|
||||
|
||||
async updateNode(id: number, updates: Partial<Node>): Promise<Node> {
|
||||
return this.updateNodeSQLite(id, updates);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
|
||||
const { title, description, content, link, type, dimensions, chunk, metadata } = updates;
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const existingRow = sqlite
|
||||
.query<{ id: number }>('SELECT id FROM nodes WHERE id = ?', [id])
|
||||
.rows[0];
|
||||
|
||||
if (!existingRow) {
|
||||
throw new Error(`Node with ID ${id} not found`);
|
||||
}
|
||||
|
||||
sqlite.transaction(() => {
|
||||
// Update node columns (only update provided fields)
|
||||
const setFields: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
if (title !== undefined) { setFields.push('title = ?'); params.push(title); }
|
||||
if (description !== undefined) { setFields.push('description = ?'); params.push(description); }
|
||||
if (content !== undefined) { setFields.push('content = ?'); params.push(content); }
|
||||
if (link !== undefined) { setFields.push('link = ?'); params.push(link); }
|
||||
if (type !== undefined) { setFields.push('type = ?'); params.push(type); }
|
||||
if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); }
|
||||
if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) {
|
||||
setFields.push('chunk_status = ?');
|
||||
params.push(updates.chunk_status ?? null);
|
||||
}
|
||||
if (metadata !== undefined) {
|
||||
setFields.push('metadata = ?');
|
||||
params.push(JSON.stringify(metadata));
|
||||
}
|
||||
|
||||
// Always update timestamp
|
||||
setFields.push('updated_at = ?');
|
||||
params.push(now, id); // id for WHERE clause
|
||||
|
||||
if (setFields.length > 1) { // More than just updated_at
|
||||
const stmt = sqlite.prepare(`UPDATE nodes SET ${setFields.join(', ')} WHERE id = ?`);
|
||||
stmt.run(...params);
|
||||
}
|
||||
|
||||
// Handle dimensions separately
|
||||
if (Array.isArray(dimensions)) {
|
||||
sqlite.prepare('DELETE FROM node_dimensions WHERE node_id = ?').run(id);
|
||||
const dimStmt = sqlite.prepare('INSERT OR IGNORE INTO node_dimensions (node_id, dimension) VALUES (?, ?)');
|
||||
for (const dim of dimensions) {
|
||||
dimStmt.run(id, dim);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Get updated node
|
||||
const updatedNode = await this.getNodeByIdSQLite(id);
|
||||
if (!updatedNode) {
|
||||
throw new Error(`Node with ID ${id} not found`);
|
||||
}
|
||||
|
||||
// Broadcast node update event
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'NODE_UPDATED',
|
||||
data: { nodeId: id, node: updatedNode }
|
||||
});
|
||||
|
||||
return updatedNode;
|
||||
}
|
||||
|
||||
async deleteNode(id: number): Promise<void> {
|
||||
return this.deleteNodeSQLite(id);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async deleteNodeSQLite(id: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.query('DELETE FROM nodes WHERE id = ?', [id]);
|
||||
|
||||
if (result.changes === 0) {
|
||||
throw new Error(`Node with ID ${id} not found`);
|
||||
}
|
||||
|
||||
// Broadcast node deletion event
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'NODE_DELETED',
|
||||
data: { nodeId: id }
|
||||
});
|
||||
}
|
||||
|
||||
// Dimension-based filtering methods
|
||||
async getNodesByDimension(dimension: string): Promise<Node[]> {
|
||||
return this.getNodes({ dimensions: [dimension] });
|
||||
}
|
||||
|
||||
async searchNodes(searchTerm: string, limit = 50): Promise<Node[]> {
|
||||
return this.getNodes({ search: searchTerm, limit });
|
||||
}
|
||||
|
||||
async getNodeCount(): Promise<number> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT COUNT(*) as count FROM nodes');
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async bulkUpdateNodes(ids: number[], updates: Partial<Node>): Promise<Node[]> {
|
||||
if (ids.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.bulkUpdateNodesSQLite(ids, updates);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async bulkUpdateNodesSQLite(ids: number[], updates: Partial<Node>): Promise<Node[]> {
|
||||
// For SQLite, use IN (SELECT value FROM json_each(?)) for safety
|
||||
const sqlite = getSQLiteClient();
|
||||
const idsJson = JSON.stringify(ids);
|
||||
|
||||
// For now, just update one by one - could optimize later
|
||||
const updatedNodes: Node[] = [];
|
||||
for (const id of ids) {
|
||||
const updated = await this.updateNodeSQLite(id, updates);
|
||||
updatedNodes.push(updated);
|
||||
}
|
||||
return updatedNodes;
|
||||
}
|
||||
|
||||
// Get all unique dimensions for UI filtering
|
||||
async getAllDimensions(): Promise<string[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT DISTINCT dimension
|
||||
FROM node_dimensions
|
||||
ORDER BY dimension
|
||||
`;
|
||||
const result = sqlite.query<{dimension: string}>(query);
|
||||
return result.rows.map(row => row.dimension);
|
||||
}
|
||||
|
||||
// Get dimension usage statistics
|
||||
async getDimensionStats(): Promise<{dimension: string, count: number}[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT dimension, COUNT(*) as count
|
||||
FROM node_dimensions
|
||||
GROUP BY dimension
|
||||
ORDER BY count DESC
|
||||
`;
|
||||
const result = sqlite.query<{dimension: string, count: number}>(query);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const nodeService = new NodeService();
|
||||
|
||||
// Legacy export for backwards compatibility during migration
|
||||
export const itemService = nodeService;
|
||||
export const ItemService = NodeService;
|
||||
@@ -0,0 +1,686 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { DatabaseError } from '@/types/database';
|
||||
|
||||
export interface SQLiteConfig {
|
||||
dbPath: string;
|
||||
vecExtensionPath: string;
|
||||
}
|
||||
|
||||
export interface SQLiteQueryResult<T = any> {
|
||||
rows: T[];
|
||||
changes?: number;
|
||||
lastInsertRowid?: number;
|
||||
}
|
||||
|
||||
class SQLiteClient {
|
||||
private static instance: SQLiteClient;
|
||||
private db: Database.Database;
|
||||
private config: SQLiteConfig;
|
||||
private readonly readOnly: boolean;
|
||||
|
||||
private constructor() {
|
||||
this.config = this.getSQLiteConfig();
|
||||
this.readOnly = process.env.SQLITE_READONLY === 'true';
|
||||
|
||||
// Initialize database connection
|
||||
const dbDirectory = path.dirname(this.config.dbPath);
|
||||
if (!this.readOnly && !fs.existsSync(dbDirectory)) {
|
||||
fs.mkdirSync(dbDirectory, { recursive: true });
|
||||
}
|
||||
this.db = this.readOnly
|
||||
? new Database(this.config.dbPath, { readonly: true, fileMustExist: true })
|
||||
: new Database(this.config.dbPath);
|
||||
|
||||
// Load sqlite-vec extension
|
||||
try {
|
||||
this.db.loadExtension(this.config.vecExtensionPath);
|
||||
console.log('SQLite vector extension loaded successfully');
|
||||
} catch (error) {
|
||||
// Do not fail hard — allow the app to run without vector features
|
||||
console.error('Warning: Failed to load vector extension:', error);
|
||||
}
|
||||
|
||||
// Configure SQLite settings
|
||||
if (this.readOnly) {
|
||||
try {
|
||||
this.db.pragma('query_only = ON');
|
||||
} catch (error) {
|
||||
console.warn('Failed to enable query_only pragma in read-only mode:', error);
|
||||
}
|
||||
} else {
|
||||
this.db.pragma('foreign_keys = ON');
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.pragma('synchronous = NORMAL');
|
||||
this.db.pragma('cache_size = 10000');
|
||||
this.db.pragma('temp_store = memory');
|
||||
this.db.pragma('busy_timeout = 5000');
|
||||
|
||||
// Ensure vector virtual tables are present and healthy
|
||||
this.ensureVectorTables();
|
||||
this.healVectorTablesIfCorrupt();
|
||||
|
||||
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
|
||||
this.ensureLoggingAndMemorySchema();
|
||||
}
|
||||
|
||||
console.log('SQLite client initialized successfully');
|
||||
}
|
||||
|
||||
private getSQLiteConfig(): SQLiteConfig {
|
||||
const dbPath = process.env.SQLITE_DB_PATH || path.join(
|
||||
process.env.HOME || '~',
|
||||
'Library/Application Support/RA-H/db/rah.sqlite'
|
||||
);
|
||||
|
||||
const vecExtensionPath = process.env.SQLITE_VEC_EXTENSION_PATH ||
|
||||
'./vendor/sqlite-extensions/vec0.dylib';
|
||||
|
||||
return {
|
||||
dbPath,
|
||||
vecExtensionPath
|
||||
};
|
||||
}
|
||||
|
||||
public static getInstance(): SQLiteClient {
|
||||
if (!SQLiteClient.instance) {
|
||||
SQLiteClient.instance = new SQLiteClient();
|
||||
}
|
||||
return SQLiteClient.instance;
|
||||
}
|
||||
|
||||
public query<T extends Record<string, any> = any>(
|
||||
sql: string,
|
||||
params?: any[]
|
||||
): SQLiteQueryResult<T> {
|
||||
try {
|
||||
const sqlLower = sql.trim().toLowerCase();
|
||||
|
||||
// Handle different query types
|
||||
if (sqlLower.startsWith('select') ||
|
||||
sqlLower.startsWith('with') ||
|
||||
sqlLower.includes('returning')) {
|
||||
// SELECT queries and queries with RETURNING clause
|
||||
const stmt = this.db.prepare(sql);
|
||||
const rows = params ? stmt.all(...params) : stmt.all();
|
||||
return { rows: rows as T[] };
|
||||
} else {
|
||||
// INSERT/UPDATE/DELETE queries without RETURNING
|
||||
const stmt = this.db.prepare(sql);
|
||||
const result = params ? stmt.run(...params) : stmt.run();
|
||||
return {
|
||||
rows: [],
|
||||
changes: result.changes,
|
||||
lastInsertRowid: Number(result.lastInsertRowid)
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('SQLite query error:', error);
|
||||
throw this.handleError(error);
|
||||
}
|
||||
}
|
||||
|
||||
public prepare(sql: string) {
|
||||
return this.db.prepare(sql);
|
||||
}
|
||||
|
||||
public transaction<T>(callback: () => T): T {
|
||||
if (this.readOnly) {
|
||||
throw {
|
||||
message: 'SQLite client is read-only',
|
||||
code: 'SQLITE_READONLY',
|
||||
details: 'Transactions are not allowed in read-only mode'
|
||||
} as DatabaseError;
|
||||
}
|
||||
// Proactively validate/repair vec vtables before any write transaction
|
||||
this.healVectorTablesIfCorrupt();
|
||||
const txn = this.db.transaction(callback);
|
||||
try {
|
||||
return txn();
|
||||
} catch (error) {
|
||||
throw this.handleError(error);
|
||||
}
|
||||
}
|
||||
|
||||
public async testConnection(): Promise<boolean> {
|
||||
try {
|
||||
const result = this.query('SELECT datetime() as current_time');
|
||||
return result.rows.length > 0;
|
||||
} catch (error) {
|
||||
console.error('SQLite connection test failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async checkVectorExtension(): Promise<boolean> {
|
||||
try {
|
||||
const result = this.query('SELECT vec_version() as version');
|
||||
return result.rows.length > 0;
|
||||
} catch (error) {
|
||||
console.error('Vector extension check failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async checkTables(): Promise<string[]> {
|
||||
try {
|
||||
const result = this.query(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
|
||||
);
|
||||
return result.rows.map(row => row.name);
|
||||
} catch (error) {
|
||||
console.error('Table check failed:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
public ensureVectorExtensions(): void {
|
||||
try {
|
||||
// Test for vec_nodes and vec_chunks; create them if missing
|
||||
const hasVecNodes = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_nodes');
|
||||
if (!hasVecNodes) {
|
||||
this.db.exec(`
|
||||
CREATE VIRTUAL TABLE vec_nodes USING vec0(
|
||||
node_id INTEGER PRIMARY KEY,
|
||||
embedding FLOAT[1536]
|
||||
);
|
||||
`);
|
||||
console.log('Created vec_nodes virtual table');
|
||||
}
|
||||
|
||||
const hasVecChunks = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_chunks');
|
||||
if (!hasVecChunks) {
|
||||
this.db.exec(`
|
||||
CREATE VIRTUAL TABLE vec_chunks USING vec0(
|
||||
chunk_id INTEGER PRIMARY KEY,
|
||||
embedding FLOAT[1536]
|
||||
);
|
||||
`);
|
||||
console.log('Created vec_chunks virtual table');
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Vector extension not available:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private ensureVectorTables(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
// Wrapper to keep existing public API stable
|
||||
this.ensureVectorExtensions();
|
||||
}
|
||||
|
||||
private ensureLoggingAndMemorySchema(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// 1) If logs table missing but legacy memory table exists, migrate
|
||||
const hasLogs = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
|
||||
const hasLegacyMemory = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memory'").get();
|
||||
if (!hasLogs && hasLegacyMemory) {
|
||||
// Drop old view to release dependency
|
||||
this.db.exec(`DROP VIEW IF EXISTS memory_v;`);
|
||||
this.db.exec(`ALTER TABLE memory RENAME TO logs;`);
|
||||
}
|
||||
|
||||
// 2) Ensure logs table exists (fresh install)
|
||||
const hasLogsNow = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
|
||||
if (!hasLogsNow) {
|
||||
this.db.exec(`
|
||||
CREATE TABLE logs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
ts TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP),
|
||||
table_name TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
row_id INTEGER NOT NULL,
|
||||
summary TEXT,
|
||||
snapshot_json TEXT
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
// Ensure nodes table has expected columns for memory nodes
|
||||
try {
|
||||
const nodeCols = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
|
||||
const ensureNodeCol = (name: string, ddl: string) => {
|
||||
if (!nodeCols.some(col => col.name === name)) {
|
||||
try {
|
||||
this.db.exec(ddl);
|
||||
} catch (colErr) {
|
||||
console.warn(`Failed to add nodes.${name}`, colErr);
|
||||
}
|
||||
}
|
||||
};
|
||||
ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;");
|
||||
ensureNodeCol('type', "ALTER TABLE nodes ADD COLUMN type TEXT;");
|
||||
} catch (nodeErr) {
|
||||
console.warn('Failed to ensure nodes columns:', nodeErr);
|
||||
}
|
||||
|
||||
// Ensure chats table tracks creation timestamp for ordering
|
||||
try {
|
||||
const chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as Array<{ name: string }>;
|
||||
if (chatCols.some(col => col.name === 'created_at')) {
|
||||
// no-op, column exists
|
||||
} else if (chatCols.length > 0) {
|
||||
this.db.exec("ALTER TABLE chats ADD COLUMN created_at TEXT DEFAULT (CURRENT_TIMESTAMP);");
|
||||
}
|
||||
} catch (chatErr) {
|
||||
console.warn('Failed to ensure chats.created_at column:', chatErr);
|
||||
}
|
||||
|
||||
// 3) Helpful indexes on logs (clean up old names first)
|
||||
this.db.exec(`
|
||||
DROP INDEX IF EXISTS idx_memory_ts;
|
||||
DROP INDEX IF EXISTS idx_memory_table_ts;
|
||||
DROP INDEX IF EXISTS idx_memory_table_row;
|
||||
CREATE INDEX IF NOT EXISTS idx_logs_ts ON logs(ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_logs_table_ts ON logs(table_name, ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_logs_table_row ON logs(table_name, row_id);
|
||||
`);
|
||||
|
||||
// 4) Recreate triggers to write to logs (use CREATE IF NOT EXISTS)
|
||||
const hasChats = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='chats'").get();
|
||||
this.db.exec(`
|
||||
DROP TRIGGER IF EXISTS trg_nodes_ai;
|
||||
DROP TRIGGER IF EXISTS trg_nodes_au;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_nodes_ai AFTER INSERT ON nodes BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('nodes', 'insert', NEW.id,
|
||||
printf('node created: %s', COALESCE(NEW.title,'')),
|
||||
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_nodes_au AFTER UPDATE ON nodes BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('nodes', 'update', NEW.id,
|
||||
printf('node updated: %s', COALESCE(NEW.title,'')),
|
||||
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
|
||||
END;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_edges_ai;
|
||||
DROP TRIGGER IF EXISTS trg_edges_au;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_edges_ai AFTER INSERT ON edges BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('edges', 'insert', NEW.id,
|
||||
printf('edge %d→%d (%s)', NEW.from_node_id, NEW.to_node_id, COALESCE(NEW.source,'')),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'from', NEW.from_node_id,
|
||||
'to', NEW.to_node_id,
|
||||
'source', NEW.source,
|
||||
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
|
||||
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
|
||||
));
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_edges_au AFTER UPDATE ON edges BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('edges', 'update', NEW.id,
|
||||
printf('edge updated %d→%d', NEW.from_node_id, NEW.to_node_id),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'from', NEW.from_node_id,
|
||||
'to', NEW.to_node_id,
|
||||
'source', NEW.source,
|
||||
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
|
||||
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
|
||||
));
|
||||
END;
|
||||
`);
|
||||
|
||||
if (hasChats) {
|
||||
this.db.exec(`
|
||||
DROP TRIGGER IF EXISTS trg_chats_ai;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_chats_ai AFTER INSERT ON chats BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('chats', 'insert', NEW.id,
|
||||
printf('chat: %s (%s)', COALESCE(NEW.helper_name,''), COALESCE(NEW.thread_id,'')),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'helper', NEW.helper_name,
|
||||
'thread', NEW.thread_id,
|
||||
'user_message', COALESCE(NEW.user_message,''),
|
||||
'assistant_message', COALESCE(NEW.assistant_message,''),
|
||||
'user_preview', substr(COALESCE(NEW.user_message,''), 1, 120),
|
||||
'assistant_preview', substr(COALESCE(NEW.assistant_message,''), 1, 120),
|
||||
'system_message', COALESCE(json_extract(NEW.metadata, '$.system_message'), ''),
|
||||
'input_tokens', COALESCE(json_extract(NEW.metadata, '$.input_tokens'), 0),
|
||||
'output_tokens', COALESCE(json_extract(NEW.metadata, '$.output_tokens'), 0),
|
||||
'cost_usd', COALESCE(json_extract(NEW.metadata, '$.estimated_cost_usd'), 0.0),
|
||||
'cache_hit', COALESCE(json_extract(NEW.metadata, '$.cache_hit'), 0),
|
||||
'model', COALESCE(json_extract(NEW.metadata, '$.model_used'), ''),
|
||||
'tools_count', COALESCE(json_extract(NEW.metadata, '$.tool_calls_count'), 0),
|
||||
'trace_id', COALESCE(json_extract(NEW.metadata, '$.trace_id'), ''),
|
||||
'voice_tts_chars', COALESCE(json_extract(NEW.metadata, '$.voice_tts_chars'), 0),
|
||||
'voice_tts_cost_usd', COALESCE(json_extract(NEW.metadata, '$.voice_tts_cost_usd'), 0),
|
||||
'voice_tts_chars_total', COALESCE(json_extract(NEW.metadata, '$.voice_tts_chars_total'), 0),
|
||||
'voice_tts_cost_usd_total', COALESCE(json_extract(NEW.metadata, '$.voice_tts_cost_usd_total'), 0),
|
||||
'voice_request_id', COALESCE(json_extract(NEW.metadata, '$.voice_request_id'), ''),
|
||||
'voice_tts_request_count', COALESCE(json_extract(NEW.metadata, '$.voice_tts_request_count'), 0)
|
||||
));
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS voice_usage (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
chat_id INTEGER,
|
||||
session_id TEXT,
|
||||
helper_name TEXT,
|
||||
request_id TEXT,
|
||||
message_id TEXT,
|
||||
voice TEXT,
|
||||
model TEXT,
|
||||
chars INTEGER,
|
||||
cost_usd REAL,
|
||||
duration_ms INTEGER,
|
||||
text_preview TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE SET NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_voice_usage_session ON voice_usage(session_id, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_voice_usage_chat ON voice_usage(chat_id);
|
||||
`);
|
||||
|
||||
// 5) Views: logs_v (drop any legacy memory_v alias)
|
||||
this.db.exec(`DROP VIEW IF EXISTS logs_v; DROP VIEW IF EXISTS memory_v;`);
|
||||
try {
|
||||
this.db.exec(`
|
||||
CREATE VIEW logs_v AS
|
||||
SELECT
|
||||
m.id,
|
||||
m.ts,
|
||||
m.table_name,
|
||||
m.action,
|
||||
m.row_id,
|
||||
m.summary,
|
||||
m.enriched_summary,
|
||||
m.snapshot_json,
|
||||
CASE WHEN m.table_name='nodes' THEN n.title END AS node_title,
|
||||
CASE WHEN m.table_name='edges' THEN nf.title END AS edge_from_title,
|
||||
CASE WHEN m.table_name='edges' THEN nt.title END AS edge_to_title,
|
||||
CASE WHEN m.table_name='chats' THEN c.helper_name END AS chat_helper,
|
||||
CASE WHEN m.table_name='chats' THEN substr(c.user_message,1,120) END AS chat_user_preview,
|
||||
CASE WHEN m.table_name='chats' THEN substr(c.assistant_message,1,120) END AS chat_assistant_preview,
|
||||
CASE WHEN m.table_name='chats' THEN c.user_message END AS chat_user_full,
|
||||
CASE WHEN m.table_name='chats' THEN c.assistant_message END AS chat_assistant_full
|
||||
FROM logs m
|
||||
LEFT JOIN nodes n ON (m.table_name='nodes' AND m.row_id = n.id)
|
||||
LEFT JOIN edges e ON (m.table_name='edges' AND m.row_id = e.id)
|
||||
LEFT JOIN nodes nf ON e.from_node_id = nf.id
|
||||
LEFT JOIN nodes nt ON e.to_node_id = nt.id
|
||||
LEFT JOIN chats c ON (m.table_name='chats' AND m.row_id = c.id);
|
||||
`);
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof Error) ||
|
||||
!/already exists/i.test(error.message || '')
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
// Do not recreate memory_v; alias has been removed.
|
||||
|
||||
// 6) Chat memory state tracking for chat-triggered memories
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS chat_memory_state (
|
||||
thread_id TEXT PRIMARY KEY,
|
||||
helper_name TEXT,
|
||||
last_processed_chat_id INTEGER DEFAULT 0,
|
||||
last_processed_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_memory_thread ON chat_memory_state(thread_id);
|
||||
`);
|
||||
|
||||
// Agent delegation table for orchestrator/worker coordination
|
||||
try {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS agent_delegations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT UNIQUE NOT NULL,
|
||||
task TEXT NOT NULL,
|
||||
context TEXT,
|
||||
expected_outcome TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
summary TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
} catch (e) {
|
||||
console.warn('Failed to ensure agent_delegations table:', e);
|
||||
}
|
||||
|
||||
// 8) Logs retention trigger (~10k most recent rows)
|
||||
try {
|
||||
this.db.exec(`
|
||||
DROP TRIGGER IF EXISTS trg_logs_prune;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_logs_prune AFTER INSERT ON logs BEGIN
|
||||
DELETE FROM logs WHERE id < NEW.id - 10000;
|
||||
END;
|
||||
`);
|
||||
} catch {}
|
||||
|
||||
// 7) Ensure agents table schema (backward compatibility)
|
||||
try {
|
||||
const agentCols = this.db.prepare('PRAGMA table_info(agents)').all() as any[];
|
||||
if (agentCols.length) {
|
||||
const hasKey = agentCols.some(col => col.name === 'key');
|
||||
const hasComponentKey = agentCols.some(col => col.name === 'component_key');
|
||||
if (!hasKey && hasComponentKey) {
|
||||
try { this.db.exec('ALTER TABLE agents RENAME COLUMN component_key TO key;'); } catch {}
|
||||
}
|
||||
|
||||
if (!agentCols.some(col => col.name === 'role')) {
|
||||
try { this.db.exec("ALTER TABLE agents ADD COLUMN role TEXT NOT NULL DEFAULT 'executor';"); } catch {}
|
||||
}
|
||||
if (!agentCols.some(col => col.name === 'memory')) {
|
||||
try { this.db.exec('ALTER TABLE agents ADD COLUMN memory TEXT;'); } catch {}
|
||||
}
|
||||
if (!agentCols.some(col => col.name === 'prompts')) {
|
||||
try { this.db.exec("ALTER TABLE agents ADD COLUMN prompts TEXT DEFAULT '[]';"); } catch {}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Agent schema ensure failed:', e);
|
||||
}
|
||||
|
||||
// 8) Ensure chats schema (remove legacy focused_memory_id, ensure agent columns)
|
||||
if (hasChats) {
|
||||
try {
|
||||
let chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as any[];
|
||||
const hasFocusedMemoryId = chatCols.some((c: any) => c.name === 'focused_memory_id');
|
||||
if (hasFocusedMemoryId) {
|
||||
console.log('Removing legacy chats.focused_memory_id column');
|
||||
let flippedForeignKeys = false;
|
||||
try {
|
||||
this.db.exec('PRAGMA foreign_keys=OFF;');
|
||||
flippedForeignKeys = true;
|
||||
this.db.exec(`
|
||||
BEGIN TRANSACTION;
|
||||
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
|
||||
CREATE TABLE chats (
|
||||
id INTEGER PRIMARY KEY,
|
||||
chat_type TEXT,
|
||||
helper_name TEXT,
|
||||
agent_type TEXT DEFAULT 'orchestrator',
|
||||
delegation_id INTEGER,
|
||||
user_message TEXT,
|
||||
assistant_message TEXT,
|
||||
thread_id TEXT,
|
||||
focused_node_id INTEGER,
|
||||
created_at TEXT DEFAULT (CURRENT_TIMESTAMP),
|
||||
metadata TEXT,
|
||||
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
|
||||
);
|
||||
INSERT INTO chats (
|
||||
id, chat_type, helper_name, agent_type, delegation_id,
|
||||
user_message, assistant_message, thread_id, focused_node_id,
|
||||
created_at, metadata
|
||||
)
|
||||
SELECT id, chat_type, helper_name, agent_type, delegation_id,
|
||||
user_message, assistant_message, thread_id, focused_node_id,
|
||||
created_at, metadata
|
||||
FROM chats_legacy_cleanup;
|
||||
DROP TABLE chats_legacy_cleanup;
|
||||
CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);
|
||||
COMMIT;
|
||||
`);
|
||||
} catch (migrationErr) {
|
||||
console.warn('Failed to migrate chats table (focused_memory_id removal):', migrationErr);
|
||||
try { this.db.exec('ROLLBACK;'); } catch {}
|
||||
} finally {
|
||||
if (flippedForeignKeys) {
|
||||
try { this.db.exec('PRAGMA foreign_keys=ON;'); } catch {}
|
||||
}
|
||||
}
|
||||
chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as any[];
|
||||
}
|
||||
|
||||
this.db.exec("CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);");
|
||||
|
||||
const ensureCol = (name: string, ddl: string) => {
|
||||
if (!chatCols.some((c: any) => c.name === name)) {
|
||||
try { this.db.exec(ddl); } catch (colErr) { console.warn(`Failed to add chats.${name}`, colErr); }
|
||||
}
|
||||
};
|
||||
ensureCol('agent_type', "ALTER TABLE chats ADD COLUMN agent_type TEXT DEFAULT 'orchestrator';");
|
||||
ensureCol('delegation_id', 'ALTER TABLE chats ADD COLUMN delegation_id INTEGER;');
|
||||
} catch (e) {
|
||||
console.warn('Failed to update chats schema:', e);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const chatColsPost = hasChats
|
||||
? this.db.prepare('PRAGMA table_info(chats)').all() as any[]
|
||||
: [];
|
||||
const stillHasFocusedMemoryId = chatColsPost.some((c: any) => c.name === 'focused_memory_id');
|
||||
if (stillHasFocusedMemoryId) {
|
||||
console.warn('Skipping legacy memory table drop because chats.focused_memory_id is still present.');
|
||||
} else {
|
||||
this.db.exec(`
|
||||
DROP TRIGGER IF EXISTS trg_episodic_prune;
|
||||
DROP TABLE IF EXISTS episodic_memory;
|
||||
DROP TABLE IF EXISTS episodic_pipeline_state;
|
||||
DROP TABLE IF EXISTS semantic_memory;
|
||||
DROP TABLE IF EXISTS semantic_pipeline_state;
|
||||
DROP TABLE IF EXISTS memory_pipeline_state;
|
||||
DROP TABLE IF EXISTS memory;
|
||||
`);
|
||||
}
|
||||
} catch (dropLegacyErr) {
|
||||
console.warn('Failed to drop legacy memory pipeline tables:', dropLegacyErr);
|
||||
}
|
||||
|
||||
// 9) Ensure dimensions table exists (v0.1.16+ schema migration)
|
||||
const hasDimensions = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='dimensions'").get();
|
||||
if (!hasDimensions) {
|
||||
console.log('Creating dimensions table for v0.1.16+ features...');
|
||||
this.db.exec(`
|
||||
CREATE TABLE dimensions (
|
||||
name TEXT PRIMARY KEY,
|
||||
description TEXT,
|
||||
is_priority INTEGER DEFAULT 0,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
// Seed default locked dimensions
|
||||
const defaultDimensions = ['research', 'ideas', 'projects', 'memory', 'preferences'];
|
||||
const insertDimension = this.db.prepare(`
|
||||
INSERT INTO dimensions (name, is_priority, updated_at)
|
||||
VALUES (?, 1, datetime('now'))
|
||||
ON CONFLICT(name) DO UPDATE SET is_priority = 1, updated_at = datetime('now')
|
||||
`);
|
||||
|
||||
for (const dimension of defaultDimensions) {
|
||||
try {
|
||||
insertDimension.run(dimension);
|
||||
} catch (e) {
|
||||
console.warn(`Failed to seed dimension '${dimension}':`, e);
|
||||
}
|
||||
}
|
||||
console.log('Dimensions table created and seeded with default locked dimensions');
|
||||
} else {
|
||||
// Check if existing dimensions table has description column
|
||||
const dimensionCols = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
|
||||
const hasDescription = dimensionCols.some(col => col.name === 'description');
|
||||
if (!hasDescription) {
|
||||
console.log('Adding description column to existing dimensions table...');
|
||||
try {
|
||||
this.db.exec('ALTER TABLE dimensions ADD COLUMN description TEXT;');
|
||||
console.log('Description column added to dimensions table');
|
||||
} catch (e) {
|
||||
console.warn('Failed to add description column to dimensions table:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Logging + memory schema ensured');
|
||||
} catch (error) {
|
||||
console.error('Failed to ensure logging/memory schema:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private healVectorTablesIfCorrupt(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
// Attempt lightweight reads to detect CORRUPT_VTAB; if detected, drop/recreate vtables
|
||||
const tryRead = (table: string) => {
|
||||
try {
|
||||
this.db.prepare(`SELECT COUNT(*) as c FROM ${table}`).get();
|
||||
} catch (e: any) {
|
||||
const msg = String(e?.message || '');
|
||||
const code = (e && e.code) ? String(e.code) : '';
|
||||
if (code === 'SQLITE_CORRUPT_VTAB' || msg.includes('database disk image is malformed') || msg.includes('CORRUPT_VTAB')) {
|
||||
console.warn(`Detected corrupted virtual table ${table} (${code || 'error'}). Recreating...`);
|
||||
try {
|
||||
this.db.exec(`DROP TABLE IF EXISTS ${table};`);
|
||||
} catch {}
|
||||
const ddl = table === 'vec_nodes'
|
||||
? `CREATE VIRTUAL TABLE vec_nodes USING vec0(node_id INTEGER PRIMARY KEY, embedding FLOAT[1536]);`
|
||||
: `CREATE VIRTUAL TABLE vec_chunks USING vec0(chunk_id INTEGER PRIMARY KEY, embedding FLOAT[1536]);`;
|
||||
try {
|
||||
this.db.exec(ddl);
|
||||
console.log(`Recreated ${table} virtual table`);
|
||||
} catch (re) {
|
||||
console.error(`Failed to recreate ${table}:`, re);
|
||||
}
|
||||
} else {
|
||||
// Other errors should bubble up normally
|
||||
// eslint-disable-next-line no-unsafe-finally
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
tryRead('vec_nodes');
|
||||
tryRead('vec_chunks');
|
||||
}
|
||||
|
||||
private handleError(error: any): DatabaseError {
|
||||
return {
|
||||
message: error.message || 'SQLite operation failed',
|
||||
code: error.code || 'SQLITE_ERROR',
|
||||
details: error
|
||||
};
|
||||
}
|
||||
|
||||
public close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance (similar to PostgreSQL client interface)
|
||||
export const sqliteDb = SQLiteClient.getInstance();
|
||||
|
||||
// Export function to get client instance
|
||||
export const getSQLiteClient = () => sqliteDb;
|
||||
|
||||
// Export class for testing
|
||||
export { SQLiteClient };
|
||||
Reference in New Issue
Block a user