diff --git a/scripts/dev/bootstrap-local.mjs b/scripts/dev/bootstrap-local.mjs index be2f090..954e047 100644 --- a/scripts/dev/bootstrap-local.mjs +++ b/scripts/dev/bootstrap-local.mjs @@ -551,12 +551,19 @@ function ensureCoreSchema(db) { `); } -function tryInitVectorTables(db, dbPath) { +function getEmbeddingDimensions(env) { + const defaultDimensions = env.EMBEDDING_PROFILE === 'openai-compatible' || env.EMBEDDING_PROFILE === 'custom' + ? '1024' + : '1536'; + return Number(env.EMBEDDING_DIMENSIONS || defaultDimensions); +} + +function tryInitVectorTables(db, dbPath, env) { const extension = process.platform === 'darwin' ? 'dylib' : process.platform === 'win32' ? 'dll' : 'so'; const extensionPath = process.env.SQLITE_VEC_EXTENSION_PATH || path.join(repoDir, 'vendor', 'sqlite-extensions', `vec0.${extension}`); - const dimensions = Number(process.env.EMBEDDING_DIMENSIONS || '1536'); + const dimensions = getEmbeddingDimensions(env); if (!Number.isInteger(dimensions) || dimensions <= 0) { - throw new Error(`Invalid EMBEDDING_DIMENSIONS="${process.env.EMBEDDING_DIMENSIONS}"`); + throw new Error(`Invalid EMBEDDING_DIMENSIONS="${env.EMBEDDING_DIMENSIONS}"`); } try { @@ -585,11 +592,12 @@ function main() { ensureEnvFile(); - const env = parseEnvFile(targetEnv); + const env = { ...parseEnvFile(targetEnv), ...process.env }; if (process.env.SQLITE_DB_PATH) { ensureEnvValue('SQLITE_DB_PATH', process.env.SQLITE_DB_PATH); + env.SQLITE_DB_PATH = process.env.SQLITE_DB_PATH; } - const dbPath = expandPath(process.env.SQLITE_DB_PATH || env.SQLITE_DB_PATH || getDefaultDbPath()); + const dbPath = expandPath(env.SQLITE_DB_PATH || getDefaultDbPath()); fs.mkdirSync(path.dirname(dbPath), { recursive: true }); if (!fs.existsSync(dbPath)) { fs.closeSync(fs.openSync(dbPath, 'w')); @@ -598,7 +606,7 @@ function main() { const db = new Database(dbPath); try { ensureCoreSchema(db); - tryInitVectorTables(db, dbPath); + tryInitVectorTables(db, dbPath, env); } finally { db.close(); } diff --git a/scripts/rebuild-embeddings.ts b/scripts/rebuild-embeddings.ts index d3ea49c..3dcdf2c 100644 --- a/scripts/rebuild-embeddings.ts +++ b/scripts/rebuild-embeddings.ts @@ -1,6 +1,7 @@ import { getSQLiteClient } from '@/services/database/sqlite-client'; import { NodeEmbedder } from '@/services/typescript/embed-nodes'; import { UniversalEmbedder } from '@/services/typescript/embed-universal'; +import { getVectorBackendType } from '@/services/vectorBackend'; async function maybeRecreateQdrantCollections() { if (process.env.VECTOR_BACKEND !== 'qdrant' || process.env.QDRANT_RECREATE_COLLECTIONS !== 'true') { @@ -30,6 +31,10 @@ async function maybeRecreateQdrantCollections() { async function main() { const sqlite = getSQLiteClient(); await maybeRecreateQdrantCollections(); + if (getVectorBackendType() === 'sqlite-vec') { + sqlite.recreateVectorTables(); + console.log('[rebuild-embeddings] Recreated sqlite-vec tables for the active embedding dimensions.'); + } const nodeRows = sqlite.query<{ id: number; source?: string | null }>(` SELECT id, source FROM nodes diff --git a/src/services/database/sqlite-client.ts b/src/services/database/sqlite-client.ts index 81e18bf..d0555c9 100644 --- a/src/services/database/sqlite-client.ts +++ b/src/services/database/sqlite-client.ts @@ -18,6 +18,7 @@ export interface SQLiteQueryResult { } type FtsSurfaceName = 'nodes' | 'chunks'; +type VectorTableName = 'vec_nodes' | 'vec_chunks'; interface IntegrityProbeResult { ok: boolean; @@ -283,33 +284,82 @@ class SQLiteClient { public ensureVectorExtensions(): void { try { const dimensions = getEmbeddingProviderInfo().dimensions; - // Test for vec_nodes and vec_chunks; create them if missing - const hasVecNodes = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_nodes'); - if (!hasVecNodes) { - this.db.exec(` - CREATE VIRTUAL TABLE vec_nodes USING vec0( - node_id INTEGER PRIMARY KEY, - embedding FLOAT[${dimensions}] - ); - `); - console.log('Created vec_nodes virtual table'); - } - - const hasVecChunks = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_chunks'); - if (!hasVecChunks) { - this.db.exec(` - CREATE VIRTUAL TABLE vec_chunks USING vec0( - chunk_id INTEGER PRIMARY KEY, - embedding FLOAT[${dimensions}] - ); - `); - console.log('Created vec_chunks virtual table'); - } + this.ensureVectorTable('vec_nodes', dimensions); + this.ensureVectorTable('vec_chunks', dimensions); } catch (error) { console.warn('Vector extension not available:', error); } } + private getVectorTableSql(tableName: VectorTableName): string | null { + const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?") + .get(tableName) as { sql?: string } | undefined; + return row?.sql || null; + } + + public getVectorTableDimensions(): Record<'nodes' | 'chunks', number | null> { + return { + nodes: this.getVectorTableDimension('vec_nodes'), + chunks: this.getVectorTableDimension('vec_chunks'), + }; + } + + private getVectorTableDimension(tableName: VectorTableName): number | null { + const sql = this.getVectorTableSql(tableName); + const match = sql?.match(/embedding\s+FLOAT\[(\d+)\]/i); + return match ? Number(match[1]) : null; + } + + private createVectorTable(tableName: VectorTableName, dimensions: number): void { + const idColumn = tableName === 'vec_nodes' ? 'node_id' : 'chunk_id'; + this.db.exec(` + CREATE VIRTUAL TABLE ${tableName} USING vec0( + ${idColumn} INTEGER PRIMARY KEY, + embedding FLOAT[${dimensions}] + ); + `); + } + + private ensureVectorTable(tableName: VectorTableName, dimensions: number): void { + const existingSql = this.getVectorTableSql(tableName); + if (!existingSql) { + this.createVectorTable(tableName, dimensions); + console.log(`Created ${tableName} virtual table`); + return; + } + + const existingDimensions = this.getVectorTableDimension(tableName); + if (existingDimensions === dimensions) { + return; + } + + const rowCount = this.countVectorRows(tableName); + if (rowCount === 0 || this.maintenanceMode) { + this.db.exec(`DROP TABLE IF EXISTS ${tableName};`); + this.createVectorTable(tableName, dimensions); + console.warn(`Recreated ${tableName} virtual table for ${dimensions} dimensions (was ${existingDimensions ?? 'unknown'}).`); + return; + } + + console.warn( + `${tableName} uses ${existingDimensions ?? 'unknown'} dimensions, but the active embedding profile requires ${dimensions}. ` + + 'Run npm run rebuild:embeddings to recreate derived vectors.' + ); + } + + public recreateVectorTables(): void { + if (this.readOnly) { + throw new Error('Cannot recreate vector tables while SQLITE_READONLY=true.'); + } + const dimensions = getEmbeddingProviderInfo().dimensions; + this.db.exec(` + DROP TABLE IF EXISTS vec_nodes; + DROP TABLE IF EXISTS vec_chunks; + `); + this.createVectorTable('vec_nodes', dimensions); + this.createVectorTable('vec_chunks', dimensions); + } + private ensureVectorTables(): void { if (this.readOnly) { return; @@ -1333,7 +1383,7 @@ class SQLiteClient { ); } - private countVectorRows(tableName: 'vec_nodes' | 'vec_chunks'): number { + private countVectorRows(tableName: VectorTableName): number { try { const exists = this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(tableName); if (!exists) return 0; diff --git a/src/services/vectorBackend/sqlite-vec-backend.ts b/src/services/vectorBackend/sqlite-vec-backend.ts index d4f1496..d1e7217 100644 --- a/src/services/vectorBackend/sqlite-vec-backend.ts +++ b/src/services/vectorBackend/sqlite-vec-backend.ts @@ -99,10 +99,25 @@ export class SqliteVecBackend implements VectorBackend { async healthCheck(): Promise { const sqlite = getSQLiteClient(); const ok = await sqlite.checkVectorExtension(); + const expectedDimensions = getEmbeddingProviderInfo().dimensions; + const tableDimensions = sqlite.getVectorTableDimensions(); + const mismatchedTables = Object.entries(tableDimensions) + .filter(([, dimensions]) => dimensions !== null && dimensions !== expectedDimensions) + .map(([table, dimensions]) => `${table}=${dimensions}`); + + if (mismatchedTables.length > 0) { + return { + ok: false, + backend: 'sqlite-vec', + dimensions: expectedDimensions, + detail: `sqlite-vec table dimensions mismatch (${mismatchedTables.join(', ')}), expected ${expectedDimensions}. Run npm run rebuild:embeddings.`, + }; + } + return { ok, backend: 'sqlite-vec', - dimensions: getEmbeddingProviderInfo().dimensions, + dimensions: expectedDimensions, detail: ok ? 'sqlite-vec extension loaded' : 'sqlite-vec extension unavailable', }; }