diff --git a/README.md b/README.md index 74d62f0..8a534aa 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ [![Watch the demo](https://img.youtube.com/vi/IA02YB8mInM/hqdefault.jpg)](https://youtu.be/IA02YB8mInM?si=WoWpNE9QZEKEukvZ) -> **Currently macOS only.** Linux and Windows support is coming. If you want to run it on Linux/Windows now, see [instructions at the bottom](#linuxwindows). +> **Cross-platform local runtime:** macOS works out of the box. Linux and Windows now support the core local/web app flow, but semantic/vector search still depends on either sqlite-vec for your platform or a later Qdrant setup. **Full documentation:** [ra-h.app/docs/open-source](https://ra-h.app/docs/open-source) @@ -33,7 +33,7 @@ Your data stays on your machine. Nothing is sent anywhere unless you configure a - **Node.js 20.18.1+** — [nodejs.org](https://nodejs.org/) - **macOS** — Works out of the box -- **Linux/Windows** — Requires building sqlite-vec manually (see below) +- **Linux/Windows** — Core app works; vector search requires sqlite-vec for your platform (see below) --- @@ -44,7 +44,7 @@ git clone https://github.com/bradwmorris/ra-h_os.git cd ra-h_os npm install npm rebuild better-sqlite3 -./scripts/dev/bootstrap-local.sh +npm run bootstrap:local npm run dev ``` @@ -178,7 +178,9 @@ See [ra-h.app/docs/open-source](https://ra-h.app/docs/open-source) for full sche | Command | What it does | |---------|--------------| +| `npm run bootstrap:local` | Create `.env.local`, create the SQLite DB, and seed the base schema | | `npm run dev` | Start the app at localhost:3000 | +| `npm run dev:local` | Alias for `npm run dev` | | `npm run build` | Production build | | `npm run type-check` | Check TypeScript | @@ -186,7 +188,9 @@ See [ra-h.app/docs/open-source](https://ra-h.app/docs/open-source) for full sche ## Linux/Windows -The repo ships with a macOS binary for sqlite-vec (`vendor/sqlite-extensions/vec0.dylib`). On Linux or Windows you need to swap it for your platform's version. +The app itself can run on Linux and Windows now. The remaining platform-specific piece is vector search. + +RA-H OS ships with a macOS sqlite-vec binary by default (`vendor/sqlite-extensions/vec0.dylib`). On Linux or Windows you need to swap it for your platform's version if you want semantic/vector search. **Linux:** @@ -206,6 +210,11 @@ The repo ships with a macOS binary for sqlite-vec (`vendor/sqlite-extensions/vec Without sqlite-vec, everything works except semantic/vector search. +If sqlite-vec is missing or fails to load: +- the app still starts +- nodes, UI, and keyword/FTS search still work +- `/api/health/vectors` reports vector search as unavailable instead of crashing + --- ## Community diff --git a/app/api/health/vectors/route.ts b/app/api/health/vectors/route.ts index 9ade409..a6034ec 100644 --- a/app/api/health/vectors/route.ts +++ b/app/api/health/vectors/route.ts @@ -5,6 +5,7 @@ import { chunkService } from '@/services/database/chunks'; export async function GET() { try { const sqlite = getSQLiteClient(); + const vectorCapability = sqlite.getVectorCapability(); // Test basic database connection const connectionTest = await sqlite.testConnection(); @@ -18,30 +19,33 @@ export async function GET() { // Check if vector extension is loaded const vectorExtensionTest = await sqlite.checkVectorExtension(); - let vectorStats = null; let chunkStats = null; - let vectorHealth = 'unknown'; + let vectorHealth = vectorCapability.available ? 'healthy' : 'unavailable'; try { - // Get chunk counts const totalChunks = await chunkService.getChunkCount(); - const chunksWithoutEmbeddings = await chunkService.getChunksWithoutEmbeddings(); - const vectorizedCount = totalChunks - chunksWithoutEmbeddings.length; - chunkStats = { total_chunks: totalChunks, - vectorized_chunks: vectorizedCount, - missing_embeddings: chunksWithoutEmbeddings.length, - coverage_percentage: totalChunks > 0 ? Math.round((vectorizedCount / totalChunks) * 100) : 0 + vectorized_chunks: null, + missing_embeddings: null, + coverage_percentage: null, }; - // Test vector table health by attempting a simple query - if (vectorExtensionTest) { + if (vectorCapability.available && vectorExtensionTest) { try { + const chunksWithoutEmbeddings = await chunkService.getChunksWithoutEmbeddings(); + const vectorizedCount = totalChunks - chunksWithoutEmbeddings.length; const result = sqlite.query('SELECT COUNT(*) as count FROM vec_chunks'); const vecCount = Number(result.rows[0].count); - + + chunkStats = { + total_chunks: totalChunks, + vectorized_chunks: vectorizedCount, + missing_embeddings: chunksWithoutEmbeddings.length, + coverage_percentage: totalChunks > 0 ? Math.round((vectorizedCount / totalChunks) * 100) : 0 + }; + vectorStats = { vec_chunks_count: vecCount, matches_chunk_embeddings: vecCount === vectorizedCount @@ -56,7 +60,12 @@ export async function GET() { }; } } else { - vectorHealth = 'extension_unavailable'; + vectorHealth = 'unavailable'; + vectorStats = { + backend: vectorCapability.backend, + extension_path: vectorCapability.extensionPath, + reason: vectorCapability.available ? null : vectorCapability.reason, + }; } } catch (error: any) { @@ -72,6 +81,7 @@ export async function GET() { data: { database_connected: connectionTest, vector_extension_loaded: vectorExtensionTest, + vector_capability: vectorCapability, vector_health: vectorHealth, chunk_stats: chunkStats, vector_stats: vectorStats, @@ -100,11 +110,11 @@ function generateRecommendations( recommendations.push('Vector tables are corrupted - restart the application to trigger automatic healing'); } - if (vectorHealth === 'extension_unavailable') { - recommendations.push('Vector extension not loaded - check sqlite-vec installation'); + if (vectorHealth === 'unavailable') { + recommendations.push('Semantic/vector search is unavailable. Install sqlite-vec for your platform or switch to Qdrant.'); } - if (chunkStats && chunkStats.coverage_percentage < 95) { + if (chunkStats && typeof chunkStats.coverage_percentage === 'number' && chunkStats.coverage_percentage < 95) { recommendations.push(`${chunkStats.missing_embeddings} chunks missing embeddings - consider running embedding generation`); } @@ -117,4 +127,4 @@ function generateRecommendations( } return recommendations; -} \ No newline at end of file +} diff --git a/package-lock.json b/package-lock.json index ef6e702..a10f5ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "@types/react": "^19.0.2", "@types/react-dom": "19.0.2", "autoprefixer": "^10.4.20", + "cross-env": "^10.0.0", "esbuild": "^0.23.1", "eslint": "^8.57.0", "eslint-config-next": "^15.1.12", @@ -165,6 +166,13 @@ "tslib": "^2.4.0" } }, + "node_modules/@epic-web/invariant": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@epic-web/invariant/-/invariant-1.0.0.tgz", + "integrity": "sha512-lrTPqgvfFQtR/eY/qkIzp98OGdNJu0m5ji3q/nJI8v3SXkRKEnWiOxMmbvcSoAIzv/cGiuvRy57k4suKQSAdwA==", + "dev": true, + "license": "MIT" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.23.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz", @@ -4643,6 +4651,24 @@ "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", "license": "MIT" }, + "node_modules/cross-env": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-10.1.0.tgz", + "integrity": "sha512-GsYosgnACZTADcmEyJctkJIoqAhHjttw7RsFrVoJNXbsWWqaq6Ym+7kZjq6mS45O0jij6vtiReppKQEtqWy6Dw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@epic-web/invariant": "^1.0.0", + "cross-spawn": "^7.0.6" + }, + "bin": { + "cross-env": "dist/bin/cross-env.js", + "cross-env-shell": "dist/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", diff --git a/package.json b/package.json index 2e666ee..3494396 100644 --- a/package.json +++ b/package.json @@ -6,12 +6,14 @@ "license": "MIT", "author": "Bradley Morris", "scripts": { - "dev": "NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next dev", - "dev:evals": "RAH_EVALS_LOG=1 NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next dev", + "bootstrap:local": "node ./scripts/dev/bootstrap-local.mjs", + "dev": "cross-env NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=--dns-result-order=ipv4first next dev", + "dev:local": "npm run dev", + "dev:evals": "cross-env RAH_EVALS_LOG=1 NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=--dns-result-order=ipv4first next dev", "clean:local": "bash ./scripts/dev/clean-local-artifacts.sh", "audit:repo": "bash ./scripts/dev/audit-repo.sh", - "build": "NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next build", - "start": "NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next start", + "build": "cross-env NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=--dns-result-order=ipv4first next build", + "start": "cross-env NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=--dns-result-order=ipv4first next start", "lint": "next lint", "type-check": "tsc --noEmit", "setup": "npm install", @@ -19,7 +21,7 @@ "sqlite:restore": "bash scripts/database/sqlite-restore.sh", "test": "vitest run", "test:watch": "vitest", - "evals": "RAH_EVALS_LOG=1 RAH_EVALS_TIMEOUT_MS=60000 tsx tests/evals/runner.ts" + "evals": "cross-env RAH_EVALS_LOG=1 RAH_EVALS_TIMEOUT_MS=60000 tsx tests/evals/runner.ts" }, "dependencies": { "@ai-sdk/openai": "^3.0.7", @@ -53,6 +55,7 @@ "@types/react": "^19.0.2", "@types/react-dom": "19.0.2", "autoprefixer": "^10.4.20", + "cross-env": "^10.0.0", "esbuild": "^0.23.1", "eslint": "^8.57.0", "eslint-config-next": "^15.1.12", diff --git a/scripts/dev/bootstrap-local.mjs b/scripts/dev/bootstrap-local.mjs new file mode 100644 index 0000000..4fdfdec --- /dev/null +++ b/scripts/dev/bootstrap-local.mjs @@ -0,0 +1,307 @@ +#!/usr/bin/env node + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import Database from 'better-sqlite3'; + +const repoDir = process.cwd(); +const envTemplate = path.join(repoDir, '.env.example'); +const targetEnv = path.join(repoDir, '.env.local'); + +function log(message) { + console.log(`[bootstrap-local] ${message}`); +} + +function getDefaultDbPath() { + const homeDir = os.homedir(); + + if (process.platform === 'win32') { + const appData = process.env.APPDATA || path.join(homeDir, 'AppData', 'Roaming'); + return path.join(appData, 'RA-H', 'db', 'rah.sqlite'); + } + + if (process.platform === 'darwin') { + return path.join(homeDir, 'Library', 'Application Support', 'RA-H', 'db', 'rah.sqlite'); + } + + return path.join( + process.env.XDG_DATA_HOME || path.join(homeDir, '.local', 'share'), + 'RA-H', + 'db', + 'rah.sqlite' + ); +} + +function parseEnvFile(filePath) { + if (!fs.existsSync(filePath)) return {}; + + return fs.readFileSync(filePath, 'utf8') + .split(/\r?\n/) + .filter(Boolean) + .reduce((acc, line) => { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) return acc; + const eqIndex = trimmed.indexOf('='); + if (eqIndex === -1) return acc; + const key = trimmed.slice(0, eqIndex).trim(); + const value = trimmed.slice(eqIndex + 1).trim(); + acc[key] = value; + return acc; + }, {}); +} + +function expandPath(rawPath) { + let value = rawPath; + + if (value.startsWith('~')) { + value = path.join(os.homedir(), value.slice(1)); + } + + value = value.replace(/\$HOME/g, os.homedir()); + value = value.replace(/%APPDATA%/g, process.env.APPDATA || path.join(os.homedir(), 'AppData', 'Roaming')); + + return path.resolve(value); +} + +function ensureEnvFile() { + if (!fs.existsSync(envTemplate)) { + throw new Error(`Missing ${envTemplate}`); + } + + if (fs.existsSync(targetEnv)) { + log('.env.local already exists; leaving it untouched.'); + return; + } + + fs.copyFileSync(envTemplate, targetEnv); + log('Created .env.local from .env.example'); +} + +function ensureCoreSchema(db) { + db.pragma('foreign_keys = ON'); + db.exec(` + CREATE TABLE IF NOT EXISTS agents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + key TEXT UNIQUE NOT NULL, + display_name TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'executor', + system_prompt TEXT NOT NULL, + available_tools TEXT NOT NULL, + model TEXT NOT NULL, + description TEXT, + enabled INTEGER DEFAULT 1, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + memory TEXT, + prompts TEXT + ); + + CREATE TABLE IF NOT EXISTS nodes ( + id INTEGER PRIMARY KEY, + title TEXT, + description TEXT, + source TEXT, + link TEXT, + event_date TEXT, + created_at TEXT, + updated_at TEXT, + metadata TEXT, + embedding BLOB, + embedding_updated_at TEXT, + embedding_text TEXT, + chunk_status TEXT DEFAULT 'not_chunked' + ); + + CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY, + node_id INTEGER NOT NULL, + chunk_idx INTEGER, + text TEXT, + created_at TEXT, + embedding_type TEXT DEFAULT 'text-embedding-3-small', + metadata TEXT, + FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_chunks_by_node ON chunks(node_id); + CREATE INDEX IF NOT EXISTS idx_chunks_by_node_idx ON chunks(node_id, chunk_idx); + + CREATE TABLE IF NOT EXISTS edges ( + id INTEGER PRIMARY KEY, + from_node_id INTEGER NOT NULL, + to_node_id INTEGER NOT NULL, + source TEXT, + created_at TEXT, + context TEXT, + FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE, + FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node_id); + CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node_id); + + CREATE TABLE IF NOT EXISTS chats ( + id INTEGER PRIMARY KEY, + chat_type TEXT, + helper_name TEXT, + agent_type TEXT DEFAULT 'orchestrator', + delegation_id INTEGER, + user_message TEXT, + assistant_message TEXT, + thread_id TEXT, + focused_node_id INTEGER, + created_at TEXT DEFAULT (CURRENT_TIMESTAMP), + metadata TEXT, + FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL + ); + CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id); + + CREATE TABLE IF NOT EXISTS node_dimensions ( + node_id INTEGER NOT NULL, + dimension TEXT NOT NULL, + PRIMARY KEY (node_id, dimension), + FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_dim_by_dimension ON node_dimensions(dimension, node_id); + CREATE INDEX IF NOT EXISTS idx_dim_by_node ON node_dimensions(node_id, dimension); + + CREATE TABLE IF NOT EXISTS dimensions ( + name TEXT PRIMARY KEY, + description TEXT, + icon TEXT, + is_priority INTEGER DEFAULT 0, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY, + ts TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP), + table_name TEXT NOT NULL, + action TEXT NOT NULL, + row_id INTEGER NOT NULL, + summary TEXT, + enriched_summary TEXT, + snapshot_json TEXT + ); + + CREATE TABLE IF NOT EXISTS voice_usage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chat_id INTEGER, + session_id TEXT, + helper_name TEXT, + request_id TEXT, + message_id TEXT, + voice TEXT, + model TEXT, + chars INTEGER, + cost_usd REAL, + duration_ms INTEGER, + text_preview TEXT, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE SET NULL + ); + CREATE INDEX IF NOT EXISTS idx_voice_usage_session ON voice_usage(session_id, created_at); + CREATE INDEX IF NOT EXISTS idx_voice_usage_chat ON voice_usage(chat_id); + `); + + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5( + title, + source, + description, + content='nodes', + content_rowid='id' + ); + CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5( + text, + content='chunks', + content_rowid='id' + ); + `); + + const now = new Date().toISOString(); + const lockedDimensions = ['research', 'ideas', 'projects', 'memory', 'preferences']; + const insertDimension = db.prepare(` + INSERT INTO dimensions (name, is_priority, updated_at) + VALUES (?, 1, ?) + ON CONFLICT(name) DO UPDATE SET is_priority = 1, updated_at = excluded.updated_at + `); + for (const dimension of lockedDimensions) { + insertDimension.run(dimension, now); + } + + const agentCount = Number(db.prepare('SELECT COUNT(*) as count FROM agents').get().count || 0); + if (agentCount === 0) { + db.prepare(` + INSERT INTO agents ( + key, display_name, role, system_prompt, available_tools, model, description, enabled, created_at, updated_at, prompts + ) VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?) + `).run( + 'ra-h', + 'ra-h', + 'orchestrator', + "You are ra-h, the main orchestrator for RA-H. Coordinate work, delegate to mini ra-hs when tasks can be isolated, and keep the conversation focused on the user's goals.", + '["queryNodes","createNode","updateNode","createEdge","queryEdge","updateEdge","searchContentEmbeddings","webSearch","think","delegateToMiniRAH"]', + 'anthropic/claude-sonnet-4.5', + 'Opinionated orchestrator agent', + now, + now, + '[{"id":"p_seed_0","name":"Summary of Focus","content":"Summarize the primary focused node clearly. Include 3–5 key points and cite [NODE:id:\\"title\\"]."},{"id":"p_seed_1","name":"Next Steps","content":"Propose 3 concrete next actions based on the focused nodes with references to [NODE:id:\\"title\\"]."}]' + ); + } +} + +function tryInitVectorTables(db, dbPath) { + const extension = process.platform === 'darwin' ? 'dylib' : process.platform === 'win32' ? 'dll' : 'so'; + const extensionPath = process.env.SQLITE_VEC_EXTENSION_PATH || path.join(repoDir, 'vendor', 'sqlite-extensions', `vec0.${extension}`); + + try { + db.loadExtension(extensionPath); + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS vec_nodes USING vec0( + node_id INTEGER PRIMARY KEY, + embedding FLOAT[1536] + ); + CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0( + chunk_id INTEGER PRIMARY KEY, + embedding FLOAT[1536] + ); + `); + log(`Initialized sqlite-vec tables using ${extensionPath}`); + } catch (error) { + log(`sqlite-vec unavailable for bootstrap (${dbPath}). Continuing without vector tables.`); + } +} + +function main() { + const major = Number(process.versions.node.split('.')[0] || '0'); + if (major < 20) { + throw new Error(`Node.js 20+ required (found ${process.version})`); + } + + ensureEnvFile(); + + const env = parseEnvFile(targetEnv); + const dbPath = expandPath(env.SQLITE_DB_PATH || getDefaultDbPath()); + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + if (!fs.existsSync(dbPath)) { + fs.closeSync(fs.openSync(dbPath, 'w')); + } + + const db = new Database(dbPath); + try { + ensureCoreSchema(db); + tryInitVectorTables(db, dbPath); + } finally { + db.close(); + } + + log(`Bootstrap complete. Database ready at ${dbPath}`); + log("Run 'npm run dev' to start the app."); +} + +try { + main(); +} catch (error) { + console.error(`[bootstrap-local] ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +} diff --git a/scripts/dev/bootstrap-local.sh b/scripts/dev/bootstrap-local.sh index b9cfea9..c49bf62 100755 --- a/scripts/dev/bootstrap-local.sh +++ b/scripts/dev/bootstrap-local.sh @@ -33,7 +33,17 @@ else fi SQLITE_DB_PATH_LINE=$(grep -E '^SQLITE_DB_PATH=' "$TARGET_ENV" | tail -n 1 || true) -DEFAULT_DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite" +case "$(uname -s)" in + Darwin) + DEFAULT_DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite" + ;; + Linux) + DEFAULT_DB_PATH="${XDG_DATA_HOME:-$HOME/.local/share}/RA-H/db/rah.sqlite" + ;; + *) + DEFAULT_DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite" + ;; +esac if [ -z "$SQLITE_DB_PATH_LINE" ]; then SQLITE_DB_PATH="$DEFAULT_DB_PATH" else diff --git a/src/services/database/index.ts b/src/services/database/index.ts index e18200b..d3243de 100644 --- a/src/services/database/index.ts +++ b/src/services/database/index.ts @@ -38,6 +38,7 @@ async function checkSQLiteDatabaseHealth(): Promise<{ const sqlite = getSQLiteClient(); const connected = await sqlite.testConnection(); + const vectorCapability = sqlite.getVectorCapability(); if (!connected) { return { connected: false, @@ -47,7 +48,7 @@ async function checkSQLiteDatabaseHealth(): Promise<{ }; } - const vectorExtension = await sqlite.checkVectorExtension(); + const vectorExtension = vectorCapability.available; // Check if main tables exist const tables = await sqlite.checkTables(); diff --git a/src/services/database/sqlite-client.ts b/src/services/database/sqlite-client.ts index f3c0407..cda87c2 100644 --- a/src/services/database/sqlite-client.ts +++ b/src/services/database/sqlite-client.ts @@ -1,7 +1,12 @@ import Database from 'better-sqlite3'; -import fs from 'fs'; -import path from 'path'; import { DatabaseError } from '@/types/database'; +import { + ensureDatabaseDirectory, + getDatabasePath, + getVecExtensionPath, + loadVecExtension, + type VectorCapability, +} from './sqlite-runtime'; export interface SQLiteConfig { dbPath: string; @@ -19,27 +24,26 @@ class SQLiteClient { private db: Database.Database; private config: SQLiteConfig; private readonly readOnly: boolean; + private readonly vectorCapability: VectorCapability; private constructor() { this.config = this.getSQLiteConfig(); this.readOnly = process.env.SQLITE_READONLY === 'true'; // Initialize database connection - const dbDirectory = path.dirname(this.config.dbPath); - if (!this.readOnly && !fs.existsSync(dbDirectory)) { - fs.mkdirSync(dbDirectory, { recursive: true }); + if (!this.readOnly) { + ensureDatabaseDirectory(this.config.dbPath); } this.db = this.readOnly ? new Database(this.config.dbPath, { readonly: true, fileMustExist: true }) : new Database(this.config.dbPath); // Load sqlite-vec extension - try { - this.db.loadExtension(this.config.vecExtensionPath); + this.vectorCapability = loadVecExtension(this.db, this.config.vecExtensionPath); + if (this.vectorCapability.available) { console.log('SQLite vector extension loaded successfully'); - } catch (error) { - // Do not fail hard — allow the app to run without vector features - console.error('Warning: Failed to load vector extension:', error); + } else { + console.warn(`Warning: ${this.vectorCapability.reason}`); } // Configure SQLite settings @@ -58,8 +62,10 @@ class SQLiteClient { this.db.pragma('busy_timeout = 5000'); // Ensure vector virtual tables are present and healthy - this.ensureVectorTables(); - this.healVectorTablesIfCorrupt(); + if (this.vectorCapability.available) { + this.ensureVectorTables(); + this.healVectorTablesIfCorrupt(); + } // Ensure logging schema (rename memory->logs if needed, create triggers/views) this.ensureLoggingAndMemorySchema(); @@ -69,17 +75,9 @@ class SQLiteClient { } private getSQLiteConfig(): SQLiteConfig { - const dbPath = process.env.SQLITE_DB_PATH || path.join( - process.env.HOME || '~', - 'Library/Application Support/RA-H/db/rah.sqlite' - ); - - const vecExtensionPath = process.env.SQLITE_VEC_EXTENSION_PATH || - './vendor/sqlite-extensions/vec0.dylib'; - return { - dbPath, - vecExtensionPath + dbPath: getDatabasePath(), + vecExtensionPath: getVecExtensionPath(), }; } @@ -134,7 +132,9 @@ class SQLiteClient { } as DatabaseError; } // Proactively validate/repair vec vtables before any write transaction - this.healVectorTablesIfCorrupt(); + if (this.vectorCapability.available) { + this.healVectorTablesIfCorrupt(); + } const txn = this.db.transaction(callback); try { return txn(); @@ -154,13 +154,11 @@ class SQLiteClient { } public async checkVectorExtension(): Promise { - try { - const result = this.query('SELECT vec_version() as version'); - return result.rows.length > 0; - } catch (error) { - console.error('Vector extension check failed:', error); - return false; - } + return this.vectorCapability.available; + } + + public getVectorCapability(): VectorCapability { + return this.vectorCapability; } public async checkTables(): Promise { @@ -176,6 +174,10 @@ class SQLiteClient { } public ensureVectorExtensions(): void { + if (!this.vectorCapability.available) { + return; + } + try { // Test for vec_nodes and vec_chunks; create them if missing const hasVecNodes = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_nodes'); @@ -205,7 +207,7 @@ class SQLiteClient { } private ensureVectorTables(): void { - if (this.readOnly) { + if (this.readOnly || !this.vectorCapability.available) { return; } // Wrapper to keep existing public API stable @@ -739,7 +741,7 @@ class SQLiteClient { } private healVectorTablesIfCorrupt(): void { - if (this.readOnly) { + if (this.readOnly || !this.vectorCapability.available) { return; } // Attempt lightweight reads to detect CORRUPT_VTAB; if detected, drop/recreate vtables diff --git a/src/services/database/sqlite-runtime.ts b/src/services/database/sqlite-runtime.ts new file mode 100644 index 0000000..8989261 --- /dev/null +++ b/src/services/database/sqlite-runtime.ts @@ -0,0 +1,120 @@ +import Database from 'better-sqlite3'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +export type VectorCapability = + | { + available: true; + backend: 'sqlite-vec'; + extensionPath: string; + detail: string; + } + | { + available: false; + backend: 'none'; + extensionPath: string; + reason: string; + }; + +const VECTOR_CAPABILITY_KEY = '__rahVectorCapability'; + +function getHomeDirectory(): string { + return os.homedir() || process.env.HOME || '~'; +} + +export function getDefaultDbPath(): string { + const homeDir = getHomeDirectory(); + + if (process.platform === 'win32') { + const appData = process.env.APPDATA || path.join(homeDir, 'AppData', 'Roaming'); + return path.join(appData, 'RA-H', 'db', 'rah.sqlite'); + } + + if (process.platform === 'darwin') { + return path.join(homeDir, 'Library', 'Application Support', 'RA-H', 'db', 'rah.sqlite'); + } + + return path.join( + process.env.XDG_DATA_HOME || path.join(homeDir, '.local', 'share'), + 'RA-H', + 'db', + 'rah.sqlite' + ); +} + +export function getDatabasePath(): string { + return process.env.SQLITE_DB_PATH || getDefaultDbPath(); +} + +export function getDefaultVecExtensionPath(): string { + const extension = process.platform === 'darwin' + ? 'dylib' + : process.platform === 'win32' + ? 'dll' + : 'so'; + + return path.join(process.cwd(), 'vendor', 'sqlite-extensions', `vec0.${extension}`); +} + +export function getVecExtensionPath(): string { + return process.env.SQLITE_VEC_EXTENSION_PATH || getDefaultVecExtensionPath(); +} + +export function ensureDatabaseDirectory(dbPath: string): void { + const dbDirectory = path.dirname(dbPath); + if (!fs.existsSync(dbDirectory)) { + fs.mkdirSync(dbDirectory, { recursive: true }); + } +} + +function buildVectorFailureReason(error: unknown, extensionPath: string): string { + const message = error instanceof Error ? error.message : String(error); + + if (process.platform === 'win32') { + return `sqlite-vec failed to load from ${extensionPath}. On Windows, install vec0.dll or switch to Qdrant. Original error: ${message}`; + } + + if (process.platform === 'linux') { + return `sqlite-vec failed to load from ${extensionPath}. Alpine/musl builds are unsupported; use Qdrant there. Original error: ${message}`; + } + + return `sqlite-vec failed to load from ${extensionPath}. Original error: ${message}`; +} + +export function loadVecExtension(db: Database.Database, extensionPath = getVecExtensionPath()): VectorCapability { + try { + db.loadExtension(extensionPath); + const capability: VectorCapability = { + available: true, + backend: 'sqlite-vec', + extensionPath, + detail: 'sqlite-vec extension loaded successfully', + }; + (db as Database.Database & Record)[VECTOR_CAPABILITY_KEY] = capability; + return capability; + } catch (error) { + const capability: VectorCapability = { + available: false, + backend: 'none', + extensionPath, + reason: buildVectorFailureReason(error, extensionPath), + }; + (db as Database.Database & Record)[VECTOR_CAPABILITY_KEY] = capability; + return capability; + } +} + +export function getDbVectorCapability(db: Database.Database): VectorCapability { + const capability = (db as Database.Database & Record)[VECTOR_CAPABILITY_KEY]; + if (capability && typeof capability === 'object' && 'available' in capability) { + return capability as VectorCapability; + } + + return { + available: false, + backend: 'none', + extensionPath: getVecExtensionPath(), + reason: 'sqlite-vec capability was not initialized for this database connection', + }; +} diff --git a/src/services/typescript/embed-nodes.ts b/src/services/typescript/embed-nodes.ts index 8fa1572..73e0426 100644 --- a/src/services/typescript/embed-nodes.ts +++ b/src/services/typescript/embed-nodes.ts @@ -8,10 +8,12 @@ import { generateText } from 'ai'; import { createOpenAI } from '@ai-sdk/openai'; import { createDatabaseConnection, + getDbVectorCapability, serializeFloat32Vector, formatEmbeddingText, batchProcess } from './sqlite-vec'; +import type { VectorCapability } from '@/services/database/sqlite-runtime'; interface NodeRecord { id: number; @@ -35,6 +37,7 @@ export class NodeEmbedder { private openaiClient: OpenAI; private openaiProvider: ReturnType; private db: ReturnType; + private readonly vectorCapability: VectorCapability; private processedCount: number = 0; private failedCount: number = 0; @@ -47,6 +50,7 @@ export class NodeEmbedder { this.openaiClient = new OpenAI({ apiKey }); this.openaiProvider = createOpenAI({ apiKey }); this.db = createDatabaseConnection(); + this.vectorCapability = getDbVectorCapability(this.db); } /** @@ -138,23 +142,18 @@ Focus on the main concepts, key relationships, and practical implications.`; const now = new Date().toISOString(); updateStmt.run(embeddingBlob, now, embeddingText, node.id); - // Update vec_nodes virtual table - try { - // Determine correct column name for primary key (node_id vs id) - // Use declared PK column from your DB schema (confirmed: node_id) - const pkCol = 'node_id'; + if (this.vectorCapability.available) { + try { + const pkCol = 'node_id'; + const deleteStmt = this.db.prepare(`DELETE FROM vec_nodes WHERE ${pkCol} = ?`); + deleteStmt.run(BigInt(node.id)); - // Delete existing entry if any - const deleteStmt = this.db.prepare(`DELETE FROM vec_nodes WHERE ${pkCol} = ?`); - deleteStmt.run(BigInt(node.id)); - - // Insert new entry (use bracketed string format compatible with sqlite-vec) - const vectorString = `[${embedding.join(',')}]`; - const insertStmt = this.db.prepare(`INSERT INTO vec_nodes (${pkCol}, embedding) VALUES (?, ?)`); - insertStmt.run(BigInt(node.id), vectorString); - } catch (vecError) { - console.warn(`Could not update vec_nodes for node ${node.id}:`, vecError); - // Continue - main embedding is still saved + const vectorString = `[${embedding.join(',')}]`; + const insertStmt = this.db.prepare(`INSERT INTO vec_nodes (${pkCol}, embedding) VALUES (?, ?)`); + insertStmt.run(BigInt(node.id), vectorString); + } catch (vecError) { + console.warn(`Could not update vec_nodes for node ${node.id}:`, vecError); + } } this.processedCount++; diff --git a/src/services/typescript/embed-universal.ts b/src/services/typescript/embed-universal.ts index 158b1b2..6655137 100644 --- a/src/services/typescript/embed-universal.ts +++ b/src/services/typescript/embed-universal.ts @@ -7,9 +7,11 @@ import OpenAI from 'openai'; import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters'; import { createDatabaseConnection, + getDbVectorCapability, serializeFloat32Vector, batchProcess } from './sqlite-vec'; +import type { VectorCapability } from '@/services/database/sqlite-runtime'; interface Node { id: number; @@ -37,6 +39,7 @@ export class UniversalEmbedder { private openaiClient: OpenAI; private db: ReturnType; private textSplitter: RecursiveCharacterTextSplitter; + private readonly vectorCapability: VectorCapability; private vecChunksInsertSQL: string | null = null; constructor() { @@ -47,6 +50,7 @@ export class UniversalEmbedder { this.openaiClient = new OpenAI({ apiKey }); this.db = createDatabaseConnection(); + this.vectorCapability = getDbVectorCapability(this.db); // Configure text splitter (same as old KMS system) this.textSplitter = new RecursiveCharacterTextSplitter({ @@ -87,12 +91,14 @@ export class UniversalEmbedder { const chunkIds = this.db.prepare('SELECT id FROM chunks WHERE node_id = ?').all(nodeId) as Array<{ id: number }>; // Delete from vec_chunks first, one by one to ensure they're removed - for (const chunk of chunkIds) { - try { - const deleteVecStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?'); - deleteVecStmt.run(BigInt(chunk.id)); - } catch (error) { - console.warn(`Could not delete vec_chunk ${chunk.id}:`, error); + if (this.vectorCapability.available) { + for (const chunk of chunkIds) { + try { + const deleteVecStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?'); + deleteVecStmt.run(BigInt(chunk.id)); + } catch (error) { + console.warn(`Could not delete vec_chunk ${chunk.id}:`, error); + } } } @@ -131,21 +137,20 @@ export class UniversalEmbedder { const chunkId = Number(result.lastInsertRowid); - // Insert into vec_chunks virtual table (use bracketed string format) - try { - const vectorString = `[${embedding.join(',')}]`; - // First try to delete any existing vec_chunk with this ID + if (this.vectorCapability.available) { try { - const deleteStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?'); - deleteStmt.run(BigInt(chunkId)); - } catch {} - - // Now insert the new embedding - const sql = 'INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)'; - const vecInsertStmt = this.db.prepare(sql); - vecInsertStmt.run(BigInt(chunkId), vectorString); - } catch (error) { - console.warn(`Could not insert into vec_chunks for chunk ${chunkId}:`, error); + const vectorString = `[${embedding.join(',')}]`; + try { + const deleteStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?'); + deleteStmt.run(BigInt(chunkId)); + } catch {} + + const sql = 'INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)'; + const vecInsertStmt = this.db.prepare(sql); + vecInsertStmt.run(BigInt(chunkId), vectorString); + } catch (error) { + console.warn(`Could not insert into vec_chunks for chunk ${chunkId}:`, error); + } } } diff --git a/src/services/typescript/sqlite-vec.ts b/src/services/typescript/sqlite-vec.ts index 11e3821..1a46d41 100644 --- a/src/services/typescript/sqlite-vec.ts +++ b/src/services/typescript/sqlite-vec.ts @@ -4,8 +4,13 @@ */ import Database from 'better-sqlite3'; -import path from 'path'; -import os from 'os'; +import { + ensureDatabaseDirectory, + getDatabasePath, + getDbVectorCapability, + getVecExtensionPath, + loadVecExtension, +} from '@/services/database/sqlite-runtime'; /** * Serialize a float array to binary format for vec0 storage @@ -30,53 +35,25 @@ export function deserializeFloat32Vector(blob: Buffer): number[] { return vector; } -/** - * Get SQLite database path from environment or default location - */ -export function getDatabasePath(): string { - const envPath = process.env.SQLITE_DB_PATH; - if (envPath) { - return envPath; - } - - // Default path: ~/Library/Application Support/RA-H/db/rah.sqlite - const homeDir = os.homedir(); - return path.join(homeDir, 'Library', 'Application Support', 'RA-H', 'db', 'rah.sqlite'); -} - -/** - * Get vec extension path from environment or default location - */ -export function getVecExtensionPath(): string { - const envPath = process.env.SQLITE_VEC_EXTENSION_PATH; - if (envPath) { - return envPath; - } - - // Default path relative to project root - return path.join(process.cwd(), 'vendor', 'sqlite-extensions', 'vec0.dylib'); -} - /** * Create database connection with vec0 extension loaded */ export function createDatabaseConnection(): Database.Database { const dbPath = getDatabasePath(); const vecPath = getVecExtensionPath(); - + ensureDatabaseDirectory(dbPath); const db = new Database(dbPath); - - // Load vec0 extension - try { - db.loadExtension(vecPath); - } catch (error) { - console.error('Warning: Could not load vec0 extension:', error); - // Continue without vector support for non-vector operations + + const capability = loadVecExtension(db, vecPath); + if (!capability.available) { + console.warn(`Warning: ${capability.reason}`); } - + return db; } +export { getDatabasePath, getDbVectorCapability, getVecExtensionPath }; + /** * Format embedding text for node metadata */ @@ -113,4 +90,4 @@ export async function batchProcess( } return results; -} \ No newline at end of file +} diff --git a/src/tools/other/sqliteQuery.ts b/src/tools/other/sqliteQuery.ts index 4f5eabb..c395536 100644 --- a/src/tools/other/sqliteQuery.ts +++ b/src/tools/other/sqliteQuery.ts @@ -2,18 +2,10 @@ import { tool } from 'ai'; import { z } from 'zod'; import { exec } from 'child_process'; import { promisify } from 'util'; -import path from 'path'; +import { getDatabasePath } from '@/services/database/sqlite-runtime'; const execAsync = promisify(exec); -// Get database path (same logic as sqlite-client.ts) -function getDatabasePath(): string { - return process.env.SQLITE_DB_PATH || path.join( - process.env.HOME || '~', - 'Library/Application Support/RA-H/db/rah.sqlite' - ); -} - // Security: Only allow SELECT statements function isReadOnlyQuery(sql: string): boolean { const normalized = sql.trim().toLowerCase();