SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stabilize cross-platform local runtime

Browse Source
This commit is contained in:
“BeeRad”
2026-03-21 17:48:48 +11:00
parent 253e0ae68f
commit 6dcf247a64
13 changed files with 605 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+13 -4
View File
@@ -13,7 +13,7 @@
[![Watch the demo](https://img.youtube.com/vi/IA02YB8mInM/hqdefault.jpg)](https://youtu.be/IA02YB8mInM?si=WoWpNE9QZEKEukvZ)
> **Currently macOS only.** Linux and Windows support is coming. If you want to run it on Linux/Windows now, see [instructions at the bottom](#linuxwindows).
> **Cross-platform local runtime:** macOS works out of the box. Linux and Windows now support the core local/web app flow, but semantic/vector search still depends on either sqlite-vec for your platform or a later Qdrant setup.
**Full documentation:** [ra-h.app/docs/open-source](https://ra-h.app/docs/open-source)
@@ -33,7 +33,7 @@ Your data stays on your machine. Nothing is sent anywhere unless you configure a
- **Node.js 20.18.1+** — [nodejs.org](https://nodejs.org/)
- **macOS** — Works out of the box
- **Linux/Windows** — Requires building sqlite-vec manually (see below)
- **Linux/Windows** — Core app works; vector search requires sqlite-vec for your platform (see below)
---
@@ -44,7 +44,7 @@ git clone https://github.com/bradwmorris/ra-h_os.git
cd ra-h_os
npm install
npm rebuild better-sqlite3
./scripts/dev/bootstrap-local.sh
npm run bootstrap:local
npm run dev
```
@@ -178,7 +178,9 @@ See [ra-h.app/docs/open-source](https://ra-h.app/docs/open-source) for full sche
| Command | What it does |
|---------|--------------|
| `npm run bootstrap:local` | Create `.env.local`, create the SQLite DB, and seed the base schema |
| `npm run dev` | Start the app at localhost:3000 |
| `npm run dev:local` | Alias for `npm run dev` |
| `npm run build` | Production build |
| `npm run type-check` | Check TypeScript |
@@ -186,7 +188,9 @@ See [ra-h.app/docs/open-source](https://ra-h.app/docs/open-source) for full sche
## Linux/Windows
The repo ships with a macOS binary for sqlite-vec (`vendor/sqlite-extensions/vec0.dylib`). On Linux or Windows you need to swap it for your platform's version.
The app itself can run on Linux and Windows now. The remaining platform-specific piece is vector search.
RA-H OS ships with a macOS sqlite-vec binary by default (`vendor/sqlite-extensions/vec0.dylib`). On Linux or Windows you need to swap it for your platform's version if you want semantic/vector search.
**Linux:**
@@ -206,6 +210,11 @@ The repo ships with a macOS binary for sqlite-vec (`vendor/sqlite-extensions/vec
Without sqlite-vec, everything works except semantic/vector search.
If sqlite-vec is missing or fails to load:
- the app still starts
- nodes, UI, and keyword/FTS search still work
- `/api/health/vectors` reports vector search as unavailable instead of crashing
---
## Community
app/api/health/vectors/route.ts
+23 -13
View File
@@ -5,6 +5,7 @@ import { chunkService } from '@/services/database/chunks';
export async function GET() {
try {
const sqlite = getSQLiteClient();
const vectorCapability = sqlite.getVectorCapability();
// Test basic database connection
const connectionTest = await sqlite.testConnection();
@@ -18,16 +19,25 @@ export async function GET() {
// Check if vector extension is loaded
const vectorExtensionTest = await sqlite.checkVectorExtension();
let vectorStats = null;
let chunkStats = null;
let vectorHealth = 'unknown';
let vectorHealth = vectorCapability.available ? 'healthy' : 'unavailable';
try {
// Get chunk counts
const totalChunks = await chunkService.getChunkCount();
chunkStats = {
total_chunks: totalChunks,
vectorized_chunks: null,
missing_embeddings: null,
coverage_percentage: null,
};
if (vectorCapability.available && vectorExtensionTest) {
try {
const chunksWithoutEmbeddings = await chunkService.getChunksWithoutEmbeddings();
const vectorizedCount = totalChunks - chunksWithoutEmbeddings.length;
const result = sqlite.query('SELECT COUNT(*) as count FROM vec_chunks');
const vecCount = Number(result.rows[0].count);
chunkStats = {
total_chunks: totalChunks,
@@ -36,12 +46,6 @@ export async function GET() {
coverage_percentage: totalChunks > 0 ? Math.round((vectorizedCount / totalChunks) * 100) : 0
};
// Test vector table health by attempting a simple query
if (vectorExtensionTest) {
try {
const result = sqlite.query('SELECT COUNT(*) as count FROM vec_chunks');
const vecCount = Number(result.rows[0].count);
vectorStats = {
vec_chunks_count: vecCount,
matches_chunk_embeddings: vecCount === vectorizedCount
@@ -56,7 +60,12 @@ export async function GET() {
};
}
} else {
vectorHealth = 'extension_unavailable';
vectorHealth = 'unavailable';
vectorStats = {
backend: vectorCapability.backend,
extension_path: vectorCapability.extensionPath,
reason: vectorCapability.available ? null : vectorCapability.reason,
};
}
} catch (error: any) {
@@ -72,6 +81,7 @@ export async function GET() {
data: {
database_connected: connectionTest,
vector_extension_loaded: vectorExtensionTest,
vector_capability: vectorCapability,
vector_health: vectorHealth,
chunk_stats: chunkStats,
vector_stats: vectorStats,
@@ -100,11 +110,11 @@ function generateRecommendations(
recommendations.push('Vector tables are corrupted - restart the application to trigger automatic healing');
}
if (vectorHealth === 'extension_unavailable') {
recommendations.push('Vector extension not loaded - check sqlite-vec installation');
if (vectorHealth === 'unavailable') {
recommendations.push('Semantic/vector search is unavailable. Install sqlite-vec for your platform or switch to Qdrant.');
}
if (chunkStats && chunkStats.coverage_percentage < 95) {
if (chunkStats && typeof chunkStats.coverage_percentage === 'number' && chunkStats.coverage_percentage < 95) {
recommendations.push(`${chunkStats.missing_embeddings} chunks missing embeddings - consider running embedding generation`);
}
package-lock.json
Generated
+26
View File
@@ -40,6 +40,7 @@
"@types/react": "^19.0.2",
"@types/react-dom": "19.0.2",
"autoprefixer": "^10.4.20",
"cross-env": "^10.0.0",
"esbuild": "^0.23.1",
"eslint": "^8.57.0",
"eslint-config-next": "^15.1.12",
@@ -165,6 +166,13 @@
"tslib": "^2.4.0"
}
},
"node_modules/@epic-web/invariant": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@epic-web/invariant/-/invariant-1.0.0.tgz",
"integrity": "sha512-lrTPqgvfFQtR/eY/qkIzp98OGdNJu0m5ji3q/nJI8v3SXkRKEnWiOxMmbvcSoAIzv/cGiuvRy57k4suKQSAdwA==",
"dev": true,
"license": "MIT"
},
"node_modules/@esbuild/aix-ppc64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz",
@@ -4643,6 +4651,24 @@
"integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==",
"license": "MIT"
},
"node_modules/cross-env": {
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/cross-env/-/cross-env-10.1.0.tgz",
"integrity": "sha512-GsYosgnACZTADcmEyJctkJIoqAhHjttw7RsFrVoJNXbsWWqaq6Ym+7kZjq6mS45O0jij6vtiReppKQEtqWy6Dw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@epic-web/invariant": "^1.0.0",
"cross-spawn": "^7.0.6"
},
"bin": {
"cross-env": "dist/bin/cross-env.js",
"cross-env-shell": "dist/bin/cross-env-shell.js"
},
"engines": {
"node": ">=20"
}
},
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
package.json
+8 -5
View File
@@ -6,12 +6,14 @@
"license": "MIT",
"author": "Bradley Morris",
"scripts": {
"dev": "NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next dev",
"dev:evals": "RAH_EVALS_LOG=1 NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next dev",
"bootstrap:local": "node ./scripts/dev/bootstrap-local.mjs",
"dev": "cross-env NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=--dns-result-order=ipv4first next dev",
"dev:local": "npm run dev",
"dev:evals": "cross-env RAH_EVALS_LOG=1 NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=--dns-result-order=ipv4first next dev",
"clean:local": "bash ./scripts/dev/clean-local-artifacts.sh",
"audit:repo": "bash ./scripts/dev/audit-repo.sh",
"build": "NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next build",
"start": "NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=\"--dns-result-order=ipv4first\" next start",
"build": "cross-env NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=--dns-result-order=ipv4first next build",
"start": "cross-env NEXT_PUBLIC_DEPLOYMENT_MODE=local NEXT_PUBLIC_ENABLE_SUBSCRIPTION_BACKEND=false NODE_OPTIONS=--dns-result-order=ipv4first next start",
"lint": "next lint",
"type-check": "tsc --noEmit",
"setup": "npm install",
@@ -19,7 +21,7 @@
"sqlite:restore": "bash scripts/database/sqlite-restore.sh",
"test": "vitest run",
"test:watch": "vitest",
"evals": "RAH_EVALS_LOG=1 RAH_EVALS_TIMEOUT_MS=60000 tsx tests/evals/runner.ts"
"evals": "cross-env RAH_EVALS_LOG=1 RAH_EVALS_TIMEOUT_MS=60000 tsx tests/evals/runner.ts"
},
"dependencies": {
"@ai-sdk/openai": "^3.0.7",
@@ -53,6 +55,7 @@
"@types/react": "^19.0.2",
"@types/react-dom": "19.0.2",
"autoprefixer": "^10.4.20",
"cross-env": "^10.0.0",
"esbuild": "^0.23.1",
"eslint": "^8.57.0",
"eslint-config-next": "^15.1.12",
scripts/dev/bootstrap-local.mjs
+307
View File
@@ -0,0 +1,307 @@
#!/usr/bin/env node
import fs from 'fs';
import os from 'os';
import path from 'path';
import Database from 'better-sqlite3';
const repoDir = process.cwd();
const envTemplate = path.join(repoDir, '.env.example');
const targetEnv = path.join(repoDir, '.env.local');
function log(message) {
console.log(`[bootstrap-local] ${message}`);
}
function getDefaultDbPath() {
const homeDir = os.homedir();
if (process.platform === 'win32') {
const appData = process.env.APPDATA || path.join(homeDir, 'AppData', 'Roaming');
return path.join(appData, 'RA-H', 'db', 'rah.sqlite');
}
if (process.platform === 'darwin') {
return path.join(homeDir, 'Library', 'Application Support', 'RA-H', 'db', 'rah.sqlite');
}
return path.join(
process.env.XDG_DATA_HOME || path.join(homeDir, '.local', 'share'),
'RA-H',
'db',
'rah.sqlite'
);
}
function parseEnvFile(filePath) {
if (!fs.existsSync(filePath)) return {};
return fs.readFileSync(filePath, 'utf8')
.split(/\r?\n/)
.filter(Boolean)
.reduce((acc, line) => {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) return acc;
const eqIndex = trimmed.indexOf('=');
if (eqIndex === -1) return acc;
const key = trimmed.slice(0, eqIndex).trim();
const value = trimmed.slice(eqIndex + 1).trim();
acc[key] = value;
return acc;
}, {});
}
function expandPath(rawPath) {
let value = rawPath;
if (value.startsWith('~')) {
value = path.join(os.homedir(), value.slice(1));
}
value = value.replace(/\$HOME/g, os.homedir());
value = value.replace(/%APPDATA%/g, process.env.APPDATA || path.join(os.homedir(), 'AppData', 'Roaming'));
return path.resolve(value);
}
function ensureEnvFile() {
if (!fs.existsSync(envTemplate)) {
throw new Error(`Missing ${envTemplate}`);
}
if (fs.existsSync(targetEnv)) {
log('.env.local already exists; leaving it untouched.');
return;
}
fs.copyFileSync(envTemplate, targetEnv);
log('Created .env.local from .env.example');
}
function ensureCoreSchema(db) {
db.pragma('foreign_keys = ON');
db.exec(`
CREATE TABLE IF NOT EXISTS agents (
id INTEGER PRIMARY KEY AUTOINCREMENT,
key TEXT UNIQUE NOT NULL,
display_name TEXT NOT NULL,
role TEXT NOT NULL DEFAULT 'executor',
system_prompt TEXT NOT NULL,
available_tools TEXT NOT NULL,
model TEXT NOT NULL,
description TEXT,
enabled INTEGER DEFAULT 1,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
memory TEXT,
prompts TEXT
);
CREATE TABLE IF NOT EXISTS nodes (
id INTEGER PRIMARY KEY,
title TEXT,
description TEXT,
source TEXT,
link TEXT,
event_date TEXT,
created_at TEXT,
updated_at TEXT,
metadata TEXT,
embedding BLOB,
embedding_updated_at TEXT,
embedding_text TEXT,
chunk_status TEXT DEFAULT 'not_chunked'
);
CREATE TABLE IF NOT EXISTS chunks (
id INTEGER PRIMARY KEY,
node_id INTEGER NOT NULL,
chunk_idx INTEGER,
text TEXT,
created_at TEXT,
embedding_type TEXT DEFAULT 'text-embedding-3-small',
metadata TEXT,
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_chunks_by_node ON chunks(node_id);
CREATE INDEX IF NOT EXISTS idx_chunks_by_node_idx ON chunks(node_id, chunk_idx);
CREATE TABLE IF NOT EXISTS edges (
id INTEGER PRIMARY KEY,
from_node_id INTEGER NOT NULL,
to_node_id INTEGER NOT NULL,
source TEXT,
created_at TEXT,
context TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node_id);
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node_id);
CREATE TABLE IF NOT EXISTS chats (
id INTEGER PRIMARY KEY,
chat_type TEXT,
helper_name TEXT,
agent_type TEXT DEFAULT 'orchestrator',
delegation_id INTEGER,
user_message TEXT,
assistant_message TEXT,
thread_id TEXT,
focused_node_id INTEGER,
created_at TEXT DEFAULT (CURRENT_TIMESTAMP),
metadata TEXT,
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);
CREATE TABLE IF NOT EXISTS node_dimensions (
node_id INTEGER NOT NULL,
dimension TEXT NOT NULL,
PRIMARY KEY (node_id, dimension),
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS idx_dim_by_dimension ON node_dimensions(dimension, node_id);
CREATE INDEX IF NOT EXISTS idx_dim_by_node ON node_dimensions(node_id, dimension);
CREATE TABLE IF NOT EXISTS dimensions (
name TEXT PRIMARY KEY,
description TEXT,
icon TEXT,
is_priority INTEGER DEFAULT 0,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY,
ts TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP),
table_name TEXT NOT NULL,
action TEXT NOT NULL,
row_id INTEGER NOT NULL,
summary TEXT,
enriched_summary TEXT,
snapshot_json TEXT
);
CREATE TABLE IF NOT EXISTS voice_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
chat_id INTEGER,
session_id TEXT,
helper_name TEXT,
request_id TEXT,
message_id TEXT,
voice TEXT,
model TEXT,
chars INTEGER,
cost_usd REAL,
duration_ms INTEGER,
text_preview TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_voice_usage_session ON voice_usage(session_id, created_at);
CREATE INDEX IF NOT EXISTS idx_voice_usage_chat ON voice_usage(chat_id);
`);
db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
title,
source,
description,
content='nodes',
content_rowid='id'
);
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
text,
content='chunks',
content_rowid='id'
);
`);
const now = new Date().toISOString();
const lockedDimensions = ['research', 'ideas', 'projects', 'memory', 'preferences'];
const insertDimension = db.prepare(`
INSERT INTO dimensions (name, is_priority, updated_at)
VALUES (?, 1, ?)
ON CONFLICT(name) DO UPDATE SET is_priority = 1, updated_at = excluded.updated_at
`);
for (const dimension of lockedDimensions) {
insertDimension.run(dimension, now);
}
const agentCount = Number(db.prepare('SELECT COUNT(*) as count FROM agents').get().count || 0);
if (agentCount === 0) {
db.prepare(`
INSERT INTO agents (
key, display_name, role, system_prompt, available_tools, model, description, enabled, created_at, updated_at, prompts
) VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
`).run(
'ra-h',
'ra-h',
'orchestrator',
"You are ra-h, the main orchestrator for RA-H. Coordinate work, delegate to mini ra-hs when tasks can be isolated, and keep the conversation focused on the user's goals.",
'["queryNodes","createNode","updateNode","createEdge","queryEdge","updateEdge","searchContentEmbeddings","webSearch","think","delegateToMiniRAH"]',
'anthropic/claude-sonnet-4.5',
'Opinionated orchestrator agent',
now,
now,
'[{"id":"p_seed_0","name":"Summary of Focus","content":"Summarize the primary focused node clearly. Include 3–5 key points and cite [NODE:id:\\"title\\"]."},{"id":"p_seed_1","name":"Next Steps","content":"Propose 3 concrete next actions based on the focused nodes with references to [NODE:id:\\"title\\"]."}]'
);
}
}
function tryInitVectorTables(db, dbPath) {
const extension = process.platform === 'darwin' ? 'dylib' : process.platform === 'win32' ? 'dll' : 'so';
const extensionPath = process.env.SQLITE_VEC_EXTENSION_PATH || path.join(repoDir, 'vendor', 'sqlite-extensions', `vec0.${extension}`);
try {
db.loadExtension(extensionPath);
db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS vec_nodes USING vec0(
node_id INTEGER PRIMARY KEY,
embedding FLOAT[1536]
);
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
chunk_id INTEGER PRIMARY KEY,
embedding FLOAT[1536]
);
`);
log(`Initialized sqlite-vec tables using ${extensionPath}`);
} catch (error) {
log(`sqlite-vec unavailable for bootstrap (${dbPath}). Continuing without vector tables.`);
}
}
function main() {
const major = Number(process.versions.node.split('.')[0] || '0');
if (major < 20) {
throw new Error(`Node.js 20+ required (found ${process.version})`);
}
ensureEnvFile();
const env = parseEnvFile(targetEnv);
const dbPath = expandPath(env.SQLITE_DB_PATH || getDefaultDbPath());
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
if (!fs.existsSync(dbPath)) {
fs.closeSync(fs.openSync(dbPath, 'w'));
}
const db = new Database(dbPath);
try {
ensureCoreSchema(db);
tryInitVectorTables(db, dbPath);
} finally {
db.close();
}
log(`Bootstrap complete. Database ready at ${dbPath}`);
log("Run 'npm run dev' to start the app.");
}
try {
main();
} catch (error) {
console.error(`[bootstrap-local] ${error instanceof Error ? error.message : String(error)}`);
process.exit(1);
}
scripts/dev/bootstrap-local.sh
+11 -1
View File
@@ -33,7 +33,17 @@ else
fi
SQLITE_DB_PATH_LINE=$(grep -E '^SQLITE_DB_PATH=' "$TARGET_ENV" | tail -n 1 || true)
DEFAULT_DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite"
case "$(uname -s)" in
Darwin)
DEFAULT_DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite"
;;
Linux)
DEFAULT_DB_PATH="${XDG_DATA_HOME:-$HOME/.local/share}/RA-H/db/rah.sqlite"
;;
*)
DEFAULT_DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite"
;;
esac
if [ -z "$SQLITE_DB_PATH_LINE" ]; then
SQLITE_DB_PATH="$DEFAULT_DB_PATH"
else
src/services/database/index.ts
+2 -1
View File
@@ -38,6 +38,7 @@ async function checkSQLiteDatabaseHealth(): Promise<{
const sqlite = getSQLiteClient();
const connected = await sqlite.testConnection();
const vectorCapability = sqlite.getVectorCapability();
if (!connected) {
return {
connected: false,
@@ -47,7 +48,7 @@ async function checkSQLiteDatabaseHealth(): Promise<{
};
}
const vectorExtension = await sqlite.checkVectorExtension();
const vectorExtension = vectorCapability.available;
// Check if main tables exist
const tables = await sqlite.checkTables();
src/services/database/sqlite-client.ts
+30 -28
View File
@@ -1,7 +1,12 @@
import Database from 'better-sqlite3';
import fs from 'fs';
import path from 'path';
import { DatabaseError } from '@/types/database';
import {
ensureDatabaseDirectory,
getDatabasePath,
getVecExtensionPath,
loadVecExtension,
type VectorCapability,
} from './sqlite-runtime';
export interface SQLiteConfig {
dbPath: string;
@@ -19,27 +24,26 @@ class SQLiteClient {
private db: Database.Database;
private config: SQLiteConfig;
private readonly readOnly: boolean;
private readonly vectorCapability: VectorCapability;
private constructor() {
this.config = this.getSQLiteConfig();
this.readOnly = process.env.SQLITE_READONLY === 'true';
// Initialize database connection
const dbDirectory = path.dirname(this.config.dbPath);
if (!this.readOnly && !fs.existsSync(dbDirectory)) {
fs.mkdirSync(dbDirectory, { recursive: true });
if (!this.readOnly) {
ensureDatabaseDirectory(this.config.dbPath);
}
this.db = this.readOnly
? new Database(this.config.dbPath, { readonly: true, fileMustExist: true })
: new Database(this.config.dbPath);
// Load sqlite-vec extension
try {
this.db.loadExtension(this.config.vecExtensionPath);
this.vectorCapability = loadVecExtension(this.db, this.config.vecExtensionPath);
if (this.vectorCapability.available) {
console.log('SQLite vector extension loaded successfully');
} catch (error) {
// Do not fail hard — allow the app to run without vector features
console.error('Warning: Failed to load vector extension:', error);
} else {
console.warn(`Warning: ${this.vectorCapability.reason}`);
}
// Configure SQLite settings
@@ -58,8 +62,10 @@ class SQLiteClient {
this.db.pragma('busy_timeout = 5000');
// Ensure vector virtual tables are present and healthy
if (this.vectorCapability.available) {
this.ensureVectorTables();
this.healVectorTablesIfCorrupt();
}
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
this.ensureLoggingAndMemorySchema();
@@ -69,17 +75,9 @@ class SQLiteClient {
}
private getSQLiteConfig(): SQLiteConfig {
const dbPath = process.env.SQLITE_DB_PATH || path.join(
process.env.HOME || '~',
'Library/Application Support/RA-H/db/rah.sqlite'
);
const vecExtensionPath = process.env.SQLITE_VEC_EXTENSION_PATH ||
'./vendor/sqlite-extensions/vec0.dylib';
return {
dbPath,
vecExtensionPath
dbPath: getDatabasePath(),
vecExtensionPath: getVecExtensionPath(),
};
}
@@ -134,7 +132,9 @@ class SQLiteClient {
} as DatabaseError;
}
// Proactively validate/repair vec vtables before any write transaction
if (this.vectorCapability.available) {
this.healVectorTablesIfCorrupt();
}
const txn = this.db.transaction(callback);
try {
return txn();
@@ -154,13 +154,11 @@ class SQLiteClient {
}
public async checkVectorExtension(): Promise<boolean> {
try {
const result = this.query('SELECT vec_version() as version');
return result.rows.length > 0;
} catch (error) {
console.error('Vector extension check failed:', error);
return false;
return this.vectorCapability.available;
}
public getVectorCapability(): VectorCapability {
return this.vectorCapability;
}
public async checkTables(): Promise<string[]> {
@@ -176,6 +174,10 @@ class SQLiteClient {
}
public ensureVectorExtensions(): void {
if (!this.vectorCapability.available) {
return;
}
try {
// Test for vec_nodes and vec_chunks; create them if missing
const hasVecNodes = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_nodes');
@@ -205,7 +207,7 @@ class SQLiteClient {
}
private ensureVectorTables(): void {
if (this.readOnly) {
if (this.readOnly || !this.vectorCapability.available) {
return;
}
// Wrapper to keep existing public API stable
@@ -739,7 +741,7 @@ class SQLiteClient {
}
private healVectorTablesIfCorrupt(): void {
if (this.readOnly) {
if (this.readOnly || !this.vectorCapability.available) {
return;
}
// Attempt lightweight reads to detect CORRUPT_VTAB; if detected, drop/recreate vtables
src/services/database/sqlite-runtime.ts
+120
View File
@@ -0,0 +1,120 @@
import Database from 'better-sqlite3';
import fs from 'fs';
import os from 'os';
import path from 'path';
export type VectorCapability =
| {
available: true;
backend: 'sqlite-vec';
extensionPath: string;
detail: string;
}
| {
available: false;
backend: 'none';
extensionPath: string;
reason: string;
};
const VECTOR_CAPABILITY_KEY = '__rahVectorCapability';
function getHomeDirectory(): string {
return os.homedir() || process.env.HOME || '~';
}
export function getDefaultDbPath(): string {
const homeDir = getHomeDirectory();
if (process.platform === 'win32') {
const appData = process.env.APPDATA || path.join(homeDir, 'AppData', 'Roaming');
return path.join(appData, 'RA-H', 'db', 'rah.sqlite');
}
if (process.platform === 'darwin') {
return path.join(homeDir, 'Library', 'Application Support', 'RA-H', 'db', 'rah.sqlite');
}
return path.join(
process.env.XDG_DATA_HOME || path.join(homeDir, '.local', 'share'),
'RA-H',
'db',
'rah.sqlite'
);
}
export function getDatabasePath(): string {
return process.env.SQLITE_DB_PATH || getDefaultDbPath();
}
export function getDefaultVecExtensionPath(): string {
const extension = process.platform === 'darwin'
? 'dylib'
: process.platform === 'win32'
? 'dll'
: 'so';
return path.join(process.cwd(), 'vendor', 'sqlite-extensions', `vec0.${extension}`);
}
export function getVecExtensionPath(): string {
return process.env.SQLITE_VEC_EXTENSION_PATH || getDefaultVecExtensionPath();
}
export function ensureDatabaseDirectory(dbPath: string): void {
const dbDirectory = path.dirname(dbPath);
if (!fs.existsSync(dbDirectory)) {
fs.mkdirSync(dbDirectory, { recursive: true });
}
}
function buildVectorFailureReason(error: unknown, extensionPath: string): string {
const message = error instanceof Error ? error.message : String(error);
if (process.platform === 'win32') {
return `sqlite-vec failed to load from ${extensionPath}. On Windows, install vec0.dll or switch to Qdrant. Original error: ${message}`;
}
if (process.platform === 'linux') {
return `sqlite-vec failed to load from ${extensionPath}. Alpine/musl builds are unsupported; use Qdrant there. Original error: ${message}`;
}
return `sqlite-vec failed to load from ${extensionPath}. Original error: ${message}`;
}
export function loadVecExtension(db: Database.Database, extensionPath = getVecExtensionPath()): VectorCapability {
try {
db.loadExtension(extensionPath);
const capability: VectorCapability = {
available: true,
backend: 'sqlite-vec',
extensionPath,
detail: 'sqlite-vec extension loaded successfully',
};
(db as Database.Database & Record<string, unknown>)[VECTOR_CAPABILITY_KEY] = capability;
return capability;
} catch (error) {
const capability: VectorCapability = {
available: false,
backend: 'none',
extensionPath,
reason: buildVectorFailureReason(error, extensionPath),
};
(db as Database.Database & Record<string, unknown>)[VECTOR_CAPABILITY_KEY] = capability;
return capability;
}
}
export function getDbVectorCapability(db: Database.Database): VectorCapability {
const capability = (db as Database.Database & Record<string, unknown>)[VECTOR_CAPABILITY_KEY];
if (capability && typeof capability === 'object' && 'available' in capability) {
return capability as VectorCapability;
}
return {
available: false,
backend: 'none',
extensionPath: getVecExtensionPath(),
reason: 'sqlite-vec capability was not initialized for this database connection',
};
}
src/services/typescript/embed-nodes.ts
+6 -7
View File
@@ -8,10 +8,12 @@ import { generateText } from 'ai';
import { createOpenAI } from '@ai-sdk/openai';
import {
createDatabaseConnection,
getDbVectorCapability,
serializeFloat32Vector,
formatEmbeddingText,
batchProcess
} from './sqlite-vec';
import type { VectorCapability } from '@/services/database/sqlite-runtime';
interface NodeRecord {
id: number;
@@ -35,6 +37,7 @@ export class NodeEmbedder {
private openaiClient: OpenAI;
private openaiProvider: ReturnType<typeof createOpenAI>;
private db: ReturnType<typeof createDatabaseConnection>;
private readonly vectorCapability: VectorCapability;
private processedCount: number = 0;
private failedCount: number = 0;
@@ -47,6 +50,7 @@ export class NodeEmbedder {
this.openaiClient = new OpenAI({ apiKey });
this.openaiProvider = createOpenAI({ apiKey });
this.db = createDatabaseConnection();
this.vectorCapability = getDbVectorCapability(this.db);
}
/**
@@ -138,23 +142,18 @@ Focus on the main concepts, key relationships, and practical implications.`;
const now = new Date().toISOString();
updateStmt.run(embeddingBlob, now, embeddingText, node.id);
// Update vec_nodes virtual table
if (this.vectorCapability.available) {
try {
// Determine correct column name for primary key (node_id vs id)
// Use declared PK column from your DB schema (confirmed: node_id)
const pkCol = 'node_id';
// Delete existing entry if any
const deleteStmt = this.db.prepare(`DELETE FROM vec_nodes WHERE ${pkCol} = ?`);
deleteStmt.run(BigInt(node.id));
// Insert new entry (use bracketed string format compatible with sqlite-vec)
const vectorString = `[${embedding.join(',')}]`;
const insertStmt = this.db.prepare(`INSERT INTO vec_nodes (${pkCol}, embedding) VALUES (?, ?)`);
insertStmt.run(BigInt(node.id), vectorString);
} catch (vecError) {
console.warn(`Could not update vec_nodes for node ${node.id}:`, vecError);
// Continue - main embedding is still saved
}
}
this.processedCount++;
src/services/typescript/embed-universal.ts
+8 -3
View File
@@ -7,9 +7,11 @@ import OpenAI from 'openai';
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
import {
createDatabaseConnection,
getDbVectorCapability,
serializeFloat32Vector,
batchProcess
} from './sqlite-vec';
import type { VectorCapability } from '@/services/database/sqlite-runtime';
interface Node {
id: number;
@@ -37,6 +39,7 @@ export class UniversalEmbedder {
private openaiClient: OpenAI;
private db: ReturnType<typeof createDatabaseConnection>;
private textSplitter: RecursiveCharacterTextSplitter;
private readonly vectorCapability: VectorCapability;
private vecChunksInsertSQL: string | null = null;
constructor() {
@@ -47,6 +50,7 @@ export class UniversalEmbedder {
this.openaiClient = new OpenAI({ apiKey });
this.db = createDatabaseConnection();
this.vectorCapability = getDbVectorCapability(this.db);
// Configure text splitter (same as old KMS system)
this.textSplitter = new RecursiveCharacterTextSplitter({
@@ -87,6 +91,7 @@ export class UniversalEmbedder {
const chunkIds = this.db.prepare('SELECT id FROM chunks WHERE node_id = ?').all(nodeId) as Array<{ id: number }>;
// Delete from vec_chunks first, one by one to ensure they're removed
if (this.vectorCapability.available) {
for (const chunk of chunkIds) {
try {
const deleteVecStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
@@ -95,6 +100,7 @@ export class UniversalEmbedder {
console.warn(`Could not delete vec_chunk ${chunk.id}:`, error);
}
}
}
// Then delete from chunks table
const deleteChunksStmt = this.db.prepare('DELETE FROM chunks WHERE node_id = ?');
@@ -131,16 +137,14 @@ export class UniversalEmbedder {
const chunkId = Number(result.lastInsertRowid);
// Insert into vec_chunks virtual table (use bracketed string format)
if (this.vectorCapability.available) {
try {
const vectorString = `[${embedding.join(',')}]`;
// First try to delete any existing vec_chunk with this ID
try {
const deleteStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
deleteStmt.run(BigInt(chunkId));
} catch {}
// Now insert the new embedding
const sql = 'INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)';
const vecInsertStmt = this.db.prepare(sql);
vecInsertStmt.run(BigInt(chunkId), vectorString);
@@ -148,6 +152,7 @@ export class UniversalEmbedder {
console.warn(`Could not insert into vec_chunks for chunk ${chunkId}:`, error);
}
}
}
/**
* Process a single node for chunking and embedding
src/services/typescript/sqlite-vec.ts
+13 -36
View File
@@ -4,8 +4,13 @@
*/
import Database from 'better-sqlite3';
import path from 'path';
import os from 'os';
import {
ensureDatabaseDirectory,
getDatabasePath,
getDbVectorCapability,
getVecExtensionPath,
loadVecExtension,
} from '@/services/database/sqlite-runtime';
/**
* Serialize a float array to binary format for vec0 storage
@@ -30,53 +35,25 @@ export function deserializeFloat32Vector(blob: Buffer): number[] {
return vector;
}
/**
* Get SQLite database path from environment or default location
*/
export function getDatabasePath(): string {
const envPath = process.env.SQLITE_DB_PATH;
if (envPath) {
return envPath;
}
// Default path: ~/Library/Application Support/RA-H/db/rah.sqlite
const homeDir = os.homedir();
return path.join(homeDir, 'Library', 'Application Support', 'RA-H', 'db', 'rah.sqlite');
}
/**
* Get vec extension path from environment or default location
*/
export function getVecExtensionPath(): string {
const envPath = process.env.SQLITE_VEC_EXTENSION_PATH;
if (envPath) {
return envPath;
}
// Default path relative to project root
return path.join(process.cwd(), 'vendor', 'sqlite-extensions', 'vec0.dylib');
}
/**
* Create database connection with vec0 extension loaded
*/
export function createDatabaseConnection(): Database.Database {
const dbPath = getDatabasePath();
const vecPath = getVecExtensionPath();
ensureDatabaseDirectory(dbPath);
const db = new Database(dbPath);
// Load vec0 extension
try {
db.loadExtension(vecPath);
} catch (error) {
console.error('Warning: Could not load vec0 extension:', error);
// Continue without vector support for non-vector operations
const capability = loadVecExtension(db, vecPath);
if (!capability.available) {
console.warn(`Warning: ${capability.reason}`);
}
return db;
}
export { getDatabasePath, getDbVectorCapability, getVecExtensionPath };
/**
* Format embedding text for node metadata
*/
src/tools/other/sqliteQuery.ts
+1 -9
View File
@@ -2,18 +2,10 @@ import { tool } from 'ai';
import { z } from 'zod';
import { exec } from 'child_process';
import { promisify } from 'util';
import path from 'path';
import { getDatabasePath } from '@/services/database/sqlite-runtime';
const execAsync = promisify(exec);
// Get database path (same logic as sqlite-client.ts)
function getDatabasePath(): string {
return process.env.SQLITE_DB_PATH || path.join(
process.env.HOME || '~',
'Library/Application Support/RA-H/db/rah.sqlite'
);
}
// Security: Only allow SELECT statements
function isReadOnlyQuery(sql: string): boolean {
const normalized = sql.trim().toLowerCase();