Merge branch 'codex/os-sqlite-fts-corruption-sync'

This commit is contained in:
“BeeRad”
2026-04-14 22:06:05 +10:00
9 changed files with 789 additions and 80 deletions
+6 -1
View File
@@ -1,10 +1,12 @@
import { NextRequest, NextResponse } from 'next/server'; import { NextRequest, NextResponse } from 'next/server';
import { nodeService } from '@/services/database'; import { nodeService } from '@/services/database';
import { getSQLiteClient } from '@/services/database/sqlite-client';
export const runtime = 'nodejs'; export const runtime = 'nodejs';
export async function GET(request: NextRequest) { export async function GET(request: NextRequest) {
try { try {
const integrity = getSQLiteClient().getIntegrityReport();
const searchParams = request.nextUrl.searchParams; const searchParams = request.nextUrl.searchParams;
const query = searchParams.get('q'); const query = searchParams.get('q');
const limit = parseInt(searchParams.get('limit') || '10'); const limit = parseInt(searchParams.get('limit') || '10');
@@ -35,7 +37,10 @@ export async function GET(request: NextRequest) {
success: true, success: true,
data: results, data: results,
count: results.length, count: results.length,
query: query.trim() query: query.trim(),
degraded: integrity.state !== 'healthy',
integrityState: integrity.state,
integritySummary: integrity.state !== 'healthy' ? integrity.summary : undefined,
}); });
} catch (error) { } catch (error) {
console.error('Error searching nodes:', error); console.error('Error searching nodes:', error);
-26
View File
@@ -220,31 +220,6 @@ function checkFtsAvailability() {
return _ftsAvailability; return _ftsAvailability;
} }
function rebuildFtsIndexes() {
const fts = checkFtsAvailability();
if (!fts.nodes && !fts.chunks) return;
const db = getDb();
if (fts.nodes) {
try {
db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')");
log('Rebuilt nodes_fts index');
} catch (err) {
log('Warning: Failed to rebuild nodes_fts:', err.message);
_ftsAvailability.nodes = false;
}
}
if (fts.chunks) {
try {
db.exec("INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild')");
log('Rebuilt chunks_fts index');
} catch (err) {
log('Warning: Failed to rebuild chunks_fts:', err.message);
_ftsAvailability.chunks = false;
}
}
}
// Security: Only allow read-only SQL statements // Security: Only allow read-only SQL statements
function isReadOnlyQuery(sql) { function isReadOnlyQuery(sql) {
const normalized = sql.trim().toLowerCase(); const normalized = sql.trim().toLowerCase();
@@ -283,7 +258,6 @@ async function main() {
try { try {
initDatabase(); initDatabase();
log('Database connected:', getDatabasePath()); log('Database connected:', getDatabasePath());
rebuildFtsIndexes();
} catch (error) { } catch (error) {
log('ERROR:', error.message); log('ERROR:', error.message);
process.exit(1); process.exit(1);
+385
View File
@@ -0,0 +1,385 @@
#!/usr/bin/env bash
set -euo pipefail
# Repair rebuildable FTS corruption by rebuilding a clean SQLite file from the
# readable canonical tables, then swapping the rebuilt file into place.
DB_PATH=${SQLITE_DB_PATH:-}
if [ -z "${DB_PATH}" ] && [ -f ".env.local" ]; then
DB_PATH=$(grep -E '^SQLITE_DB_PATH=' .env.local | sed 's/^SQLITE_DB_PATH=//' | sed 's/^"\(.*\)"$/\1/') || true
fi
if [ -z "${DB_PATH}" ]; then
DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite"
fi
if [ ! -f "$DB_PATH" ]; then
echo "Error: Resolved DB not found: $DB_PATH" >&2
exit 1
fi
DB_DIR="$(dirname "$DB_PATH")"
DB_NAME="$(basename "$DB_PATH")"
TS=$(date +"%Y%m%d_%H%M%S")
RAW_BACKUP_DIR="$DB_DIR/working/fts_repair_${TS}"
REBUILT_DB="$DB_DIR/${DB_NAME}.rebuilt.${TS}"
QUOTED_DB_PATH=${DB_PATH//\'/\'\'}
mkdir -p "$RAW_BACKUP_DIR"
cp -p "$DB_PATH" "$RAW_BACKUP_DIR/$DB_NAME"
if [ -f "${DB_PATH}-wal" ]; then
cp -p "${DB_PATH}-wal" "$RAW_BACKUP_DIR/${DB_NAME}-wal"
fi
if [ -f "${DB_PATH}-shm" ]; then
cp -p "${DB_PATH}-shm" "$RAW_BACKUP_DIR/${DB_NAME}-shm"
fi
echo "Raw backup saved to: $RAW_BACKUP_DIR"
echo "Preflight source probes:"
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM nodes;" | sed 's/^/ nodes: /'
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM edges;" | sed 's/^/ edges: /'
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM chunks;" | sed 's/^/ chunks: /'
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM nodes_fts;" | sed 's/^/ nodes_fts: /' || true
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM chunks_fts;" | sed 's/^/ chunks_fts: /' || true
rm -f "$REBUILT_DB" "${REBUILT_DB}-wal" "${REBUILT_DB}-shm"
sqlite3 "$REBUILT_DB" <<SQL
PRAGMA journal_mode = DELETE;
PRAGMA synchronous = NORMAL;
PRAGMA foreign_keys = OFF;
ATTACH DATABASE '$QUOTED_DB_PATH' AS source;
CREATE TABLE schema_version (
version INTEGER PRIMARY KEY,
applied_at TEXT DEFAULT CURRENT_TIMESTAMP,
description TEXT
);
CREATE TABLE agents (
id INTEGER PRIMARY KEY AUTOINCREMENT,
key TEXT UNIQUE NOT NULL,
display_name TEXT NOT NULL,
role TEXT NOT NULL DEFAULT 'executor',
system_prompt TEXT NOT NULL,
available_tools TEXT NOT NULL,
model TEXT NOT NULL,
description TEXT,
enabled INTEGER DEFAULT 1,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
memory TEXT,
prompts TEXT
);
CREATE TABLE contexts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL,
icon TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE nodes (
id INTEGER PRIMARY KEY,
title TEXT,
description TEXT,
source TEXT,
link TEXT,
event_date TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
embedding BLOB,
embedding_updated_at TEXT,
embedding_text TEXT,
chunk_status TEXT DEFAULT 'not_chunked',
context_id INTEGER,
FOREIGN KEY (context_id) REFERENCES contexts(id) ON DELETE SET NULL
);
CREATE TABLE edges (
id INTEGER PRIMARY KEY,
from_node_id INTEGER NOT NULL,
to_node_id INTEGER NOT NULL,
source TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
CREATE TABLE chunks (
id INTEGER PRIMARY KEY,
node_id INTEGER NOT NULL,
chunk_idx INTEGER,
text TEXT NOT NULL,
embedding BLOB,
embedding_type TEXT DEFAULT 'openai',
metadata TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
CREATE TABLE chats (
id INTEGER PRIMARY KEY,
chat_type TEXT,
helper_name TEXT,
agent_type TEXT DEFAULT 'orchestrator',
delegation_id INTEGER,
user_message TEXT,
assistant_message TEXT,
thread_id TEXT,
focused_node_id INTEGER,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
);
CREATE TABLE logs (
id INTEGER PRIMARY KEY,
ts TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
table_name TEXT NOT NULL,
action TEXT NOT NULL,
row_id INTEGER NOT NULL,
summary TEXT,
snapshot_json TEXT,
enriched_summary TEXT
);
CREATE TABLE voice_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
chat_id INTEGER,
session_id TEXT,
helper_name TEXT,
request_id TEXT,
message_id TEXT,
voice TEXT,
model TEXT,
chars INTEGER,
cost_usd REAL,
duration_ms INTEGER,
text_preview TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE SET NULL
);
CREATE TABLE dimension_migration_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
migrated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
dimension_count INTEGER NOT NULL,
assignment_count INTEGER NOT NULL,
payload TEXT
);
INSERT INTO schema_version SELECT version, applied_at, description FROM source.schema_version;
INSERT INTO agents SELECT id, key, display_name, role, system_prompt, available_tools, model, description, enabled, created_at, updated_at, memory, prompts FROM source.agents;
INSERT INTO contexts SELECT id, name, description, icon, created_at, updated_at FROM source.contexts;
INSERT INTO nodes SELECT id, title, description, source, link, event_date, created_at, updated_at, metadata, embedding, embedding_updated_at, embedding_text, chunk_status, context_id FROM source.nodes;
INSERT INTO edges SELECT id, from_node_id, to_node_id, source, created_at, context, explanation FROM source.edges;
INSERT INTO chunks SELECT id, node_id, chunk_idx, text, embedding, embedding_type, metadata, created_at FROM source.chunks;
INSERT INTO chats SELECT id, chat_type, helper_name, agent_type, delegation_id, user_message, assistant_message, thread_id, focused_node_id, created_at, metadata FROM source.chats;
INSERT INTO logs SELECT id, ts, table_name, action, row_id, summary, snapshot_json, enriched_summary FROM source.logs;
INSERT INTO voice_usage SELECT id, chat_id, session_id, helper_name, request_id, message_id, voice, model, chars, cost_usd, duration_ms, text_preview, created_at FROM source.voice_usage;
INSERT INTO dimension_migration_snapshots SELECT id, migrated_at, dimension_count, assignment_count, payload FROM source.dimension_migration_snapshots;
CREATE UNIQUE INDEX idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
CREATE INDEX idx_edges_from ON edges(from_node_id);
CREATE INDEX idx_edges_to ON edges(to_node_id);
CREATE INDEX idx_nodes_updated_at ON nodes(updated_at DESC);
CREATE INDEX idx_nodes_context_id ON nodes(context_id);
CREATE INDEX idx_chunks_node_id ON chunks(node_id);
CREATE INDEX idx_chunks_by_node ON chunks(node_id);
CREATE INDEX idx_chunks_by_node_idx ON chunks(node_id, chunk_idx);
CREATE INDEX idx_chats_thread ON chats(thread_id);
CREATE INDEX idx_chats_created_at ON chats(created_at DESC);
CREATE INDEX idx_logs_ts ON logs(ts);
CREATE INDEX idx_logs_table_ts ON logs(table_name, ts);
CREATE INDEX idx_logs_table_row ON logs(table_name, row_id);
CREATE INDEX idx_voice_usage_session ON voice_usage(session_id, created_at);
CREATE INDEX idx_voice_usage_chat ON voice_usage(chat_id);
CREATE VIEW nodes_v AS
SELECT n.id,
n.title,
n.description,
n.source,
n.link,
n.event_date,
n.metadata,
n.created_at,
n.updated_at
FROM nodes n;
CREATE VIEW logs_v AS
SELECT
m.id,
m.ts,
m.table_name,
m.action,
m.row_id,
m.summary,
m.enriched_summary,
m.snapshot_json,
CASE WHEN m.table_name='nodes' THEN n.title END AS node_title,
CASE WHEN m.table_name='edges' THEN nf.title END AS edge_from_title,
CASE WHEN m.table_name='edges' THEN nt.title END AS edge_to_title,
CASE WHEN m.table_name='chats' THEN c.helper_name END AS chat_helper,
CASE WHEN m.table_name='chats' THEN substr(c.user_message,1,120) END AS chat_user_preview,
CASE WHEN m.table_name='chats' THEN substr(c.assistant_message,1,120) END AS chat_assistant_preview,
CASE WHEN m.table_name='chats' THEN c.user_message END AS chat_user_full,
CASE WHEN m.table_name='chats' THEN c.assistant_message END AS chat_assistant_full
FROM logs m
LEFT JOIN nodes n ON (m.table_name='nodes' AND m.row_id = n.id)
LEFT JOIN edges e ON (m.table_name='edges' AND m.row_id = e.id)
LEFT JOIN nodes nf ON e.from_node_id = nf.id
LEFT JOIN nodes nt ON e.to_node_id = nt.id
LEFT JOIN chats c ON (m.table_name='chats' AND m.row_id = c.id);
CREATE VIRTUAL TABLE nodes_fts USING fts5(
title,
source,
description,
content='nodes',
content_rowid='id'
);
CREATE VIRTUAL TABLE chunks_fts USING fts5(
text,
content='chunks',
content_rowid='id'
);
INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');
INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');
CREATE TRIGGER nodes_fts_ai AFTER INSERT ON nodes BEGIN
INSERT INTO nodes_fts(rowid, title, source, description)
VALUES (new.id, new.title, new.source, new.description);
END;
CREATE TRIGGER nodes_fts_ad AFTER DELETE ON nodes BEGIN
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
VALUES('delete', old.id, old.title, old.source, old.description);
END;
CREATE TRIGGER nodes_fts_au AFTER UPDATE OF title, source, description ON nodes BEGIN
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
VALUES('delete', old.id, old.title, old.source, old.description);
INSERT INTO nodes_fts(rowid, title, source, description)
VALUES (new.id, new.title, new.source, new.description);
END;
CREATE TRIGGER chunks_fts_ai AFTER INSERT ON chunks BEGIN
INSERT INTO chunks_fts(rowid, text)
VALUES (new.id, new.text);
END;
CREATE TRIGGER chunks_fts_ad AFTER DELETE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text)
VALUES('delete', old.id, old.text);
END;
CREATE TRIGGER chunks_fts_au AFTER UPDATE OF text ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text)
VALUES('delete', old.id, old.text);
INSERT INTO chunks_fts(rowid, text)
VALUES (new.id, new.text);
END;
CREATE TRIGGER trg_nodes_ai AFTER INSERT ON nodes BEGIN
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
VALUES('nodes', 'insert', NEW.id,
printf('node created: %s', COALESCE(NEW.title,'')),
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
END;
CREATE TRIGGER trg_nodes_au AFTER UPDATE ON nodes BEGIN
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
VALUES('nodes', 'update', NEW.id,
printf('node updated: %s', COALESCE(NEW.title,'')),
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
END;
CREATE TRIGGER trg_edges_ai AFTER INSERT ON edges BEGIN
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
VALUES('edges', 'insert', NEW.id,
printf('edge %d→%d (%s)', NEW.from_node_id, NEW.to_node_id, COALESCE(NEW.source,'')),
json_object(
'id', NEW.id,
'from', NEW.from_node_id,
'to', NEW.to_node_id,
'source', NEW.source,
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
));
END;
CREATE TRIGGER trg_edges_au AFTER UPDATE ON edges BEGIN
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
VALUES('edges', 'update', NEW.id,
printf('edge updated %d→%d', NEW.from_node_id, NEW.to_node_id),
json_object(
'id', NEW.id,
'from', NEW.from_node_id,
'to', NEW.to_node_id,
'source', NEW.source,
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
));
END;
CREATE TRIGGER trg_edges_update_nodes_on_insert
AFTER INSERT ON edges
BEGIN
UPDATE nodes SET updated_at = strftime('%Y-%m-%dT%H:%M:%f', 'now') || 'Z' WHERE id = NEW.from_node_id;
UPDATE nodes SET updated_at = strftime('%Y-%m-%dT%H:%M:%f', 'now') || 'Z' WHERE id = NEW.to_node_id;
END;
CREATE TRIGGER trg_chats_ai AFTER INSERT ON chats BEGIN
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
VALUES('chats', 'insert', NEW.id,
printf('chat: %s (%s)', COALESCE(NEW.helper_name,''), COALESCE(NEW.thread_id,'')),
json_object(
'id', NEW.id,
'helper', NEW.helper_name,
'thread', NEW.thread_id,
'user_preview', substr(COALESCE(NEW.user_message,''), 1, 120),
'assistant_preview', substr(COALESCE(NEW.assistant_message,''), 1, 120)
));
END;
CREATE TRIGGER trg_logs_prune AFTER INSERT ON logs BEGIN
DELETE FROM logs WHERE id < NEW.id - 10000;
END;
DETACH DATABASE source;
PRAGMA foreign_keys = ON;
PRAGMA journal_mode = WAL;
PRAGMA optimize;
SQL
echo "Rebuilt DB written to: $REBUILT_DB"
echo "Post-rebuild probes:"
sqlite3 "$REBUILT_DB" "PRAGMA quick_check;" | sed 's/^/ quick_check: /'
sqlite3 "$REBUILT_DB" "PRAGMA integrity_check;" | sed 's/^/ integrity_check: /'
sqlite3 "$REBUILT_DB" "SELECT COUNT(*) FROM nodes_fts;" | sed 's/^/ nodes_fts: /'
sqlite3 "$REBUILT_DB" "SELECT COUNT(*) FROM chunks_fts;" | sed 's/^/ chunks_fts: /'
mv "$DB_PATH" "${DB_PATH}.corrupt.${TS}"
if [ -f "${DB_PATH}-wal" ]; then
mv "${DB_PATH}-wal" "${DB_PATH}-wal.corrupt.${TS}"
fi
if [ -f "${DB_PATH}-shm" ]; then
mv "${DB_PATH}-shm" "${DB_PATH}-shm.corrupt.${TS}"
fi
mv "$REBUILT_DB" "$DB_PATH"
rm -f "${DB_PATH}-wal" "${DB_PATH}-shm"
rm -f "${REBUILT_DB}-wal" "${REBUILT_DB}-shm"
echo "Swapped rebuilt DB into place: $DB_PATH"
echo "Original live files preserved with suffix .corrupt.${TS}"
+2 -4
View File
@@ -420,10 +420,7 @@ export class ChunkService {
matchCount: number, matchCount: number,
nodeIds?: number[] nodeIds?: number[]
): RankedChunk[] { ): RankedChunk[] {
const ftsExists = sqlite.prepare( if (!sqlite.canUseFtsTable('chunks')) return [];
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks_fts'"
).get();
if (!ftsExists) return [];
const ftsQuery = sanitizeFtsQuery(query); const ftsQuery = sanitizeFtsQuery(query);
if (!ftsQuery) return []; if (!ftsQuery) return [];
@@ -461,6 +458,7 @@ export class ChunkService {
return result.rows; return result.rows;
} catch (error) { } catch (error) {
sqlite.disableFtsTable('chunks', 'chunks_fts query failed during chunk search', error);
console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error); console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error);
return []; return [];
} }
+67 -10
View File
@@ -1,3 +1,5 @@
import type { DatabaseIntegrityReport } from './sqlite-client';
// Service instances // Service instances
export { nodeService, NodeService } from './nodes'; export { nodeService, NodeService } from './nodes';
export { chunkService, ChunkService } from './chunks'; export { chunkService, ChunkService } from './chunks';
@@ -8,13 +10,31 @@ export { contextService, ContextService } from './contextService';
// Types // Types
export * from '@/types/database'; export * from '@/types/database';
// Health check utility export interface DatabaseHealthStatus {
export async function checkDatabaseHealth(): Promise<{
connected: boolean; connected: boolean;
vectorExtension: boolean; vectorExtension: boolean;
tablesExist: boolean; tablesExist: boolean;
quickCheckOk: boolean;
integrityCheckOk: boolean;
foreignKeyViolations: number;
lostAndFoundExists: boolean;
ftsTables: {
nodes: boolean;
chunks: boolean;
};
vecTables: {
nodes: boolean;
chunks: boolean;
};
integrityState: DatabaseIntegrityReport['state'];
repairableFtsTables: DatabaseIntegrityReport['repairableFtsTables'];
canRepairFts: boolean;
summary: string;
error?: string; error?: string;
}> { }
// Health check utility
export async function checkDatabaseHealth(): Promise<DatabaseHealthStatus> {
try { try {
return checkSQLiteDatabaseHealth(); return checkSQLiteDatabaseHealth();
} catch (error) { } catch (error) {
@@ -22,20 +42,26 @@ export async function checkDatabaseHealth(): Promise<{
connected: false, connected: false,
vectorExtension: false, vectorExtension: false,
tablesExist: false, tablesExist: false,
quickCheckOk: false,
integrityCheckOk: false,
foreignKeyViolations: -1,
lostAndFoundExists: false,
ftsTables: { nodes: false, chunks: false },
vecTables: { nodes: false, chunks: false },
integrityState: 'corrupt',
repairableFtsTables: [],
canRepairFts: false,
summary: 'Database health check failed before classification.',
error: error instanceof Error ? error.message : 'Unknown error' error: error instanceof Error ? error.message : 'Unknown error'
}; };
} }
} }
async function checkSQLiteDatabaseHealth(): Promise<{ async function checkSQLiteDatabaseHealth(): Promise<DatabaseHealthStatus> {
connected: boolean;
vectorExtension: boolean;
tablesExist: boolean;
error?: string;
}> {
try { try {
const { getSQLiteClient } = await import('./sqlite-client'); const { getSQLiteClient } = await import('./sqlite-client');
const sqlite = getSQLiteClient(); const sqlite = getSQLiteClient();
const integrity = sqlite.getIntegrityReport(true);
const connected = await sqlite.testConnection(); const connected = await sqlite.testConnection();
if (!connected) { if (!connected) {
@@ -43,6 +69,16 @@ async function checkSQLiteDatabaseHealth(): Promise<{
connected: false, connected: false,
vectorExtension: false, vectorExtension: false,
tablesExist: false, tablesExist: false,
quickCheckOk: false,
integrityCheckOk: false,
foreignKeyViolations: -1,
lostAndFoundExists: false,
ftsTables: { nodes: false, chunks: false },
vecTables: { nodes: false, chunks: false },
integrityState: integrity.state,
repairableFtsTables: integrity.repairableFtsTables,
canRepairFts: integrity.canRepairFts,
summary: integrity.summary,
error: 'SQLite connection failed' error: 'SQLite connection failed'
}; };
} }
@@ -57,13 +93,34 @@ async function checkSQLiteDatabaseHealth(): Promise<{
return { return {
connected, connected,
vectorExtension, vectorExtension,
tablesExist tablesExist,
quickCheckOk: integrity.quickCheck.ok,
integrityCheckOk: integrity.integrityCheck.ok,
foreignKeyViolations: integrity.foreignKeyViolations,
lostAndFoundExists: integrity.lostAndFoundExists,
ftsTables: integrity.ftsTables,
vecTables: integrity.vecTables,
integrityState: integrity.state,
repairableFtsTables: integrity.repairableFtsTables,
canRepairFts: integrity.canRepairFts,
summary: integrity.summary,
error: integrity.error,
}; };
} catch (error) { } catch (error) {
return { return {
connected: false, connected: false,
vectorExtension: false, vectorExtension: false,
tablesExist: false, tablesExist: false,
quickCheckOk: false,
integrityCheckOk: false,
foreignKeyViolations: -1,
lostAndFoundExists: false,
ftsTables: { nodes: false, chunks: false },
vecTables: { nodes: false, chunks: false },
integrityState: 'corrupt',
repairableFtsTables: [],
canRepairFts: false,
summary: 'Database health check failed during execution.',
error: error instanceof Error ? error.message : 'Unknown error' error: error instanceof Error ? error.message : 'Unknown error'
}; };
} }
+5 -12
View File
@@ -498,14 +498,11 @@ export class NodeService {
if (!search) return 0; if (!search) return 0;
const ftsQuery = sanitizeFtsQuery(search); const ftsQuery = sanitizeFtsQuery(search);
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
).get();
const { clauses, params } = this.buildNodeFilterClauses(filters); const { clauses, params } = this.buildNodeFilterClauses(filters);
if (ftsExists && ftsQuery) { if (sqlite.canUseFtsTable('nodes') && ftsQuery) {
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
try { try {
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const result = sqlite.query<{ total: number }>(` const result = sqlite.query<{ total: number }>(`
WITH matched_nodes AS ( WITH matched_nodes AS (
SELECT rowid SELECT rowid
@@ -520,7 +517,7 @@ export class NodeService {
return Number(result.rows[0]?.total ?? 0); return Number(result.rows[0]?.total ?? 0);
} catch (error) { } catch (error) {
sqlite.disableNodesFts('nodes_fts query failed during count search', error); sqlite.disableFtsTable('nodes', 'nodes_fts query failed during count search', error);
} }
} }
@@ -549,11 +546,7 @@ export class NodeService {
): NodeSearchRow[] { ): NodeSearchRow[] {
const ftsQuery = sanitizeFtsQuery(search); const ftsQuery = sanitizeFtsQuery(search);
if (!ftsQuery) return []; if (!ftsQuery) return [];
if (!sqlite.canUseFtsTable('nodes')) return [];
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
).get();
if (!ftsExists) return [];
const { clauses, params } = this.buildNodeFilterClauses(filters); const { clauses, params } = this.buildNodeFilterClauses(filters);
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : ''; const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
@@ -584,7 +577,7 @@ export class NodeService {
return result.rows; return result.rows;
} catch (error) { } catch (error) {
sqlite.disableNodesFts('nodes_fts query failed during node search', error); sqlite.disableFtsTable('nodes', 'nodes_fts query failed during node search', error);
return []; return [];
} }
} }
+305 -27
View File
@@ -19,14 +19,49 @@ export interface SQLiteQueryResult<T = any> {
lastInsertRowid?: number; lastInsertRowid?: number;
} }
type FtsSurfaceName = 'nodes' | 'chunks';
interface IntegrityProbeResult {
ok: boolean;
error?: string;
code?: string;
}
export interface DatabaseIntegrityReport {
state: 'healthy' | 'degraded_fts' | 'corrupt';
connected: boolean;
quickCheck: IntegrityProbeResult;
integrityCheck: IntegrityProbeResult;
baseTables: Record<'nodes' | 'edges' | 'chunks', IntegrityProbeResult>;
ftsTables: Record<FtsSurfaceName, boolean>;
ftsProbeResults: Record<FtsSurfaceName, IntegrityProbeResult>;
repairableFtsTables: FtsSurfaceName[];
canRepairFts: boolean;
foreignKeyViolations: number;
lostAndFoundExists: boolean;
vecTables: {
nodes: boolean;
chunks: boolean;
};
summary: string;
error?: string;
}
class SQLiteClient { class SQLiteClient {
private static instance: SQLiteClient; private static instance: SQLiteClient;
private db: Database.Database; private db: Database.Database;
private config: SQLiteConfig; private config: SQLiteConfig;
private readonly readOnly: boolean; private readonly readOnly: boolean;
private readonly vectorCapability: VectorCapability; private readonly vectorCapability: VectorCapability;
private nodesFtsUsable = true; private integrityReport: DatabaseIntegrityReport | null = null;
private nodesFtsDisabledReason: string | null = null; private readonly ftsUsable: Record<FtsSurfaceName, boolean> = {
nodes: true,
chunks: true,
};
private readonly ftsDisabledReason: Record<FtsSurfaceName, string | null> = {
nodes: null,
chunks: null,
};
private constructor() { private constructor() {
this.config = this.getSQLiteConfig(); this.config = this.getSQLiteConfig();
@@ -73,6 +108,16 @@ class SQLiteClient {
// Ensure logging schema (rename memory->logs if needed, create triggers/views) // Ensure logging schema (rename memory->logs if needed, create triggers/views)
this.ensureLoggingAndMemorySchema(); this.ensureLoggingAndMemorySchema();
this.ensureContextsSchema(); this.ensureContextsSchema();
this.integrityReport = this.inspectIntegrity();
if (this.integrityReport.state === 'healthy') {
this.ensureFtsTables();
this.integrityReport = this.inspectIntegrity();
} else {
console.warn(
`[SQLiteIntegrity] Skipping startup FTS mutation because database is ${this.integrityReport.state}: ${this.integrityReport.summary}`
);
}
} }
console.log('SQLite client initialized successfully'); console.log('SQLite client initialized successfully');
@@ -118,6 +163,7 @@ class SQLiteClient {
}; };
} }
} catch (error) { } catch (error) {
this.refreshIntegrityReportForCorruptionError(error);
console.error('SQLite query error:', error); console.error('SQLite query error:', error);
throw this.handleError(error); throw this.handleError(error);
} }
@@ -143,6 +189,7 @@ class SQLiteClient {
try { try {
return txn(); return txn();
} catch (error) { } catch (error) {
this.refreshIntegrityReportForCorruptionError(error);
throw this.handleError(error); throw this.handleError(error);
} }
} }
@@ -166,22 +213,11 @@ class SQLiteClient {
} }
public isNodesFtsUsable(): boolean { public isNodesFtsUsable(): boolean {
return this.nodesFtsUsable; return this.canUseFtsTable('nodes');
} }
public disableNodesFts(reason: string, error?: unknown): void { public disableNodesFts(reason: string, error?: unknown): void {
this.nodesFtsUsable = false; this.disableFtsTable('nodes', reason, error);
if (this.nodesFtsDisabledReason === reason) {
return;
}
this.nodesFtsDisabledReason = reason;
if (error && !this.isSqliteCorruptError(error)) {
console.warn(`[SQLite] nodes_fts disabled: ${reason}`, error);
return;
}
console.warn(`[SQLite] nodes_fts disabled: ${reason}. Falling back to LIKE search for this database session.`);
} }
public async checkTables(): Promise<string[]> { public async checkTables(): Promise<string[]> {
@@ -229,6 +265,25 @@ class SQLiteClient {
} }
} }
public canUseFtsTable(tableName: FtsSurfaceName): boolean {
return this.ftsUsable[tableName] && this.getIntegrityReport().ftsTables[tableName];
}
public disableFtsTable(tableName: FtsSurfaceName, reason: string, error?: unknown): void {
this.ftsUsable[tableName] = false;
if (this.ftsDisabledReason[tableName] === reason) {
return;
}
this.ftsDisabledReason[tableName] = reason;
if (error && !this.isSqliteCorruptError(error)) {
console.warn(`[SQLite] ${tableName}_fts disabled: ${reason}`, error);
return;
}
console.warn(`[SQLite] ${tableName}_fts disabled: ${reason}. Falling back to non-FTS behavior for this database session.`);
}
private ensureVectorTables(): void { private ensureVectorTables(): void {
if (this.readOnly || !this.vectorCapability.available) { if (this.readOnly || !this.vectorCapability.available) {
return; return;
@@ -1073,17 +1128,6 @@ class SQLiteClient {
} catch {} } catch {}
} }
// Only create nodes_fts when it does not exist yet.
// Do not rewrite existing FTS tables on startup in the OS app.
try {
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined;
if (!ftsCheck?.sql) {
this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');");
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
}
} catch (ftsErr) {
console.warn('Failed to initialize nodes_fts:', ftsErr);
}
} catch (schemaErr) { } catch (schemaErr) {
console.warn('Final schema pass migration error:', schemaErr); console.warn('Final schema pass migration error:', schemaErr);
} }
@@ -1183,6 +1227,237 @@ class SQLiteClient {
tryRead('vec_chunks'); tryRead('vec_chunks');
} }
private ensureFtsTables(): void {
if (this.readOnly) {
return;
}
try {
const ensureFts = (
tableName: 'nodes_fts' | 'chunks_fts',
createSql: string,
triggerSql: string,
rebuildSql: string,
) => {
const existing = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?").get(tableName) as { sql?: string } | undefined;
if (!existing?.sql) {
this.db.exec(createSql);
this.db.exec(rebuildSql);
}
this.db.exec(triggerSql);
};
ensureFts(
'nodes_fts',
"CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');",
`
DROP TRIGGER IF EXISTS nodes_fts_ai;
DROP TRIGGER IF EXISTS nodes_fts_ad;
DROP TRIGGER IF EXISTS nodes_fts_au;
CREATE TRIGGER nodes_fts_ai AFTER INSERT ON nodes BEGIN
INSERT INTO nodes_fts(rowid, title, source, description)
VALUES (new.id, new.title, new.source, new.description);
END;
CREATE TRIGGER nodes_fts_ad AFTER DELETE ON nodes BEGIN
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
VALUES('delete', old.id, old.title, old.source, old.description);
END;
CREATE TRIGGER nodes_fts_au AFTER UPDATE OF title, source, description ON nodes BEGIN
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
VALUES('delete', old.id, old.title, old.source, old.description);
INSERT INTO nodes_fts(rowid, title, source, description)
VALUES (new.id, new.title, new.source, new.description);
END;
`,
"INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');",
);
ensureFts(
'chunks_fts',
"CREATE VIRTUAL TABLE chunks_fts USING fts5(text, content='chunks', content_rowid='id');",
`
DROP TRIGGER IF EXISTS chunks_fts_ai;
DROP TRIGGER IF EXISTS chunks_fts_ad;
DROP TRIGGER IF EXISTS chunks_fts_au;
CREATE TRIGGER chunks_fts_ai AFTER INSERT ON chunks BEGIN
INSERT INTO chunks_fts(rowid, text)
VALUES (new.id, new.text);
END;
CREATE TRIGGER chunks_fts_ad AFTER DELETE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text)
VALUES('delete', old.id, old.text);
END;
CREATE TRIGGER chunks_fts_au AFTER UPDATE OF text ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text)
VALUES('delete', old.id, old.text);
INSERT INTO chunks_fts(rowid, text)
VALUES (new.id, new.text);
END;
`,
"INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');",
);
} catch (error) {
console.warn('Failed to ensure FTS tables:', error);
}
}
private buildProbeError(error: unknown): IntegrityProbeResult {
const message = error instanceof Error ? error.message : String(error);
const code =
typeof error === 'object' &&
error !== null &&
'code' in error &&
typeof (error as { code?: unknown }).code === 'string'
? String((error as { code: string }).code)
: undefined;
return { ok: false, error: message, code };
}
private runIntegrityProbe(fn: () => void): IntegrityProbeResult {
try {
fn();
return { ok: true };
} catch (error) {
return this.buildProbeError(error);
}
}
private refreshIntegrityReportForCorruptionError(error: unknown): void {
if (this.isSqliteCorruptError(error)) {
this.getIntegrityReport(true);
}
}
private inspectIntegrity(): DatabaseIntegrityReport {
const connected = this.runIntegrityProbe(() => {
this.db.prepare('SELECT 1').get();
}).ok;
const quickCheck = this.runIntegrityProbe(() => {
const rows = this.db.prepare('PRAGMA quick_check').pluck().all() as string[];
if (!(rows.length === 1 && rows[0] === 'ok')) {
throw new Error(rows.join('\n') || 'quick_check failed');
}
});
const integrityCheck = this.runIntegrityProbe(() => {
const rows = this.db.prepare('PRAGMA integrity_check').pluck().all() as string[];
if (!(rows.length === 1 && rows[0] === 'ok')) {
throw new Error(rows.join('\n') || 'integrity_check failed');
}
});
const baseTables: Record<'nodes' | 'edges' | 'chunks', IntegrityProbeResult> = {
nodes: this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM nodes').pluck().get();
}),
edges: this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM edges').pluck().get();
}),
chunks: this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
}),
};
const hasTable = (name: string) =>
Boolean(this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name));
const ftsSchemaExists: Record<FtsSurfaceName, boolean> = {
nodes: hasTable('nodes_fts'),
chunks: hasTable('chunks_fts'),
};
const ftsProbeResults: Record<FtsSurfaceName, IntegrityProbeResult> = {
nodes: ftsSchemaExists.nodes
? this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM nodes_fts').pluck().get();
})
: { ok: false, error: 'nodes_fts missing' },
chunks: ftsSchemaExists.chunks
? this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM chunks_fts').pluck().get();
})
: { ok: false, error: 'chunks_fts missing' },
};
const ftsTables = {
nodes: ftsSchemaExists.nodes && ftsProbeResults.nodes.ok,
chunks: ftsSchemaExists.chunks && ftsProbeResults.chunks.ok,
};
let foreignKeyViolations = -1;
if (baseTables.nodes.ok && baseTables.edges.ok) {
const foreignKeyProbe = this.runIntegrityProbe(() => {
foreignKeyViolations = Number(
(this.db.prepare('SELECT COUNT(*) FROM pragma_foreign_key_check').pluck().get() as number | undefined) ?? 0
);
});
if (!foreignKeyProbe.ok) {
foreignKeyViolations = -1;
}
}
const lostAndFoundExists = hasTable('lost_and_found');
const vecTables = {
nodes: hasTable('vec_nodes'),
chunks: hasTable('vec_chunks'),
};
const baseTablesReadable = Object.values(baseTables).every(probe => probe.ok);
const repairableFtsTables = (Object.entries(ftsProbeResults) as Array<[FtsSurfaceName, IntegrityProbeResult]>)
.filter(([name, probe]) => ftsSchemaExists[name] && !probe.ok)
.map(([name]) => name);
let state: DatabaseIntegrityReport['state'] = 'healthy';
if (!quickCheck.ok || !integrityCheck.ok) {
state = baseTablesReadable && repairableFtsTables.length > 0 ? 'degraded_fts' : 'corrupt';
} else if (!baseTablesReadable) {
state = 'corrupt';
} else if (repairableFtsTables.length > 0) {
state = 'degraded_fts';
}
let summary = 'Database integrity checks passed.';
if (state === 'degraded_fts') {
summary = `Rebuildable FTS surfaces are degraded: ${repairableFtsTables.join(', ')}. Base tables are still readable.`;
} else if (state === 'corrupt') {
summary = 'Canonical database integrity checks are failing. Treat the database as corrupted.';
}
const firstError =
quickCheck.error ||
integrityCheck.error ||
Object.values(baseTables).find(probe => !probe.ok)?.error ||
Object.values(ftsProbeResults).find(probe => !probe.ok)?.error;
return {
state,
connected,
quickCheck,
integrityCheck,
baseTables,
ftsTables,
ftsProbeResults,
repairableFtsTables,
canRepairFts: state === 'degraded_fts' && repairableFtsTables.length > 0,
foreignKeyViolations,
lostAndFoundExists,
vecTables,
summary,
error: firstError,
};
}
public getIntegrityReport(forceRefresh = false): DatabaseIntegrityReport {
if (!this.integrityReport || forceRefresh) {
this.integrityReport = this.inspectIntegrity();
}
return this.integrityReport;
}
private handleError(error: any): DatabaseError { private handleError(error: any): DatabaseError {
return { return {
message: error.message || 'SQLite operation failed', message: error.message || 'SQLite operation failed',
@@ -1201,7 +1476,10 @@ class SQLiteClient {
} }
const sqliteError = error as Error & { code?: string }; const sqliteError = error as Error & { code?: string };
return sqliteError.code === 'SQLITE_CORRUPT' || /database disk image is malformed/i.test(sqliteError.message || ''); return (
sqliteError.code?.includes('CORRUPT') === true ||
/database disk image is malformed/i.test(sqliteError.message || '')
);
} }
} }
+8
View File
@@ -1,5 +1,6 @@
import { embedNodeContent } from '@/services/embedding/ingestion'; import { embedNodeContent } from '@/services/embedding/ingestion';
import { nodeService } from '@/services/database'; import { nodeService } from '@/services/database';
import { getSQLiteClient } from '@/services/database/sqlite-client';
interface AutoEmbedTask { interface AutoEmbedTask {
nodeId: number; nodeId: number;
@@ -87,6 +88,13 @@ export class AutoEmbedQueue {
} }
private async executeTask(task: AutoEmbedTask) { private async executeTask(task: AutoEmbedTask) {
const sqlite = getSQLiteClient();
const integrity = sqlite.getIntegrityReport();
if (!sqlite.canUseFtsTable('chunks')) {
console.warn('[AutoEmbedQueue] Skipping chunk write because chunks FTS is degraded:', integrity.summary);
return;
}
const node = await nodeService.getNodeById(task.nodeId); const node = await nodeService.getNodeById(task.nodeId);
if (!node) { if (!node) {
console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId); console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId);
+11
View File
@@ -1,4 +1,5 @@
import { nodeService } from '@/services/database'; import { nodeService } from '@/services/database';
import { getSQLiteClient } from '@/services/database/sqlite-client';
import { NodeEmbedder } from '@/services/typescript/embed-nodes'; import { NodeEmbedder } from '@/services/typescript/embed-nodes';
import { UniversalEmbedder } from '@/services/typescript/embed-universal'; import { UniversalEmbedder } from '@/services/typescript/embed-universal';
import type { Node } from '@/types/database'; import type { Node } from '@/types/database';
@@ -53,6 +54,16 @@ async function updateChunkStatus(nodeId: number, status: Node['chunk_status']) {
} }
async function runChunkEmbedding(nodeId: number): Promise<EmbeddingResult> { async function runChunkEmbedding(nodeId: number): Promise<EmbeddingResult> {
const sqlite = getSQLiteClient();
const integrity = sqlite.getIntegrityReport();
if (!sqlite.canUseFtsTable('chunks')) {
await updateChunkStatus(nodeId, 'error');
return {
success: false,
error: `Chunk indexing is disabled while the chunks FTS surface is degraded. ${integrity.summary}`
};
}
const embedder = new UniversalEmbedder(); const embedder = new UniversalEmbedder();
try { try {
await updateChunkStatus(nodeId, 'chunking'); await updateChunkStatus(nodeId, 'chunking');