SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: port sqlite fts corruption containment

Browse Source 
- remove unsafe standalone startup FTS rebuild behavior
- add integrity classification and degraded FTS handling in the OS database runtime
- add clean-db FTS repair script for rebuildable corruption recovery

Generated with Claude Code
This commit is contained in:
“BeeRad”
2026-04-14 22:05:54 +10:00
parent 581bd3341d
commit 91ce23309b
9 changed files with 789 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/database/chunks.ts
+2 -4
View File
@@ -420,10 +420,7 @@ export class ChunkService {
matchCount: number,
nodeIds?: number[]
): RankedChunk[] {
const ftsExists = sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks_fts'"
).get();
if (!ftsExists) return [];
if (!sqlite.canUseFtsTable('chunks')) return [];
const ftsQuery = sanitizeFtsQuery(query);
if (!ftsQuery) return [];
@@ -461,6 +458,7 @@ export class ChunkService {
return result.rows;
} catch (error) {
sqlite.disableFtsTable('chunks', 'chunks_fts query failed during chunk search', error);
console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error);
return [];
}
src/services/database/index.ts
+67 -10
View File
@@ -1,3 +1,5 @@
import type { DatabaseIntegrityReport } from './sqlite-client';
// Service instances
export { nodeService, NodeService } from './nodes';
export { chunkService, ChunkService } from './chunks';
@@ -8,13 +10,31 @@ export { contextService, ContextService } from './contextService';
// Types
export * from '@/types/database';
// Health check utility
export async function checkDatabaseHealth(): Promise<{
export interface DatabaseHealthStatus {
connected: boolean;
vectorExtension: boolean;
tablesExist: boolean;
quickCheckOk: boolean;
integrityCheckOk: boolean;
foreignKeyViolations: number;
lostAndFoundExists: boolean;
ftsTables: {
nodes: boolean;
chunks: boolean;
};
vecTables: {
nodes: boolean;
chunks: boolean;
};
integrityState: DatabaseIntegrityReport['state'];
repairableFtsTables: DatabaseIntegrityReport['repairableFtsTables'];
canRepairFts: boolean;
summary: string;
error?: string;
}> {
}
// Health check utility
export async function checkDatabaseHealth(): Promise<DatabaseHealthStatus> {
try {
return checkSQLiteDatabaseHealth();
} catch (error) {
@@ -22,20 +42,26 @@ export async function checkDatabaseHealth(): Promise<{
connected: false,
vectorExtension: false,
tablesExist: false,
quickCheckOk: false,
integrityCheckOk: false,
foreignKeyViolations: -1,
lostAndFoundExists: false,
ftsTables: { nodes: false, chunks: false },
vecTables: { nodes: false, chunks: false },
integrityState: 'corrupt',
repairableFtsTables: [],
canRepairFts: false,
summary: 'Database health check failed before classification.',
error: error instanceof Error ? error.message : 'Unknown error'
};
}
}
async function checkSQLiteDatabaseHealth(): Promise<{
connected: boolean;
vectorExtension: boolean;
tablesExist: boolean;
error?: string;
}> {
async function checkSQLiteDatabaseHealth(): Promise<DatabaseHealthStatus> {
try {
const { getSQLiteClient } = await import('./sqlite-client');
const sqlite = getSQLiteClient();
const integrity = sqlite.getIntegrityReport(true);
const connected = await sqlite.testConnection();
if (!connected) {
@@ -43,6 +69,16 @@ async function checkSQLiteDatabaseHealth(): Promise<{
connected: false,
vectorExtension: false,
tablesExist: false,
quickCheckOk: false,
integrityCheckOk: false,
foreignKeyViolations: -1,
lostAndFoundExists: false,
ftsTables: { nodes: false, chunks: false },
vecTables: { nodes: false, chunks: false },
integrityState: integrity.state,
repairableFtsTables: integrity.repairableFtsTables,
canRepairFts: integrity.canRepairFts,
summary: integrity.summary,
error: 'SQLite connection failed'
};
}
@@ -57,13 +93,34 @@ async function checkSQLiteDatabaseHealth(): Promise<{
return {
connected,
vectorExtension,
tablesExist
tablesExist,
quickCheckOk: integrity.quickCheck.ok,
integrityCheckOk: integrity.integrityCheck.ok,
foreignKeyViolations: integrity.foreignKeyViolations,
lostAndFoundExists: integrity.lostAndFoundExists,
ftsTables: integrity.ftsTables,
vecTables: integrity.vecTables,
integrityState: integrity.state,
repairableFtsTables: integrity.repairableFtsTables,
canRepairFts: integrity.canRepairFts,
summary: integrity.summary,
error: integrity.error,
};
} catch (error) {
return {
connected: false,
vectorExtension: false,
tablesExist: false,
quickCheckOk: false,
integrityCheckOk: false,
foreignKeyViolations: -1,
lostAndFoundExists: false,
ftsTables: { nodes: false, chunks: false },
vecTables: { nodes: false, chunks: false },
integrityState: 'corrupt',
repairableFtsTables: [],
canRepairFts: false,
summary: 'Database health check failed during execution.',
error: error instanceof Error ? error.message : 'Unknown error'
};
}
src/services/database/nodes.ts
+5 -12
View File
@@ -498,14 +498,11 @@ export class NodeService {
if (!search) return 0;
const ftsQuery = sanitizeFtsQuery(search);
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
).get();
const { clauses, params } = this.buildNodeFilterClauses(filters);
if (ftsExists && ftsQuery) {
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
if (sqlite.canUseFtsTable('nodes') && ftsQuery) {
try {
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const result = sqlite.query<{ total: number }>(`
WITH matched_nodes AS (
SELECT rowid
@@ -520,7 +517,7 @@ export class NodeService {
return Number(result.rows[0]?.total ?? 0);
} catch (error) {
sqlite.disableNodesFts('nodes_fts query failed during count search', error);
sqlite.disableFtsTable('nodes', 'nodes_fts query failed during count search', error);
}
}
@@ -549,11 +546,7 @@ export class NodeService {
): NodeSearchRow[] {
const ftsQuery = sanitizeFtsQuery(search);
if (!ftsQuery) return [];
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
).get();
if (!ftsExists) return [];
if (!sqlite.canUseFtsTable('nodes')) return [];
const { clauses, params } = this.buildNodeFilterClauses(filters);
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
@@ -584,7 +577,7 @@ export class NodeService {
return result.rows;
} catch (error) {
sqlite.disableNodesFts('nodes_fts query failed during node search', error);
sqlite.disableFtsTable('nodes', 'nodes_fts query failed during node search', error);
return [];
}
}
src/services/database/sqlite-client.ts
+305 -27
View File
@@ -19,14 +19,49 @@ export interface SQLiteQueryResult<T = any> {
lastInsertRowid?: number;
}
type FtsSurfaceName = 'nodes' | 'chunks';
interface IntegrityProbeResult {
ok: boolean;
error?: string;
code?: string;
}
export interface DatabaseIntegrityReport {
state: 'healthy' | 'degraded_fts' | 'corrupt';
connected: boolean;
quickCheck: IntegrityProbeResult;
integrityCheck: IntegrityProbeResult;
baseTables: Record<'nodes' | 'edges' | 'chunks', IntegrityProbeResult>;
ftsTables: Record<FtsSurfaceName, boolean>;
ftsProbeResults: Record<FtsSurfaceName, IntegrityProbeResult>;
repairableFtsTables: FtsSurfaceName[];
canRepairFts: boolean;
foreignKeyViolations: number;
lostAndFoundExists: boolean;
vecTables: {
nodes: boolean;
chunks: boolean;
};
summary: string;
error?: string;
}
class SQLiteClient {
private static instance: SQLiteClient;
private db: Database.Database;
private config: SQLiteConfig;
private readonly readOnly: boolean;
private readonly vectorCapability: VectorCapability;
private nodesFtsUsable = true;
private nodesFtsDisabledReason: string | null = null;
private integrityReport: DatabaseIntegrityReport | null = null;
private readonly ftsUsable: Record<FtsSurfaceName, boolean> = {
nodes: true,
chunks: true,
};
private readonly ftsDisabledReason: Record<FtsSurfaceName, string | null> = {
nodes: null,
chunks: null,
};
private constructor() {
this.config = this.getSQLiteConfig();
@@ -73,6 +108,16 @@ class SQLiteClient {
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
this.ensureLoggingAndMemorySchema();
this.ensureContextsSchema();
this.integrityReport = this.inspectIntegrity();
if (this.integrityReport.state === 'healthy') {
this.ensureFtsTables();
this.integrityReport = this.inspectIntegrity();
} else {
console.warn(
`[SQLiteIntegrity] Skipping startup FTS mutation because database is ${this.integrityReport.state}: ${this.integrityReport.summary}`
);
}
}
console.log('SQLite client initialized successfully');
@@ -118,6 +163,7 @@ class SQLiteClient {
};
}
} catch (error) {
this.refreshIntegrityReportForCorruptionError(error);
console.error('SQLite query error:', error);
throw this.handleError(error);
}
@@ -143,6 +189,7 @@ class SQLiteClient {
try {
return txn();
} catch (error) {
this.refreshIntegrityReportForCorruptionError(error);
throw this.handleError(error);
}
}
@@ -166,22 +213,11 @@ class SQLiteClient {
}
public isNodesFtsUsable(): boolean {
return this.nodesFtsUsable;
return this.canUseFtsTable('nodes');
}
public disableNodesFts(reason: string, error?: unknown): void {
this.nodesFtsUsable = false;
if (this.nodesFtsDisabledReason === reason) {
return;
}
this.nodesFtsDisabledReason = reason;
if (error && !this.isSqliteCorruptError(error)) {
console.warn(`[SQLite] nodes_fts disabled: ${reason}`, error);
return;
}
console.warn(`[SQLite] nodes_fts disabled: ${reason}. Falling back to LIKE search for this database session.`);
this.disableFtsTable('nodes', reason, error);
}
public async checkTables(): Promise<string[]> {
@@ -229,6 +265,25 @@ class SQLiteClient {
}
}
public canUseFtsTable(tableName: FtsSurfaceName): boolean {
return this.ftsUsable[tableName] && this.getIntegrityReport().ftsTables[tableName];
}
public disableFtsTable(tableName: FtsSurfaceName, reason: string, error?: unknown): void {
this.ftsUsable[tableName] = false;
if (this.ftsDisabledReason[tableName] === reason) {
return;
}
this.ftsDisabledReason[tableName] = reason;
if (error && !this.isSqliteCorruptError(error)) {
console.warn(`[SQLite] ${tableName}_fts disabled: ${reason}`, error);
return;
}
console.warn(`[SQLite] ${tableName}_fts disabled: ${reason}. Falling back to non-FTS behavior for this database session.`);
}
private ensureVectorTables(): void {
if (this.readOnly || !this.vectorCapability.available) {
return;
@@ -1073,17 +1128,6 @@ class SQLiteClient {
} catch {}
}
// Only create nodes_fts when it does not exist yet.
// Do not rewrite existing FTS tables on startup in the OS app.
try {
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined;
if (!ftsCheck?.sql) {
this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');");
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
}
} catch (ftsErr) {
console.warn('Failed to initialize nodes_fts:', ftsErr);
}
} catch (schemaErr) {
console.warn('Final schema pass migration error:', schemaErr);
}
@@ -1183,6 +1227,237 @@ class SQLiteClient {
tryRead('vec_chunks');
}
private ensureFtsTables(): void {
if (this.readOnly) {
return;
}
try {
const ensureFts = (
tableName: 'nodes_fts' | 'chunks_fts',
createSql: string,
triggerSql: string,
rebuildSql: string,
) => {
const existing = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?").get(tableName) as { sql?: string } | undefined;
if (!existing?.sql) {
this.db.exec(createSql);
this.db.exec(rebuildSql);
}
this.db.exec(triggerSql);
};
ensureFts(
'nodes_fts',
"CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');",
`
DROP TRIGGER IF EXISTS nodes_fts_ai;
DROP TRIGGER IF EXISTS nodes_fts_ad;
DROP TRIGGER IF EXISTS nodes_fts_au;
CREATE TRIGGER nodes_fts_ai AFTER INSERT ON nodes BEGIN
INSERT INTO nodes_fts(rowid, title, source, description)
VALUES (new.id, new.title, new.source, new.description);
END;
CREATE TRIGGER nodes_fts_ad AFTER DELETE ON nodes BEGIN
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
VALUES('delete', old.id, old.title, old.source, old.description);
END;
CREATE TRIGGER nodes_fts_au AFTER UPDATE OF title, source, description ON nodes BEGIN
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
VALUES('delete', old.id, old.title, old.source, old.description);
INSERT INTO nodes_fts(rowid, title, source, description)
VALUES (new.id, new.title, new.source, new.description);
END;
`,
"INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');",
);
ensureFts(
'chunks_fts',
"CREATE VIRTUAL TABLE chunks_fts USING fts5(text, content='chunks', content_rowid='id');",
`
DROP TRIGGER IF EXISTS chunks_fts_ai;
DROP TRIGGER IF EXISTS chunks_fts_ad;
DROP TRIGGER IF EXISTS chunks_fts_au;
CREATE TRIGGER chunks_fts_ai AFTER INSERT ON chunks BEGIN
INSERT INTO chunks_fts(rowid, text)
VALUES (new.id, new.text);
END;
CREATE TRIGGER chunks_fts_ad AFTER DELETE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text)
VALUES('delete', old.id, old.text);
END;
CREATE TRIGGER chunks_fts_au AFTER UPDATE OF text ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text)
VALUES('delete', old.id, old.text);
INSERT INTO chunks_fts(rowid, text)
VALUES (new.id, new.text);
END;
`,
"INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');",
);
} catch (error) {
console.warn('Failed to ensure FTS tables:', error);
}
}
private buildProbeError(error: unknown): IntegrityProbeResult {
const message = error instanceof Error ? error.message : String(error);
const code =
typeof error === 'object' &&
error !== null &&
'code' in error &&
typeof (error as { code?: unknown }).code === 'string'
? String((error as { code: string }).code)
: undefined;
return { ok: false, error: message, code };
}
private runIntegrityProbe(fn: () => void): IntegrityProbeResult {
try {
fn();
return { ok: true };
} catch (error) {
return this.buildProbeError(error);
}
}
private refreshIntegrityReportForCorruptionError(error: unknown): void {
if (this.isSqliteCorruptError(error)) {
this.getIntegrityReport(true);
}
}
private inspectIntegrity(): DatabaseIntegrityReport {
const connected = this.runIntegrityProbe(() => {
this.db.prepare('SELECT 1').get();
}).ok;
const quickCheck = this.runIntegrityProbe(() => {
const rows = this.db.prepare('PRAGMA quick_check').pluck().all() as string[];
if (!(rows.length === 1 && rows[0] === 'ok')) {
throw new Error(rows.join('\n') || 'quick_check failed');
}
});
const integrityCheck = this.runIntegrityProbe(() => {
const rows = this.db.prepare('PRAGMA integrity_check').pluck().all() as string[];
if (!(rows.length === 1 && rows[0] === 'ok')) {
throw new Error(rows.join('\n') || 'integrity_check failed');
}
});
const baseTables: Record<'nodes' | 'edges' | 'chunks', IntegrityProbeResult> = {
nodes: this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM nodes').pluck().get();
}),
edges: this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM edges').pluck().get();
}),
chunks: this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
}),
};
const hasTable = (name: string) =>
Boolean(this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name));
const ftsSchemaExists: Record<FtsSurfaceName, boolean> = {
nodes: hasTable('nodes_fts'),
chunks: hasTable('chunks_fts'),
};
const ftsProbeResults: Record<FtsSurfaceName, IntegrityProbeResult> = {
nodes: ftsSchemaExists.nodes
? this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM nodes_fts').pluck().get();
})
: { ok: false, error: 'nodes_fts missing' },
chunks: ftsSchemaExists.chunks
? this.runIntegrityProbe(() => {
this.db.prepare('SELECT COUNT(*) FROM chunks_fts').pluck().get();
})
: { ok: false, error: 'chunks_fts missing' },
};
const ftsTables = {
nodes: ftsSchemaExists.nodes && ftsProbeResults.nodes.ok,
chunks: ftsSchemaExists.chunks && ftsProbeResults.chunks.ok,
};
let foreignKeyViolations = -1;
if (baseTables.nodes.ok && baseTables.edges.ok) {
const foreignKeyProbe = this.runIntegrityProbe(() => {
foreignKeyViolations = Number(
(this.db.prepare('SELECT COUNT(*) FROM pragma_foreign_key_check').pluck().get() as number | undefined) ?? 0
);
});
if (!foreignKeyProbe.ok) {
foreignKeyViolations = -1;
}
}
const lostAndFoundExists = hasTable('lost_and_found');
const vecTables = {
nodes: hasTable('vec_nodes'),
chunks: hasTable('vec_chunks'),
};
const baseTablesReadable = Object.values(baseTables).every(probe => probe.ok);
const repairableFtsTables = (Object.entries(ftsProbeResults) as Array<[FtsSurfaceName, IntegrityProbeResult]>)
.filter(([name, probe]) => ftsSchemaExists[name] && !probe.ok)
.map(([name]) => name);
let state: DatabaseIntegrityReport['state'] = 'healthy';
if (!quickCheck.ok || !integrityCheck.ok) {
state = baseTablesReadable && repairableFtsTables.length > 0 ? 'degraded_fts' : 'corrupt';
} else if (!baseTablesReadable) {
state = 'corrupt';
} else if (repairableFtsTables.length > 0) {
state = 'degraded_fts';
}
let summary = 'Database integrity checks passed.';
if (state === 'degraded_fts') {
summary = `Rebuildable FTS surfaces are degraded: ${repairableFtsTables.join(', ')}. Base tables are still readable.`;
} else if (state === 'corrupt') {
summary = 'Canonical database integrity checks are failing. Treat the database as corrupted.';
}
const firstError =
quickCheck.error ||
integrityCheck.error ||
Object.values(baseTables).find(probe => !probe.ok)?.error ||
Object.values(ftsProbeResults).find(probe => !probe.ok)?.error;
return {
state,
connected,
quickCheck,
integrityCheck,
baseTables,
ftsTables,
ftsProbeResults,
repairableFtsTables,
canRepairFts: state === 'degraded_fts' && repairableFtsTables.length > 0,
foreignKeyViolations,
lostAndFoundExists,
vecTables,
summary,
error: firstError,
};
}
public getIntegrityReport(forceRefresh = false): DatabaseIntegrityReport {
if (!this.integrityReport || forceRefresh) {
this.integrityReport = this.inspectIntegrity();
}
return this.integrityReport;
}
private handleError(error: any): DatabaseError {
return {
message: error.message || 'SQLite operation failed',
@@ -1201,7 +1476,10 @@ class SQLiteClient {
}
const sqliteError = error as Error & { code?: string };
return sqliteError.code === 'SQLITE_CORRUPT' || /database disk image is malformed/i.test(sqliteError.message || '');
return (
sqliteError.code?.includes('CORRUPT') === true ||
/database disk image is malformed/i.test(sqliteError.message || '')
);
}
}
src/services/embedding/autoEmbedQueue.ts
+8
View File
@@ -1,5 +1,6 @@
import { embedNodeContent } from '@/services/embedding/ingestion';
import { nodeService } from '@/services/database';
import { getSQLiteClient } from '@/services/database/sqlite-client';
interface AutoEmbedTask {
nodeId: number;
@@ -87,6 +88,13 @@ export class AutoEmbedQueue {
}
private async executeTask(task: AutoEmbedTask) {
const sqlite = getSQLiteClient();
const integrity = sqlite.getIntegrityReport();
if (!sqlite.canUseFtsTable('chunks')) {
console.warn('[AutoEmbedQueue] Skipping chunk write because chunks FTS is degraded:', integrity.summary);
return;
}
const node = await nodeService.getNodeById(task.nodeId);
if (!node) {
console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId);
src/services/embedding/ingestion.ts
+11
View File
@@ -1,4 +1,5 @@
import { nodeService } from '@/services/database';
import { getSQLiteClient } from '@/services/database/sqlite-client';
import { NodeEmbedder } from '@/services/typescript/embed-nodes';
import { UniversalEmbedder } from '@/services/typescript/embed-universal';
import type { Node } from '@/types/database';
@@ -53,6 +54,16 @@ async function updateChunkStatus(nodeId: number, status: Node['chunk_status']) {
}
async function runChunkEmbedding(nodeId: number): Promise<EmbeddingResult> {
const sqlite = getSQLiteClient();
const integrity = sqlite.getIntegrityReport();
if (!sqlite.canUseFtsTable('chunks')) {
await updateChunkStatus(nodeId, 'error');
return {
success: false,
error: `Chunk indexing is disabled while the chunks FTS surface is degraded. ${integrity.summary}`
};
}
const embedder = new UniversalEmbedder();
try {
await updateChunkStatus(nodeId, 'chunking');