Merge branch 'codex/os-sqlite-fts-corruption-sync'
This commit is contained in:
@@ -1,10 +1,12 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { nodeService } from '@/services/database';
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const integrity = getSQLiteClient().getIntegrityReport();
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const query = searchParams.get('q');
|
||||
const limit = parseInt(searchParams.get('limit') || '10');
|
||||
@@ -35,7 +37,10 @@ export async function GET(request: NextRequest) {
|
||||
success: true,
|
||||
data: results,
|
||||
count: results.length,
|
||||
query: query.trim()
|
||||
query: query.trim(),
|
||||
degraded: integrity.state !== 'healthy',
|
||||
integrityState: integrity.state,
|
||||
integritySummary: integrity.state !== 'healthy' ? integrity.summary : undefined,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error searching nodes:', error);
|
||||
|
||||
@@ -220,31 +220,6 @@ function checkFtsAvailability() {
|
||||
return _ftsAvailability;
|
||||
}
|
||||
|
||||
function rebuildFtsIndexes() {
|
||||
const fts = checkFtsAvailability();
|
||||
if (!fts.nodes && !fts.chunks) return;
|
||||
|
||||
const db = getDb();
|
||||
if (fts.nodes) {
|
||||
try {
|
||||
db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')");
|
||||
log('Rebuilt nodes_fts index');
|
||||
} catch (err) {
|
||||
log('Warning: Failed to rebuild nodes_fts:', err.message);
|
||||
_ftsAvailability.nodes = false;
|
||||
}
|
||||
}
|
||||
if (fts.chunks) {
|
||||
try {
|
||||
db.exec("INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild')");
|
||||
log('Rebuilt chunks_fts index');
|
||||
} catch (err) {
|
||||
log('Warning: Failed to rebuild chunks_fts:', err.message);
|
||||
_ftsAvailability.chunks = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Security: Only allow read-only SQL statements
|
||||
function isReadOnlyQuery(sql) {
|
||||
const normalized = sql.trim().toLowerCase();
|
||||
@@ -283,7 +258,6 @@ async function main() {
|
||||
try {
|
||||
initDatabase();
|
||||
log('Database connected:', getDatabasePath());
|
||||
rebuildFtsIndexes();
|
||||
} catch (error) {
|
||||
log('ERROR:', error.message);
|
||||
process.exit(1);
|
||||
|
||||
Executable
+385
@@ -0,0 +1,385 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Repair rebuildable FTS corruption by rebuilding a clean SQLite file from the
|
||||
# readable canonical tables, then swapping the rebuilt file into place.
|
||||
|
||||
DB_PATH=${SQLITE_DB_PATH:-}
|
||||
if [ -z "${DB_PATH}" ] && [ -f ".env.local" ]; then
|
||||
DB_PATH=$(grep -E '^SQLITE_DB_PATH=' .env.local | sed 's/^SQLITE_DB_PATH=//' | sed 's/^"\(.*\)"$/\1/') || true
|
||||
fi
|
||||
if [ -z "${DB_PATH}" ]; then
|
||||
DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite"
|
||||
fi
|
||||
|
||||
if [ ! -f "$DB_PATH" ]; then
|
||||
echo "Error: Resolved DB not found: $DB_PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DB_DIR="$(dirname "$DB_PATH")"
|
||||
DB_NAME="$(basename "$DB_PATH")"
|
||||
TS=$(date +"%Y%m%d_%H%M%S")
|
||||
RAW_BACKUP_DIR="$DB_DIR/working/fts_repair_${TS}"
|
||||
REBUILT_DB="$DB_DIR/${DB_NAME}.rebuilt.${TS}"
|
||||
QUOTED_DB_PATH=${DB_PATH//\'/\'\'}
|
||||
|
||||
mkdir -p "$RAW_BACKUP_DIR"
|
||||
|
||||
cp -p "$DB_PATH" "$RAW_BACKUP_DIR/$DB_NAME"
|
||||
if [ -f "${DB_PATH}-wal" ]; then
|
||||
cp -p "${DB_PATH}-wal" "$RAW_BACKUP_DIR/${DB_NAME}-wal"
|
||||
fi
|
||||
if [ -f "${DB_PATH}-shm" ]; then
|
||||
cp -p "${DB_PATH}-shm" "$RAW_BACKUP_DIR/${DB_NAME}-shm"
|
||||
fi
|
||||
|
||||
echo "Raw backup saved to: $RAW_BACKUP_DIR"
|
||||
|
||||
echo "Preflight source probes:"
|
||||
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM nodes;" | sed 's/^/ nodes: /'
|
||||
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM edges;" | sed 's/^/ edges: /'
|
||||
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM chunks;" | sed 's/^/ chunks: /'
|
||||
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM nodes_fts;" | sed 's/^/ nodes_fts: /' || true
|
||||
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM chunks_fts;" | sed 's/^/ chunks_fts: /' || true
|
||||
|
||||
rm -f "$REBUILT_DB" "${REBUILT_DB}-wal" "${REBUILT_DB}-shm"
|
||||
|
||||
sqlite3 "$REBUILT_DB" <<SQL
|
||||
PRAGMA journal_mode = DELETE;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
PRAGMA foreign_keys = OFF;
|
||||
ATTACH DATABASE '$QUOTED_DB_PATH' AS source;
|
||||
|
||||
CREATE TABLE schema_version (
|
||||
version INTEGER PRIMARY KEY,
|
||||
applied_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
description TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE agents (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
key TEXT UNIQUE NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT 'executor',
|
||||
system_prompt TEXT NOT NULL,
|
||||
available_tools TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
description TEXT,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
memory TEXT,
|
||||
prompts TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE contexts (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
icon TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE nodes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
source TEXT,
|
||||
link TEXT,
|
||||
event_date TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT,
|
||||
embedding BLOB,
|
||||
embedding_updated_at TEXT,
|
||||
embedding_text TEXT,
|
||||
chunk_status TEXT DEFAULT 'not_chunked',
|
||||
context_id INTEGER,
|
||||
FOREIGN KEY (context_id) REFERENCES contexts(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE TABLE edges (
|
||||
id INTEGER PRIMARY KEY,
|
||||
from_node_id INTEGER NOT NULL,
|
||||
to_node_id INTEGER NOT NULL,
|
||||
source TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
context TEXT,
|
||||
explanation TEXT,
|
||||
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE chunks (
|
||||
id INTEGER PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
chunk_idx INTEGER,
|
||||
text TEXT NOT NULL,
|
||||
embedding BLOB,
|
||||
embedding_type TEXT DEFAULT 'openai',
|
||||
metadata TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id INTEGER PRIMARY KEY,
|
||||
chat_type TEXT,
|
||||
helper_name TEXT,
|
||||
agent_type TEXT DEFAULT 'orchestrator',
|
||||
delegation_id INTEGER,
|
||||
user_message TEXT,
|
||||
assistant_message TEXT,
|
||||
thread_id TEXT,
|
||||
focused_node_id INTEGER,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT,
|
||||
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE TABLE logs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
ts TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
table_name TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
row_id INTEGER NOT NULL,
|
||||
summary TEXT,
|
||||
snapshot_json TEXT,
|
||||
enriched_summary TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE voice_usage (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
chat_id INTEGER,
|
||||
session_id TEXT,
|
||||
helper_name TEXT,
|
||||
request_id TEXT,
|
||||
message_id TEXT,
|
||||
voice TEXT,
|
||||
model TEXT,
|
||||
chars INTEGER,
|
||||
cost_usd REAL,
|
||||
duration_ms INTEGER,
|
||||
text_preview TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE TABLE dimension_migration_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
migrated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
dimension_count INTEGER NOT NULL,
|
||||
assignment_count INTEGER NOT NULL,
|
||||
payload TEXT
|
||||
);
|
||||
|
||||
INSERT INTO schema_version SELECT version, applied_at, description FROM source.schema_version;
|
||||
INSERT INTO agents SELECT id, key, display_name, role, system_prompt, available_tools, model, description, enabled, created_at, updated_at, memory, prompts FROM source.agents;
|
||||
INSERT INTO contexts SELECT id, name, description, icon, created_at, updated_at FROM source.contexts;
|
||||
INSERT INTO nodes SELECT id, title, description, source, link, event_date, created_at, updated_at, metadata, embedding, embedding_updated_at, embedding_text, chunk_status, context_id FROM source.nodes;
|
||||
INSERT INTO edges SELECT id, from_node_id, to_node_id, source, created_at, context, explanation FROM source.edges;
|
||||
INSERT INTO chunks SELECT id, node_id, chunk_idx, text, embedding, embedding_type, metadata, created_at FROM source.chunks;
|
||||
INSERT INTO chats SELECT id, chat_type, helper_name, agent_type, delegation_id, user_message, assistant_message, thread_id, focused_node_id, created_at, metadata FROM source.chats;
|
||||
INSERT INTO logs SELECT id, ts, table_name, action, row_id, summary, snapshot_json, enriched_summary FROM source.logs;
|
||||
INSERT INTO voice_usage SELECT id, chat_id, session_id, helper_name, request_id, message_id, voice, model, chars, cost_usd, duration_ms, text_preview, created_at FROM source.voice_usage;
|
||||
INSERT INTO dimension_migration_snapshots SELECT id, migrated_at, dimension_count, assignment_count, payload FROM source.dimension_migration_snapshots;
|
||||
|
||||
CREATE UNIQUE INDEX idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
|
||||
CREATE INDEX idx_edges_from ON edges(from_node_id);
|
||||
CREATE INDEX idx_edges_to ON edges(to_node_id);
|
||||
CREATE INDEX idx_nodes_updated_at ON nodes(updated_at DESC);
|
||||
CREATE INDEX idx_nodes_context_id ON nodes(context_id);
|
||||
CREATE INDEX idx_chunks_node_id ON chunks(node_id);
|
||||
CREATE INDEX idx_chunks_by_node ON chunks(node_id);
|
||||
CREATE INDEX idx_chunks_by_node_idx ON chunks(node_id, chunk_idx);
|
||||
CREATE INDEX idx_chats_thread ON chats(thread_id);
|
||||
CREATE INDEX idx_chats_created_at ON chats(created_at DESC);
|
||||
CREATE INDEX idx_logs_ts ON logs(ts);
|
||||
CREATE INDEX idx_logs_table_ts ON logs(table_name, ts);
|
||||
CREATE INDEX idx_logs_table_row ON logs(table_name, row_id);
|
||||
CREATE INDEX idx_voice_usage_session ON voice_usage(session_id, created_at);
|
||||
CREATE INDEX idx_voice_usage_chat ON voice_usage(chat_id);
|
||||
|
||||
CREATE VIEW nodes_v AS
|
||||
SELECT n.id,
|
||||
n.title,
|
||||
n.description,
|
||||
n.source,
|
||||
n.link,
|
||||
n.event_date,
|
||||
n.metadata,
|
||||
n.created_at,
|
||||
n.updated_at
|
||||
FROM nodes n;
|
||||
|
||||
CREATE VIEW logs_v AS
|
||||
SELECT
|
||||
m.id,
|
||||
m.ts,
|
||||
m.table_name,
|
||||
m.action,
|
||||
m.row_id,
|
||||
m.summary,
|
||||
m.enriched_summary,
|
||||
m.snapshot_json,
|
||||
CASE WHEN m.table_name='nodes' THEN n.title END AS node_title,
|
||||
CASE WHEN m.table_name='edges' THEN nf.title END AS edge_from_title,
|
||||
CASE WHEN m.table_name='edges' THEN nt.title END AS edge_to_title,
|
||||
CASE WHEN m.table_name='chats' THEN c.helper_name END AS chat_helper,
|
||||
CASE WHEN m.table_name='chats' THEN substr(c.user_message,1,120) END AS chat_user_preview,
|
||||
CASE WHEN m.table_name='chats' THEN substr(c.assistant_message,1,120) END AS chat_assistant_preview,
|
||||
CASE WHEN m.table_name='chats' THEN c.user_message END AS chat_user_full,
|
||||
CASE WHEN m.table_name='chats' THEN c.assistant_message END AS chat_assistant_full
|
||||
FROM logs m
|
||||
LEFT JOIN nodes n ON (m.table_name='nodes' AND m.row_id = n.id)
|
||||
LEFT JOIN edges e ON (m.table_name='edges' AND m.row_id = e.id)
|
||||
LEFT JOIN nodes nf ON e.from_node_id = nf.id
|
||||
LEFT JOIN nodes nt ON e.to_node_id = nt.id
|
||||
LEFT JOIN chats c ON (m.table_name='chats' AND m.row_id = c.id);
|
||||
|
||||
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
||||
title,
|
||||
source,
|
||||
description,
|
||||
content='nodes',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE chunks_fts USING fts5(
|
||||
text,
|
||||
content='chunks',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');
|
||||
INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');
|
||||
|
||||
CREATE TRIGGER nodes_fts_ai AFTER INSERT ON nodes BEGIN
|
||||
INSERT INTO nodes_fts(rowid, title, source, description)
|
||||
VALUES (new.id, new.title, new.source, new.description);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER nodes_fts_ad AFTER DELETE ON nodes BEGIN
|
||||
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
|
||||
VALUES('delete', old.id, old.title, old.source, old.description);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER nodes_fts_au AFTER UPDATE OF title, source, description ON nodes BEGIN
|
||||
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
|
||||
VALUES('delete', old.id, old.title, old.source, old.description);
|
||||
INSERT INTO nodes_fts(rowid, title, source, description)
|
||||
VALUES (new.id, new.title, new.source, new.description);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER chunks_fts_ai AFTER INSERT ON chunks BEGIN
|
||||
INSERT INTO chunks_fts(rowid, text)
|
||||
VALUES (new.id, new.text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER chunks_fts_ad AFTER DELETE ON chunks BEGIN
|
||||
INSERT INTO chunks_fts(chunks_fts, rowid, text)
|
||||
VALUES('delete', old.id, old.text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER chunks_fts_au AFTER UPDATE OF text ON chunks BEGIN
|
||||
INSERT INTO chunks_fts(chunks_fts, rowid, text)
|
||||
VALUES('delete', old.id, old.text);
|
||||
INSERT INTO chunks_fts(rowid, text)
|
||||
VALUES (new.id, new.text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER trg_nodes_ai AFTER INSERT ON nodes BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('nodes', 'insert', NEW.id,
|
||||
printf('node created: %s', COALESCE(NEW.title,'')),
|
||||
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
|
||||
END;
|
||||
|
||||
CREATE TRIGGER trg_nodes_au AFTER UPDATE ON nodes BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('nodes', 'update', NEW.id,
|
||||
printf('node updated: %s', COALESCE(NEW.title,'')),
|
||||
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
|
||||
END;
|
||||
|
||||
CREATE TRIGGER trg_edges_ai AFTER INSERT ON edges BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('edges', 'insert', NEW.id,
|
||||
printf('edge %d→%d (%s)', NEW.from_node_id, NEW.to_node_id, COALESCE(NEW.source,'')),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'from', NEW.from_node_id,
|
||||
'to', NEW.to_node_id,
|
||||
'source', NEW.source,
|
||||
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
|
||||
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
|
||||
));
|
||||
END;
|
||||
|
||||
CREATE TRIGGER trg_edges_au AFTER UPDATE ON edges BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('edges', 'update', NEW.id,
|
||||
printf('edge updated %d→%d', NEW.from_node_id, NEW.to_node_id),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'from', NEW.from_node_id,
|
||||
'to', NEW.to_node_id,
|
||||
'source', NEW.source,
|
||||
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
|
||||
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
|
||||
));
|
||||
END;
|
||||
|
||||
CREATE TRIGGER trg_edges_update_nodes_on_insert
|
||||
AFTER INSERT ON edges
|
||||
BEGIN
|
||||
UPDATE nodes SET updated_at = strftime('%Y-%m-%dT%H:%M:%f', 'now') || 'Z' WHERE id = NEW.from_node_id;
|
||||
UPDATE nodes SET updated_at = strftime('%Y-%m-%dT%H:%M:%f', 'now') || 'Z' WHERE id = NEW.to_node_id;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER trg_chats_ai AFTER INSERT ON chats BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('chats', 'insert', NEW.id,
|
||||
printf('chat: %s (%s)', COALESCE(NEW.helper_name,''), COALESCE(NEW.thread_id,'')),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'helper', NEW.helper_name,
|
||||
'thread', NEW.thread_id,
|
||||
'user_preview', substr(COALESCE(NEW.user_message,''), 1, 120),
|
||||
'assistant_preview', substr(COALESCE(NEW.assistant_message,''), 1, 120)
|
||||
));
|
||||
END;
|
||||
|
||||
CREATE TRIGGER trg_logs_prune AFTER INSERT ON logs BEGIN
|
||||
DELETE FROM logs WHERE id < NEW.id - 10000;
|
||||
END;
|
||||
|
||||
DETACH DATABASE source;
|
||||
PRAGMA foreign_keys = ON;
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA optimize;
|
||||
SQL
|
||||
|
||||
echo "Rebuilt DB written to: $REBUILT_DB"
|
||||
|
||||
echo "Post-rebuild probes:"
|
||||
sqlite3 "$REBUILT_DB" "PRAGMA quick_check;" | sed 's/^/ quick_check: /'
|
||||
sqlite3 "$REBUILT_DB" "PRAGMA integrity_check;" | sed 's/^/ integrity_check: /'
|
||||
sqlite3 "$REBUILT_DB" "SELECT COUNT(*) FROM nodes_fts;" | sed 's/^/ nodes_fts: /'
|
||||
sqlite3 "$REBUILT_DB" "SELECT COUNT(*) FROM chunks_fts;" | sed 's/^/ chunks_fts: /'
|
||||
|
||||
mv "$DB_PATH" "${DB_PATH}.corrupt.${TS}"
|
||||
if [ -f "${DB_PATH}-wal" ]; then
|
||||
mv "${DB_PATH}-wal" "${DB_PATH}-wal.corrupt.${TS}"
|
||||
fi
|
||||
if [ -f "${DB_PATH}-shm" ]; then
|
||||
mv "${DB_PATH}-shm" "${DB_PATH}-shm.corrupt.${TS}"
|
||||
fi
|
||||
mv "$REBUILT_DB" "$DB_PATH"
|
||||
rm -f "${DB_PATH}-wal" "${DB_PATH}-shm"
|
||||
rm -f "${REBUILT_DB}-wal" "${REBUILT_DB}-shm"
|
||||
|
||||
echo "Swapped rebuilt DB into place: $DB_PATH"
|
||||
echo "Original live files preserved with suffix .corrupt.${TS}"
|
||||
@@ -420,10 +420,7 @@ export class ChunkService {
|
||||
matchCount: number,
|
||||
nodeIds?: number[]
|
||||
): RankedChunk[] {
|
||||
const ftsExists = sqlite.prepare(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks_fts'"
|
||||
).get();
|
||||
if (!ftsExists) return [];
|
||||
if (!sqlite.canUseFtsTable('chunks')) return [];
|
||||
|
||||
const ftsQuery = sanitizeFtsQuery(query);
|
||||
if (!ftsQuery) return [];
|
||||
@@ -461,6 +458,7 @@ export class ChunkService {
|
||||
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
sqlite.disableFtsTable('chunks', 'chunks_fts query failed during chunk search', error);
|
||||
console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error);
|
||||
return [];
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { DatabaseIntegrityReport } from './sqlite-client';
|
||||
|
||||
// Service instances
|
||||
export { nodeService, NodeService } from './nodes';
|
||||
export { chunkService, ChunkService } from './chunks';
|
||||
@@ -8,13 +10,31 @@ export { contextService, ContextService } from './contextService';
|
||||
// Types
|
||||
export * from '@/types/database';
|
||||
|
||||
// Health check utility
|
||||
export async function checkDatabaseHealth(): Promise<{
|
||||
export interface DatabaseHealthStatus {
|
||||
connected: boolean;
|
||||
vectorExtension: boolean;
|
||||
tablesExist: boolean;
|
||||
quickCheckOk: boolean;
|
||||
integrityCheckOk: boolean;
|
||||
foreignKeyViolations: number;
|
||||
lostAndFoundExists: boolean;
|
||||
ftsTables: {
|
||||
nodes: boolean;
|
||||
chunks: boolean;
|
||||
};
|
||||
vecTables: {
|
||||
nodes: boolean;
|
||||
chunks: boolean;
|
||||
};
|
||||
integrityState: DatabaseIntegrityReport['state'];
|
||||
repairableFtsTables: DatabaseIntegrityReport['repairableFtsTables'];
|
||||
canRepairFts: boolean;
|
||||
summary: string;
|
||||
error?: string;
|
||||
}> {
|
||||
}
|
||||
|
||||
// Health check utility
|
||||
export async function checkDatabaseHealth(): Promise<DatabaseHealthStatus> {
|
||||
try {
|
||||
return checkSQLiteDatabaseHealth();
|
||||
} catch (error) {
|
||||
@@ -22,20 +42,26 @@ export async function checkDatabaseHealth(): Promise<{
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
quickCheckOk: false,
|
||||
integrityCheckOk: false,
|
||||
foreignKeyViolations: -1,
|
||||
lostAndFoundExists: false,
|
||||
ftsTables: { nodes: false, chunks: false },
|
||||
vecTables: { nodes: false, chunks: false },
|
||||
integrityState: 'corrupt',
|
||||
repairableFtsTables: [],
|
||||
canRepairFts: false,
|
||||
summary: 'Database health check failed before classification.',
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function checkSQLiteDatabaseHealth(): Promise<{
|
||||
connected: boolean;
|
||||
vectorExtension: boolean;
|
||||
tablesExist: boolean;
|
||||
error?: string;
|
||||
}> {
|
||||
async function checkSQLiteDatabaseHealth(): Promise<DatabaseHealthStatus> {
|
||||
try {
|
||||
const { getSQLiteClient } = await import('./sqlite-client');
|
||||
const sqlite = getSQLiteClient();
|
||||
const integrity = sqlite.getIntegrityReport(true);
|
||||
|
||||
const connected = await sqlite.testConnection();
|
||||
if (!connected) {
|
||||
@@ -43,6 +69,16 @@ async function checkSQLiteDatabaseHealth(): Promise<{
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
quickCheckOk: false,
|
||||
integrityCheckOk: false,
|
||||
foreignKeyViolations: -1,
|
||||
lostAndFoundExists: false,
|
||||
ftsTables: { nodes: false, chunks: false },
|
||||
vecTables: { nodes: false, chunks: false },
|
||||
integrityState: integrity.state,
|
||||
repairableFtsTables: integrity.repairableFtsTables,
|
||||
canRepairFts: integrity.canRepairFts,
|
||||
summary: integrity.summary,
|
||||
error: 'SQLite connection failed'
|
||||
};
|
||||
}
|
||||
@@ -57,13 +93,34 @@ async function checkSQLiteDatabaseHealth(): Promise<{
|
||||
return {
|
||||
connected,
|
||||
vectorExtension,
|
||||
tablesExist
|
||||
tablesExist,
|
||||
quickCheckOk: integrity.quickCheck.ok,
|
||||
integrityCheckOk: integrity.integrityCheck.ok,
|
||||
foreignKeyViolations: integrity.foreignKeyViolations,
|
||||
lostAndFoundExists: integrity.lostAndFoundExists,
|
||||
ftsTables: integrity.ftsTables,
|
||||
vecTables: integrity.vecTables,
|
||||
integrityState: integrity.state,
|
||||
repairableFtsTables: integrity.repairableFtsTables,
|
||||
canRepairFts: integrity.canRepairFts,
|
||||
summary: integrity.summary,
|
||||
error: integrity.error,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
quickCheckOk: false,
|
||||
integrityCheckOk: false,
|
||||
foreignKeyViolations: -1,
|
||||
lostAndFoundExists: false,
|
||||
ftsTables: { nodes: false, chunks: false },
|
||||
vecTables: { nodes: false, chunks: false },
|
||||
integrityState: 'corrupt',
|
||||
repairableFtsTables: [],
|
||||
canRepairFts: false,
|
||||
summary: 'Database health check failed during execution.',
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
};
|
||||
}
|
||||
|
||||
@@ -498,14 +498,11 @@ export class NodeService {
|
||||
if (!search) return 0;
|
||||
|
||||
const ftsQuery = sanitizeFtsQuery(search);
|
||||
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
|
||||
).get();
|
||||
const { clauses, params } = this.buildNodeFilterClauses(filters);
|
||||
|
||||
if (ftsExists && ftsQuery) {
|
||||
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
if (sqlite.canUseFtsTable('nodes') && ftsQuery) {
|
||||
try {
|
||||
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
const result = sqlite.query<{ total: number }>(`
|
||||
WITH matched_nodes AS (
|
||||
SELECT rowid
|
||||
@@ -520,7 +517,7 @@ export class NodeService {
|
||||
|
||||
return Number(result.rows[0]?.total ?? 0);
|
||||
} catch (error) {
|
||||
sqlite.disableNodesFts('nodes_fts query failed during count search', error);
|
||||
sqlite.disableFtsTable('nodes', 'nodes_fts query failed during count search', error);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -549,11 +546,7 @@ export class NodeService {
|
||||
): NodeSearchRow[] {
|
||||
const ftsQuery = sanitizeFtsQuery(search);
|
||||
if (!ftsQuery) return [];
|
||||
|
||||
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
|
||||
).get();
|
||||
if (!ftsExists) return [];
|
||||
if (!sqlite.canUseFtsTable('nodes')) return [];
|
||||
|
||||
const { clauses, params } = this.buildNodeFilterClauses(filters);
|
||||
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||
@@ -584,7 +577,7 @@ export class NodeService {
|
||||
|
||||
return result.rows;
|
||||
} catch (error) {
|
||||
sqlite.disableNodesFts('nodes_fts query failed during node search', error);
|
||||
sqlite.disableFtsTable('nodes', 'nodes_fts query failed during node search', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,14 +19,49 @@ export interface SQLiteQueryResult<T = any> {
|
||||
lastInsertRowid?: number;
|
||||
}
|
||||
|
||||
type FtsSurfaceName = 'nodes' | 'chunks';
|
||||
|
||||
interface IntegrityProbeResult {
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
code?: string;
|
||||
}
|
||||
|
||||
export interface DatabaseIntegrityReport {
|
||||
state: 'healthy' | 'degraded_fts' | 'corrupt';
|
||||
connected: boolean;
|
||||
quickCheck: IntegrityProbeResult;
|
||||
integrityCheck: IntegrityProbeResult;
|
||||
baseTables: Record<'nodes' | 'edges' | 'chunks', IntegrityProbeResult>;
|
||||
ftsTables: Record<FtsSurfaceName, boolean>;
|
||||
ftsProbeResults: Record<FtsSurfaceName, IntegrityProbeResult>;
|
||||
repairableFtsTables: FtsSurfaceName[];
|
||||
canRepairFts: boolean;
|
||||
foreignKeyViolations: number;
|
||||
lostAndFoundExists: boolean;
|
||||
vecTables: {
|
||||
nodes: boolean;
|
||||
chunks: boolean;
|
||||
};
|
||||
summary: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
class SQLiteClient {
|
||||
private static instance: SQLiteClient;
|
||||
private db: Database.Database;
|
||||
private config: SQLiteConfig;
|
||||
private readonly readOnly: boolean;
|
||||
private readonly vectorCapability: VectorCapability;
|
||||
private nodesFtsUsable = true;
|
||||
private nodesFtsDisabledReason: string | null = null;
|
||||
private integrityReport: DatabaseIntegrityReport | null = null;
|
||||
private readonly ftsUsable: Record<FtsSurfaceName, boolean> = {
|
||||
nodes: true,
|
||||
chunks: true,
|
||||
};
|
||||
private readonly ftsDisabledReason: Record<FtsSurfaceName, string | null> = {
|
||||
nodes: null,
|
||||
chunks: null,
|
||||
};
|
||||
|
||||
private constructor() {
|
||||
this.config = this.getSQLiteConfig();
|
||||
@@ -73,6 +108,16 @@ class SQLiteClient {
|
||||
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
|
||||
this.ensureLoggingAndMemorySchema();
|
||||
this.ensureContextsSchema();
|
||||
this.integrityReport = this.inspectIntegrity();
|
||||
|
||||
if (this.integrityReport.state === 'healthy') {
|
||||
this.ensureFtsTables();
|
||||
this.integrityReport = this.inspectIntegrity();
|
||||
} else {
|
||||
console.warn(
|
||||
`[SQLiteIntegrity] Skipping startup FTS mutation because database is ${this.integrityReport.state}: ${this.integrityReport.summary}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('SQLite client initialized successfully');
|
||||
@@ -118,6 +163,7 @@ class SQLiteClient {
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
this.refreshIntegrityReportForCorruptionError(error);
|
||||
console.error('SQLite query error:', error);
|
||||
throw this.handleError(error);
|
||||
}
|
||||
@@ -143,6 +189,7 @@ class SQLiteClient {
|
||||
try {
|
||||
return txn();
|
||||
} catch (error) {
|
||||
this.refreshIntegrityReportForCorruptionError(error);
|
||||
throw this.handleError(error);
|
||||
}
|
||||
}
|
||||
@@ -166,22 +213,11 @@ class SQLiteClient {
|
||||
}
|
||||
|
||||
public isNodesFtsUsable(): boolean {
|
||||
return this.nodesFtsUsable;
|
||||
return this.canUseFtsTable('nodes');
|
||||
}
|
||||
|
||||
public disableNodesFts(reason: string, error?: unknown): void {
|
||||
this.nodesFtsUsable = false;
|
||||
if (this.nodesFtsDisabledReason === reason) {
|
||||
return;
|
||||
}
|
||||
this.nodesFtsDisabledReason = reason;
|
||||
|
||||
if (error && !this.isSqliteCorruptError(error)) {
|
||||
console.warn(`[SQLite] nodes_fts disabled: ${reason}`, error);
|
||||
return;
|
||||
}
|
||||
|
||||
console.warn(`[SQLite] nodes_fts disabled: ${reason}. Falling back to LIKE search for this database session.`);
|
||||
this.disableFtsTable('nodes', reason, error);
|
||||
}
|
||||
|
||||
public async checkTables(): Promise<string[]> {
|
||||
@@ -229,6 +265,25 @@ class SQLiteClient {
|
||||
}
|
||||
}
|
||||
|
||||
public canUseFtsTable(tableName: FtsSurfaceName): boolean {
|
||||
return this.ftsUsable[tableName] && this.getIntegrityReport().ftsTables[tableName];
|
||||
}
|
||||
|
||||
public disableFtsTable(tableName: FtsSurfaceName, reason: string, error?: unknown): void {
|
||||
this.ftsUsable[tableName] = false;
|
||||
if (this.ftsDisabledReason[tableName] === reason) {
|
||||
return;
|
||||
}
|
||||
this.ftsDisabledReason[tableName] = reason;
|
||||
|
||||
if (error && !this.isSqliteCorruptError(error)) {
|
||||
console.warn(`[SQLite] ${tableName}_fts disabled: ${reason}`, error);
|
||||
return;
|
||||
}
|
||||
|
||||
console.warn(`[SQLite] ${tableName}_fts disabled: ${reason}. Falling back to non-FTS behavior for this database session.`);
|
||||
}
|
||||
|
||||
private ensureVectorTables(): void {
|
||||
if (this.readOnly || !this.vectorCapability.available) {
|
||||
return;
|
||||
@@ -1073,17 +1128,6 @@ class SQLiteClient {
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Only create nodes_fts when it does not exist yet.
|
||||
// Do not rewrite existing FTS tables on startup in the OS app.
|
||||
try {
|
||||
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined;
|
||||
if (!ftsCheck?.sql) {
|
||||
this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');");
|
||||
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
|
||||
}
|
||||
} catch (ftsErr) {
|
||||
console.warn('Failed to initialize nodes_fts:', ftsErr);
|
||||
}
|
||||
} catch (schemaErr) {
|
||||
console.warn('Final schema pass migration error:', schemaErr);
|
||||
}
|
||||
@@ -1183,6 +1227,237 @@ class SQLiteClient {
|
||||
tryRead('vec_chunks');
|
||||
}
|
||||
|
||||
private ensureFtsTables(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const ensureFts = (
|
||||
tableName: 'nodes_fts' | 'chunks_fts',
|
||||
createSql: string,
|
||||
triggerSql: string,
|
||||
rebuildSql: string,
|
||||
) => {
|
||||
const existing = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?").get(tableName) as { sql?: string } | undefined;
|
||||
|
||||
if (!existing?.sql) {
|
||||
this.db.exec(createSql);
|
||||
this.db.exec(rebuildSql);
|
||||
}
|
||||
|
||||
this.db.exec(triggerSql);
|
||||
};
|
||||
|
||||
ensureFts(
|
||||
'nodes_fts',
|
||||
"CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');",
|
||||
`
|
||||
DROP TRIGGER IF EXISTS nodes_fts_ai;
|
||||
DROP TRIGGER IF EXISTS nodes_fts_ad;
|
||||
DROP TRIGGER IF EXISTS nodes_fts_au;
|
||||
CREATE TRIGGER nodes_fts_ai AFTER INSERT ON nodes BEGIN
|
||||
INSERT INTO nodes_fts(rowid, title, source, description)
|
||||
VALUES (new.id, new.title, new.source, new.description);
|
||||
END;
|
||||
CREATE TRIGGER nodes_fts_ad AFTER DELETE ON nodes BEGIN
|
||||
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
|
||||
VALUES('delete', old.id, old.title, old.source, old.description);
|
||||
END;
|
||||
CREATE TRIGGER nodes_fts_au AFTER UPDATE OF title, source, description ON nodes BEGIN
|
||||
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
|
||||
VALUES('delete', old.id, old.title, old.source, old.description);
|
||||
INSERT INTO nodes_fts(rowid, title, source, description)
|
||||
VALUES (new.id, new.title, new.source, new.description);
|
||||
END;
|
||||
`,
|
||||
"INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');",
|
||||
);
|
||||
|
||||
ensureFts(
|
||||
'chunks_fts',
|
||||
"CREATE VIRTUAL TABLE chunks_fts USING fts5(text, content='chunks', content_rowid='id');",
|
||||
`
|
||||
DROP TRIGGER IF EXISTS chunks_fts_ai;
|
||||
DROP TRIGGER IF EXISTS chunks_fts_ad;
|
||||
DROP TRIGGER IF EXISTS chunks_fts_au;
|
||||
CREATE TRIGGER chunks_fts_ai AFTER INSERT ON chunks BEGIN
|
||||
INSERT INTO chunks_fts(rowid, text)
|
||||
VALUES (new.id, new.text);
|
||||
END;
|
||||
CREATE TRIGGER chunks_fts_ad AFTER DELETE ON chunks BEGIN
|
||||
INSERT INTO chunks_fts(chunks_fts, rowid, text)
|
||||
VALUES('delete', old.id, old.text);
|
||||
END;
|
||||
CREATE TRIGGER chunks_fts_au AFTER UPDATE OF text ON chunks BEGIN
|
||||
INSERT INTO chunks_fts(chunks_fts, rowid, text)
|
||||
VALUES('delete', old.id, old.text);
|
||||
INSERT INTO chunks_fts(rowid, text)
|
||||
VALUES (new.id, new.text);
|
||||
END;
|
||||
`,
|
||||
"INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');",
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn('Failed to ensure FTS tables:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private buildProbeError(error: unknown): IntegrityProbeResult {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const code =
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
typeof (error as { code?: unknown }).code === 'string'
|
||||
? String((error as { code: string }).code)
|
||||
: undefined;
|
||||
|
||||
return { ok: false, error: message, code };
|
||||
}
|
||||
|
||||
private runIntegrityProbe(fn: () => void): IntegrityProbeResult {
|
||||
try {
|
||||
fn();
|
||||
return { ok: true };
|
||||
} catch (error) {
|
||||
return this.buildProbeError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private refreshIntegrityReportForCorruptionError(error: unknown): void {
|
||||
if (this.isSqliteCorruptError(error)) {
|
||||
this.getIntegrityReport(true);
|
||||
}
|
||||
}
|
||||
|
||||
private inspectIntegrity(): DatabaseIntegrityReport {
|
||||
const connected = this.runIntegrityProbe(() => {
|
||||
this.db.prepare('SELECT 1').get();
|
||||
}).ok;
|
||||
|
||||
const quickCheck = this.runIntegrityProbe(() => {
|
||||
const rows = this.db.prepare('PRAGMA quick_check').pluck().all() as string[];
|
||||
if (!(rows.length === 1 && rows[0] === 'ok')) {
|
||||
throw new Error(rows.join('\n') || 'quick_check failed');
|
||||
}
|
||||
});
|
||||
|
||||
const integrityCheck = this.runIntegrityProbe(() => {
|
||||
const rows = this.db.prepare('PRAGMA integrity_check').pluck().all() as string[];
|
||||
if (!(rows.length === 1 && rows[0] === 'ok')) {
|
||||
throw new Error(rows.join('\n') || 'integrity_check failed');
|
||||
}
|
||||
});
|
||||
|
||||
const baseTables: Record<'nodes' | 'edges' | 'chunks', IntegrityProbeResult> = {
|
||||
nodes: this.runIntegrityProbe(() => {
|
||||
this.db.prepare('SELECT COUNT(*) FROM nodes').pluck().get();
|
||||
}),
|
||||
edges: this.runIntegrityProbe(() => {
|
||||
this.db.prepare('SELECT COUNT(*) FROM edges').pluck().get();
|
||||
}),
|
||||
chunks: this.runIntegrityProbe(() => {
|
||||
this.db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
|
||||
}),
|
||||
};
|
||||
|
||||
const hasTable = (name: string) =>
|
||||
Boolean(this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name));
|
||||
|
||||
const ftsSchemaExists: Record<FtsSurfaceName, boolean> = {
|
||||
nodes: hasTable('nodes_fts'),
|
||||
chunks: hasTable('chunks_fts'),
|
||||
};
|
||||
|
||||
const ftsProbeResults: Record<FtsSurfaceName, IntegrityProbeResult> = {
|
||||
nodes: ftsSchemaExists.nodes
|
||||
? this.runIntegrityProbe(() => {
|
||||
this.db.prepare('SELECT COUNT(*) FROM nodes_fts').pluck().get();
|
||||
})
|
||||
: { ok: false, error: 'nodes_fts missing' },
|
||||
chunks: ftsSchemaExists.chunks
|
||||
? this.runIntegrityProbe(() => {
|
||||
this.db.prepare('SELECT COUNT(*) FROM chunks_fts').pluck().get();
|
||||
})
|
||||
: { ok: false, error: 'chunks_fts missing' },
|
||||
};
|
||||
|
||||
const ftsTables = {
|
||||
nodes: ftsSchemaExists.nodes && ftsProbeResults.nodes.ok,
|
||||
chunks: ftsSchemaExists.chunks && ftsProbeResults.chunks.ok,
|
||||
};
|
||||
|
||||
let foreignKeyViolations = -1;
|
||||
if (baseTables.nodes.ok && baseTables.edges.ok) {
|
||||
const foreignKeyProbe = this.runIntegrityProbe(() => {
|
||||
foreignKeyViolations = Number(
|
||||
(this.db.prepare('SELECT COUNT(*) FROM pragma_foreign_key_check').pluck().get() as number | undefined) ?? 0
|
||||
);
|
||||
});
|
||||
if (!foreignKeyProbe.ok) {
|
||||
foreignKeyViolations = -1;
|
||||
}
|
||||
}
|
||||
|
||||
const lostAndFoundExists = hasTable('lost_and_found');
|
||||
const vecTables = {
|
||||
nodes: hasTable('vec_nodes'),
|
||||
chunks: hasTable('vec_chunks'),
|
||||
};
|
||||
|
||||
const baseTablesReadable = Object.values(baseTables).every(probe => probe.ok);
|
||||
const repairableFtsTables = (Object.entries(ftsProbeResults) as Array<[FtsSurfaceName, IntegrityProbeResult]>)
|
||||
.filter(([name, probe]) => ftsSchemaExists[name] && !probe.ok)
|
||||
.map(([name]) => name);
|
||||
|
||||
let state: DatabaseIntegrityReport['state'] = 'healthy';
|
||||
if (!quickCheck.ok || !integrityCheck.ok) {
|
||||
state = baseTablesReadable && repairableFtsTables.length > 0 ? 'degraded_fts' : 'corrupt';
|
||||
} else if (!baseTablesReadable) {
|
||||
state = 'corrupt';
|
||||
} else if (repairableFtsTables.length > 0) {
|
||||
state = 'degraded_fts';
|
||||
}
|
||||
|
||||
let summary = 'Database integrity checks passed.';
|
||||
if (state === 'degraded_fts') {
|
||||
summary = `Rebuildable FTS surfaces are degraded: ${repairableFtsTables.join(', ')}. Base tables are still readable.`;
|
||||
} else if (state === 'corrupt') {
|
||||
summary = 'Canonical database integrity checks are failing. Treat the database as corrupted.';
|
||||
}
|
||||
|
||||
const firstError =
|
||||
quickCheck.error ||
|
||||
integrityCheck.error ||
|
||||
Object.values(baseTables).find(probe => !probe.ok)?.error ||
|
||||
Object.values(ftsProbeResults).find(probe => !probe.ok)?.error;
|
||||
|
||||
return {
|
||||
state,
|
||||
connected,
|
||||
quickCheck,
|
||||
integrityCheck,
|
||||
baseTables,
|
||||
ftsTables,
|
||||
ftsProbeResults,
|
||||
repairableFtsTables,
|
||||
canRepairFts: state === 'degraded_fts' && repairableFtsTables.length > 0,
|
||||
foreignKeyViolations,
|
||||
lostAndFoundExists,
|
||||
vecTables,
|
||||
summary,
|
||||
error: firstError,
|
||||
};
|
||||
}
|
||||
|
||||
public getIntegrityReport(forceRefresh = false): DatabaseIntegrityReport {
|
||||
if (!this.integrityReport || forceRefresh) {
|
||||
this.integrityReport = this.inspectIntegrity();
|
||||
}
|
||||
return this.integrityReport;
|
||||
}
|
||||
|
||||
private handleError(error: any): DatabaseError {
|
||||
return {
|
||||
message: error.message || 'SQLite operation failed',
|
||||
@@ -1201,7 +1476,10 @@ class SQLiteClient {
|
||||
}
|
||||
|
||||
const sqliteError = error as Error & { code?: string };
|
||||
return sqliteError.code === 'SQLITE_CORRUPT' || /database disk image is malformed/i.test(sqliteError.message || '');
|
||||
return (
|
||||
sqliteError.code?.includes('CORRUPT') === true ||
|
||||
/database disk image is malformed/i.test(sqliteError.message || '')
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { embedNodeContent } from '@/services/embedding/ingestion';
|
||||
import { nodeService } from '@/services/database';
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
|
||||
interface AutoEmbedTask {
|
||||
nodeId: number;
|
||||
@@ -87,6 +88,13 @@ export class AutoEmbedQueue {
|
||||
}
|
||||
|
||||
private async executeTask(task: AutoEmbedTask) {
|
||||
const sqlite = getSQLiteClient();
|
||||
const integrity = sqlite.getIntegrityReport();
|
||||
if (!sqlite.canUseFtsTable('chunks')) {
|
||||
console.warn('[AutoEmbedQueue] Skipping chunk write because chunks FTS is degraded:', integrity.summary);
|
||||
return;
|
||||
}
|
||||
|
||||
const node = await nodeService.getNodeById(task.nodeId);
|
||||
if (!node) {
|
||||
console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { nodeService } from '@/services/database';
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
import { NodeEmbedder } from '@/services/typescript/embed-nodes';
|
||||
import { UniversalEmbedder } from '@/services/typescript/embed-universal';
|
||||
import type { Node } from '@/types/database';
|
||||
@@ -53,6 +54,16 @@ async function updateChunkStatus(nodeId: number, status: Node['chunk_status']) {
|
||||
}
|
||||
|
||||
async function runChunkEmbedding(nodeId: number): Promise<EmbeddingResult> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const integrity = sqlite.getIntegrityReport();
|
||||
if (!sqlite.canUseFtsTable('chunks')) {
|
||||
await updateChunkStatus(nodeId, 'error');
|
||||
return {
|
||||
success: false,
|
||||
error: `Chunk indexing is disabled while the chunks FTS surface is degraded. ${integrity.summary}`
|
||||
};
|
||||
}
|
||||
|
||||
const embedder = new UniversalEmbedder();
|
||||
try {
|
||||
await updateChunkStatus(nodeId, 'chunking');
|
||||
|
||||
Reference in New Issue
Block a user