Merge branch 'codex/os-sqlite-fts-corruption-sync'
This commit is contained in:
@@ -1,10 +1,12 @@
|
|||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
import { nodeService } from '@/services/database';
|
import { nodeService } from '@/services/database';
|
||||||
|
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||||
|
|
||||||
export const runtime = 'nodejs';
|
export const runtime = 'nodejs';
|
||||||
|
|
||||||
export async function GET(request: NextRequest) {
|
export async function GET(request: NextRequest) {
|
||||||
try {
|
try {
|
||||||
|
const integrity = getSQLiteClient().getIntegrityReport();
|
||||||
const searchParams = request.nextUrl.searchParams;
|
const searchParams = request.nextUrl.searchParams;
|
||||||
const query = searchParams.get('q');
|
const query = searchParams.get('q');
|
||||||
const limit = parseInt(searchParams.get('limit') || '10');
|
const limit = parseInt(searchParams.get('limit') || '10');
|
||||||
@@ -35,7 +37,10 @@ export async function GET(request: NextRequest) {
|
|||||||
success: true,
|
success: true,
|
||||||
data: results,
|
data: results,
|
||||||
count: results.length,
|
count: results.length,
|
||||||
query: query.trim()
|
query: query.trim(),
|
||||||
|
degraded: integrity.state !== 'healthy',
|
||||||
|
integrityState: integrity.state,
|
||||||
|
integritySummary: integrity.state !== 'healthy' ? integrity.summary : undefined,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error searching nodes:', error);
|
console.error('Error searching nodes:', error);
|
||||||
|
|||||||
@@ -220,31 +220,6 @@ function checkFtsAvailability() {
|
|||||||
return _ftsAvailability;
|
return _ftsAvailability;
|
||||||
}
|
}
|
||||||
|
|
||||||
function rebuildFtsIndexes() {
|
|
||||||
const fts = checkFtsAvailability();
|
|
||||||
if (!fts.nodes && !fts.chunks) return;
|
|
||||||
|
|
||||||
const db = getDb();
|
|
||||||
if (fts.nodes) {
|
|
||||||
try {
|
|
||||||
db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')");
|
|
||||||
log('Rebuilt nodes_fts index');
|
|
||||||
} catch (err) {
|
|
||||||
log('Warning: Failed to rebuild nodes_fts:', err.message);
|
|
||||||
_ftsAvailability.nodes = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (fts.chunks) {
|
|
||||||
try {
|
|
||||||
db.exec("INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild')");
|
|
||||||
log('Rebuilt chunks_fts index');
|
|
||||||
} catch (err) {
|
|
||||||
log('Warning: Failed to rebuild chunks_fts:', err.message);
|
|
||||||
_ftsAvailability.chunks = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Security: Only allow read-only SQL statements
|
// Security: Only allow read-only SQL statements
|
||||||
function isReadOnlyQuery(sql) {
|
function isReadOnlyQuery(sql) {
|
||||||
const normalized = sql.trim().toLowerCase();
|
const normalized = sql.trim().toLowerCase();
|
||||||
@@ -283,7 +258,6 @@ async function main() {
|
|||||||
try {
|
try {
|
||||||
initDatabase();
|
initDatabase();
|
||||||
log('Database connected:', getDatabasePath());
|
log('Database connected:', getDatabasePath());
|
||||||
rebuildFtsIndexes();
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log('ERROR:', error.message);
|
log('ERROR:', error.message);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
|
|||||||
Executable
+385
@@ -0,0 +1,385 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Repair rebuildable FTS corruption by rebuilding a clean SQLite file from the
|
||||||
|
# readable canonical tables, then swapping the rebuilt file into place.
|
||||||
|
|
||||||
|
DB_PATH=${SQLITE_DB_PATH:-}
|
||||||
|
if [ -z "${DB_PATH}" ] && [ -f ".env.local" ]; then
|
||||||
|
DB_PATH=$(grep -E '^SQLITE_DB_PATH=' .env.local | sed 's/^SQLITE_DB_PATH=//' | sed 's/^"\(.*\)"$/\1/') || true
|
||||||
|
fi
|
||||||
|
if [ -z "${DB_PATH}" ]; then
|
||||||
|
DB_PATH="$HOME/Library/Application Support/RA-H/db/rah.sqlite"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "$DB_PATH" ]; then
|
||||||
|
echo "Error: Resolved DB not found: $DB_PATH" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
DB_DIR="$(dirname "$DB_PATH")"
|
||||||
|
DB_NAME="$(basename "$DB_PATH")"
|
||||||
|
TS=$(date +"%Y%m%d_%H%M%S")
|
||||||
|
RAW_BACKUP_DIR="$DB_DIR/working/fts_repair_${TS}"
|
||||||
|
REBUILT_DB="$DB_DIR/${DB_NAME}.rebuilt.${TS}"
|
||||||
|
QUOTED_DB_PATH=${DB_PATH//\'/\'\'}
|
||||||
|
|
||||||
|
mkdir -p "$RAW_BACKUP_DIR"
|
||||||
|
|
||||||
|
cp -p "$DB_PATH" "$RAW_BACKUP_DIR/$DB_NAME"
|
||||||
|
if [ -f "${DB_PATH}-wal" ]; then
|
||||||
|
cp -p "${DB_PATH}-wal" "$RAW_BACKUP_DIR/${DB_NAME}-wal"
|
||||||
|
fi
|
||||||
|
if [ -f "${DB_PATH}-shm" ]; then
|
||||||
|
cp -p "${DB_PATH}-shm" "$RAW_BACKUP_DIR/${DB_NAME}-shm"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Raw backup saved to: $RAW_BACKUP_DIR"
|
||||||
|
|
||||||
|
echo "Preflight source probes:"
|
||||||
|
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM nodes;" | sed 's/^/ nodes: /'
|
||||||
|
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM edges;" | sed 's/^/ edges: /'
|
||||||
|
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM chunks;" | sed 's/^/ chunks: /'
|
||||||
|
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM nodes_fts;" | sed 's/^/ nodes_fts: /' || true
|
||||||
|
sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM chunks_fts;" | sed 's/^/ chunks_fts: /' || true
|
||||||
|
|
||||||
|
rm -f "$REBUILT_DB" "${REBUILT_DB}-wal" "${REBUILT_DB}-shm"
|
||||||
|
|
||||||
|
sqlite3 "$REBUILT_DB" <<SQL
|
||||||
|
PRAGMA journal_mode = DELETE;
|
||||||
|
PRAGMA synchronous = NORMAL;
|
||||||
|
PRAGMA foreign_keys = OFF;
|
||||||
|
ATTACH DATABASE '$QUOTED_DB_PATH' AS source;
|
||||||
|
|
||||||
|
CREATE TABLE schema_version (
|
||||||
|
version INTEGER PRIMARY KEY,
|
||||||
|
applied_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
description TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE agents (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
key TEXT UNIQUE NOT NULL,
|
||||||
|
display_name TEXT NOT NULL,
|
||||||
|
role TEXT NOT NULL DEFAULT 'executor',
|
||||||
|
system_prompt TEXT NOT NULL,
|
||||||
|
available_tools TEXT NOT NULL,
|
||||||
|
model TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
enabled INTEGER DEFAULT 1,
|
||||||
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
memory TEXT,
|
||||||
|
prompts TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE contexts (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
description TEXT NOT NULL,
|
||||||
|
icon TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE nodes (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
title TEXT,
|
||||||
|
description TEXT,
|
||||||
|
source TEXT,
|
||||||
|
link TEXT,
|
||||||
|
event_date TEXT,
|
||||||
|
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
metadata TEXT,
|
||||||
|
embedding BLOB,
|
||||||
|
embedding_updated_at TEXT,
|
||||||
|
embedding_text TEXT,
|
||||||
|
chunk_status TEXT DEFAULT 'not_chunked',
|
||||||
|
context_id INTEGER,
|
||||||
|
FOREIGN KEY (context_id) REFERENCES contexts(id) ON DELETE SET NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE edges (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
from_node_id INTEGER NOT NULL,
|
||||||
|
to_node_id INTEGER NOT NULL,
|
||||||
|
source TEXT,
|
||||||
|
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
context TEXT,
|
||||||
|
explanation TEXT,
|
||||||
|
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chunks (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
node_id INTEGER NOT NULL,
|
||||||
|
chunk_idx INTEGER,
|
||||||
|
text TEXT NOT NULL,
|
||||||
|
embedding BLOB,
|
||||||
|
embedding_type TEXT DEFAULT 'openai',
|
||||||
|
metadata TEXT,
|
||||||
|
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chats (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
chat_type TEXT,
|
||||||
|
helper_name TEXT,
|
||||||
|
agent_type TEXT DEFAULT 'orchestrator',
|
||||||
|
delegation_id INTEGER,
|
||||||
|
user_message TEXT,
|
||||||
|
assistant_message TEXT,
|
||||||
|
thread_id TEXT,
|
||||||
|
focused_node_id INTEGER,
|
||||||
|
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
metadata TEXT,
|
||||||
|
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE logs (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
ts TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
table_name TEXT NOT NULL,
|
||||||
|
action TEXT NOT NULL,
|
||||||
|
row_id INTEGER NOT NULL,
|
||||||
|
summary TEXT,
|
||||||
|
snapshot_json TEXT,
|
||||||
|
enriched_summary TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE voice_usage (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
chat_id INTEGER,
|
||||||
|
session_id TEXT,
|
||||||
|
helper_name TEXT,
|
||||||
|
request_id TEXT,
|
||||||
|
message_id TEXT,
|
||||||
|
voice TEXT,
|
||||||
|
model TEXT,
|
||||||
|
chars INTEGER,
|
||||||
|
cost_usd REAL,
|
||||||
|
duration_ms INTEGER,
|
||||||
|
text_preview TEXT,
|
||||||
|
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE SET NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE dimension_migration_snapshots (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
migrated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
dimension_count INTEGER NOT NULL,
|
||||||
|
assignment_count INTEGER NOT NULL,
|
||||||
|
payload TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO schema_version SELECT version, applied_at, description FROM source.schema_version;
|
||||||
|
INSERT INTO agents SELECT id, key, display_name, role, system_prompt, available_tools, model, description, enabled, created_at, updated_at, memory, prompts FROM source.agents;
|
||||||
|
INSERT INTO contexts SELECT id, name, description, icon, created_at, updated_at FROM source.contexts;
|
||||||
|
INSERT INTO nodes SELECT id, title, description, source, link, event_date, created_at, updated_at, metadata, embedding, embedding_updated_at, embedding_text, chunk_status, context_id FROM source.nodes;
|
||||||
|
INSERT INTO edges SELECT id, from_node_id, to_node_id, source, created_at, context, explanation FROM source.edges;
|
||||||
|
INSERT INTO chunks SELECT id, node_id, chunk_idx, text, embedding, embedding_type, metadata, created_at FROM source.chunks;
|
||||||
|
INSERT INTO chats SELECT id, chat_type, helper_name, agent_type, delegation_id, user_message, assistant_message, thread_id, focused_node_id, created_at, metadata FROM source.chats;
|
||||||
|
INSERT INTO logs SELECT id, ts, table_name, action, row_id, summary, snapshot_json, enriched_summary FROM source.logs;
|
||||||
|
INSERT INTO voice_usage SELECT id, chat_id, session_id, helper_name, request_id, message_id, voice, model, chars, cost_usd, duration_ms, text_preview, created_at FROM source.voice_usage;
|
||||||
|
INSERT INTO dimension_migration_snapshots SELECT id, migrated_at, dimension_count, assignment_count, payload FROM source.dimension_migration_snapshots;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
|
||||||
|
CREATE INDEX idx_edges_from ON edges(from_node_id);
|
||||||
|
CREATE INDEX idx_edges_to ON edges(to_node_id);
|
||||||
|
CREATE INDEX idx_nodes_updated_at ON nodes(updated_at DESC);
|
||||||
|
CREATE INDEX idx_nodes_context_id ON nodes(context_id);
|
||||||
|
CREATE INDEX idx_chunks_node_id ON chunks(node_id);
|
||||||
|
CREATE INDEX idx_chunks_by_node ON chunks(node_id);
|
||||||
|
CREATE INDEX idx_chunks_by_node_idx ON chunks(node_id, chunk_idx);
|
||||||
|
CREATE INDEX idx_chats_thread ON chats(thread_id);
|
||||||
|
CREATE INDEX idx_chats_created_at ON chats(created_at DESC);
|
||||||
|
CREATE INDEX idx_logs_ts ON logs(ts);
|
||||||
|
CREATE INDEX idx_logs_table_ts ON logs(table_name, ts);
|
||||||
|
CREATE INDEX idx_logs_table_row ON logs(table_name, row_id);
|
||||||
|
CREATE INDEX idx_voice_usage_session ON voice_usage(session_id, created_at);
|
||||||
|
CREATE INDEX idx_voice_usage_chat ON voice_usage(chat_id);
|
||||||
|
|
||||||
|
CREATE VIEW nodes_v AS
|
||||||
|
SELECT n.id,
|
||||||
|
n.title,
|
||||||
|
n.description,
|
||||||
|
n.source,
|
||||||
|
n.link,
|
||||||
|
n.event_date,
|
||||||
|
n.metadata,
|
||||||
|
n.created_at,
|
||||||
|
n.updated_at
|
||||||
|
FROM nodes n;
|
||||||
|
|
||||||
|
CREATE VIEW logs_v AS
|
||||||
|
SELECT
|
||||||
|
m.id,
|
||||||
|
m.ts,
|
||||||
|
m.table_name,
|
||||||
|
m.action,
|
||||||
|
m.row_id,
|
||||||
|
m.summary,
|
||||||
|
m.enriched_summary,
|
||||||
|
m.snapshot_json,
|
||||||
|
CASE WHEN m.table_name='nodes' THEN n.title END AS node_title,
|
||||||
|
CASE WHEN m.table_name='edges' THEN nf.title END AS edge_from_title,
|
||||||
|
CASE WHEN m.table_name='edges' THEN nt.title END AS edge_to_title,
|
||||||
|
CASE WHEN m.table_name='chats' THEN c.helper_name END AS chat_helper,
|
||||||
|
CASE WHEN m.table_name='chats' THEN substr(c.user_message,1,120) END AS chat_user_preview,
|
||||||
|
CASE WHEN m.table_name='chats' THEN substr(c.assistant_message,1,120) END AS chat_assistant_preview,
|
||||||
|
CASE WHEN m.table_name='chats' THEN c.user_message END AS chat_user_full,
|
||||||
|
CASE WHEN m.table_name='chats' THEN c.assistant_message END AS chat_assistant_full
|
||||||
|
FROM logs m
|
||||||
|
LEFT JOIN nodes n ON (m.table_name='nodes' AND m.row_id = n.id)
|
||||||
|
LEFT JOIN edges e ON (m.table_name='edges' AND m.row_id = e.id)
|
||||||
|
LEFT JOIN nodes nf ON e.from_node_id = nf.id
|
||||||
|
LEFT JOIN nodes nt ON e.to_node_id = nt.id
|
||||||
|
LEFT JOIN chats c ON (m.table_name='chats' AND m.row_id = c.id);
|
||||||
|
|
||||||
|
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
||||||
|
title,
|
||||||
|
source,
|
||||||
|
description,
|
||||||
|
content='nodes',
|
||||||
|
content_rowid='id'
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE VIRTUAL TABLE chunks_fts USING fts5(
|
||||||
|
text,
|
||||||
|
content='chunks',
|
||||||
|
content_rowid='id'
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');
|
||||||
|
INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');
|
||||||
|
|
||||||
|
CREATE TRIGGER nodes_fts_ai AFTER INSERT ON nodes BEGIN
|
||||||
|
INSERT INTO nodes_fts(rowid, title, source, description)
|
||||||
|
VALUES (new.id, new.title, new.source, new.description);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER nodes_fts_ad AFTER DELETE ON nodes BEGIN
|
||||||
|
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
|
||||||
|
VALUES('delete', old.id, old.title, old.source, old.description);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER nodes_fts_au AFTER UPDATE OF title, source, description ON nodes BEGIN
|
||||||
|
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
|
||||||
|
VALUES('delete', old.id, old.title, old.source, old.description);
|
||||||
|
INSERT INTO nodes_fts(rowid, title, source, description)
|
||||||
|
VALUES (new.id, new.title, new.source, new.description);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER chunks_fts_ai AFTER INSERT ON chunks BEGIN
|
||||||
|
INSERT INTO chunks_fts(rowid, text)
|
||||||
|
VALUES (new.id, new.text);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER chunks_fts_ad AFTER DELETE ON chunks BEGIN
|
||||||
|
INSERT INTO chunks_fts(chunks_fts, rowid, text)
|
||||||
|
VALUES('delete', old.id, old.text);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER chunks_fts_au AFTER UPDATE OF text ON chunks BEGIN
|
||||||
|
INSERT INTO chunks_fts(chunks_fts, rowid, text)
|
||||||
|
VALUES('delete', old.id, old.text);
|
||||||
|
INSERT INTO chunks_fts(rowid, text)
|
||||||
|
VALUES (new.id, new.text);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_nodes_ai AFTER INSERT ON nodes BEGIN
|
||||||
|
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||||
|
VALUES('nodes', 'insert', NEW.id,
|
||||||
|
printf('node created: %s', COALESCE(NEW.title,'')),
|
||||||
|
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_nodes_au AFTER UPDATE ON nodes BEGIN
|
||||||
|
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||||
|
VALUES('nodes', 'update', NEW.id,
|
||||||
|
printf('node updated: %s', COALESCE(NEW.title,'')),
|
||||||
|
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_edges_ai AFTER INSERT ON edges BEGIN
|
||||||
|
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||||
|
VALUES('edges', 'insert', NEW.id,
|
||||||
|
printf('edge %d→%d (%s)', NEW.from_node_id, NEW.to_node_id, COALESCE(NEW.source,'')),
|
||||||
|
json_object(
|
||||||
|
'id', NEW.id,
|
||||||
|
'from', NEW.from_node_id,
|
||||||
|
'to', NEW.to_node_id,
|
||||||
|
'source', NEW.source,
|
||||||
|
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
|
||||||
|
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
|
||||||
|
));
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_edges_au AFTER UPDATE ON edges BEGIN
|
||||||
|
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||||
|
VALUES('edges', 'update', NEW.id,
|
||||||
|
printf('edge updated %d→%d', NEW.from_node_id, NEW.to_node_id),
|
||||||
|
json_object(
|
||||||
|
'id', NEW.id,
|
||||||
|
'from', NEW.from_node_id,
|
||||||
|
'to', NEW.to_node_id,
|
||||||
|
'source', NEW.source,
|
||||||
|
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
|
||||||
|
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
|
||||||
|
));
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_edges_update_nodes_on_insert
|
||||||
|
AFTER INSERT ON edges
|
||||||
|
BEGIN
|
||||||
|
UPDATE nodes SET updated_at = strftime('%Y-%m-%dT%H:%M:%f', 'now') || 'Z' WHERE id = NEW.from_node_id;
|
||||||
|
UPDATE nodes SET updated_at = strftime('%Y-%m-%dT%H:%M:%f', 'now') || 'Z' WHERE id = NEW.to_node_id;
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_chats_ai AFTER INSERT ON chats BEGIN
|
||||||
|
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||||
|
VALUES('chats', 'insert', NEW.id,
|
||||||
|
printf('chat: %s (%s)', COALESCE(NEW.helper_name,''), COALESCE(NEW.thread_id,'')),
|
||||||
|
json_object(
|
||||||
|
'id', NEW.id,
|
||||||
|
'helper', NEW.helper_name,
|
||||||
|
'thread', NEW.thread_id,
|
||||||
|
'user_preview', substr(COALESCE(NEW.user_message,''), 1, 120),
|
||||||
|
'assistant_preview', substr(COALESCE(NEW.assistant_message,''), 1, 120)
|
||||||
|
));
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_logs_prune AFTER INSERT ON logs BEGIN
|
||||||
|
DELETE FROM logs WHERE id < NEW.id - 10000;
|
||||||
|
END;
|
||||||
|
|
||||||
|
DETACH DATABASE source;
|
||||||
|
PRAGMA foreign_keys = ON;
|
||||||
|
PRAGMA journal_mode = WAL;
|
||||||
|
PRAGMA optimize;
|
||||||
|
SQL
|
||||||
|
|
||||||
|
echo "Rebuilt DB written to: $REBUILT_DB"
|
||||||
|
|
||||||
|
echo "Post-rebuild probes:"
|
||||||
|
sqlite3 "$REBUILT_DB" "PRAGMA quick_check;" | sed 's/^/ quick_check: /'
|
||||||
|
sqlite3 "$REBUILT_DB" "PRAGMA integrity_check;" | sed 's/^/ integrity_check: /'
|
||||||
|
sqlite3 "$REBUILT_DB" "SELECT COUNT(*) FROM nodes_fts;" | sed 's/^/ nodes_fts: /'
|
||||||
|
sqlite3 "$REBUILT_DB" "SELECT COUNT(*) FROM chunks_fts;" | sed 's/^/ chunks_fts: /'
|
||||||
|
|
||||||
|
mv "$DB_PATH" "${DB_PATH}.corrupt.${TS}"
|
||||||
|
if [ -f "${DB_PATH}-wal" ]; then
|
||||||
|
mv "${DB_PATH}-wal" "${DB_PATH}-wal.corrupt.${TS}"
|
||||||
|
fi
|
||||||
|
if [ -f "${DB_PATH}-shm" ]; then
|
||||||
|
mv "${DB_PATH}-shm" "${DB_PATH}-shm.corrupt.${TS}"
|
||||||
|
fi
|
||||||
|
mv "$REBUILT_DB" "$DB_PATH"
|
||||||
|
rm -f "${DB_PATH}-wal" "${DB_PATH}-shm"
|
||||||
|
rm -f "${REBUILT_DB}-wal" "${REBUILT_DB}-shm"
|
||||||
|
|
||||||
|
echo "Swapped rebuilt DB into place: $DB_PATH"
|
||||||
|
echo "Original live files preserved with suffix .corrupt.${TS}"
|
||||||
@@ -420,10 +420,7 @@ export class ChunkService {
|
|||||||
matchCount: number,
|
matchCount: number,
|
||||||
nodeIds?: number[]
|
nodeIds?: number[]
|
||||||
): RankedChunk[] {
|
): RankedChunk[] {
|
||||||
const ftsExists = sqlite.prepare(
|
if (!sqlite.canUseFtsTable('chunks')) return [];
|
||||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks_fts'"
|
|
||||||
).get();
|
|
||||||
if (!ftsExists) return [];
|
|
||||||
|
|
||||||
const ftsQuery = sanitizeFtsQuery(query);
|
const ftsQuery = sanitizeFtsQuery(query);
|
||||||
if (!ftsQuery) return [];
|
if (!ftsQuery) return [];
|
||||||
@@ -461,6 +458,7 @@ export class ChunkService {
|
|||||||
|
|
||||||
return result.rows;
|
return result.rows;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
sqlite.disableFtsTable('chunks', 'chunks_fts query failed during chunk search', error);
|
||||||
console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error);
|
console.warn('[ChunkSearch] FTS chunk search failed, falling back to LIKE:', error);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import type { DatabaseIntegrityReport } from './sqlite-client';
|
||||||
|
|
||||||
// Service instances
|
// Service instances
|
||||||
export { nodeService, NodeService } from './nodes';
|
export { nodeService, NodeService } from './nodes';
|
||||||
export { chunkService, ChunkService } from './chunks';
|
export { chunkService, ChunkService } from './chunks';
|
||||||
@@ -8,13 +10,31 @@ export { contextService, ContextService } from './contextService';
|
|||||||
// Types
|
// Types
|
||||||
export * from '@/types/database';
|
export * from '@/types/database';
|
||||||
|
|
||||||
// Health check utility
|
export interface DatabaseHealthStatus {
|
||||||
export async function checkDatabaseHealth(): Promise<{
|
|
||||||
connected: boolean;
|
connected: boolean;
|
||||||
vectorExtension: boolean;
|
vectorExtension: boolean;
|
||||||
tablesExist: boolean;
|
tablesExist: boolean;
|
||||||
|
quickCheckOk: boolean;
|
||||||
|
integrityCheckOk: boolean;
|
||||||
|
foreignKeyViolations: number;
|
||||||
|
lostAndFoundExists: boolean;
|
||||||
|
ftsTables: {
|
||||||
|
nodes: boolean;
|
||||||
|
chunks: boolean;
|
||||||
|
};
|
||||||
|
vecTables: {
|
||||||
|
nodes: boolean;
|
||||||
|
chunks: boolean;
|
||||||
|
};
|
||||||
|
integrityState: DatabaseIntegrityReport['state'];
|
||||||
|
repairableFtsTables: DatabaseIntegrityReport['repairableFtsTables'];
|
||||||
|
canRepairFts: boolean;
|
||||||
|
summary: string;
|
||||||
error?: string;
|
error?: string;
|
||||||
}> {
|
}
|
||||||
|
|
||||||
|
// Health check utility
|
||||||
|
export async function checkDatabaseHealth(): Promise<DatabaseHealthStatus> {
|
||||||
try {
|
try {
|
||||||
return checkSQLiteDatabaseHealth();
|
return checkSQLiteDatabaseHealth();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -22,20 +42,26 @@ export async function checkDatabaseHealth(): Promise<{
|
|||||||
connected: false,
|
connected: false,
|
||||||
vectorExtension: false,
|
vectorExtension: false,
|
||||||
tablesExist: false,
|
tablesExist: false,
|
||||||
|
quickCheckOk: false,
|
||||||
|
integrityCheckOk: false,
|
||||||
|
foreignKeyViolations: -1,
|
||||||
|
lostAndFoundExists: false,
|
||||||
|
ftsTables: { nodes: false, chunks: false },
|
||||||
|
vecTables: { nodes: false, chunks: false },
|
||||||
|
integrityState: 'corrupt',
|
||||||
|
repairableFtsTables: [],
|
||||||
|
canRepairFts: false,
|
||||||
|
summary: 'Database health check failed before classification.',
|
||||||
error: error instanceof Error ? error.message : 'Unknown error'
|
error: error instanceof Error ? error.message : 'Unknown error'
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function checkSQLiteDatabaseHealth(): Promise<{
|
async function checkSQLiteDatabaseHealth(): Promise<DatabaseHealthStatus> {
|
||||||
connected: boolean;
|
|
||||||
vectorExtension: boolean;
|
|
||||||
tablesExist: boolean;
|
|
||||||
error?: string;
|
|
||||||
}> {
|
|
||||||
try {
|
try {
|
||||||
const { getSQLiteClient } = await import('./sqlite-client');
|
const { getSQLiteClient } = await import('./sqlite-client');
|
||||||
const sqlite = getSQLiteClient();
|
const sqlite = getSQLiteClient();
|
||||||
|
const integrity = sqlite.getIntegrityReport(true);
|
||||||
|
|
||||||
const connected = await sqlite.testConnection();
|
const connected = await sqlite.testConnection();
|
||||||
if (!connected) {
|
if (!connected) {
|
||||||
@@ -43,6 +69,16 @@ async function checkSQLiteDatabaseHealth(): Promise<{
|
|||||||
connected: false,
|
connected: false,
|
||||||
vectorExtension: false,
|
vectorExtension: false,
|
||||||
tablesExist: false,
|
tablesExist: false,
|
||||||
|
quickCheckOk: false,
|
||||||
|
integrityCheckOk: false,
|
||||||
|
foreignKeyViolations: -1,
|
||||||
|
lostAndFoundExists: false,
|
||||||
|
ftsTables: { nodes: false, chunks: false },
|
||||||
|
vecTables: { nodes: false, chunks: false },
|
||||||
|
integrityState: integrity.state,
|
||||||
|
repairableFtsTables: integrity.repairableFtsTables,
|
||||||
|
canRepairFts: integrity.canRepairFts,
|
||||||
|
summary: integrity.summary,
|
||||||
error: 'SQLite connection failed'
|
error: 'SQLite connection failed'
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -57,13 +93,34 @@ async function checkSQLiteDatabaseHealth(): Promise<{
|
|||||||
return {
|
return {
|
||||||
connected,
|
connected,
|
||||||
vectorExtension,
|
vectorExtension,
|
||||||
tablesExist
|
tablesExist,
|
||||||
|
quickCheckOk: integrity.quickCheck.ok,
|
||||||
|
integrityCheckOk: integrity.integrityCheck.ok,
|
||||||
|
foreignKeyViolations: integrity.foreignKeyViolations,
|
||||||
|
lostAndFoundExists: integrity.lostAndFoundExists,
|
||||||
|
ftsTables: integrity.ftsTables,
|
||||||
|
vecTables: integrity.vecTables,
|
||||||
|
integrityState: integrity.state,
|
||||||
|
repairableFtsTables: integrity.repairableFtsTables,
|
||||||
|
canRepairFts: integrity.canRepairFts,
|
||||||
|
summary: integrity.summary,
|
||||||
|
error: integrity.error,
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return {
|
return {
|
||||||
connected: false,
|
connected: false,
|
||||||
vectorExtension: false,
|
vectorExtension: false,
|
||||||
tablesExist: false,
|
tablesExist: false,
|
||||||
|
quickCheckOk: false,
|
||||||
|
integrityCheckOk: false,
|
||||||
|
foreignKeyViolations: -1,
|
||||||
|
lostAndFoundExists: false,
|
||||||
|
ftsTables: { nodes: false, chunks: false },
|
||||||
|
vecTables: { nodes: false, chunks: false },
|
||||||
|
integrityState: 'corrupt',
|
||||||
|
repairableFtsTables: [],
|
||||||
|
canRepairFts: false,
|
||||||
|
summary: 'Database health check failed during execution.',
|
||||||
error: error instanceof Error ? error.message : 'Unknown error'
|
error: error instanceof Error ? error.message : 'Unknown error'
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -498,14 +498,11 @@ export class NodeService {
|
|||||||
if (!search) return 0;
|
if (!search) return 0;
|
||||||
|
|
||||||
const ftsQuery = sanitizeFtsQuery(search);
|
const ftsQuery = sanitizeFtsQuery(search);
|
||||||
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
|
|
||||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
|
|
||||||
).get();
|
|
||||||
const { clauses, params } = this.buildNodeFilterClauses(filters);
|
const { clauses, params } = this.buildNodeFilterClauses(filters);
|
||||||
|
|
||||||
if (ftsExists && ftsQuery) {
|
if (sqlite.canUseFtsTable('nodes') && ftsQuery) {
|
||||||
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
|
||||||
try {
|
try {
|
||||||
|
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||||
const result = sqlite.query<{ total: number }>(`
|
const result = sqlite.query<{ total: number }>(`
|
||||||
WITH matched_nodes AS (
|
WITH matched_nodes AS (
|
||||||
SELECT rowid
|
SELECT rowid
|
||||||
@@ -520,7 +517,7 @@ export class NodeService {
|
|||||||
|
|
||||||
return Number(result.rows[0]?.total ?? 0);
|
return Number(result.rows[0]?.total ?? 0);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
sqlite.disableNodesFts('nodes_fts query failed during count search', error);
|
sqlite.disableFtsTable('nodes', 'nodes_fts query failed during count search', error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -549,11 +546,7 @@ export class NodeService {
|
|||||||
): NodeSearchRow[] {
|
): NodeSearchRow[] {
|
||||||
const ftsQuery = sanitizeFtsQuery(search);
|
const ftsQuery = sanitizeFtsQuery(search);
|
||||||
if (!ftsQuery) return [];
|
if (!ftsQuery) return [];
|
||||||
|
if (!sqlite.canUseFtsTable('nodes')) return [];
|
||||||
const ftsExists = sqlite.isNodesFtsUsable() && sqlite.prepare(
|
|
||||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='nodes_fts'"
|
|
||||||
).get();
|
|
||||||
if (!ftsExists) return [];
|
|
||||||
|
|
||||||
const { clauses, params } = this.buildNodeFilterClauses(filters);
|
const { clauses, params } = this.buildNodeFilterClauses(filters);
|
||||||
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
const whereClauses = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
|
||||||
@@ -584,7 +577,7 @@ export class NodeService {
|
|||||||
|
|
||||||
return result.rows;
|
return result.rows;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
sqlite.disableNodesFts('nodes_fts query failed during node search', error);
|
sqlite.disableFtsTable('nodes', 'nodes_fts query failed during node search', error);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,14 +19,49 @@ export interface SQLiteQueryResult<T = any> {
|
|||||||
lastInsertRowid?: number;
|
lastInsertRowid?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type FtsSurfaceName = 'nodes' | 'chunks';
|
||||||
|
|
||||||
|
interface IntegrityProbeResult {
|
||||||
|
ok: boolean;
|
||||||
|
error?: string;
|
||||||
|
code?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DatabaseIntegrityReport {
|
||||||
|
state: 'healthy' | 'degraded_fts' | 'corrupt';
|
||||||
|
connected: boolean;
|
||||||
|
quickCheck: IntegrityProbeResult;
|
||||||
|
integrityCheck: IntegrityProbeResult;
|
||||||
|
baseTables: Record<'nodes' | 'edges' | 'chunks', IntegrityProbeResult>;
|
||||||
|
ftsTables: Record<FtsSurfaceName, boolean>;
|
||||||
|
ftsProbeResults: Record<FtsSurfaceName, IntegrityProbeResult>;
|
||||||
|
repairableFtsTables: FtsSurfaceName[];
|
||||||
|
canRepairFts: boolean;
|
||||||
|
foreignKeyViolations: number;
|
||||||
|
lostAndFoundExists: boolean;
|
||||||
|
vecTables: {
|
||||||
|
nodes: boolean;
|
||||||
|
chunks: boolean;
|
||||||
|
};
|
||||||
|
summary: string;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
class SQLiteClient {
|
class SQLiteClient {
|
||||||
private static instance: SQLiteClient;
|
private static instance: SQLiteClient;
|
||||||
private db: Database.Database;
|
private db: Database.Database;
|
||||||
private config: SQLiteConfig;
|
private config: SQLiteConfig;
|
||||||
private readonly readOnly: boolean;
|
private readonly readOnly: boolean;
|
||||||
private readonly vectorCapability: VectorCapability;
|
private readonly vectorCapability: VectorCapability;
|
||||||
private nodesFtsUsable = true;
|
private integrityReport: DatabaseIntegrityReport | null = null;
|
||||||
private nodesFtsDisabledReason: string | null = null;
|
private readonly ftsUsable: Record<FtsSurfaceName, boolean> = {
|
||||||
|
nodes: true,
|
||||||
|
chunks: true,
|
||||||
|
};
|
||||||
|
private readonly ftsDisabledReason: Record<FtsSurfaceName, string | null> = {
|
||||||
|
nodes: null,
|
||||||
|
chunks: null,
|
||||||
|
};
|
||||||
|
|
||||||
private constructor() {
|
private constructor() {
|
||||||
this.config = this.getSQLiteConfig();
|
this.config = this.getSQLiteConfig();
|
||||||
@@ -73,6 +108,16 @@ class SQLiteClient {
|
|||||||
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
|
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
|
||||||
this.ensureLoggingAndMemorySchema();
|
this.ensureLoggingAndMemorySchema();
|
||||||
this.ensureContextsSchema();
|
this.ensureContextsSchema();
|
||||||
|
this.integrityReport = this.inspectIntegrity();
|
||||||
|
|
||||||
|
if (this.integrityReport.state === 'healthy') {
|
||||||
|
this.ensureFtsTables();
|
||||||
|
this.integrityReport = this.inspectIntegrity();
|
||||||
|
} else {
|
||||||
|
console.warn(
|
||||||
|
`[SQLiteIntegrity] Skipping startup FTS mutation because database is ${this.integrityReport.state}: ${this.integrityReport.summary}`
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('SQLite client initialized successfully');
|
console.log('SQLite client initialized successfully');
|
||||||
@@ -118,6 +163,7 @@ class SQLiteClient {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
this.refreshIntegrityReportForCorruptionError(error);
|
||||||
console.error('SQLite query error:', error);
|
console.error('SQLite query error:', error);
|
||||||
throw this.handleError(error);
|
throw this.handleError(error);
|
||||||
}
|
}
|
||||||
@@ -143,6 +189,7 @@ class SQLiteClient {
|
|||||||
try {
|
try {
|
||||||
return txn();
|
return txn();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
this.refreshIntegrityReportForCorruptionError(error);
|
||||||
throw this.handleError(error);
|
throw this.handleError(error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -166,22 +213,11 @@ class SQLiteClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public isNodesFtsUsable(): boolean {
|
public isNodesFtsUsable(): boolean {
|
||||||
return this.nodesFtsUsable;
|
return this.canUseFtsTable('nodes');
|
||||||
}
|
}
|
||||||
|
|
||||||
public disableNodesFts(reason: string, error?: unknown): void {
|
public disableNodesFts(reason: string, error?: unknown): void {
|
||||||
this.nodesFtsUsable = false;
|
this.disableFtsTable('nodes', reason, error);
|
||||||
if (this.nodesFtsDisabledReason === reason) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.nodesFtsDisabledReason = reason;
|
|
||||||
|
|
||||||
if (error && !this.isSqliteCorruptError(error)) {
|
|
||||||
console.warn(`[SQLite] nodes_fts disabled: ${reason}`, error);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.warn(`[SQLite] nodes_fts disabled: ${reason}. Falling back to LIKE search for this database session.`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async checkTables(): Promise<string[]> {
|
public async checkTables(): Promise<string[]> {
|
||||||
@@ -229,6 +265,25 @@ class SQLiteClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public canUseFtsTable(tableName: FtsSurfaceName): boolean {
|
||||||
|
return this.ftsUsable[tableName] && this.getIntegrityReport().ftsTables[tableName];
|
||||||
|
}
|
||||||
|
|
||||||
|
public disableFtsTable(tableName: FtsSurfaceName, reason: string, error?: unknown): void {
|
||||||
|
this.ftsUsable[tableName] = false;
|
||||||
|
if (this.ftsDisabledReason[tableName] === reason) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.ftsDisabledReason[tableName] = reason;
|
||||||
|
|
||||||
|
if (error && !this.isSqliteCorruptError(error)) {
|
||||||
|
console.warn(`[SQLite] ${tableName}_fts disabled: ${reason}`, error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.warn(`[SQLite] ${tableName}_fts disabled: ${reason}. Falling back to non-FTS behavior for this database session.`);
|
||||||
|
}
|
||||||
|
|
||||||
private ensureVectorTables(): void {
|
private ensureVectorTables(): void {
|
||||||
if (this.readOnly || !this.vectorCapability.available) {
|
if (this.readOnly || !this.vectorCapability.available) {
|
||||||
return;
|
return;
|
||||||
@@ -1073,17 +1128,6 @@ class SQLiteClient {
|
|||||||
} catch {}
|
} catch {}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only create nodes_fts when it does not exist yet.
|
|
||||||
// Do not rewrite existing FTS tables on startup in the OS app.
|
|
||||||
try {
|
|
||||||
const ftsCheck = this.db.prepare("SELECT sql FROM sqlite_master WHERE name='nodes_fts'").get() as { sql?: string } | undefined;
|
|
||||||
if (!ftsCheck?.sql) {
|
|
||||||
this.db.exec("CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');");
|
|
||||||
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');");
|
|
||||||
}
|
|
||||||
} catch (ftsErr) {
|
|
||||||
console.warn('Failed to initialize nodes_fts:', ftsErr);
|
|
||||||
}
|
|
||||||
} catch (schemaErr) {
|
} catch (schemaErr) {
|
||||||
console.warn('Final schema pass migration error:', schemaErr);
|
console.warn('Final schema pass migration error:', schemaErr);
|
||||||
}
|
}
|
||||||
@@ -1183,6 +1227,237 @@ class SQLiteClient {
|
|||||||
tryRead('vec_chunks');
|
tryRead('vec_chunks');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private ensureFtsTables(): void {
|
||||||
|
if (this.readOnly) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ensureFts = (
|
||||||
|
tableName: 'nodes_fts' | 'chunks_fts',
|
||||||
|
createSql: string,
|
||||||
|
triggerSql: string,
|
||||||
|
rebuildSql: string,
|
||||||
|
) => {
|
||||||
|
const existing = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?").get(tableName) as { sql?: string } | undefined;
|
||||||
|
|
||||||
|
if (!existing?.sql) {
|
||||||
|
this.db.exec(createSql);
|
||||||
|
this.db.exec(rebuildSql);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.db.exec(triggerSql);
|
||||||
|
};
|
||||||
|
|
||||||
|
ensureFts(
|
||||||
|
'nodes_fts',
|
||||||
|
"CREATE VIRTUAL TABLE nodes_fts USING fts5(title, source, description, content='nodes', content_rowid='id');",
|
||||||
|
`
|
||||||
|
DROP TRIGGER IF EXISTS nodes_fts_ai;
|
||||||
|
DROP TRIGGER IF EXISTS nodes_fts_ad;
|
||||||
|
DROP TRIGGER IF EXISTS nodes_fts_au;
|
||||||
|
CREATE TRIGGER nodes_fts_ai AFTER INSERT ON nodes BEGIN
|
||||||
|
INSERT INTO nodes_fts(rowid, title, source, description)
|
||||||
|
VALUES (new.id, new.title, new.source, new.description);
|
||||||
|
END;
|
||||||
|
CREATE TRIGGER nodes_fts_ad AFTER DELETE ON nodes BEGIN
|
||||||
|
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
|
||||||
|
VALUES('delete', old.id, old.title, old.source, old.description);
|
||||||
|
END;
|
||||||
|
CREATE TRIGGER nodes_fts_au AFTER UPDATE OF title, source, description ON nodes BEGIN
|
||||||
|
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
|
||||||
|
VALUES('delete', old.id, old.title, old.source, old.description);
|
||||||
|
INSERT INTO nodes_fts(rowid, title, source, description)
|
||||||
|
VALUES (new.id, new.title, new.source, new.description);
|
||||||
|
END;
|
||||||
|
`,
|
||||||
|
"INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');",
|
||||||
|
);
|
||||||
|
|
||||||
|
ensureFts(
|
||||||
|
'chunks_fts',
|
||||||
|
"CREATE VIRTUAL TABLE chunks_fts USING fts5(text, content='chunks', content_rowid='id');",
|
||||||
|
`
|
||||||
|
DROP TRIGGER IF EXISTS chunks_fts_ai;
|
||||||
|
DROP TRIGGER IF EXISTS chunks_fts_ad;
|
||||||
|
DROP TRIGGER IF EXISTS chunks_fts_au;
|
||||||
|
CREATE TRIGGER chunks_fts_ai AFTER INSERT ON chunks BEGIN
|
||||||
|
INSERT INTO chunks_fts(rowid, text)
|
||||||
|
VALUES (new.id, new.text);
|
||||||
|
END;
|
||||||
|
CREATE TRIGGER chunks_fts_ad AFTER DELETE ON chunks BEGIN
|
||||||
|
INSERT INTO chunks_fts(chunks_fts, rowid, text)
|
||||||
|
VALUES('delete', old.id, old.text);
|
||||||
|
END;
|
||||||
|
CREATE TRIGGER chunks_fts_au AFTER UPDATE OF text ON chunks BEGIN
|
||||||
|
INSERT INTO chunks_fts(chunks_fts, rowid, text)
|
||||||
|
VALUES('delete', old.id, old.text);
|
||||||
|
INSERT INTO chunks_fts(rowid, text)
|
||||||
|
VALUES (new.id, new.text);
|
||||||
|
END;
|
||||||
|
`,
|
||||||
|
"INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');",
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('Failed to ensure FTS tables:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildProbeError(error: unknown): IntegrityProbeResult {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
const code =
|
||||||
|
typeof error === 'object' &&
|
||||||
|
error !== null &&
|
||||||
|
'code' in error &&
|
||||||
|
typeof (error as { code?: unknown }).code === 'string'
|
||||||
|
? String((error as { code: string }).code)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
return { ok: false, error: message, code };
|
||||||
|
}
|
||||||
|
|
||||||
|
private runIntegrityProbe(fn: () => void): IntegrityProbeResult {
|
||||||
|
try {
|
||||||
|
fn();
|
||||||
|
return { ok: true };
|
||||||
|
} catch (error) {
|
||||||
|
return this.buildProbeError(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private refreshIntegrityReportForCorruptionError(error: unknown): void {
|
||||||
|
if (this.isSqliteCorruptError(error)) {
|
||||||
|
this.getIntegrityReport(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private inspectIntegrity(): DatabaseIntegrityReport {
|
||||||
|
const connected = this.runIntegrityProbe(() => {
|
||||||
|
this.db.prepare('SELECT 1').get();
|
||||||
|
}).ok;
|
||||||
|
|
||||||
|
const quickCheck = this.runIntegrityProbe(() => {
|
||||||
|
const rows = this.db.prepare('PRAGMA quick_check').pluck().all() as string[];
|
||||||
|
if (!(rows.length === 1 && rows[0] === 'ok')) {
|
||||||
|
throw new Error(rows.join('\n') || 'quick_check failed');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const integrityCheck = this.runIntegrityProbe(() => {
|
||||||
|
const rows = this.db.prepare('PRAGMA integrity_check').pluck().all() as string[];
|
||||||
|
if (!(rows.length === 1 && rows[0] === 'ok')) {
|
||||||
|
throw new Error(rows.join('\n') || 'integrity_check failed');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const baseTables: Record<'nodes' | 'edges' | 'chunks', IntegrityProbeResult> = {
|
||||||
|
nodes: this.runIntegrityProbe(() => {
|
||||||
|
this.db.prepare('SELECT COUNT(*) FROM nodes').pluck().get();
|
||||||
|
}),
|
||||||
|
edges: this.runIntegrityProbe(() => {
|
||||||
|
this.db.prepare('SELECT COUNT(*) FROM edges').pluck().get();
|
||||||
|
}),
|
||||||
|
chunks: this.runIntegrityProbe(() => {
|
||||||
|
this.db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const hasTable = (name: string) =>
|
||||||
|
Boolean(this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name));
|
||||||
|
|
||||||
|
const ftsSchemaExists: Record<FtsSurfaceName, boolean> = {
|
||||||
|
nodes: hasTable('nodes_fts'),
|
||||||
|
chunks: hasTable('chunks_fts'),
|
||||||
|
};
|
||||||
|
|
||||||
|
const ftsProbeResults: Record<FtsSurfaceName, IntegrityProbeResult> = {
|
||||||
|
nodes: ftsSchemaExists.nodes
|
||||||
|
? this.runIntegrityProbe(() => {
|
||||||
|
this.db.prepare('SELECT COUNT(*) FROM nodes_fts').pluck().get();
|
||||||
|
})
|
||||||
|
: { ok: false, error: 'nodes_fts missing' },
|
||||||
|
chunks: ftsSchemaExists.chunks
|
||||||
|
? this.runIntegrityProbe(() => {
|
||||||
|
this.db.prepare('SELECT COUNT(*) FROM chunks_fts').pluck().get();
|
||||||
|
})
|
||||||
|
: { ok: false, error: 'chunks_fts missing' },
|
||||||
|
};
|
||||||
|
|
||||||
|
const ftsTables = {
|
||||||
|
nodes: ftsSchemaExists.nodes && ftsProbeResults.nodes.ok,
|
||||||
|
chunks: ftsSchemaExists.chunks && ftsProbeResults.chunks.ok,
|
||||||
|
};
|
||||||
|
|
||||||
|
let foreignKeyViolations = -1;
|
||||||
|
if (baseTables.nodes.ok && baseTables.edges.ok) {
|
||||||
|
const foreignKeyProbe = this.runIntegrityProbe(() => {
|
||||||
|
foreignKeyViolations = Number(
|
||||||
|
(this.db.prepare('SELECT COUNT(*) FROM pragma_foreign_key_check').pluck().get() as number | undefined) ?? 0
|
||||||
|
);
|
||||||
|
});
|
||||||
|
if (!foreignKeyProbe.ok) {
|
||||||
|
foreignKeyViolations = -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const lostAndFoundExists = hasTable('lost_and_found');
|
||||||
|
const vecTables = {
|
||||||
|
nodes: hasTable('vec_nodes'),
|
||||||
|
chunks: hasTable('vec_chunks'),
|
||||||
|
};
|
||||||
|
|
||||||
|
const baseTablesReadable = Object.values(baseTables).every(probe => probe.ok);
|
||||||
|
const repairableFtsTables = (Object.entries(ftsProbeResults) as Array<[FtsSurfaceName, IntegrityProbeResult]>)
|
||||||
|
.filter(([name, probe]) => ftsSchemaExists[name] && !probe.ok)
|
||||||
|
.map(([name]) => name);
|
||||||
|
|
||||||
|
let state: DatabaseIntegrityReport['state'] = 'healthy';
|
||||||
|
if (!quickCheck.ok || !integrityCheck.ok) {
|
||||||
|
state = baseTablesReadable && repairableFtsTables.length > 0 ? 'degraded_fts' : 'corrupt';
|
||||||
|
} else if (!baseTablesReadable) {
|
||||||
|
state = 'corrupt';
|
||||||
|
} else if (repairableFtsTables.length > 0) {
|
||||||
|
state = 'degraded_fts';
|
||||||
|
}
|
||||||
|
|
||||||
|
let summary = 'Database integrity checks passed.';
|
||||||
|
if (state === 'degraded_fts') {
|
||||||
|
summary = `Rebuildable FTS surfaces are degraded: ${repairableFtsTables.join(', ')}. Base tables are still readable.`;
|
||||||
|
} else if (state === 'corrupt') {
|
||||||
|
summary = 'Canonical database integrity checks are failing. Treat the database as corrupted.';
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstError =
|
||||||
|
quickCheck.error ||
|
||||||
|
integrityCheck.error ||
|
||||||
|
Object.values(baseTables).find(probe => !probe.ok)?.error ||
|
||||||
|
Object.values(ftsProbeResults).find(probe => !probe.ok)?.error;
|
||||||
|
|
||||||
|
return {
|
||||||
|
state,
|
||||||
|
connected,
|
||||||
|
quickCheck,
|
||||||
|
integrityCheck,
|
||||||
|
baseTables,
|
||||||
|
ftsTables,
|
||||||
|
ftsProbeResults,
|
||||||
|
repairableFtsTables,
|
||||||
|
canRepairFts: state === 'degraded_fts' && repairableFtsTables.length > 0,
|
||||||
|
foreignKeyViolations,
|
||||||
|
lostAndFoundExists,
|
||||||
|
vecTables,
|
||||||
|
summary,
|
||||||
|
error: firstError,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public getIntegrityReport(forceRefresh = false): DatabaseIntegrityReport {
|
||||||
|
if (!this.integrityReport || forceRefresh) {
|
||||||
|
this.integrityReport = this.inspectIntegrity();
|
||||||
|
}
|
||||||
|
return this.integrityReport;
|
||||||
|
}
|
||||||
|
|
||||||
private handleError(error: any): DatabaseError {
|
private handleError(error: any): DatabaseError {
|
||||||
return {
|
return {
|
||||||
message: error.message || 'SQLite operation failed',
|
message: error.message || 'SQLite operation failed',
|
||||||
@@ -1201,7 +1476,10 @@ class SQLiteClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const sqliteError = error as Error & { code?: string };
|
const sqliteError = error as Error & { code?: string };
|
||||||
return sqliteError.code === 'SQLITE_CORRUPT' || /database disk image is malformed/i.test(sqliteError.message || '');
|
return (
|
||||||
|
sqliteError.code?.includes('CORRUPT') === true ||
|
||||||
|
/database disk image is malformed/i.test(sqliteError.message || '')
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { embedNodeContent } from '@/services/embedding/ingestion';
|
import { embedNodeContent } from '@/services/embedding/ingestion';
|
||||||
import { nodeService } from '@/services/database';
|
import { nodeService } from '@/services/database';
|
||||||
|
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||||
|
|
||||||
interface AutoEmbedTask {
|
interface AutoEmbedTask {
|
||||||
nodeId: number;
|
nodeId: number;
|
||||||
@@ -87,6 +88,13 @@ export class AutoEmbedQueue {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async executeTask(task: AutoEmbedTask) {
|
private async executeTask(task: AutoEmbedTask) {
|
||||||
|
const sqlite = getSQLiteClient();
|
||||||
|
const integrity = sqlite.getIntegrityReport();
|
||||||
|
if (!sqlite.canUseFtsTable('chunks')) {
|
||||||
|
console.warn('[AutoEmbedQueue] Skipping chunk write because chunks FTS is degraded:', integrity.summary);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const node = await nodeService.getNodeById(task.nodeId);
|
const node = await nodeService.getNodeById(task.nodeId);
|
||||||
if (!node) {
|
if (!node) {
|
||||||
console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId);
|
console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId);
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { nodeService } from '@/services/database';
|
import { nodeService } from '@/services/database';
|
||||||
|
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||||
import { NodeEmbedder } from '@/services/typescript/embed-nodes';
|
import { NodeEmbedder } from '@/services/typescript/embed-nodes';
|
||||||
import { UniversalEmbedder } from '@/services/typescript/embed-universal';
|
import { UniversalEmbedder } from '@/services/typescript/embed-universal';
|
||||||
import type { Node } from '@/types/database';
|
import type { Node } from '@/types/database';
|
||||||
@@ -53,6 +54,16 @@ async function updateChunkStatus(nodeId: number, status: Node['chunk_status']) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function runChunkEmbedding(nodeId: number): Promise<EmbeddingResult> {
|
async function runChunkEmbedding(nodeId: number): Promise<EmbeddingResult> {
|
||||||
|
const sqlite = getSQLiteClient();
|
||||||
|
const integrity = sqlite.getIntegrityReport();
|
||||||
|
if (!sqlite.canUseFtsTable('chunks')) {
|
||||||
|
await updateChunkStatus(nodeId, 'error');
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: `Chunk indexing is disabled while the chunks FTS surface is degraded. ${integrity.summary}`
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const embedder = new UniversalEmbedder();
|
const embedder = new UniversalEmbedder();
|
||||||
try {
|
try {
|
||||||
await updateChunkStatus(nodeId, 'chunking');
|
await updateChunkStatus(nodeId, 'chunking');
|
||||||
|
|||||||
Reference in New Issue
Block a user