SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: harden standalone mcp and chunk recovery

Browse Source
This commit is contained in:
“BeeRad”
2026-04-16 09:53:56 +10:00
parent 097c4c61bb
commit c2f880d957
15 changed files with 222 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/mcp-server-standalone/services/nodeService.js
+6 -76
View File
@@ -158,67 +158,8 @@ function sanitizeTitle(title) {
return clean.slice(0, 160);
}
const CHUNK_SIZE = 1000;
const CHUNK_OVERLAP = 200;
function splitSourceIntoChunks(sourceText) {
const text = String(sourceText || '').trim();
if (!text) return [];
const chunks = [];
let start = 0;
while (start < text.length) {
const end = Math.min(start + CHUNK_SIZE, text.length);
const chunkText = text.slice(start, end).trim();
if (chunkText) {
chunks.push({
text: chunkText,
start_char: start,
end_char: end
});
}
if (end >= text.length) break;
start = Math.max(end - CHUNK_OVERLAP, start + 1);
}
return chunks;
}
function replaceNodeChunks(db, nodeId, title, sourceText) {
db.prepare('DELETE FROM chunks WHERE node_id = ?').run(nodeId);
const chunks = splitSourceIntoChunks(sourceText);
if (chunks.length === 0) {
return 0;
}
const now = new Date().toISOString();
const insertChunk = db.prepare(`
INSERT INTO chunks (node_id, chunk_idx, text, embedding_type, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?)
`);
chunks.forEach((chunk, index) => {
insertChunk.run(
nodeId,
index,
chunk.text,
'text-embedding-3-small',
JSON.stringify({
node_id: nodeId,
chunk_index: index,
start_char: chunk.start_char,
end_char: chunk.end_char,
title
}),
now
);
});
return chunks.length;
function getChunkStatusForSource(sourceText) {
return normalizeString(sourceText) ? 'not_chunked' : null;
}
/**
@@ -243,7 +184,7 @@ function createNode(nodeData) {
const sourceToStore = source ?? ([title, description].filter(Boolean).join('\n\n').trim() || null);
const effectiveContextId = context_id ?? null;
const hasSource = !!normalizeString(sourceToStore);
const chunkStatus = getChunkStatusForSource(sourceToStore);
const nodeId = transaction(() => {
const stmt = db.prepare(`
@@ -258,19 +199,13 @@ function createNode(nodeData) {
link ?? null,
event_date ?? null,
JSON.stringify(canonicalMetadata),
hasSource ? 'chunked' : null,
chunkStatus,
effectiveContextId ?? null,
now,
now
);
const id = Number(result.lastInsertRowid);
if (hasSource) {
replaceNodeChunks(db, id, title, sourceToStore);
}
return id;
return Number(result.lastInsertRowid);
});
return getNodeById(nodeId);
@@ -294,7 +229,6 @@ function updateNode(id, updates, options = {}) {
? buildCanonicalMetadata({ existing: existing.metadata, metadata })
: undefined;
const sourceWasProvided = Object.prototype.hasOwnProperty.call(updates, 'source');
const updatedTitle = title !== undefined ? title : existing.title;
const normalizedSource = sourceWasProvided ? normalizeString(source) : undefined;
transaction(() => {
@@ -323,7 +257,7 @@ function updateNode(id, updates, options = {}) {
}
if (sourceWasProvided) {
setFields.push('chunk_status = ?');
params.push(normalizedSource ? 'chunked' : null);
params.push(getChunkStatusForSource(normalizedSource));
}
if (Object.prototype.hasOwnProperty.call(updates, 'context_id')) {
setFields.push('context_id = ?');
@@ -344,10 +278,6 @@ function updateNode(id, updates, options = {}) {
stmt.run(...params);
}
if (sourceWasProvided) {
replaceNodeChunks(db, id, updatedTitle, normalizedSource || '');
}
});
return getNodeById(id);
apps/mcp-server-standalone/services/sqlite-client.js
+51 -153
View File
@@ -27,6 +27,51 @@ function getDatabasePath() {
let db = null;
function getExistingColumnNames(db, tableName) {
return db.prepare(`PRAGMA table_info(${tableName})`).all().map(c => c.name);
}
function validateExistingRahSchema(db) {
const requiredTables = ['contexts', 'nodes', 'edges', 'chunks'];
const missingTables = requiredTables.filter(
tableName => !db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name = ?").get(tableName)
);
if (missingTables.length > 0) {
throw new Error(
`[RA-H MCP] Database is missing required tables: ${missingTables.join(', ')}. ` +
'MCP startup will not create or migrate tables. Open the app and repair the database first.'
);
}
const requiredNodeColumns = [
'title', 'description', 'source', 'link', 'event_date', 'metadata',
'embedding', 'embedding_updated_at', 'embedding_text', 'chunk_status', 'context_id',
'created_at', 'updated_at'
];
const requiredContextColumns = ['name', 'description', 'icon', 'created_at', 'updated_at'];
const requiredEdgeColumns = ['from_node_id', 'to_node_id', 'source', 'created_at', 'context', 'explanation'];
const requiredChunkColumns = ['node_id', 'chunk_idx', 'text', 'embedding_type', 'metadata', 'created_at'];
const schemaChecks = [
['nodes', requiredNodeColumns],
['contexts', requiredContextColumns],
['edges', requiredEdgeColumns],
['chunks', requiredChunkColumns],
];
for (const [tableName, requiredColumns] of schemaChecks) {
const existingColumns = getExistingColumnNames(db, tableName);
const missingColumns = requiredColumns.filter(columnName => !existingColumns.includes(columnName));
if (missingColumns.length > 0) {
throw new Error(
`[RA-H MCP] Database table ${tableName} is missing required columns: ${missingColumns.join(', ')}. ` +
'MCP startup will not migrate schema. Open the app and repair the database first.'
);
}
}
}
/**
* Initialize the database connection.
* Call this once at startup.
@@ -38,168 +83,21 @@ function initDatabase() {
const dbPath = getDatabasePath();
// Auto-create database if it doesn't exist
if (!fs.existsSync(dbPath)) {
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
db = new Database(dbPath);
console.error('[RA-H] Creating new database at:', dbPath);
console.error('[RA-H] Database created successfully');
} else {
db = new Database(dbPath);
throw new Error(
`[RA-H MCP] Database not found at ${dbPath}. MCP startup will not create a new database. Open RA-H first to create or repair the app database.`
);
}
db = new Database(dbPath);
// Configure SQLite for performance
db.pragma('journal_mode = WAL');
db.pragma('synchronous = NORMAL');
db.pragma('cache_size = 5000');
db.pragma('busy_timeout = 5000');
ensureCoreSchema(db);
// Migrations for existing databases
const edgeCols = db.prepare('PRAGMA table_info(edges)').all().map(c => c.name);
const nodeCols = db.prepare('PRAGMA table_info(nodes)').all().map(c => c.name);
const contextCols = db.prepare('PRAGMA table_info(contexts)').all().map(c => c.name);
let hasSourceColumn = nodeCols.includes('source');
if (!hasSourceColumn) {
db.exec('ALTER TABLE nodes ADD COLUMN source TEXT;');
hasSourceColumn = true;
console.error('[RA-H] Migrated nodes: added source column');
}
if (nodeCols.includes('content')) {
db.exec(`
UPDATE nodes
SET source = content
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND content IS NOT NULL
AND LENGTH(TRIM(content)) > 0;
`);
}
if (nodeCols.includes('notes')) {
db.exec(`
UPDATE nodes
SET source = notes
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND notes IS NOT NULL
AND LENGTH(TRIM(notes)) > 0;
`);
}
if (nodeCols.includes('chunk')) {
db.exec(`
UPDATE nodes
SET source = chunk
WHERE (source IS NULL OR LENGTH(TRIM(source)) = 0)
AND chunk IS NOT NULL
AND LENGTH(TRIM(chunk)) > 0;
`);
}
if (hasSourceColumn) {
db.exec(`
UPDATE nodes
SET source = title || CASE
WHEN description IS NOT NULL AND LENGTH(TRIM(description)) > 0
THEN char(10) || char(10) || description
ELSE ''
END
WHERE source IS NULL OR LENGTH(TRIM(source)) = 0;
`);
}
if (!edgeCols.includes('explanation')) {
db.exec('ALTER TABLE edges ADD COLUMN explanation TEXT;');
try {
db.exec(`
UPDATE edges SET explanation = json_extract(context, '$.explanation')
WHERE explanation IS NULL AND json_extract(context, '$.explanation') IS NOT NULL;
`);
} catch {}
console.error('[RA-H] Migrated edges: added explanation column');
}
if (!nodeCols.includes('context_id')) {
db.exec('ALTER TABLE nodes ADD COLUMN context_id INTEGER REFERENCES contexts(id) ON DELETE SET NULL;');
console.error('[RA-H] Migrated nodes: added context_id column');
}
if (!contextCols.includes('description')) {
db.exec("ALTER TABLE contexts ADD COLUMN description TEXT NOT NULL DEFAULT '';");
}
if (!contextCols.includes('icon')) {
db.exec('ALTER TABLE contexts ADD COLUMN icon TEXT;');
}
if (!contextCols.includes('created_at')) {
db.exec("ALTER TABLE contexts ADD COLUMN created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
}
if (!contextCols.includes('updated_at')) {
db.exec("ALTER TABLE contexts ADD COLUMN updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
}
db.exec(`
UPDATE contexts
SET description = COALESCE(NULLIF(TRIM(description), ''), name)
WHERE description IS NULL OR LENGTH(TRIM(description)) = 0;
`);
db.exec(`
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized
ON contexts(LOWER(TRIM(name)));
CREATE INDEX IF NOT EXISTS idx_nodes_context_id ON nodes(context_id);
`);
db.exec(`
CREATE TABLE IF NOT EXISTS dimension_migration_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
migrated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
dimension_count INTEGER NOT NULL,
assignment_count INTEGER NOT NULL,
payload TEXT
);
`);
const hasLegacyDimensions = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='dimensions'").get();
const hasLegacyNodeDimensions = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='node_dimensions'").get();
if (hasLegacyDimensions || hasLegacyNodeDimensions) {
const snapshotCount = Number((db.prepare('SELECT COUNT(*) as count FROM dimension_migration_snapshots').get() || {}).count || 0);
if (snapshotCount === 0) {
const dimensionCount = hasLegacyDimensions
? Number((db.prepare('SELECT COUNT(*) as count FROM dimensions').get() || {}).count || 0)
: 0;
const assignmentCount = hasLegacyNodeDimensions
? Number((db.prepare('SELECT COUNT(*) as count FROM node_dimensions').get() || {}).count || 0)
: 0;
const payload = hasLegacyNodeDimensions
? ((db.prepare(`
SELECT COALESCE(
json_group_array(
json_object(
'node_id', nd.node_id,
'dimension', nd.dimension,
'description', d.description,
'icon', d.icon,
'is_priority', d.is_priority
)
),
'[]'
) AS payload
FROM node_dimensions nd
LEFT JOIN dimensions d ON d.name = nd.dimension
`).get() || {}).payload || '[]')
: '[]';
db.prepare(`
INSERT INTO dimension_migration_snapshots (dimension_count, assignment_count, payload)
VALUES (?, ?, ?)
`).run(dimensionCount, assignmentCount, payload);
}
db.exec(`
DROP INDEX IF EXISTS idx_dim_by_dimension;
DROP INDEX IF EXISTS idx_dim_by_node;
DROP TABLE IF EXISTS node_dimensions;
DROP TABLE IF EXISTS dimensions;
`);
}
validateExistingRahSchema(db);
return db;
}