sync: harden standalone mcp and chunk recovery
This commit is contained in:
@@ -158,67 +158,8 @@ function sanitizeTitle(title) {
|
||||
return clean.slice(0, 160);
|
||||
}
|
||||
|
||||
const CHUNK_SIZE = 1000;
|
||||
const CHUNK_OVERLAP = 200;
|
||||
|
||||
function splitSourceIntoChunks(sourceText) {
|
||||
const text = String(sourceText || '').trim();
|
||||
if (!text) return [];
|
||||
|
||||
const chunks = [];
|
||||
let start = 0;
|
||||
|
||||
while (start < text.length) {
|
||||
const end = Math.min(start + CHUNK_SIZE, text.length);
|
||||
const chunkText = text.slice(start, end).trim();
|
||||
|
||||
if (chunkText) {
|
||||
chunks.push({
|
||||
text: chunkText,
|
||||
start_char: start,
|
||||
end_char: end
|
||||
});
|
||||
}
|
||||
|
||||
if (end >= text.length) break;
|
||||
start = Math.max(end - CHUNK_OVERLAP, start + 1);
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function replaceNodeChunks(db, nodeId, title, sourceText) {
|
||||
db.prepare('DELETE FROM chunks WHERE node_id = ?').run(nodeId);
|
||||
|
||||
const chunks = splitSourceIntoChunks(sourceText);
|
||||
if (chunks.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const insertChunk = db.prepare(`
|
||||
INSERT INTO chunks (node_id, chunk_idx, text, embedding_type, metadata, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
chunks.forEach((chunk, index) => {
|
||||
insertChunk.run(
|
||||
nodeId,
|
||||
index,
|
||||
chunk.text,
|
||||
'text-embedding-3-small',
|
||||
JSON.stringify({
|
||||
node_id: nodeId,
|
||||
chunk_index: index,
|
||||
start_char: chunk.start_char,
|
||||
end_char: chunk.end_char,
|
||||
title
|
||||
}),
|
||||
now
|
||||
);
|
||||
});
|
||||
|
||||
return chunks.length;
|
||||
function getChunkStatusForSource(sourceText) {
|
||||
return normalizeString(sourceText) ? 'not_chunked' : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -243,7 +184,7 @@ function createNode(nodeData) {
|
||||
|
||||
const sourceToStore = source ?? ([title, description].filter(Boolean).join('\n\n').trim() || null);
|
||||
const effectiveContextId = context_id ?? null;
|
||||
const hasSource = !!normalizeString(sourceToStore);
|
||||
const chunkStatus = getChunkStatusForSource(sourceToStore);
|
||||
|
||||
const nodeId = transaction(() => {
|
||||
const stmt = db.prepare(`
|
||||
@@ -258,19 +199,13 @@ function createNode(nodeData) {
|
||||
link ?? null,
|
||||
event_date ?? null,
|
||||
JSON.stringify(canonicalMetadata),
|
||||
hasSource ? 'chunked' : null,
|
||||
chunkStatus,
|
||||
effectiveContextId ?? null,
|
||||
now,
|
||||
now
|
||||
);
|
||||
|
||||
const id = Number(result.lastInsertRowid);
|
||||
|
||||
if (hasSource) {
|
||||
replaceNodeChunks(db, id, title, sourceToStore);
|
||||
}
|
||||
|
||||
return id;
|
||||
return Number(result.lastInsertRowid);
|
||||
});
|
||||
|
||||
return getNodeById(nodeId);
|
||||
@@ -294,7 +229,6 @@ function updateNode(id, updates, options = {}) {
|
||||
? buildCanonicalMetadata({ existing: existing.metadata, metadata })
|
||||
: undefined;
|
||||
const sourceWasProvided = Object.prototype.hasOwnProperty.call(updates, 'source');
|
||||
const updatedTitle = title !== undefined ? title : existing.title;
|
||||
const normalizedSource = sourceWasProvided ? normalizeString(source) : undefined;
|
||||
|
||||
transaction(() => {
|
||||
@@ -323,7 +257,7 @@ function updateNode(id, updates, options = {}) {
|
||||
}
|
||||
if (sourceWasProvided) {
|
||||
setFields.push('chunk_status = ?');
|
||||
params.push(normalizedSource ? 'chunked' : null);
|
||||
params.push(getChunkStatusForSource(normalizedSource));
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(updates, 'context_id')) {
|
||||
setFields.push('context_id = ?');
|
||||
@@ -344,10 +278,6 @@ function updateNode(id, updates, options = {}) {
|
||||
stmt.run(...params);
|
||||
}
|
||||
|
||||
if (sourceWasProvided) {
|
||||
replaceNodeChunks(db, id, updatedTitle, normalizedSource || '');
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
return getNodeById(id);
|
||||
|
||||
Reference in New Issue
Block a user