SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: sync open-source context and capsule removal

Browse Source 
- remove legacy contexts surfaces from the app, MCP bridge, standalone server, and docs
- keep getContext and retrieveQueryContext while aligning the simplified graph-only contract
- harden dev startup migration behavior and disable the accidental chat surface in open source

Generated with Codex
This commit is contained in:
“BeeRad”
2026-04-17 12:34:51 +10:00
parent 97eeb0789f
commit 07754f5030
87 changed files with 2688 additions and 4861 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/README.md
+2 -2
View File
@@ -24,7 +24,7 @@ npm run backup
```
- Creates timestamped backup in `/backups/` folder
- Example: `rah_backup_20250902_102846.sql`
- Includes all nodes, chunks, edges, contexts, and migration snapshots
- Includes all nodes, chunks, edges, logs, chats, voice usage, and migration snapshots
### Restore from Backup
```bash
@@ -59,7 +59,7 @@ node scripts/helpers/delete-helper.js "helper-id"
- All nodes (42,000+ knowledge items)
- Content chunks and embeddings
- Node connections/edges
- Contexts, metadata, and migration snapshots
- Metadata, logs, chats, voice usage, and migration snapshots
- Database schema and indexes
## Notes
scripts/database/sqlite-ensure-app-schema.sh
+82 -238
View File
@@ -110,7 +110,7 @@ VALUES (
SQL
fi
echo "Ensuring core tables exist (nodes, chunks, edges, chats, contexts)..."
echo "Ensuring core tables exist (nodes, chunks, edges, chats)..."
if ! has_table nodes; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
@@ -132,20 +132,6 @@ CREATE TABLE nodes (
SQL
fi
if ! has_table contexts; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE TABLE contexts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL,
icon TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
SQL
fi
if ! has_table chunks; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE TABLE chunks (
@@ -172,7 +158,6 @@ CREATE TABLE edges (
source TEXT,
created_at TEXT,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
@@ -181,186 +166,14 @@ CREATE INDEX idx_edges_to ON edges(to_node_id);
SQL
fi
if has_table edges; then
NEEDS_EDGE_REWRITE=0
if ! has_col edges from_node_id || ! has_col edges to_node_id || ! has_col edges source || ! has_col edges created_at || ! has_col edges context; then
NEEDS_EDGE_REWRITE=1
fi
if has_col edges from_id || has_col edges to_id || has_col edges description || has_col edges updated_at; then
NEEDS_EDGE_REWRITE=1
fi
if [ "$NEEDS_EDGE_REWRITE" = "1" ]; then
echo "Migrating legacy edges table to canonical schema"
FROM_EXPR="NULL"
if has_col edges from_node_id; then
FROM_EXPR="from_node_id"
elif has_col edges from_id; then
FROM_EXPR="from_id"
fi
TO_EXPR="NULL"
if has_col edges to_node_id; then
TO_EXPR="to_node_id"
elif has_col edges to_id; then
TO_EXPR="to_id"
fi
SOURCE_EXPR="'legacy'"
if has_col edges source; then
SOURCE_EXPR="source"
fi
CREATED_AT_EXPR="CURRENT_TIMESTAMP"
if has_col edges created_at; then
CREATED_AT_EXPR="created_at"
fi
CONTEXT_EXPR="NULL"
if has_col edges context; then
CONTEXT_EXPR="context"
fi
EXPLANATION_EXPR="NULL"
if has_col edges explanation; then
EXPLANATION_EXPR="explanation"
elif has_col edges description; then
EXPLANATION_EXPR="description"
elif has_col edges context; then
EXPLANATION_EXPR="CASE WHEN json_valid(context) THEN json_extract(context, '\$.explanation') ELSE NULL END"
fi
"$SQLITE_BIN" "$DB_PATH" <<SQL
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
DROP INDEX IF EXISTS idx_edges_from;
DROP INDEX IF EXISTS idx_edges_to;
ALTER TABLE edges RENAME TO edges_legacy_migration;
CREATE TABLE edges (
id INTEGER PRIMARY KEY,
from_node_id INTEGER NOT NULL,
to_node_id INTEGER NOT NULL,
source TEXT,
created_at TEXT,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
INSERT INTO edges (id, from_node_id, to_node_id, source, created_at, context, explanation)
SELECT
id,
${FROM_EXPR},
${TO_EXPR},
${SOURCE_EXPR},
COALESCE(${CREATED_AT_EXPR}, CURRENT_TIMESTAMP),
${CONTEXT_EXPR},
${EXPLANATION_EXPR}
FROM edges_legacy_migration
WHERE ${FROM_EXPR} IS NOT NULL
AND ${TO_EXPR} IS NOT NULL;
DROP TABLE edges_legacy_migration;
COMMIT;
PRAGMA foreign_keys=ON;
SQL
fi
if ! has_col edges explanation; then
echo "Adding edges.explanation"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE edges ADD COLUMN explanation TEXT;"
if has_col edges context; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
UPDATE edges
SET explanation = CASE
WHEN json_valid(context) THEN json_extract(context, '$.explanation')
ELSE explanation
END
WHERE explanation IS NULL
AND context IS NOT NULL;
SQL
fi
fi
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node_id);
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node_id);
SQL
fi
echo "Dropping legacy episodic/semantic memory tables if they exist..."
if has_table chats; then
NEEDS_CHAT_REWRITE=0
if has_col chats focused_memory_id; then
NEEDS_CHAT_REWRITE=1
fi
for required_col in chat_type helper_name agent_type delegation_id user_message assistant_message thread_id focused_node_id created_at metadata; do
if ! has_col chats "$required_col"; then
NEEDS_CHAT_REWRITE=1
break
fi
done
if [ "$NEEDS_CHAT_REWRITE" = "1" ]; then
echo " Migrating legacy chats table to canonical schema"
CHAT_TYPE_EXPR="NULL"
if has_col chats chat_type; then
CHAT_TYPE_EXPR="chat_type"
fi
HELPER_NAME_EXPR="NULL"
if has_col chats helper_name; then
HELPER_NAME_EXPR="helper_name"
elif has_col chats title; then
HELPER_NAME_EXPR="title"
fi
AGENT_TYPE_EXPR="'orchestrator'"
if has_col chats agent_type; then
AGENT_TYPE_EXPR="COALESCE(agent_type, 'orchestrator')"
fi
DELEGATION_ID_EXPR="NULL"
if has_col chats delegation_id; then
DELEGATION_ID_EXPR="delegation_id"
fi
USER_MESSAGE_EXPR="NULL"
if has_col chats user_message; then
USER_MESSAGE_EXPR="user_message"
fi
ASSISTANT_MESSAGE_EXPR="NULL"
if has_col chats assistant_message; then
ASSISTANT_MESSAGE_EXPR="assistant_message"
fi
THREAD_ID_EXPR="NULL"
if has_col chats thread_id; then
THREAD_ID_EXPR="thread_id"
fi
FOCUSED_NODE_ID_EXPR="NULL"
if has_col chats focused_node_id; then
FOCUSED_NODE_ID_EXPR="focused_node_id"
fi
CREATED_AT_CHAT_EXPR="CURRENT_TIMESTAMP"
if has_col chats created_at; then
CREATED_AT_CHAT_EXPR="created_at"
fi
METADATA_CHAT_EXPR="NULL"
if has_col chats metadata; then
METADATA_CHAT_EXPR="metadata"
fi
"$SQLITE_BIN" "$DB_PATH" <<SQL
if "$SQLITE_BIN" "$DB_PATH" -json "PRAGMA table_info(chats);" | grep -q 'focused_memory_id'; then
echo " Removing legacy chats.focused_memory_id column"
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
DROP INDEX IF EXISTS idx_chats_thread;
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
CREATE TABLE chats (
id INTEGER PRIMARY KEY,
@@ -381,17 +194,9 @@ INSERT INTO chats (
user_message, assistant_message, thread_id, focused_node_id,
created_at, metadata
)
SELECT id,
${CHAT_TYPE_EXPR},
${HELPER_NAME_EXPR},
${AGENT_TYPE_EXPR},
${DELEGATION_ID_EXPR},
${USER_MESSAGE_EXPR},
${ASSISTANT_MESSAGE_EXPR},
${THREAD_ID_EXPR},
${FOCUSED_NODE_ID_EXPR},
COALESCE(${CREATED_AT_CHAT_EXPR}, CURRENT_TIMESTAMP),
${METADATA_CHAT_EXPR}
SELECT id, chat_type, helper_name, agent_type, delegation_id,
user_message, assistant_message, thread_id, focused_node_id,
created_at, metadata
FROM chats_legacy_cleanup;
DROP TABLE chats_legacy_cleanup;
CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);
@@ -465,44 +270,8 @@ if has_table nodes; then
echo "Adding nodes.source"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN source TEXT;"
fi
if ! has_col nodes chunk_status; then
echo "Adding nodes.chunk_status"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN chunk_status TEXT DEFAULT 'not_chunked';"
fi
if ! has_col nodes context_id; then
echo "Adding nodes.context_id"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN context_id INTEGER REFERENCES contexts(id) ON DELETE SET NULL;"
fi
fi
if has_table contexts; then
if ! has_col contexts description; then
echo "Adding contexts.description"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE contexts ADD COLUMN description TEXT NOT NULL DEFAULT '';"
fi
if ! has_col contexts icon; then
echo "Adding contexts.icon"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE contexts ADD COLUMN icon TEXT;"
fi
if ! has_col contexts created_at; then
echo "Adding contexts.created_at"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE contexts ADD COLUMN created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;"
fi
if ! has_col contexts updated_at; then
echo "Adding contexts.updated_at"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE contexts ADD COLUMN updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;"
fi
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
UPDATE contexts
SET description = COALESCE(NULLIF(TRIM(description), ''), name)
WHERE description IS NULL OR LENGTH(TRIM(description)) = 0;
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
SQL
fi
"$SQLITE_BIN" "$DB_PATH" "CREATE INDEX IF NOT EXISTS idx_nodes_context_id ON nodes(context_id);"
# --- Additive migrations (do first) ---
# Add event_date column (2026-02-15)
@@ -602,6 +371,18 @@ if has_table edges && has_col edges user_feedback; then
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE edges DROP COLUMN user_feedback;"
fi
# Add explanation column to edges if missing
if has_table edges && ! has_col edges explanation; then
echo "Adding edges.explanation"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE edges ADD COLUMN explanation TEXT;"
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
UPDATE edges
SET explanation = json_extract(context, '$.explanation')
WHERE explanation IS NULL
AND json_extract(context, '$.explanation') IS NOT NULL;
SQL
fi
if has_table chunks; then
if ! has_col chunks embedding_type; then
echo "Adding chunks.embedding_type"
@@ -669,6 +450,69 @@ SELECT n.id,
FROM nodes n;
SQL
echo "Ensuring FTS tables and triggers exist..."
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
DROP TRIGGER IF EXISTS nodes_fts_ai;
DROP TRIGGER IF EXISTS nodes_fts_ad;
DROP TRIGGER IF EXISTS nodes_fts_au;
DROP TRIGGER IF EXISTS chunks_fts_ai;
DROP TRIGGER IF EXISTS chunks_fts_ad;
DROP TRIGGER IF EXISTS chunks_fts_au;
DROP TABLE IF EXISTS nodes_fts;
DROP TABLE IF EXISTS chunks_fts;
CREATE VIRTUAL TABLE nodes_fts USING fts5(
title,
source,
description,
content='nodes',
content_rowid='id'
);
CREATE VIRTUAL TABLE chunks_fts USING fts5(
text,
content='chunks',
content_rowid='id'
);
INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild');
INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild');
CREATE TRIGGER nodes_fts_ai AFTER INSERT ON nodes BEGIN
INSERT INTO nodes_fts(rowid, title, source, description)
VALUES (new.id, new.title, new.source, new.description);
END;
CREATE TRIGGER nodes_fts_ad AFTER DELETE ON nodes BEGIN
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
VALUES('delete', old.id, old.title, old.source, old.description);
END;
CREATE TRIGGER nodes_fts_au AFTER UPDATE OF title, source, description ON nodes BEGIN
INSERT INTO nodes_fts(nodes_fts, rowid, title, source, description)
VALUES('delete', old.id, old.title, old.source, old.description);
INSERT INTO nodes_fts(rowid, title, source, description)
VALUES (new.id, new.title, new.source, new.description);
END;
CREATE TRIGGER chunks_fts_ai AFTER INSERT ON chunks BEGIN
INSERT INTO chunks_fts(rowid, text)
VALUES (new.id, new.text);
END;
CREATE TRIGGER chunks_fts_ad AFTER DELETE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text)
VALUES('delete', old.id, old.text);
END;
CREATE TRIGGER chunks_fts_au AFTER UPDATE OF text ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text)
VALUES('delete', old.id, old.text);
INSERT INTO chunks_fts(rowid, text)
VALUES (new.id, new.text);
END;
SQL
echo "Ensuring logs table and triggers exist (migrating from memory if needed)..."
# migrate memory -> logs if needed
scripts/database/sqlite-repair-fts.sh
+4 -19
View File
@@ -113,15 +113,6 @@ CREATE TABLE agents (
prompts TEXT
);
CREATE TABLE contexts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL,
icon TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE nodes (
id INTEGER PRIMARY KEY,
title TEXT,
@@ -135,9 +126,7 @@ CREATE TABLE nodes (
embedding BLOB,
embedding_updated_at TEXT,
embedding_text TEXT,
chunk_status TEXT DEFAULT 'not_chunked',
context_id INTEGER,
FOREIGN KEY (context_id) REFERENCES contexts(id) ON DELETE SET NULL
chunk_status TEXT DEFAULT 'not_chunked'
);
CREATE TABLE edges (
@@ -157,8 +146,7 @@ CREATE TABLE chunks (
node_id INTEGER NOT NULL,
chunk_idx INTEGER,
text TEXT NOT NULL,
embedding BLOB,
embedding_type TEXT DEFAULT 'openai',
embedding_type TEXT DEFAULT 'text-embedding-3-small',
metadata TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
@@ -217,20 +205,17 @@ CREATE TABLE dimension_migration_snapshots (
INSERT INTO schema_version SELECT version, applied_at, description FROM source.schema_version;
INSERT INTO agents SELECT id, key, display_name, role, system_prompt, available_tools, model, description, enabled, created_at, updated_at, memory, prompts FROM source.agents;
INSERT INTO contexts SELECT id, name, description, icon, created_at, updated_at FROM source.contexts;
INSERT INTO nodes SELECT id, title, description, source, link, event_date, created_at, updated_at, metadata, embedding, embedding_updated_at, embedding_text, chunk_status, context_id FROM source.nodes;
INSERT INTO nodes SELECT id, title, description, source, link, event_date, created_at, updated_at, metadata, embedding, embedding_updated_at, embedding_text, chunk_status FROM source.nodes;
INSERT INTO edges SELECT id, from_node_id, to_node_id, source, created_at, context, explanation FROM source.edges;
INSERT INTO chunks SELECT id, node_id, chunk_idx, text, embedding, embedding_type, metadata, created_at FROM source.chunks;
INSERT INTO chunks SELECT id, node_id, chunk_idx, text, embedding_type, metadata, created_at FROM source.chunks;
INSERT INTO chats SELECT id, chat_type, helper_name, agent_type, delegation_id, user_message, assistant_message, thread_id, focused_node_id, created_at, metadata FROM source.chats;
INSERT INTO logs SELECT id, ts, table_name, action, row_id, summary, snapshot_json, enriched_summary FROM source.logs;
INSERT INTO voice_usage SELECT id, chat_id, session_id, helper_name, request_id, message_id, voice, model, chars, cost_usd, duration_ms, text_preview, created_at FROM source.voice_usage;
INSERT INTO dimension_migration_snapshots SELECT id, migrated_at, dimension_count, assignment_count, payload FROM source.dimension_migration_snapshots;
CREATE UNIQUE INDEX idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
CREATE INDEX idx_edges_from ON edges(from_node_id);
CREATE INDEX idx_edges_to ON edges(to_node_id);
CREATE INDEX idx_nodes_updated_at ON nodes(updated_at DESC);
CREATE INDEX idx_nodes_context_id ON nodes(context_id);
CREATE INDEX idx_chunks_node_id ON chunks(node_id);
CREATE INDEX idx_chunks_by_node ON chunks(node_id);
CREATE INDEX idx_chunks_by_node_idx ON chunks(node_id, chunk_idx);
scripts/dev/bootstrap-local.mjs
-35
View File
@@ -153,16 +153,6 @@ function ensureCoreSchema(db) {
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
);
CREATE TABLE IF NOT EXISTS contexts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
icon TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY,
ts TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP),
@@ -408,31 +398,6 @@ function ensureCoreSchema(db) {
if (!hasNodeCol('chunk_status')) {
db.exec("ALTER TABLE nodes ADD COLUMN chunk_status TEXT DEFAULT 'not_chunked';");
}
if (!hasNodeCol('context_id')) {
db.exec('ALTER TABLE nodes ADD COLUMN context_id INTEGER REFERENCES contexts(id) ON DELETE SET NULL;');
}
const contextCols = db.prepare('PRAGMA table_info(contexts)').all().map(col => col.name);
const hasContextCol = (name) => contextCols.includes(name);
if (!hasContextCol('description')) {
db.exec("ALTER TABLE contexts ADD COLUMN description TEXT NOT NULL DEFAULT '';");
}
if (!hasContextCol('icon')) {
db.exec('ALTER TABLE contexts ADD COLUMN icon TEXT;');
}
if (!hasContextCol('created_at')) {
db.exec("ALTER TABLE contexts ADD COLUMN created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
}
if (!hasContextCol('updated_at')) {
db.exec("ALTER TABLE contexts ADD COLUMN updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
}
db.exec(`
UPDATE contexts
SET description = COALESCE(NULLIF(TRIM(description), ''), name)
WHERE description IS NULL OR LENGTH(TRIM(description)) = 0;
CREATE INDEX IF NOT EXISTS idx_nodes_context_id ON nodes(context_id);
`);
if (hasNodeCol('content')) {
db.exec(`