SyslogSolution/ra-h-os
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: port holistic node refinement contract

Browse Source
This commit is contained in:
“BeeRad”
2026-04-11 21:37:52 +10:00
parent 35f9ecf89c
commit 3ae46245ec
119 changed files with 6596 additions and 10982 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/README.md
+4 -4
View File
@@ -13,7 +13,7 @@ scripts/
│ ├── duplicate-helper.js # Create new helpers
│ └── delete-helper.js # Remove helpers
└── migrations/ # Completed one-time migrations
└── cleanup-dimensions.sql
└── (historical migration artifacts)
```
## Database Scripts
@@ -24,7 +24,7 @@ npm run backup
```
- Creates timestamped backup in `/backups/` folder
- Example: `rah_backup_20250902_102846.sql`
- Includes all nodes, chunks, edges, dimensions
- Includes all nodes, chunks, edges, contexts, and migration snapshots
### Restore from Backup
```bash
@@ -59,10 +59,10 @@ node scripts/helpers/delete-helper.js "helper-id"
- All nodes (42,000+ knowledge items)
- Content chunks and embeddings
- Node connections/edges
- Dimensions and metadata
- Contexts, metadata, and migration snapshots
- Database schema and indexes
## Notes
- Backups typically ~250MB for 40k+ nodes
- Store backups safely - they contain your entire knowledge base
- Helper scripts are integrated with the application - modify with caution
- Helper scripts are integrated with the application - modify with caution
scripts/database/sqlite-ensure-app-schema.sh
+344 -50
View File
@@ -110,7 +110,7 @@ VALUES (
SQL
fi
echo "Ensuring core tables exist (nodes, chunks, edges, chats, node_dimensions)..."
echo "Ensuring core tables exist (nodes, chunks, edges, chats, contexts)..."
if ! has_table nodes; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
@@ -132,6 +132,20 @@ CREATE TABLE nodes (
SQL
fi
if ! has_table contexts; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE TABLE contexts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL,
icon TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
SQL
fi
if ! has_table chunks; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE TABLE chunks (
@@ -158,6 +172,7 @@ CREATE TABLE edges (
source TEXT,
created_at TEXT,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
@@ -166,14 +181,186 @@ CREATE INDEX idx_edges_to ON edges(to_node_id);
SQL
fi
if has_table edges; then
NEEDS_EDGE_REWRITE=0
if ! has_col edges from_node_id || ! has_col edges to_node_id || ! has_col edges source || ! has_col edges created_at || ! has_col edges context; then
NEEDS_EDGE_REWRITE=1
fi
if has_col edges from_id || has_col edges to_id || has_col edges description || has_col edges updated_at; then
NEEDS_EDGE_REWRITE=1
fi
if [ "$NEEDS_EDGE_REWRITE" = "1" ]; then
echo "Migrating legacy edges table to canonical schema"
FROM_EXPR="NULL"
if has_col edges from_node_id; then
FROM_EXPR="from_node_id"
elif has_col edges from_id; then
FROM_EXPR="from_id"
fi
TO_EXPR="NULL"
if has_col edges to_node_id; then
TO_EXPR="to_node_id"
elif has_col edges to_id; then
TO_EXPR="to_id"
fi
SOURCE_EXPR="'legacy'"
if has_col edges source; then
SOURCE_EXPR="source"
fi
CREATED_AT_EXPR="CURRENT_TIMESTAMP"
if has_col edges created_at; then
CREATED_AT_EXPR="created_at"
fi
CONTEXT_EXPR="NULL"
if has_col edges context; then
CONTEXT_EXPR="context"
fi
EXPLANATION_EXPR="NULL"
if has_col edges explanation; then
EXPLANATION_EXPR="explanation"
elif has_col edges description; then
EXPLANATION_EXPR="description"
elif has_col edges context; then
EXPLANATION_EXPR="CASE WHEN json_valid(context) THEN json_extract(context, '\$.explanation') ELSE NULL END"
fi
"$SQLITE_BIN" "$DB_PATH" <<SQL
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
DROP INDEX IF EXISTS idx_edges_from;
DROP INDEX IF EXISTS idx_edges_to;
ALTER TABLE edges RENAME TO edges_legacy_migration;
CREATE TABLE edges (
id INTEGER PRIMARY KEY,
from_node_id INTEGER NOT NULL,
to_node_id INTEGER NOT NULL,
source TEXT,
created_at TEXT,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
INSERT INTO edges (id, from_node_id, to_node_id, source, created_at, context, explanation)
SELECT
id,
${FROM_EXPR},
${TO_EXPR},
${SOURCE_EXPR},
COALESCE(${CREATED_AT_EXPR}, CURRENT_TIMESTAMP),
${CONTEXT_EXPR},
${EXPLANATION_EXPR}
FROM edges_legacy_migration
WHERE ${FROM_EXPR} IS NOT NULL
AND ${TO_EXPR} IS NOT NULL;
DROP TABLE edges_legacy_migration;
COMMIT;
PRAGMA foreign_keys=ON;
SQL
fi
if ! has_col edges explanation; then
echo "Adding edges.explanation"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE edges ADD COLUMN explanation TEXT;"
if has_col edges context; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
UPDATE edges
SET explanation = CASE
WHEN json_valid(context) THEN json_extract(context, '$.explanation')
ELSE explanation
END
WHERE explanation IS NULL
AND context IS NOT NULL;
SQL
fi
fi
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node_id);
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node_id);
SQL
fi
echo "Dropping legacy episodic/semantic memory tables if they exist..."
if has_table chats; then
if "$SQLITE_BIN" "$DB_PATH" -json "PRAGMA table_info(chats);" | grep -q 'focused_memory_id'; then
echo " Removing legacy chats.focused_memory_id column"
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
NEEDS_CHAT_REWRITE=0
if has_col chats focused_memory_id; then
NEEDS_CHAT_REWRITE=1
fi
for required_col in chat_type helper_name agent_type delegation_id user_message assistant_message thread_id focused_node_id created_at metadata; do
if ! has_col chats "$required_col"; then
NEEDS_CHAT_REWRITE=1
break
fi
done
if [ "$NEEDS_CHAT_REWRITE" = "1" ]; then
echo " Migrating legacy chats table to canonical schema"
CHAT_TYPE_EXPR="NULL"
if has_col chats chat_type; then
CHAT_TYPE_EXPR="chat_type"
fi
HELPER_NAME_EXPR="NULL"
if has_col chats helper_name; then
HELPER_NAME_EXPR="helper_name"
elif has_col chats title; then
HELPER_NAME_EXPR="title"
fi
AGENT_TYPE_EXPR="'orchestrator'"
if has_col chats agent_type; then
AGENT_TYPE_EXPR="COALESCE(agent_type, 'orchestrator')"
fi
DELEGATION_ID_EXPR="NULL"
if has_col chats delegation_id; then
DELEGATION_ID_EXPR="delegation_id"
fi
USER_MESSAGE_EXPR="NULL"
if has_col chats user_message; then
USER_MESSAGE_EXPR="user_message"
fi
ASSISTANT_MESSAGE_EXPR="NULL"
if has_col chats assistant_message; then
ASSISTANT_MESSAGE_EXPR="assistant_message"
fi
THREAD_ID_EXPR="NULL"
if has_col chats thread_id; then
THREAD_ID_EXPR="thread_id"
fi
FOCUSED_NODE_ID_EXPR="NULL"
if has_col chats focused_node_id; then
FOCUSED_NODE_ID_EXPR="focused_node_id"
fi
CREATED_AT_CHAT_EXPR="CURRENT_TIMESTAMP"
if has_col chats created_at; then
CREATED_AT_CHAT_EXPR="created_at"
fi
METADATA_CHAT_EXPR="NULL"
if has_col chats metadata; then
METADATA_CHAT_EXPR="metadata"
fi
"$SQLITE_BIN" "$DB_PATH" <<SQL
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
DROP INDEX IF EXISTS idx_chats_thread;
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
CREATE TABLE chats (
id INTEGER PRIMARY KEY,
@@ -194,9 +381,17 @@ INSERT INTO chats (
user_message, assistant_message, thread_id, focused_node_id,
created_at, metadata
)
SELECT id, chat_type, helper_name, agent_type, delegation_id,
user_message, assistant_message, thread_id, focused_node_id,
created_at, metadata
SELECT id,
${CHAT_TYPE_EXPR},
${HELPER_NAME_EXPR},
${AGENT_TYPE_EXPR},
${DELEGATION_ID_EXPR},
${USER_MESSAGE_EXPR},
${ASSISTANT_MESSAGE_EXPR},
${THREAD_ID_EXPR},
${FOCUSED_NODE_ID_EXPR},
COALESCE(${CREATED_AT_CHAT_EXPR}, CURRENT_TIMESTAMP),
${METADATA_CHAT_EXPR}
FROM chats_legacy_cleanup;
DROP TABLE chats_legacy_cleanup;
CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);
@@ -250,25 +445,11 @@ if has_table chats && ! has_col chats delegation_id; then
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE chats ADD COLUMN delegation_id INTEGER;"
fi
# chat_memory_state removed in final schema pass
# Drop dead chat_memory_state table (orphaned from removed memory pipeline)
echo "Dropping dead chat_memory_state table if present..."
"$SQLITE_BIN" "$DB_PATH" "DROP INDEX IF EXISTS idx_chat_memory_thread;"
"$SQLITE_BIN" "$DB_PATH" "DROP TABLE IF EXISTS chat_memory_state;"
echo "Dropping legacy agent_delegations table if present..."
"$SQLITE_BIN" "$DB_PATH" "DROP TABLE IF EXISTS agent_delegations;"
if ! has_table node_dimensions; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE TABLE node_dimensions (
node_id INTEGER NOT NULL,
dimension TEXT NOT NULL,
PRIMARY KEY (node_id, dimension),
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
) WITHOUT ROWID;
CREATE INDEX idx_dim_by_dimension ON node_dimensions(dimension, node_id);
CREATE INDEX idx_dim_by_node ON node_dimensions(node_id, dimension);
SQL
fi
echo "Checking/adding missing columns..."
if has_table nodes; then
@@ -284,7 +465,50 @@ if has_table nodes; then
echo "Adding nodes.source"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN source TEXT;"
fi
# is_pinned removed in final schema pass
if ! has_col nodes chunk_status; then
echo "Adding nodes.chunk_status"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN chunk_status TEXT DEFAULT 'not_chunked';"
fi
if ! has_col nodes context_id; then
echo "Adding nodes.context_id"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN context_id INTEGER REFERENCES contexts(id) ON DELETE SET NULL;"
fi
fi
if has_table contexts; then
if ! has_col contexts description; then
echo "Adding contexts.description"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE contexts ADD COLUMN description TEXT NOT NULL DEFAULT '';"
fi
if ! has_col contexts icon; then
echo "Adding contexts.icon"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE contexts ADD COLUMN icon TEXT;"
fi
if ! has_col contexts created_at; then
echo "Adding contexts.created_at"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE contexts ADD COLUMN created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;"
fi
if ! has_col contexts updated_at; then
echo "Adding contexts.updated_at"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE contexts ADD COLUMN updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;"
fi
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
UPDATE contexts
SET description = COALESCE(NULLIF(TRIM(description), ''), name)
WHERE description IS NULL OR LENGTH(TRIM(description)) = 0;
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
SQL
fi
"$SQLITE_BIN" "$DB_PATH" "CREATE INDEX IF NOT EXISTS idx_nodes_context_id ON nodes(context_id);"
# --- Additive migrations (do first) ---
# Add event_date column (2026-02-15)
if has_table nodes && ! has_col nodes event_date; then
echo "Adding nodes.event_date"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes ADD COLUMN event_date TEXT;"
fi
if has_table nodes && has_col nodes source; then
@@ -346,6 +570,38 @@ WHERE source IS NOT NULL
SQL
fi
# Backfill event_date from metadata.published_date where available
if has_table nodes && has_col nodes event_date; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
UPDATE nodes
SET event_date = json_extract(metadata, '$.published_date')
WHERE event_date IS NULL
AND json_extract(metadata, '$.published_date') IS NOT NULL;
SQL
fi
# --- Destructive migrations (do last, SQLite 3.35+ required) ---
# Drop deprecated nodes.type column
if has_table nodes && has_col nodes type; then
echo "Dropping deprecated nodes.type"
"$SQLITE_BIN" "$DB_PATH" "DROP INDEX IF EXISTS idx_nodes_type;"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes DROP COLUMN type;"
fi
# Drop deprecated nodes.is_pinned column
if has_table nodes && has_col nodes is_pinned; then
echo "Dropping deprecated nodes.is_pinned"
"$SQLITE_BIN" "$DB_PATH" "DROP INDEX IF EXISTS idx_nodes_pinned;"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE nodes DROP COLUMN is_pinned;"
fi
# Drop dead edges.user_feedback column
if has_table edges && has_col edges user_feedback; then
echo "Dropping dead edges.user_feedback"
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE edges DROP COLUMN user_feedback;"
fi
if has_table chunks; then
if ! has_col chunks embedding_type; then
echo "Adding chunks.embedding_type"
@@ -357,28 +613,44 @@ if has_table chunks; then
fi
fi
echo "Ensuring dimensions table exists..."
if ! has_table dimensions; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE TABLE dimensions (
name TEXT PRIMARY KEY,
description TEXT,
icon TEXT,
is_priority INTEGER DEFAULT 0,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
echo "Dropping legacy dimension tables after snapshot..."
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE TABLE IF NOT EXISTS dimension_migration_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
migrated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
dimension_count INTEGER NOT NULL,
assignment_count INTEGER NOT NULL,
payload TEXT
);
SQL
fi
if has_table dimensions; then
echo "Seeding default locked dimensions..."
for dimension in research ideas projects memory preferences; do
if has_table dimensions || has_table node_dimensions; then
SNAPSHOT_COUNT=$("$SQLITE_BIN" -readonly "$DB_PATH" "SELECT COUNT(*) FROM dimension_migration_snapshots;" 2>/dev/null || echo 0)
if [ "${SNAPSHOT_COUNT:-0}" = "0" ]; then
DIM_COUNT=0
ASSIGN_COUNT=0
PAYLOAD='[]'
if has_table dimensions; then
DIM_COUNT=$("$SQLITE_BIN" -readonly "$DB_PATH" "SELECT COUNT(*) FROM dimensions;" 2>/dev/null || echo 0)
fi
if has_table node_dimensions; then
ASSIGN_COUNT=$("$SQLITE_BIN" -readonly "$DB_PATH" "SELECT COUNT(*) FROM node_dimensions;" 2>/dev/null || echo 0)
PAYLOAD=$("$SQLITE_BIN" -readonly "$DB_PATH" "SELECT COALESCE(json_group_array(json_object('node_id', nd.node_id, 'dimension', nd.dimension, 'description', d.description, 'icon', d.icon, 'is_priority', d.is_priority)), '[]') FROM node_dimensions nd LEFT JOIN dimensions d ON d.name = nd.dimension;" 2>/dev/null || echo '[]')
fi
"$SQLITE_BIN" "$DB_PATH" <<SQL
INSERT INTO dimensions (name, is_priority, updated_at)
VALUES ('$dimension', 1, datetime('now'))
ON CONFLICT(name) DO UPDATE SET is_priority = 1, updated_at = datetime('now');
INSERT INTO dimension_migration_snapshots (dimension_count, assignment_count, payload)
VALUES (${DIM_COUNT:-0}, ${ASSIGN_COUNT:-0}, '${PAYLOAD//\'/''}');
SQL
fi
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
DROP INDEX IF EXISTS idx_dim_by_dimension;
DROP INDEX IF EXISTS idx_dim_by_node;
DROP TABLE IF EXISTS node_dimensions;
DROP TABLE IF EXISTS dimensions;
SQL
done
fi
echo "Refreshing helper view nodes_v..."
@@ -393,10 +665,7 @@ SELECT n.id,
n.event_date,
n.metadata,
n.created_at,
n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d
WHERE d.node_id = n.id), '[]') AS dimensions_json
n.updated_at
FROM nodes n;
SQL
@@ -441,6 +710,13 @@ CREATE INDEX IF NOT EXISTS idx_logs_table_ts ON logs(table_name, ts);
CREATE INDEX IF NOT EXISTS idx_logs_table_row ON logs(table_name, row_id);
SQL
# Performance indexes for common query patterns
echo "Ensuring performance indexes..."
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE INDEX IF NOT EXISTS idx_nodes_updated_at ON nodes(updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_chats_created_at ON chats(created_at DESC);
SQL
# triggers for nodes (drop/recreate to ensure enriched payloads)
if has_table nodes; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
@@ -604,11 +880,29 @@ if has_table helpers && has_col helpers fluid_context; then
"$SQLITE_BIN" "$DB_PATH" "ALTER TABLE helpers DROP COLUMN fluid_context;" || true
fi
echo "Ensuring performance indexes exist..."
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE INDEX IF NOT EXISTS idx_nodes_updated_at ON nodes(updated_at DESC);
CREATE INDEX IF NOT EXISTS idx_chats_created_at ON chats(created_at DESC);
echo "Ensuring voice_usage table exists..."
if ! has_table voice_usage; then
"$SQLITE_BIN" "$DB_PATH" <<'SQL'
CREATE TABLE voice_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
chat_id INTEGER,
session_id TEXT,
helper_name TEXT,
request_id TEXT,
message_id TEXT,
voice TEXT,
model TEXT,
chars INTEGER,
cost_usd REAL,
duration_ms INTEGER,
text_preview TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_voice_usage_session ON voice_usage(session_id, created_at);
CREATE INDEX IF NOT EXISTS idx_voice_usage_chat ON voice_usage(chat_id);
SQL
fi
echo "Running VACUUM and ANALYZE..."
"$SQLITE_BIN" "$DB_PATH" "VACUUM; ANALYZE;"
scripts/dev/bootstrap-local.mjs
+250 -30
View File
@@ -133,11 +133,10 @@ function ensureCoreSchema(db) {
source TEXT,
created_at TEXT,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node_id);
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node_id);
CREATE TABLE IF NOT EXISTS chats (
id INTEGER PRIMARY KEY,
@@ -153,24 +152,16 @@ function ensureCoreSchema(db) {
metadata TEXT,
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);
CREATE TABLE IF NOT EXISTS node_dimensions (
node_id INTEGER NOT NULL,
dimension TEXT NOT NULL,
PRIMARY KEY (node_id, dimension),
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS idx_dim_by_dimension ON node_dimensions(dimension, node_id);
CREATE INDEX IF NOT EXISTS idx_dim_by_node ON node_dimensions(node_id, dimension);
CREATE TABLE IF NOT EXISTS dimensions (
name TEXT PRIMARY KEY,
description TEXT,
CREATE TABLE IF NOT EXISTS contexts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
icon TEXT,
is_priority INTEGER DEFAULT 0,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_contexts_name_normalized ON contexts(LOWER(TRIM(name)));
CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY,
@@ -219,15 +210,6 @@ function ensureCoreSchema(db) {
`);
const now = new Date().toISOString();
const lockedDimensions = ['research', 'ideas', 'projects', 'memory', 'preferences'];
const insertDimension = db.prepare(`
INSERT INTO dimensions (name, is_priority, updated_at)
VALUES (?, 1, ?)
ON CONFLICT(name) DO UPDATE SET is_priority = 1, updated_at = excluded.updated_at
`);
for (const dimension of lockedDimensions) {
insertDimension.run(dimension, now);
}
const agentCount = Number(db.prepare('SELECT COUNT(*) as count FROM agents').get().count || 0);
if (agentCount === 0) {
@@ -249,6 +231,165 @@ function ensureCoreSchema(db) {
);
}
const edgeCols = db.prepare('PRAGMA table_info(edges)').all().map(col => col.name);
const hasEdgeCol = (name) => edgeCols.includes(name);
const needsLegacyEdgeRewrite =
!hasEdgeCol('from_node_id') ||
!hasEdgeCol('to_node_id') ||
!hasEdgeCol('source') ||
!hasEdgeCol('created_at') ||
!hasEdgeCol('context') ||
hasEdgeCol('from_id') ||
hasEdgeCol('to_id') ||
hasEdgeCol('description') ||
hasEdgeCol('updated_at');
if (needsLegacyEdgeRewrite) {
const fromExpr = hasEdgeCol('from_node_id')
? 'from_node_id'
: hasEdgeCol('from_id')
? 'from_id'
: 'NULL';
const toExpr = hasEdgeCol('to_node_id')
? 'to_node_id'
: hasEdgeCol('to_id')
? 'to_id'
: 'NULL';
const sourceExpr = hasEdgeCol('source') ? 'source' : "'legacy'";
const createdAtExpr = hasEdgeCol('created_at') ? 'created_at' : 'CURRENT_TIMESTAMP';
const contextExpr = hasEdgeCol('context') ? 'context' : 'NULL';
const explanationExpr = hasEdgeCol('explanation')
? 'explanation'
: hasEdgeCol('description')
? 'description'
: hasEdgeCol('context')
? "CASE WHEN json_valid(context) THEN json_extract(context, '$.explanation') ELSE NULL END"
: 'NULL';
console.log('Migrating legacy edges table to canonical schema');
db.exec('PRAGMA foreign_keys=OFF;');
db.exec(`
BEGIN TRANSACTION;
DROP INDEX IF EXISTS idx_edges_from;
DROP INDEX IF EXISTS idx_edges_to;
ALTER TABLE edges RENAME TO edges_legacy_migration;
CREATE TABLE edges (
id INTEGER PRIMARY KEY,
from_node_id INTEGER NOT NULL,
to_node_id INTEGER NOT NULL,
source TEXT,
created_at TEXT,
context TEXT,
explanation TEXT,
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
);
INSERT INTO edges (id, from_node_id, to_node_id, source, created_at, context, explanation)
SELECT
id,
${fromExpr},
${toExpr},
${sourceExpr},
COALESCE(${createdAtExpr}, CURRENT_TIMESTAMP),
${contextExpr},
${explanationExpr}
FROM edges_legacy_migration
WHERE ${fromExpr} IS NOT NULL
AND ${toExpr} IS NOT NULL;
DROP TABLE edges_legacy_migration;
COMMIT;
`);
db.exec('PRAGMA foreign_keys=ON;');
}
const refreshedEdgeCols = db.prepare('PRAGMA table_info(edges)').all().map(col => col.name);
if (!refreshedEdgeCols.includes('explanation')) {
db.exec('ALTER TABLE edges ADD COLUMN explanation TEXT;');
db.exec(`
UPDATE edges
SET explanation = CASE
WHEN json_valid(context) THEN json_extract(context, '$.explanation')
ELSE explanation
END
WHERE explanation IS NULL
AND context IS NOT NULL;
`);
}
db.exec(`
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node_id);
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node_id);
`);
const chatCols = db.prepare('PRAGMA table_info(chats)').all().map(col => col.name);
const hasChatCol = (name) => chatCols.includes(name);
const needsLegacyChatRewrite =
hasChatCol('focused_memory_id') ||
['chat_type', 'helper_name', 'agent_type', 'delegation_id', 'user_message', 'assistant_message', 'thread_id', 'focused_node_id', 'created_at', 'metadata']
.some((name) => !hasChatCol(name));
if (needsLegacyChatRewrite) {
const chatTypeExpr = hasChatCol('chat_type') ? 'chat_type' : 'NULL';
const helperNameExpr = hasChatCol('helper_name')
? 'helper_name'
: hasChatCol('title')
? 'title'
: 'NULL';
const agentTypeExpr = hasChatCol('agent_type')
? "COALESCE(agent_type, 'orchestrator')"
: "'orchestrator'";
const delegationIdExpr = hasChatCol('delegation_id') ? 'delegation_id' : 'NULL';
const userMessageExpr = hasChatCol('user_message') ? 'user_message' : 'NULL';
const assistantMessageExpr = hasChatCol('assistant_message') ? 'assistant_message' : 'NULL';
const threadIdExpr = hasChatCol('thread_id') ? 'thread_id' : 'NULL';
const focusedNodeIdExpr = hasChatCol('focused_node_id') ? 'focused_node_id' : 'NULL';
const createdAtChatExpr = hasChatCol('created_at') ? 'created_at' : 'CURRENT_TIMESTAMP';
const metadataChatExpr = hasChatCol('metadata') ? 'metadata' : 'NULL';
console.log('Migrating legacy chats table to canonical schema');
db.exec('PRAGMA foreign_keys=OFF;');
db.exec(`
BEGIN TRANSACTION;
DROP INDEX IF EXISTS idx_chats_thread;
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
CREATE TABLE chats (
id INTEGER PRIMARY KEY,
chat_type TEXT,
helper_name TEXT,
agent_type TEXT DEFAULT 'orchestrator',
delegation_id INTEGER,
user_message TEXT,
assistant_message TEXT,
thread_id TEXT,
focused_node_id INTEGER,
created_at TEXT DEFAULT (CURRENT_TIMESTAMP),
metadata TEXT,
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
);
INSERT INTO chats (
id, chat_type, helper_name, agent_type, delegation_id,
user_message, assistant_message, thread_id, focused_node_id,
created_at, metadata
)
SELECT
id,
${chatTypeExpr},
${helperNameExpr},
${agentTypeExpr},
${delegationIdExpr},
${userMessageExpr},
${assistantMessageExpr},
${threadIdExpr},
${focusedNodeIdExpr},
COALESCE(${createdAtChatExpr}, CURRENT_TIMESTAMP),
${metadataChatExpr}
FROM chats_legacy_cleanup;
DROP TABLE chats_legacy_cleanup;
COMMIT;
`);
db.exec('PRAGMA foreign_keys=ON;');
}
db.exec("CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);");
const nodeCols = db.prepare('PRAGMA table_info(nodes)').all().map(col => col.name);
const hasNodeCol = (name) => nodeCols.includes(name);
@@ -264,6 +405,34 @@ function ensureCoreSchema(db) {
if (!hasNodeCol('event_date')) {
db.exec('ALTER TABLE nodes ADD COLUMN event_date TEXT;');
}
if (!hasNodeCol('chunk_status')) {
db.exec("ALTER TABLE nodes ADD COLUMN chunk_status TEXT DEFAULT 'not_chunked';");
}
if (!hasNodeCol('context_id')) {
db.exec('ALTER TABLE nodes ADD COLUMN context_id INTEGER REFERENCES contexts(id) ON DELETE SET NULL;');
}
const contextCols = db.prepare('PRAGMA table_info(contexts)').all().map(col => col.name);
const hasContextCol = (name) => contextCols.includes(name);
if (!hasContextCol('description')) {
db.exec("ALTER TABLE contexts ADD COLUMN description TEXT NOT NULL DEFAULT '';");
}
if (!hasContextCol('icon')) {
db.exec('ALTER TABLE contexts ADD COLUMN icon TEXT;');
}
if (!hasContextCol('created_at')) {
db.exec("ALTER TABLE contexts ADD COLUMN created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
}
if (!hasContextCol('updated_at')) {
db.exec("ALTER TABLE contexts ADD COLUMN updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP;");
}
db.exec(`
UPDATE contexts
SET description = COALESCE(NULLIF(TRIM(description), ''), name)
WHERE description IS NULL OR LENGTH(TRIM(description)) = 0;
CREATE INDEX IF NOT EXISTS idx_nodes_context_id ON nodes(context_id);
`);
if (hasNodeCol('content')) {
db.exec(`
@@ -315,6 +484,60 @@ function ensureCoreSchema(db) {
AND (chunk_status IS NULL OR chunk_status != 'chunked');
`);
db.exec(`
CREATE TABLE IF NOT EXISTS dimension_migration_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
migrated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
dimension_count INTEGER NOT NULL,
assignment_count INTEGER NOT NULL,
payload TEXT
);
`);
const hasLegacyDimensions = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='dimensions'").get();
const hasLegacyNodeDimensions = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='node_dimensions'").get();
if (hasLegacyDimensions || hasLegacyNodeDimensions) {
const existingSnapshotCount = Number(db.prepare('SELECT COUNT(*) as count FROM dimension_migration_snapshots').get().count || 0);
if (existingSnapshotCount === 0) {
const dimensionCount = hasLegacyDimensions
? Number(db.prepare('SELECT COUNT(*) as count FROM dimensions').get().count || 0)
: 0;
const assignmentCount = hasLegacyNodeDimensions
? Number(db.prepare('SELECT COUNT(*) as count FROM node_dimensions').get().count || 0)
: 0;
const payload = hasLegacyNodeDimensions
? (db.prepare(`
SELECT COALESCE(
json_group_array(
json_object(
'node_id', nd.node_id,
'dimension', nd.dimension,
'description', d.description,
'icon', d.icon,
'is_priority', d.is_priority
)
),
'[]'
) AS payload
FROM node_dimensions nd
LEFT JOIN dimensions d ON d.name = nd.dimension
`).get().payload || '[]')
: '[]';
db.prepare(`
INSERT INTO dimension_migration_snapshots (dimension_count, assignment_count, payload)
VALUES (?, ?, ?)
`).run(dimensionCount, assignmentCount, payload);
}
db.exec(`
DROP INDEX IF EXISTS idx_dim_by_dimension;
DROP INDEX IF EXISTS idx_dim_by_node;
DROP TABLE IF EXISTS node_dimensions;
DROP TABLE IF EXISTS dimensions;
`);
}
db.exec('DROP VIEW IF EXISTS nodes_v;');
db.exec(`
CREATE VIEW nodes_v AS
@@ -326,10 +549,7 @@ function ensureCoreSchema(db) {
n.event_date,
n.metadata,
n.created_at,
n.updated_at,
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
FROM node_dimensions d
WHERE d.node_id = n.id), '[]') AS dimensions_json
n.updated_at
FROM nodes n;
`);
}