Initial commit: RA-H Open Source Edition
Local-first knowledge management system with BYO API keys. Features: - 3-panel UI (Nodes | Focus | Helpers) - SQLite + sqlite-vec for vector search - Agent system (Easy/Hard mode orchestrators) - Content extraction (YouTube, PDF, web) - Integrate workflow for connection discovery - Dimension system with auto-assignment Tech stack: - Next.js 15 + TypeScript + Tailwind CSS - Anthropic (Claude) + OpenAI (GPT) via Vercel AI SDK Setup: npm install && npm rebuild better-sqlite3 scripts/dev/bootstrap-local.sh npm run dev MIT License
This commit is contained in:
“BeeRad”
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
# Export all PostgreSQL data to CSV files for SQLite migration
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "Starting PostgreSQL data export..."
|
||||
|
||||
# Ensure tmp directory exists
|
||||
mkdir -p tmp/migrate
|
||||
|
||||
# Database connection string
|
||||
DB_URL="postgresql://rah_user:rah_password@localhost:5432/rah_db"
|
||||
|
||||
# Export nodes
|
||||
echo "Exporting nodes..."
|
||||
psql "$DB_URL" -c "\COPY (SELECT id, title, content, link, created_at, updated_at FROM nodes ORDER BY id) TO STDOUT WITH CSV HEADER" > tmp/migrate/nodes.csv
|
||||
|
||||
# Export dimensions (unnest array into rows)
|
||||
echo "Exporting dimensions (array to rows)..."
|
||||
psql "$DB_URL" -c "\COPY (SELECT n.id AS node_id, TRIM(unnest(n.dimensions)) AS dimension FROM nodes n WHERE n.dimensions IS NOT NULL AND array_length(n.dimensions, 1) > 0 ORDER BY n.id) TO STDOUT WITH CSV HEADER" > tmp/migrate/node_dimensions.csv
|
||||
|
||||
# Export chunks (without embeddings)
|
||||
echo "Exporting chunks..."
|
||||
psql "$DB_URL" -c "\COPY (SELECT id, node_id, chunk_idx, text, created_at FROM chunks ORDER BY id) TO STDOUT WITH CSV HEADER" > tmp/migrate/chunks.csv
|
||||
|
||||
# Export embeddings as CSV text (handle NULL embeddings)
|
||||
echo "Exporting embeddings..."
|
||||
psql "$DB_URL" -c "\COPY (SELECT id AS chunk_id, CASE WHEN embedding IS NOT NULL THEN trim(both '[]' from embedding::text) ELSE NULL END AS embedding_csv FROM chunks WHERE embedding IS NOT NULL ORDER BY id) TO STDOUT WITH CSV HEADER" > tmp/migrate/chunk_embeddings.csv
|
||||
|
||||
# Export edges
|
||||
echo "Exporting edges..."
|
||||
psql "$DB_URL" -c "\COPY (SELECT id, from_node_id, to_node_id, source, created_at FROM edges ORDER BY id) TO STDOUT WITH CSV HEADER" > tmp/migrate/edges.csv
|
||||
|
||||
# Export chats (for completeness)
|
||||
echo "Exporting chats..."
|
||||
psql "$DB_URL" -c "\COPY (SELECT id, chat_type, helper_name, user_message, assistant_message, thread_id, focused_node_id, created_at FROM chats ORDER BY id) TO STDOUT WITH CSV HEADER" > tmp/migrate/chats.csv
|
||||
|
||||
# Get counts for verification
|
||||
echo ""
|
||||
echo "Export complete! Counts:"
|
||||
echo "------------------------"
|
||||
echo -n "Nodes: "
|
||||
psql "$DB_URL" -t -c "SELECT COUNT(*) FROM nodes"
|
||||
echo -n "Dimensions: "
|
||||
psql "$DB_URL" -t -c "SELECT COUNT(*) FROM (SELECT unnest(dimensions) FROM nodes WHERE dimensions IS NOT NULL) AS d"
|
||||
echo -n "Chunks: "
|
||||
psql "$DB_URL" -t -c "SELECT COUNT(*) FROM chunks"
|
||||
echo -n "Chunks with embeddings: "
|
||||
psql "$DB_URL" -t -c "SELECT COUNT(*) FROM chunks WHERE embedding IS NOT NULL"
|
||||
echo -n "Edges: "
|
||||
psql "$DB_URL" -t -c "SELECT COUNT(*) FROM edges"
|
||||
echo -n "Chats: "
|
||||
psql "$DB_URL" -t -c "SELECT COUNT(*) FROM chats"
|
||||
|
||||
echo ""
|
||||
echo "Files exported to tmp/migrate/"
|
||||
echo "Next step: Run ./scripts/migrate/import_to_sqlite.sh"
|
||||
Executable
+148
@@ -0,0 +1,148 @@
|
||||
#!/bin/bash
|
||||
# Simplified import without FTS complications
|
||||
|
||||
set -e
|
||||
|
||||
echo "Starting simplified SQLite import..."
|
||||
|
||||
# Remove old database
|
||||
rm -f rah_trial.db
|
||||
|
||||
# Create basic schema first
|
||||
sqlite3 rah_trial.db <<'SQL'
|
||||
-- Basic settings
|
||||
PRAGMA foreign_keys = ON;
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
PRAGMA cache_size = -200000;
|
||||
PRAGMA temp_store = MEMORY;
|
||||
PRAGMA mmap_size = 268435456;
|
||||
|
||||
-- Create tables without FTS first
|
||||
CREATE TABLE nodes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
title TEXT,
|
||||
content TEXT,
|
||||
link TEXT,
|
||||
created_at TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE node_dimensions (
|
||||
node_id INTEGER NOT NULL,
|
||||
dimension TEXT NOT NULL,
|
||||
PRIMARY KEY (node_id, dimension),
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE chunks (
|
||||
id INTEGER PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
chunk_idx INTEGER,
|
||||
text TEXT,
|
||||
created_at TEXT,
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE chunk_embeddings (
|
||||
chunk_id INTEGER PRIMARY KEY,
|
||||
embedding TEXT NOT NULL,
|
||||
FOREIGN KEY (chunk_id) REFERENCES chunks(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE edges (
|
||||
id INTEGER PRIMARY KEY,
|
||||
from_node_id INTEGER NOT NULL,
|
||||
to_node_id INTEGER NOT NULL,
|
||||
source TEXT,
|
||||
created_at TEXT,
|
||||
FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (to_node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id INTEGER PRIMARY KEY,
|
||||
chat_type TEXT,
|
||||
helper_name TEXT,
|
||||
user_message TEXT,
|
||||
assistant_message TEXT,
|
||||
thread_id TEXT,
|
||||
focused_node_id INTEGER,
|
||||
created_at TEXT,
|
||||
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Create indexes
|
||||
CREATE INDEX idx_dim_by_dimension ON node_dimensions(dimension, node_id);
|
||||
CREATE INDEX idx_dim_by_node ON node_dimensions(node_id, dimension);
|
||||
CREATE INDEX idx_chunks_by_node ON chunks(node_id);
|
||||
CREATE INDEX idx_chunks_by_node_idx ON chunks(node_id, chunk_idx);
|
||||
CREATE INDEX idx_edges_from ON edges(from_node_id);
|
||||
CREATE INDEX idx_edges_to ON edges(to_node_id);
|
||||
CREATE INDEX idx_chats_thread ON chats(thread_id);
|
||||
SQL
|
||||
|
||||
echo "Schema created. Importing data..."
|
||||
|
||||
# Import CSVs with proper handling
|
||||
sqlite3 rah_trial.db <<'SQL'
|
||||
.mode csv
|
||||
|
||||
-- Import nodes
|
||||
.import --skip 1 tmp/migrate/nodes.csv nodes
|
||||
|
||||
-- Import dimensions
|
||||
.import --skip 1 tmp/migrate/node_dimensions.csv node_dimensions
|
||||
|
||||
-- Import chunks
|
||||
.import --skip 1 tmp/migrate/chunks.csv chunks
|
||||
|
||||
-- Import embeddings
|
||||
.import --skip 1 tmp/migrate/chunk_embeddings.csv chunk_embeddings
|
||||
|
||||
-- Import edges (skip header)
|
||||
.import --skip 1 tmp/migrate/edges.csv edges
|
||||
|
||||
-- Import chats
|
||||
.import --skip 1 tmp/migrate/chats.csv chats
|
||||
|
||||
-- Add FTS after data is loaded
|
||||
CREATE VIRTUAL TABLE chunks_fts USING fts5(
|
||||
text,
|
||||
content=chunks,
|
||||
content_rowid=id,
|
||||
tokenize='porter ascii'
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
||||
title,
|
||||
content,
|
||||
content=nodes,
|
||||
content_rowid=id,
|
||||
tokenize='porter ascii'
|
||||
);
|
||||
|
||||
-- Populate FTS
|
||||
INSERT INTO chunks_fts(text) SELECT text FROM chunks;
|
||||
INSERT INTO nodes_fts(title, content) SELECT title, content FROM nodes;
|
||||
|
||||
-- Analyze for optimization
|
||||
ANALYZE;
|
||||
SQL
|
||||
|
||||
# Verify counts
|
||||
echo ""
|
||||
echo "Verifying import..."
|
||||
sqlite3 rah_trial.db <<'SQL'
|
||||
.mode list
|
||||
SELECT 'Nodes: ' || COUNT(*) FROM nodes;
|
||||
SELECT 'Chunks: ' || COUNT(*) FROM chunks;
|
||||
SELECT 'Embeddings: ' || COUNT(*) FROM chunk_embeddings;
|
||||
SELECT 'Dimensions: ' || COUNT(*) FROM node_dimensions;
|
||||
SELECT 'Edges: ' || COUNT(*) FROM edges;
|
||||
SELECT 'Chats: ' || COUNT(*) FROM chats;
|
||||
SELECT 'Database size: ' || ROUND(page_count * page_size / 1024.0 / 1024.0, 2) || ' MB' FROM pragma_page_count(), pragma_page_size();
|
||||
SQL
|
||||
|
||||
echo ""
|
||||
echo "✅ Import complete!"
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/bin/bash
|
||||
# Import CSV data into SQLite database
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "Starting SQLite database import..."
|
||||
echo "================================"
|
||||
|
||||
# Remove old database if exists
|
||||
if [ -f "rah_trial.db" ]; then
|
||||
echo "Removing existing rah_trial.db..."
|
||||
rm -f rah_trial.db
|
||||
fi
|
||||
|
||||
# Create database with schema
|
||||
echo "Creating database with optimized schema..."
|
||||
sqlite3 rah_trial.db < scripts/migrate/sqlite_schema.sql
|
||||
|
||||
# Import CSV files
|
||||
echo ""
|
||||
echo "Importing data from CSV files..."
|
||||
echo "--------------------------------"
|
||||
|
||||
sqlite3 rah_trial.db <<'EOF'
|
||||
.mode csv
|
||||
.headers on
|
||||
|
||||
-- Import nodes (handle embedded quotes/newlines)
|
||||
.print "Importing nodes..."
|
||||
.separator ","
|
||||
.import tmp/migrate/nodes.csv nodes_temp
|
||||
INSERT INTO nodes SELECT * FROM nodes_temp WHERE id != 'id';
|
||||
DROP TABLE nodes_temp;
|
||||
|
||||
-- Import node dimensions
|
||||
.print "Importing dimensions..."
|
||||
.import tmp/migrate/node_dimensions.csv node_dimensions
|
||||
|
||||
-- Import chunks
|
||||
.print "Importing chunks..."
|
||||
.separator ","
|
||||
.import tmp/migrate/chunks.csv chunks_temp
|
||||
INSERT INTO chunks SELECT * FROM chunks_temp WHERE id != 'id';
|
||||
DROP TABLE chunks_temp;
|
||||
|
||||
-- Import embeddings
|
||||
.print "Importing embeddings (this may take a moment)..."
|
||||
.import tmp/migrate/chunk_embeddings.csv chunk_embeddings
|
||||
|
||||
-- Import edges
|
||||
.print "Importing edges..."
|
||||
.import tmp/migrate/edges.csv edges
|
||||
|
||||
-- Import chats
|
||||
.print "Importing chats..."
|
||||
.separator ","
|
||||
.import tmp/migrate/chats.csv chats_temp
|
||||
INSERT INTO chats SELECT * FROM chats_temp WHERE id != 'id';
|
||||
DROP TABLE chats_temp;
|
||||
|
||||
-- Populate FTS indexes
|
||||
.print ""
|
||||
.print "Building full-text search indexes..."
|
||||
INSERT INTO chunks_fts(rowid, text) SELECT id, text FROM chunks;
|
||||
INSERT INTO nodes_fts(rowid, title, content) SELECT id, title, content FROM nodes;
|
||||
|
||||
-- Run ANALYZE for query optimization
|
||||
.print "Analyzing tables for query optimization..."
|
||||
ANALYZE;
|
||||
|
||||
-- Verify imports with counts
|
||||
.print ""
|
||||
.print "Import complete! Verifying counts:"
|
||||
.print "===================================="
|
||||
SELECT printf('Nodes: %d', COUNT(*)) FROM nodes;
|
||||
SELECT printf('Node dimensions: %d', COUNT(*)) FROM node_dimensions;
|
||||
SELECT printf('Unique dimensions: %d', COUNT(DISTINCT dimension)) FROM node_dimensions;
|
||||
SELECT printf('Chunks: %d', COUNT(*)) FROM chunks;
|
||||
SELECT printf('Chunks with embeddings: %d', COUNT(*)) FROM chunk_embeddings;
|
||||
SELECT printf('Edges: %d', COUNT(*)) FROM edges;
|
||||
SELECT printf('Chats: %d', COUNT(*)) FROM chats;
|
||||
SELECT printf('FTS chunks indexed: %d', COUNT(*)) FROM chunks_fts;
|
||||
SELECT printf('FTS nodes indexed: %d', COUNT(*)) FROM nodes_fts;
|
||||
|
||||
-- Database statistics
|
||||
.print ""
|
||||
.print "Database Statistics:"
|
||||
.print "===================="
|
||||
SELECT printf('Database size: %.2f MB', page_count * page_size / 1024.0 / 1024.0) FROM pragma_page_count(), pragma_page_size();
|
||||
SELECT printf('Cache size: %.0f MB', cache_size * -1 / 1000.0) FROM pragma_cache_size();
|
||||
SELECT printf('Journal mode: %s', journal_mode) FROM pragma_journal_mode();
|
||||
|
||||
-- Sample data verification
|
||||
.print ""
|
||||
.print "Sample data (first 3 nodes):"
|
||||
.print "============================="
|
||||
.mode column
|
||||
.width 10 40
|
||||
SELECT id, substr(title, 1, 40) as title FROM nodes LIMIT 3;
|
||||
|
||||
.print ""
|
||||
.print "Sample dimensions for node 5:"
|
||||
.print "=============================="
|
||||
SELECT dimension FROM node_dimensions WHERE node_id = 5 LIMIT 5;
|
||||
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
echo "✅ SQLite database created successfully: rah_trial.db"
|
||||
echo ""
|
||||
echo "Database is ready for testing!"
|
||||
echo "Next steps:"
|
||||
echo " 1. Install sqlite-vec extension for vector search (optional)"
|
||||
echo " 2. Run performance benchmarks"
|
||||
echo " 3. Compare with PostgreSQL baseline"
|
||||
Reference in New Issue
Block a user