Initial commit: RA-H Open Source Edition
Local-first knowledge management system with BYO API keys. Features: - 3-panel UI (Nodes | Focus | Helpers) - SQLite + sqlite-vec for vector search - Agent system (Easy/Hard mode orchestrators) - Content extraction (YouTube, PDF, web) - Integrate workflow for connection discovery - Dimension system with auto-assignment Tech stack: - Next.js 15 + TypeScript + Tailwind CSS - Anthropic (Claude) + OpenAI (GPT) via Vercel AI SDK Setup: npm install && npm rebuild better-sqlite3 scripts/dev/bootstrap-local.sh npm run dev MIT License
This commit is contained in:
“BeeRad”
@@ -0,0 +1,113 @@
|
||||
import { embedNodeContent } from '@/services/embedding/ingestion';
|
||||
import { nodeService } from '@/services/database';
|
||||
|
||||
interface AutoEmbedTask {
|
||||
nodeId: number;
|
||||
force?: boolean;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_COOLDOWN_MS = 2 * 60 * 1000; // 2 minutes between automatic runs per node
|
||||
|
||||
export class AutoEmbedQueue {
|
||||
private readonly queue: number[] = [];
|
||||
private readonly pendingTasks = new Map<number, AutoEmbedTask>();
|
||||
private readonly running = new Set<number>();
|
||||
private readonly lastRunAt = new Map<number, number>();
|
||||
private readonly maxConcurrent = 1;
|
||||
private readonly cooldownMs = DEFAULT_COOLDOWN_MS;
|
||||
|
||||
enqueue(nodeId: number, task: Omit<AutoEmbedTask, 'nodeId'> = {}): boolean {
|
||||
const existing = this.pendingTasks.get(nodeId);
|
||||
if (!existing) {
|
||||
this.pendingTasks.set(nodeId, { nodeId, ...task });
|
||||
this.queue.push(nodeId);
|
||||
} else {
|
||||
existing.force = existing.force || task.force;
|
||||
existing.reason = existing.reason || task.reason;
|
||||
}
|
||||
|
||||
this.processQueue();
|
||||
return true;
|
||||
}
|
||||
|
||||
private processQueue() {
|
||||
if (this.running.size >= this.maxConcurrent) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextId = this.queue.shift();
|
||||
if (typeof nextId !== 'number') {
|
||||
return;
|
||||
}
|
||||
|
||||
const task = this.pendingTasks.get(nextId);
|
||||
if (!task) {
|
||||
// Task was removed; try next
|
||||
this.processQueue();
|
||||
return;
|
||||
}
|
||||
this.pendingTasks.delete(nextId);
|
||||
|
||||
const now = Date.now();
|
||||
const lastRun = this.lastRunAt.get(task.nodeId);
|
||||
if (!task.force && lastRun && now - lastRun < this.cooldownMs) {
|
||||
const delay = this.cooldownMs - (now - lastRun);
|
||||
setTimeout(() => this.enqueue(task.nodeId, task), delay);
|
||||
this.processQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
this.running.add(task.nodeId);
|
||||
this.executeTask(task)
|
||||
.catch(error => {
|
||||
console.error('[AutoEmbedQueue] Task failed', task.nodeId, error);
|
||||
})
|
||||
.finally(() => {
|
||||
this.running.delete(task.nodeId);
|
||||
this.lastRunAt.set(task.nodeId, Date.now());
|
||||
if (this.queue.length > 0) {
|
||||
setTimeout(() => this.processQueue(), 10);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async executeTask(task: AutoEmbedTask) {
|
||||
const node = await nodeService.getNodeById(task.nodeId);
|
||||
if (!node) {
|
||||
console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
const chunkText = node.chunk?.trim();
|
||||
if (!chunkText) {
|
||||
console.warn('[AutoEmbedQueue] Node has no chunk content, skipping', task.nodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!task.force && node.chunk_status === 'chunked') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.chunk_status === 'chunking' && !task.force) {
|
||||
console.log('[AutoEmbedQueue] Node already chunking, skipping duplicate run', task.nodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`🔄 [AutoEmbedQueue] Embedding node ${task.nodeId}${task.reason ? ` (${task.reason})` : ''}`);
|
||||
const result = await embedNodeContent(task.nodeId);
|
||||
if (!result.success) {
|
||||
console.error('[AutoEmbedQueue] Embedding failed', task.nodeId, result.error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var autoEmbedQueue: AutoEmbedQueue | undefined;
|
||||
}
|
||||
|
||||
export const autoEmbedQueue = globalThis.autoEmbedQueue ?? new AutoEmbedQueue();
|
||||
if (!globalThis.autoEmbedQueue) {
|
||||
globalThis.autoEmbedQueue = autoEmbedQueue;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export const AUTO_EMBED_MIN_CHARS = 200;
|
||||
|
||||
export function hasSufficientContent(text?: string | null): boolean {
|
||||
if (typeof text !== 'string') {
|
||||
return false;
|
||||
}
|
||||
return text.trim().length >= AUTO_EMBED_MIN_CHARS;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { nodeService } from '@/services/database';
|
||||
import { NodeEmbedder } from '@/services/typescript/embed-nodes';
|
||||
import { UniversalEmbedder } from '@/services/typescript/embed-universal';
|
||||
import type { Node } from '@/types/database';
|
||||
|
||||
export interface EmbeddingStageStatus {
|
||||
status: 'pending' | 'completed' | 'failed' | 'skipped';
|
||||
message: string;
|
||||
chunks_created?: number;
|
||||
}
|
||||
|
||||
export interface EmbeddingPipelineResult {
|
||||
success: boolean;
|
||||
error?: string;
|
||||
node_embedding: EmbeddingStageStatus;
|
||||
chunk_embeddings: EmbeddingStageStatus;
|
||||
overall_status: 'pending' | 'fully_embedded' | 'partially_embedded' | 'no_content' | 'failed';
|
||||
}
|
||||
|
||||
interface EmbeddingResult {
|
||||
success: boolean;
|
||||
error?: string;
|
||||
output?: string;
|
||||
}
|
||||
|
||||
async function runNodeEmbedding(nodeId: number): Promise<EmbeddingResult> {
|
||||
const embedder = new NodeEmbedder();
|
||||
try {
|
||||
const result = await embedder.embedNodes({ nodeId });
|
||||
if (result.processed > 0) {
|
||||
return { success: true, output: `Embedded ${result.processed} nodes` };
|
||||
}
|
||||
if (result.failed > 0) {
|
||||
return { success: false, error: 'Failed to embed node' };
|
||||
}
|
||||
return { success: true, output: 'Node already has embedding' };
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Embedding failed'
|
||||
};
|
||||
} finally {
|
||||
embedder.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function updateChunkStatus(nodeId: number, status: Node['chunk_status']) {
|
||||
try {
|
||||
await nodeService.updateNode(nodeId, { chunk_status: status });
|
||||
} catch (error) {
|
||||
console.warn('Failed to update chunk_status for node', nodeId, status, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function runChunkEmbedding(nodeId: number): Promise<EmbeddingResult> {
|
||||
const embedder = new UniversalEmbedder();
|
||||
try {
|
||||
await updateChunkStatus(nodeId, 'chunking');
|
||||
const result = await embedder.processNode({ nodeId });
|
||||
return {
|
||||
success: true,
|
||||
output: `Stored ${result.chunks} chunks`
|
||||
};
|
||||
} catch (error) {
|
||||
await updateChunkStatus(nodeId, 'error');
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Chunking failed'
|
||||
};
|
||||
} finally {
|
||||
embedder.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function embedNodeContent(nodeId: number): Promise<EmbeddingPipelineResult> {
|
||||
const node = await nodeService.getNodeById(nodeId);
|
||||
if (!node) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Node not found',
|
||||
node_embedding: { status: 'failed', message: 'Node not found' },
|
||||
chunk_embeddings: { status: 'failed', message: 'Node not found' },
|
||||
overall_status: 'failed'
|
||||
};
|
||||
}
|
||||
|
||||
const results = {
|
||||
node_embedding: { status: 'pending', message: '' } as EmbeddingStageStatus,
|
||||
chunk_embeddings: { status: 'pending', message: '', chunks_created: 0 } as EmbeddingStageStatus,
|
||||
overall_status: 'pending' as EmbeddingPipelineResult['overall_status']
|
||||
};
|
||||
|
||||
const nodeResult = await runNodeEmbedding(nodeId);
|
||||
if (nodeResult.success) {
|
||||
results.node_embedding = {
|
||||
status: 'completed',
|
||||
message: nodeResult.output || 'Node metadata embedded successfully'
|
||||
};
|
||||
} else {
|
||||
results.node_embedding = {
|
||||
status: 'failed',
|
||||
message: nodeResult.error || 'Failed to embed node metadata'
|
||||
};
|
||||
}
|
||||
|
||||
if (!node.chunk || !node.chunk.trim()) {
|
||||
results.chunk_embeddings = {
|
||||
status: 'skipped',
|
||||
message: 'No chunk content to embed',
|
||||
chunks_created: 0
|
||||
};
|
||||
} else {
|
||||
const chunkResult = await runChunkEmbedding(nodeId);
|
||||
if (chunkResult.success) {
|
||||
const chunkMatch = chunkResult.output?.match(/Stored (\d+) chunks/);
|
||||
const chunksCreated = chunkMatch ? parseInt(chunkMatch[1], 10) : 0;
|
||||
results.chunk_embeddings = {
|
||||
status: 'completed',
|
||||
message: chunkResult.output || 'Chunk content embedded successfully',
|
||||
chunks_created: chunksCreated
|
||||
};
|
||||
} else {
|
||||
results.chunk_embeddings = {
|
||||
status: 'failed',
|
||||
message: chunkResult.error || 'Failed to embed chunk content'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const nodeSuccess = results.node_embedding.status === 'completed';
|
||||
const chunkSuccess = results.chunk_embeddings.status === 'completed';
|
||||
const chunkSkipped = results.chunk_embeddings.status === 'skipped';
|
||||
|
||||
if (nodeSuccess && (chunkSuccess || chunkSkipped)) {
|
||||
results.overall_status = chunkSkipped ? 'no_content' : 'fully_embedded';
|
||||
if (chunkSuccess) {
|
||||
await updateChunkStatus(nodeId, 'chunked');
|
||||
}
|
||||
} else if (nodeSuccess || chunkSuccess) {
|
||||
results.overall_status = 'partially_embedded';
|
||||
} else {
|
||||
results.overall_status = 'failed';
|
||||
}
|
||||
|
||||
const errorParts: string[] = [];
|
||||
if (results.node_embedding.status === 'failed') {
|
||||
errorParts.push(`node: ${results.node_embedding.message}`);
|
||||
}
|
||||
if (results.chunk_embeddings.status === 'failed') {
|
||||
errorParts.push(`chunks: ${results.chunk_embeddings.message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: results.overall_status !== 'failed',
|
||||
error: errorParts.length ? errorParts.join('; ') : undefined,
|
||||
...results
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user