Initial commit: RA-H Open Source Edition
Local-first knowledge management system with BYO API keys. Features: - 3-panel UI (Nodes | Focus | Helpers) - SQLite + sqlite-vec for vector search - Agent system (Easy/Hard mode orchestrators) - Content extraction (YouTube, PDF, web) - Integrate workflow for connection discovery - Dimension system with auto-assignment Tech stack: - Next.js 15 + TypeScript + Tailwind CSS - Anthropic (Claude) + OpenAI (GPT) via Vercel AI SDK Setup: npm install && npm rebuild better-sqlite3 scripts/dev/bootstrap-local.sh npm run dev MIT License
This commit is contained in:
“BeeRad”
@@ -0,0 +1,243 @@
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
import { eventBroadcaster } from '@/services/events';
|
||||
|
||||
export type DelegationStatus = 'queued' | 'in_progress' | 'completed' | 'failed';
|
||||
export type DelegationAgentType = 'mini' | 'wise-rah';
|
||||
|
||||
export interface AgentDelegation {
|
||||
id: number;
|
||||
sessionId: string;
|
||||
task: string;
|
||||
context: string[];
|
||||
expectedOutcome?: string | null;
|
||||
status: DelegationStatus;
|
||||
summary?: string | null;
|
||||
agentType: DelegationAgentType;
|
||||
supabaseToken?: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
function rowToDelegation(row: any): AgentDelegation {
|
||||
return {
|
||||
id: row.id,
|
||||
sessionId: row.session_id,
|
||||
task: row.task,
|
||||
context: (() => {
|
||||
try {
|
||||
return row.context ? JSON.parse(row.context) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
expectedOutcome: row.expected_outcome,
|
||||
status: row.status as DelegationStatus,
|
||||
summary: row.summary,
|
||||
agentType: (row.agent_type || 'mini') as DelegationAgentType,
|
||||
supabaseToken: row.supabase_token ?? null,
|
||||
// SQLite CURRENT_TIMESTAMP is UTC, append 'Z' to parse correctly as UTC
|
||||
createdAt: row.created_at ? row.created_at.replace(' ', 'T') + 'Z' : row.created_at,
|
||||
updatedAt: row.updated_at ? row.updated_at.replace(' ', 'T') + 'Z' : row.updated_at,
|
||||
};
|
||||
}
|
||||
|
||||
function ensureTable() {
|
||||
const db = getSQLiteClient();
|
||||
db.query(`
|
||||
CREATE TABLE IF NOT EXISTS agent_delegations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT UNIQUE NOT NULL,
|
||||
task TEXT NOT NULL,
|
||||
context TEXT,
|
||||
expected_outcome TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
summary TEXT,
|
||||
agent_type TEXT NOT NULL DEFAULT 'mini',
|
||||
supabase_token TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
`);
|
||||
|
||||
// Add agent_type column if it doesn't exist (migration)
|
||||
try {
|
||||
const stmt = db.prepare('SELECT 1 FROM agent_delegations LIMIT 0');
|
||||
const tableExists = stmt !== null;
|
||||
|
||||
if (tableExists) {
|
||||
// Try to add the column, ignore if it already exists
|
||||
try {
|
||||
db.prepare(`ALTER TABLE agent_delegations ADD COLUMN agent_type TEXT NOT NULL DEFAULT 'mini'`).run();
|
||||
console.log('✅ Added agent_type column to agent_delegations table');
|
||||
} catch (alterError: any) {
|
||||
// Column already exists, ignore
|
||||
if (!alterError.message?.includes('duplicate column')) {
|
||||
console.warn('Migration warning:', alterError.message);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
db.prepare(`ALTER TABLE agent_delegations ADD COLUMN supabase_token TEXT`).run();
|
||||
console.log('✅ Added supabase_token column to agent_delegations table');
|
||||
} catch (alterError: any) {
|
||||
if (!alterError.message?.includes('duplicate column')) {
|
||||
console.warn('Migration warning:', alterError.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
// Table doesn't exist yet or other error - it will be created with the columns
|
||||
console.log('Table creation will include agent_type and supabase_token columns');
|
||||
}
|
||||
}
|
||||
|
||||
export class AgentDelegationService {
|
||||
static createDelegation(input: {
|
||||
task: string;
|
||||
context?: string[];
|
||||
expectedOutcome?: string | null;
|
||||
agentType?: DelegationAgentType;
|
||||
supabaseToken?: string | null;
|
||||
}): AgentDelegation {
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
const sessionId = `delegation_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
const contextJson = JSON.stringify(input.context ?? []);
|
||||
const agentType = input.agentType || 'mini';
|
||||
db.prepare(
|
||||
`INSERT INTO agent_delegations (session_id, task, context, expected_outcome, status, agent_type, supabase_token)
|
||||
VALUES (?, ?, ?, ?, 'queued', ?, ?)`
|
||||
).run(
|
||||
sessionId,
|
||||
input.task,
|
||||
contextJson,
|
||||
input.expectedOutcome ?? null,
|
||||
agentType,
|
||||
input.supabaseToken ?? null
|
||||
);
|
||||
|
||||
const row = db
|
||||
.prepare('SELECT * FROM agent_delegations WHERE session_id = ?')
|
||||
.get(sessionId);
|
||||
|
||||
const delegation = rowToDelegation(row);
|
||||
eventBroadcaster.broadcast({ type: 'AGENT_DELEGATION_CREATED', data: { delegation } });
|
||||
return delegation;
|
||||
}
|
||||
|
||||
static markInProgress(sessionId: string): AgentDelegation | null {
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
db.prepare(
|
||||
`UPDATE agent_delegations
|
||||
SET status = 'in_progress', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE session_id = ? AND status = 'queued'`
|
||||
).run(sessionId);
|
||||
const row = db.prepare('SELECT * FROM agent_delegations WHERE session_id = ?').get(sessionId);
|
||||
if (!row) return null;
|
||||
const delegation = rowToDelegation(row);
|
||||
eventBroadcaster.broadcast({ type: 'AGENT_DELEGATION_UPDATED', data: { delegation } });
|
||||
return delegation;
|
||||
}
|
||||
|
||||
static touchDelegation(sessionId: string): void {
|
||||
// Update the timestamp to prevent cleanup from killing active delegations
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
db.prepare(
|
||||
`UPDATE agent_delegations SET updated_at = CURRENT_TIMESTAMP WHERE session_id = ? AND status = 'in_progress'`
|
||||
).run(sessionId);
|
||||
}
|
||||
|
||||
static completeDelegation(sessionId: string, summary: string, status: DelegationStatus = 'completed'): AgentDelegation | null {
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
db.prepare(
|
||||
`UPDATE agent_delegations
|
||||
SET status = ?, summary = ?, updated_at = CURRENT_TIMESTAMP, supabase_token = NULL
|
||||
WHERE session_id = ?`
|
||||
).run(status, summary, sessionId);
|
||||
const row = db.prepare('SELECT * FROM agent_delegations WHERE session_id = ?').get(sessionId);
|
||||
if (!row) return null;
|
||||
const delegation = rowToDelegation(row);
|
||||
eventBroadcaster.broadcast({ type: 'AGENT_DELEGATION_UPDATED', data: { delegation } });
|
||||
return delegation;
|
||||
}
|
||||
|
||||
static getBySessionId(sessionId: string): AgentDelegation | null {
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
const row = db.prepare('SELECT * FROM agent_delegations WHERE session_id = ?').get(sessionId);
|
||||
return row ? rowToDelegation(row) : null;
|
||||
}
|
||||
|
||||
static getDelegation(sessionId: string): AgentDelegation | null {
|
||||
return this.getBySessionId(sessionId);
|
||||
}
|
||||
|
||||
static listActive({ includeCompleted = true, limit = 100 }: { includeCompleted?: boolean; limit?: number } = {}): AgentDelegation[] {
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
// Load all delegations - user closes them manually from UI
|
||||
const rows = includeCompleted
|
||||
? db.prepare(
|
||||
`SELECT * FROM agent_delegations
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT ?`
|
||||
).all(limit)
|
||||
: db.prepare(
|
||||
`SELECT * FROM agent_delegations
|
||||
WHERE status IN ('queued','in_progress')
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT ?`
|
||||
).all(limit);
|
||||
return rows.map(rowToDelegation);
|
||||
}
|
||||
|
||||
static listRecent(limit = 20): AgentDelegation[] {
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
const rows = db.prepare(
|
||||
`SELECT * FROM agent_delegations
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ?`
|
||||
).all(limit);
|
||||
return rows.map(rowToDelegation);
|
||||
}
|
||||
|
||||
static deleteDelegation(sessionId: string): boolean {
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
const result = db.prepare('DELETE FROM agent_delegations WHERE session_id = ?').run(sessionId);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
static cleanupStaleDelegations(timeoutMinutes = 15): number {
|
||||
ensureTable();
|
||||
const db = getSQLiteClient();
|
||||
const cutoffTime = new Date(Date.now() - timeoutMinutes * 60 * 1000).toISOString();
|
||||
|
||||
const result = db.prepare(`
|
||||
UPDATE agent_delegations
|
||||
SET status = 'failed',
|
||||
summary = 'Task timed out (exceeded ${timeoutMinutes} minutes)',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE status = 'in_progress'
|
||||
AND updated_at < ?
|
||||
`).run(cutoffTime);
|
||||
|
||||
const affectedCount = result.changes || 0;
|
||||
|
||||
if (affectedCount > 0) {
|
||||
const rows = db.prepare(
|
||||
`SELECT * FROM agent_delegations WHERE status = 'failed' AND summary LIKE 'Task timed out%'`
|
||||
).all();
|
||||
rows.forEach(row => {
|
||||
const delegation = rowToDelegation(row);
|
||||
eventBroadcaster.broadcast({ type: 'AGENT_DELEGATION_UPDATED', data: { delegation } });
|
||||
});
|
||||
}
|
||||
|
||||
return affectedCount;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,428 @@
|
||||
import { generateText, CoreMessage } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { AgentDelegationService, DelegationStatus } from '@/services/agents/delegation';
|
||||
import { MINI_RAH_SYSTEM_PROMPT } from '@/config/prompts/rah-mini';
|
||||
import { getToolsForRole } from '@/tools/infrastructure/registry';
|
||||
import { ChatLoggingMiddleware } from '@/services/chat/middleware';
|
||||
import { calculateCost } from '@/services/analytics/pricing';
|
||||
import { UsageData } from '@/types/analytics';
|
||||
import { summarizeToolExecution } from '@/services/agents/toolResultUtils';
|
||||
import { edgeService } from '@/services/database/edges';
|
||||
import { RequestContext } from '@/services/context/requestContext';
|
||||
|
||||
interface CapsuleNodeJSON {
|
||||
id: number;
|
||||
}
|
||||
|
||||
interface DelegationCapsuleJSON {
|
||||
version?: number;
|
||||
primary?: CapsuleNodeJSON | null;
|
||||
secondary?: CapsuleNodeJSON[];
|
||||
referenced?: CapsuleNodeJSON[];
|
||||
}
|
||||
|
||||
interface SummaryValidation {
|
||||
status: 'ok' | 'failed';
|
||||
reason?: string;
|
||||
sourcesUsed: number[];
|
||||
}
|
||||
|
||||
function extractCapsuleFromContext(context: string[]): { capsule?: DelegationCapsuleJSON; nodeIds: number[]; version?: number } {
|
||||
const entry = context.find(item => typeof item === 'string' && item.startsWith('CAPSULE_JSON::'));
|
||||
if (!entry) {
|
||||
return { nodeIds: [] };
|
||||
}
|
||||
const json = entry.substring('CAPSULE_JSON::'.length);
|
||||
try {
|
||||
const capsule = JSON.parse(json) as DelegationCapsuleJSON & { version?: number };
|
||||
const nodeIds = new Set<number>();
|
||||
const pushId = (value?: CapsuleNodeJSON | null) => {
|
||||
if (!value) return;
|
||||
if (typeof value.id === 'number') {
|
||||
nodeIds.add(value.id);
|
||||
}
|
||||
};
|
||||
pushId(capsule.primary ?? null);
|
||||
(capsule.secondary ?? []).forEach(pushId);
|
||||
(capsule.referenced ?? []).forEach(pushId);
|
||||
return {
|
||||
capsule,
|
||||
nodeIds: Array.from(nodeIds),
|
||||
version: typeof capsule.version === 'number' ? capsule.version : undefined,
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn('MiniRAHExecutor: failed to parse delegation capsule', error);
|
||||
return { nodeIds: [] };
|
||||
}
|
||||
}
|
||||
|
||||
function parseSourcesLine(summary: string): { line?: string; ids: number[] } {
|
||||
const lines = summary
|
||||
.split(/\n+/)
|
||||
.map(line => line.trim())
|
||||
.filter(Boolean);
|
||||
const contextLine = lines.find(line => line.toLowerCase().startsWith('context sources used:'));
|
||||
if (!contextLine) {
|
||||
return { ids: [] };
|
||||
}
|
||||
const matches = contextLine.match(/\d+/g);
|
||||
const ids = matches ? matches.map(id => Number(id)).filter(id => Number.isFinite(id)) : [];
|
||||
return { line: contextLine, ids: Array.from(new Set(ids)) };
|
||||
}
|
||||
|
||||
function validateMiniSummary(summary: string, expectedNodeIds: number[]): SummaryValidation {
|
||||
const trimmed = summary.trim();
|
||||
if (!trimmed) {
|
||||
return { status: 'failed', reason: 'Worker returned an empty summary.', sourcesUsed: [] };
|
||||
}
|
||||
|
||||
const lines = trimmed.split(/\n+/).map(line => line.trim()).filter(Boolean);
|
||||
const resultIndex = lines.findIndex(line => line.toLowerCase().startsWith('result:'));
|
||||
if (resultIndex === -1) {
|
||||
return { status: 'failed', reason: 'Missing or empty Result line in worker summary.', sourcesUsed: [] };
|
||||
}
|
||||
|
||||
const resultLine = lines[resultIndex];
|
||||
const resultContent = resultLine.slice('result:'.length).trim();
|
||||
if (!resultContent) {
|
||||
const nextLine = lines[resultIndex + 1];
|
||||
if (!nextLine || /^(task:|actions:|node:|context sources used:|follow-up:)/i.test(nextLine)) {
|
||||
return { status: 'failed', reason: 'Missing or empty Result line in worker summary.', sourcesUsed: [] };
|
||||
}
|
||||
}
|
||||
|
||||
const followUpLine = lines.find(line => line.toLowerCase().startsWith('follow-up:'));
|
||||
if (!followUpLine) {
|
||||
return { status: 'failed', reason: 'Missing Follow-up line in worker summary.', sourcesUsed: [] };
|
||||
}
|
||||
|
||||
const { line: contextLine, ids } = parseSourcesLine(summary);
|
||||
if (!contextLine) {
|
||||
return { status: 'failed', reason: 'Missing "Context sources used" line. Workers must list node IDs they referenced.', sourcesUsed: [] };
|
||||
}
|
||||
if (expectedNodeIds.length > 0 && ids.length === 0) {
|
||||
return { status: 'failed', reason: 'Worker did not cite any node IDs even though a capsule was provided.', sourcesUsed: [] };
|
||||
}
|
||||
|
||||
return { status: 'ok', sourcesUsed: ids };
|
||||
}
|
||||
|
||||
export interface MiniRAHExecutionInput {
|
||||
sessionId: string;
|
||||
task: string;
|
||||
context: string[];
|
||||
expectedOutcome?: string | null;
|
||||
traceId?: string;
|
||||
parentChatId?: number;
|
||||
workflowKey?: string;
|
||||
workflowNodeId?: number;
|
||||
}
|
||||
|
||||
export class MiniRAHExecutor {
|
||||
static async execute({ sessionId, task, context, expectedOutcome, traceId, parentChatId, workflowKey, workflowNodeId }: MiniRAHExecutionInput) {
|
||||
try {
|
||||
const requestContext = RequestContext.get();
|
||||
const delegateKey =
|
||||
requestContext.apiKeys?.openai ||
|
||||
process.env.RAH_DELEGATE_OPENAI_API_KEY ||
|
||||
process.env.OPENAI_API_KEY;
|
||||
if (!delegateKey) {
|
||||
throw new Error('RAH_DELEGATE_OPENAI_API_KEY (or OPENAI_API_KEY) is not set.');
|
||||
}
|
||||
|
||||
AgentDelegationService.markInProgress(sessionId);
|
||||
|
||||
const promptSections = [
|
||||
`Task: ${task}`,
|
||||
context.length ? `Context:\n- ${context.join('\n- ')}` : undefined,
|
||||
expectedOutcome ? `Expected outcome: ${expectedOutcome}` : undefined,
|
||||
'Return only the final summary the orchestrator should see.'
|
||||
].filter(Boolean);
|
||||
|
||||
const openaiProvider = createOpenAI({ apiKey: delegateKey });
|
||||
const executorTools = getToolsForRole('executor');
|
||||
const capturedSummaries: string[] = [];
|
||||
const toolsUsedInSession: string[] = [];
|
||||
const integrateAllowed = workflowKey === 'integrate'
|
||||
? new Set(['createEdge', 'updateEdge', 'updateNode'])
|
||||
: null;
|
||||
const wrappedTools = Object.fromEntries(
|
||||
Object.entries(executorTools)
|
||||
.filter(([name]) => {
|
||||
if (integrateAllowed) {
|
||||
return integrateAllowed.has(name);
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.map(([name, tool]) => {
|
||||
const wrapped = {
|
||||
...tool,
|
||||
async execute(params: any, context: any) {
|
||||
if (!toolsUsedInSession.includes(name)) {
|
||||
toolsUsedInSession.push(name);
|
||||
}
|
||||
if (name === 'createEdge' && params && typeof params === 'object' && 'from_node_id' in params && 'to_node_id' in params) {
|
||||
const fromId = Number(params.from_node_id);
|
||||
const toId = Number(params.to_node_id);
|
||||
if (Number.isFinite(fromId) && Number.isFinite(toId)) {
|
||||
const exists = await edgeService.edgeExists(fromId, toId);
|
||||
if (exists) {
|
||||
const skipSummary = `Edge already exists between node ${fromId} and node ${toId}; skipping createEdge.`;
|
||||
capturedSummaries.push(skipSummary);
|
||||
return {
|
||||
success: true,
|
||||
skipped: true,
|
||||
message: skipSummary,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const result = await tool.execute(params, context);
|
||||
const summary = summarizeToolExecution(name, params, result);
|
||||
if (summary) {
|
||||
capturedSummaries.push(summary);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
return [name, wrapped];
|
||||
})
|
||||
);
|
||||
|
||||
if ('delegateToMiniRAH' in wrappedTools) {
|
||||
console.warn('MiniRAHExecutor: delegateToMiniRAH detected in executor toolset. Removing to enforce single-level delegation.');
|
||||
delete wrappedTools.delegateToMiniRAH;
|
||||
}
|
||||
|
||||
const userPrompt = promptSections.join('\n\n');
|
||||
const messages: CoreMessage[] = [{ role: 'user', content: userPrompt }];
|
||||
|
||||
const maxIterations = 6;
|
||||
let rawSummary = '';
|
||||
let lastFinishReason: string | undefined;
|
||||
let lastToolCalls: any[] | undefined;
|
||||
|
||||
let totalInputTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
let totalTokens = 0;
|
||||
|
||||
// Track if extraction tool was called (for Quick Add - should only call once)
|
||||
const isQuickAddTask = task.toLowerCase().includes('quick add');
|
||||
let extractionToolCalled = false;
|
||||
|
||||
for (let iteration = 0; iteration < maxIterations; iteration++) {
|
||||
const response = await generateText({
|
||||
model: openaiProvider('gpt-4o-mini'),
|
||||
system: MINI_RAH_SYSTEM_PROMPT,
|
||||
messages,
|
||||
tools: wrappedTools,
|
||||
});
|
||||
|
||||
const usage = response.usage;
|
||||
if (usage) {
|
||||
const inputTokens = (usage as any).promptTokens || usage.inputTokens || 0;
|
||||
const outputTokens = (usage as any).completionTokens || usage.outputTokens || 0;
|
||||
const combinedTotal = (usage as any).totalTokens || usage.totalTokens || inputTokens + outputTokens;
|
||||
totalInputTokens += inputTokens;
|
||||
totalOutputTokens += outputTokens;
|
||||
totalTokens += combinedTotal;
|
||||
}
|
||||
|
||||
lastFinishReason = response.finishReason;
|
||||
lastToolCalls = response.toolCalls ?? [];
|
||||
|
||||
if (response.finishReason === 'tool-calls' && response.toolCalls && response.toolCalls.length > 0) {
|
||||
messages.push({
|
||||
role: 'assistant',
|
||||
content: response.toolCalls.map(call => ({
|
||||
type: 'tool-call' as const,
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
input: (call as any).input ?? (call as any).args,
|
||||
})),
|
||||
});
|
||||
|
||||
const toolResults: Array<{ type: 'tool-result'; toolCallId: string; toolName: string; output: { type: 'text' | 'error-text'; value: string } }> = [];
|
||||
|
||||
for (const call of response.toolCalls) {
|
||||
const callInput = (call as any).input ?? (call as any).args;
|
||||
const tool = wrappedTools[call.toolName];
|
||||
const isExtractionToolCall = isQuickAddTask && ['youtubeExtract', 'websiteExtract', 'paperExtract'].includes(call.toolName);
|
||||
|
||||
if (isExtractionToolCall && extractionToolCalled) {
|
||||
const skipMessage = `Extraction already completed; skipping duplicate ${call.toolName} request.`;
|
||||
console.warn(`[MiniRAHExecutor] ${skipMessage}`);
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output: { type: 'text', value: skipMessage },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!tool) {
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output: { type: 'error-text', value: `Tool ${call.toolName} is not available.` },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const toolResult = await tool.execute(callInput, {});
|
||||
const summary = summarizeToolExecution(call.toolName, callInput, toolResult);
|
||||
const value = summary || `${call.toolName} completed.`;
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output: { type: 'text', value },
|
||||
});
|
||||
|
||||
// For Quick Add: stop after first successful extraction to prevent duplicates
|
||||
if (isExtractionToolCall) {
|
||||
const success = typeof toolResult === 'object' && toolResult !== null && (toolResult as any).success !== false;
|
||||
if (success) {
|
||||
console.log(`[MiniRAHExecutor] Quick Add extraction succeeded, forcing summary generation to prevent duplicate calls`);
|
||||
extractionToolCalled = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Tool execution failed';
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output: { type: 'error-text', value: message },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
messages.push({ role: 'tool', content: toolResults });
|
||||
|
||||
// If Quick Add extraction succeeded, request final summary and exit loop
|
||||
if (extractionToolCalled) {
|
||||
console.log('[MiniRAHExecutor] Requesting final summary after Quick Add extraction');
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: 'Extraction completed successfully. Provide your final summary now using the required format (Task/Actions/Result/Node/Context sources used/Follow-up).'
|
||||
});
|
||||
const summaryResponse = await generateText({
|
||||
model: openaiProvider('gpt-4o-mini'),
|
||||
system: MINI_RAH_SYSTEM_PROMPT,
|
||||
messages,
|
||||
tools: {}, // No tools - summary only
|
||||
});
|
||||
rawSummary = summaryResponse.text?.trim() || 'Extraction completed successfully.';
|
||||
break;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
rawSummary = typeof response.text === 'string' ? response.text.trim() : '';
|
||||
if (!rawSummary) {
|
||||
console.warn('[MiniRAHExecutor] Worker returned empty summary.', {
|
||||
finishReason: response.finishReason,
|
||||
toolCalls: response.toolCalls,
|
||||
text: response.text,
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (!rawSummary) {
|
||||
console.warn('[MiniRAHExecutor] No summary after tool loop.', {
|
||||
lastFinishReason,
|
||||
lastToolCalls,
|
||||
});
|
||||
}
|
||||
|
||||
const fallbackSummary = capturedSummaries.length > 0
|
||||
? capturedSummaries[capturedSummaries.length - 1]
|
||||
: 'Completed the delegated task (worker returned no additional summary).';
|
||||
const initialSummary = rawSummary.length > 0 ? rawSummary : fallbackSummary;
|
||||
|
||||
const capsuleInfo = extractCapsuleFromContext(context);
|
||||
const validation = validateMiniSummary(initialSummary, capsuleInfo.nodeIds);
|
||||
const validationStatus: DelegationStatus = validation.status === 'ok' ? 'completed' : 'failed';
|
||||
if (validation.status === 'failed') {
|
||||
console.warn('[MiniRAHExecutor] summary validation failed.', {
|
||||
reason: validation.reason,
|
||||
initialSummary,
|
||||
});
|
||||
}
|
||||
|
||||
const finalSummary = validation.status === 'ok'
|
||||
? initialSummary
|
||||
: `${initialSummary}\nValidation: ${validation.reason}`;
|
||||
|
||||
console.log('[MiniRAHExecutor] summary:', finalSummary);
|
||||
|
||||
// Calculate cost and log to chats table
|
||||
if (totalInputTokens > 0 || totalOutputTokens > 0 || totalTokens > 0) {
|
||||
const effectiveTotalTokens = totalTokens > 0 ? totalTokens : totalInputTokens + totalOutputTokens;
|
||||
|
||||
const costResult = calculateCost({
|
||||
inputTokens: totalInputTokens,
|
||||
outputTokens: totalOutputTokens,
|
||||
modelId: 'gpt-4o-mini',
|
||||
});
|
||||
|
||||
const usageData: UsageData = {
|
||||
inputTokens: totalInputTokens,
|
||||
outputTokens: totalOutputTokens,
|
||||
totalTokens: effectiveTotalTokens,
|
||||
estimatedCostUsd: costResult.totalCostUsd,
|
||||
modelUsed: 'gpt-4o-mini',
|
||||
provider: 'openai',
|
||||
toolsUsed: toolsUsedInSession.length > 0 ? toolsUsedInSession : undefined,
|
||||
toolCallsCount: toolsUsedInSession.length > 0 ? toolsUsedInSession.length : undefined,
|
||||
traceId,
|
||||
parentChatId,
|
||||
workflowKey,
|
||||
workflowNodeId,
|
||||
capsuleVersion: capsuleInfo.version,
|
||||
contextSourcesUsed: validation.sourcesUsed.length > 0 ? validation.sourcesUsed : undefined,
|
||||
validationStatus: validation.status,
|
||||
validationMessage: validation.reason,
|
||||
fallbackAction: validation.status === 'failed' ? 'Review context capsule, hydrate nodes manually, then re-delegate with clarified instructions.' : undefined,
|
||||
};
|
||||
|
||||
const delegation = AgentDelegationService.getDelegation(sessionId);
|
||||
const delegationId = delegation?.id;
|
||||
|
||||
await ChatLoggingMiddleware.logChatInteraction(
|
||||
task,
|
||||
finalSummary,
|
||||
{
|
||||
helperName: 'mini-rah',
|
||||
agentType: 'executor',
|
||||
delegationId: delegationId ?? null,
|
||||
sessionId,
|
||||
usageData,
|
||||
traceId,
|
||||
parentChatId,
|
||||
workflowKey,
|
||||
workflowNodeId,
|
||||
systemMessage: MINI_RAH_SYSTEM_PROMPT,
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
console.log(`💰 [MiniRAHExecutor] Cost: $${costResult.totalCostUsd.toFixed(6)} (${effectiveTotalTokens} tokens)`);
|
||||
}
|
||||
|
||||
return AgentDelegationService.completeDelegation(sessionId, finalSummary, validationStatus);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Unknown delegation error';
|
||||
AgentDelegationService.completeDelegation(sessionId, `Mini ra-h failed: ${message}`, 'failed');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
import { AgentDelegationService } from './delegation';
|
||||
import { summarizeToolExecution } from './toolResultUtils';
|
||||
import { youtubeExtractTool } from '@/tools/other/youtubeExtract';
|
||||
import { websiteExtractTool } from '@/tools/other/websiteExtract';
|
||||
import { paperExtractTool } from '@/tools/other/paperExtract';
|
||||
import { formatNodeForChat } from '@/tools/infrastructure/nodeFormatter';
|
||||
import { summarizeTranscript } from './transcriptSummarizer';
|
||||
|
||||
export type QuickAddMode = 'link' | 'note' | 'chat';
|
||||
|
||||
export type QuickAddInputType = 'youtube' | 'website' | 'pdf' | 'note' | 'chat';
|
||||
|
||||
export interface QuickAddInput {
|
||||
rawInput: string;
|
||||
mode?: QuickAddMode;
|
||||
}
|
||||
|
||||
function isLikelyChatTranscript(raw: string): boolean {
|
||||
const newlineCount = (raw.match(/\n/g)?.length ?? 0);
|
||||
if (newlineCount >= 3 && raw.length > 300) return true;
|
||||
if (/\b\d{1,2}:\d{2}\b/.test(raw) && newlineCount >= 1) return true;
|
||||
if (/You said:|ChatGPT said:|Claude said:|Assistant:|User:/i.test(raw)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function detectInputType(raw: string, mode?: QuickAddMode): QuickAddInputType {
|
||||
if (mode === 'chat') return 'chat';
|
||||
if (mode === 'note') return 'note';
|
||||
|
||||
const input = raw.trim();
|
||||
if (/youtu(\.be|be\.com)/i.test(input)) return 'youtube';
|
||||
if (/\.pdf($|\?)/i.test(input) || /arxiv\.org\//i.test(input)) return 'pdf';
|
||||
if (/^https?:\/\//i.test(input)) return 'website';
|
||||
if (!mode && isLikelyChatTranscript(input)) return 'chat';
|
||||
return 'note';
|
||||
}
|
||||
|
||||
function buildTaskPrompt(type: QuickAddInputType, input: string): string {
|
||||
switch (type) {
|
||||
case 'youtube':
|
||||
return `Quick Add: extract YouTube video and create node → ${input}`;
|
||||
case 'website':
|
||||
return `Quick Add: extract webpage and create node → ${input}`;
|
||||
case 'pdf':
|
||||
return `Quick Add: extract PDF and create node → ${input}`;
|
||||
case 'note':
|
||||
return `Quick Add note: create a node from this text with no dimensions → ${input}`;
|
||||
case 'chat':
|
||||
return `Quick Add: import chat transcript and summarize → ${input.slice(0, 120)}${input.length > 120 ? '…' : ''}`;
|
||||
}
|
||||
}
|
||||
|
||||
function buildExpectedOutcome(type: QuickAddInputType): string {
|
||||
if (type === 'note') {
|
||||
return 'Call createNode with the user input as content. Use a descriptive title extracted from the first few words. Set dimensions to an empty array. Return a brief confirmation like "Created note [NODE:id:title]"';
|
||||
}
|
||||
if (type === 'chat') {
|
||||
return 'Store the entire transcript in chunk_content, summarize the conversation into content, and return a confirmation like "Created chat transcript node [NODE:id:title]"';
|
||||
}
|
||||
return 'Use the appropriate extraction tool (youtubeExtract, websiteExtract, or paperExtract) to create the node. Return the standard extraction summary.';
|
||||
}
|
||||
|
||||
type ExtractionQuickAddType = Extract<QuickAddInputType, 'youtube' | 'website' | 'pdf'>;
|
||||
|
||||
const EXTRACTION_TOOL_MAP = {
|
||||
youtube: { toolName: 'youtubeExtract' as const, execute: youtubeExtractTool.execute },
|
||||
website: { toolName: 'websiteExtract' as const, execute: websiteExtractTool.execute },
|
||||
pdf: { toolName: 'paperExtract' as const, execute: paperExtractTool.execute },
|
||||
};
|
||||
|
||||
interface SummaryParts {
|
||||
task: string;
|
||||
action: string;
|
||||
resultMessage: string;
|
||||
nodeReference: string;
|
||||
}
|
||||
|
||||
interface ExtractionToolResultData {
|
||||
nodeId?: number;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
interface ExtractionToolResult {
|
||||
success?: boolean;
|
||||
error?: string;
|
||||
data?: ExtractionToolResultData | null;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
interface CreateNodeResponse {
|
||||
success?: boolean;
|
||||
data?: { id?: number; title?: string } | null;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function buildStructuredSummary({ task, action, resultMessage, nodeReference }: SummaryParts): string {
|
||||
const normalizedResult = resultMessage?.trim().length ? resultMessage.trim() : `${action} completed.`;
|
||||
const normalizedNode = nodeReference || 'None';
|
||||
return [
|
||||
`Task: ${task}`,
|
||||
`Actions: ${action}`,
|
||||
`Result: ${normalizedResult}`,
|
||||
`Node: ${normalizedNode}`,
|
||||
'Context sources used: None',
|
||||
'Follow-up: None',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function deriveNoteTitle(raw: string): string {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) {
|
||||
return 'Quick Add Note';
|
||||
}
|
||||
const sentenceMatch = trimmed.match(/^(.{1,120}?)([.!?]\s|\n|$)/);
|
||||
const candidate = sentenceMatch ? sentenceMatch[1] : trimmed.slice(0, 120);
|
||||
const title = candidate.replace(/\s+/g, ' ').trim();
|
||||
return title.length >= trimmed.length || title.length <= 120 ? title : `${title.slice(0, 117)}…`;
|
||||
}
|
||||
|
||||
function deriveChatTitle(raw: string, summarySubject?: string): string {
|
||||
if (summarySubject && summarySubject.trim().length > 0) {
|
||||
return summarySubject.trim();
|
||||
}
|
||||
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return 'Chat Transcript';
|
||||
const firstLine = trimmed.split('\n')[0];
|
||||
const cleaned = firstLine.replace(/You said:|ChatGPT said:|Claude said:/gi, '').trim();
|
||||
if (!cleaned) return 'Chat Transcript';
|
||||
return cleaned.length > 120 ? `${cleaned.slice(0, 117)}…` : cleaned;
|
||||
}
|
||||
|
||||
function isExtractionToolResult(value: unknown): value is ExtractionToolResult {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const candidate = value as Record<string, unknown>;
|
||||
if ('success' in candidate && typeof candidate.success !== 'boolean' && candidate.success !== undefined) {
|
||||
return false;
|
||||
}
|
||||
if ('error' in candidate && typeof candidate.error !== 'string' && candidate.error !== undefined && candidate.error !== null) {
|
||||
return false;
|
||||
}
|
||||
if ('data' in candidate && candidate.data !== undefined && candidate.data !== null && typeof candidate.data !== 'object') {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function isCreateNodeResponse(value: unknown): value is CreateNodeResponse {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const candidate = value as Record<string, unknown>;
|
||||
if ('success' in candidate && typeof candidate.success !== 'boolean' && candidate.success !== undefined) {
|
||||
return false;
|
||||
}
|
||||
if ('error' in candidate && typeof candidate.error !== 'string' && candidate.error !== undefined && candidate.error !== null) {
|
||||
return false;
|
||||
}
|
||||
if ('data' in candidate && candidate.data !== undefined && candidate.data !== null && typeof candidate.data !== 'object') {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function handleExtractionQuickAdd(type: ExtractionQuickAddType, url: string, task: string): Promise<string> {
|
||||
const { toolName, execute } = EXTRACTION_TOOL_MAP[type];
|
||||
if (!execute) {
|
||||
throw new Error(`Tool ${toolName} does not have an execute function`);
|
||||
}
|
||||
const rawResult = await execute({ url }, { toolCallId: 'quickadd-extract', messages: [] });
|
||||
|
||||
if (!isExtractionToolResult(rawResult)) {
|
||||
throw new Error(`Unexpected response from ${toolName}`);
|
||||
}
|
||||
|
||||
const toolResult = rawResult;
|
||||
|
||||
if (!toolResult || toolResult.success === false) {
|
||||
const errorMessage = toolResult?.error || `Failed to execute ${toolName}`;
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
const summaryLine = summarizeToolExecution(toolName, { url }, toolResult);
|
||||
const nodeId = toolResult.data?.nodeId;
|
||||
const nodeTitle = typeof toolResult.data?.title === 'string' && toolResult.data.title.trim().length > 0
|
||||
? toolResult.data.title.trim()
|
||||
: nodeId ? `Node ${nodeId}` : 'Created node';
|
||||
const nodeReference = nodeId ? formatNodeForChat({ id: nodeId, title: nodeTitle }) : 'None';
|
||||
|
||||
return buildStructuredSummary({
|
||||
task,
|
||||
action: toolName,
|
||||
resultMessage: summaryLine,
|
||||
nodeReference,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleNoteQuickAdd(rawInput: string, task: string): Promise<string> {
|
||||
const content = rawInput.trim();
|
||||
if (!content) {
|
||||
throw new Error('Input is required to create a note');
|
||||
}
|
||||
|
||||
const title = deriveNoteTitle(content);
|
||||
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
title,
|
||||
content,
|
||||
dimensions: [],
|
||||
metadata: {
|
||||
source: 'quick-add-note',
|
||||
refined_at: new Date().toISOString(),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const rawResult = await response.json();
|
||||
|
||||
if (!isCreateNodeResponse(rawResult)) {
|
||||
throw new Error('Unexpected response from node creation');
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(rawResult?.error || 'Failed to create note');
|
||||
}
|
||||
|
||||
const nodeId = rawResult?.data?.id;
|
||||
const nodeReference = nodeId ? formatNodeForChat({ id: nodeId, title }) : 'None';
|
||||
const resultMessage = nodeId ? `Created note ${nodeReference}.` : 'Created note.';
|
||||
|
||||
return buildStructuredSummary({
|
||||
task,
|
||||
action: 'createNode',
|
||||
resultMessage,
|
||||
nodeReference,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleChatTranscriptQuickAdd(rawInput: string, task: string): Promise<string> {
|
||||
const transcript = rawInput.trim();
|
||||
if (!transcript) {
|
||||
throw new Error('Input is required to import a chat transcript');
|
||||
}
|
||||
|
||||
const summaryResult = await summarizeTranscript(transcript);
|
||||
const baseSummary = summaryResult.summary?.trim() || 'Captured chat transcript. Review the raw transcript for full detail.';
|
||||
|
||||
const intentLine = summaryResult.intent?.trim();
|
||||
const progressLine = summaryResult.progress?.trim();
|
||||
const stickingPoints = summaryResult.stickingPoints || [];
|
||||
|
||||
const highlightSection = (summaryResult.highlights?.length ?? 0) > 0
|
||||
? ['Highlights:', ...summaryResult.highlights!.map((item) => `- ${item}`)].join('\n')
|
||||
: null;
|
||||
|
||||
const followUpSection = (summaryResult.openQuestions?.length ?? 0) > 0
|
||||
? ['Open Questions:', ...summaryResult.openQuestions!.map((item) => `- ${item}`)].join('\n')
|
||||
: null;
|
||||
|
||||
const stickingSection = stickingPoints.length > 0
|
||||
? ['Where things felt stuck:', ...stickingPoints.map((item) => `- ${item}`)].join('\n')
|
||||
: null;
|
||||
|
||||
const contentParts = [
|
||||
`Overview:\n${baseSummary}`,
|
||||
];
|
||||
|
||||
if (intentLine) {
|
||||
contentParts.push(`What you were trying to do:\n${intentLine}`);
|
||||
}
|
||||
if (progressLine) {
|
||||
contentParts.push(`Where you made progress:\n${progressLine}`);
|
||||
}
|
||||
if (stickingSection) {
|
||||
contentParts.push(stickingSection);
|
||||
}
|
||||
if (highlightSection) contentParts.push(highlightSection);
|
||||
if (followUpSection) contentParts.push(followUpSection);
|
||||
const content = contentParts.join('\n\n');
|
||||
|
||||
const title = deriveChatTitle(transcript, summaryResult.subject);
|
||||
const wordCount = transcript.split(/\s+/).filter(Boolean).length;
|
||||
|
||||
const metadata = {
|
||||
source: 'quick-add-chat',
|
||||
summary_subject: summaryResult.subject,
|
||||
summary_intent: summaryResult.intent,
|
||||
summary_progress: summaryResult.progress,
|
||||
highlights: summaryResult.highlights ?? [],
|
||||
open_questions: summaryResult.openQuestions ?? [],
|
||||
participants: summaryResult.participants ?? [],
|
||||
sticking_points: summaryResult.stickingPoints ?? [],
|
||||
transcript_length_chars: transcript.length,
|
||||
transcript_length_words: wordCount,
|
||||
transcript_truncated_for_summary: summaryResult.truncated ?? false,
|
||||
summary_generated_at: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'}/api/nodes`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
title,
|
||||
content,
|
||||
chunk: transcript,
|
||||
dimensions: [],
|
||||
metadata,
|
||||
}),
|
||||
});
|
||||
|
||||
const rawResult = await response.json();
|
||||
|
||||
if (!isCreateNodeResponse(rawResult)) {
|
||||
throw new Error('Unexpected response from node creation');
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(rawResult?.error || 'Failed to create chat transcript node');
|
||||
}
|
||||
|
||||
const nodeId = rawResult?.data?.id;
|
||||
const nodeReference = nodeId ? formatNodeForChat({ id: nodeId, title }) : 'None';
|
||||
const resultMessage = nodeId ? `Created chat transcript ${nodeReference}.` : 'Created chat transcript.';
|
||||
|
||||
return buildStructuredSummary({
|
||||
task,
|
||||
action: 'chatTranscriptImport',
|
||||
resultMessage,
|
||||
nodeReference,
|
||||
});
|
||||
}
|
||||
|
||||
export async function enqueueQuickAdd({ rawInput, mode }: QuickAddInput) {
|
||||
const inputType = detectInputType(rawInput, mode);
|
||||
const context: string[] = (inputType === 'note' || inputType === 'chat') ? [] : [rawInput];
|
||||
const task = buildTaskPrompt(inputType, rawInput);
|
||||
const expectedOutcome = buildExpectedOutcome(inputType);
|
||||
|
||||
const delegation = AgentDelegationService.createDelegation({
|
||||
task,
|
||||
context,
|
||||
expectedOutcome,
|
||||
});
|
||||
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
AgentDelegationService.markInProgress(delegation.sessionId);
|
||||
|
||||
let summary: string;
|
||||
if (inputType === 'note') {
|
||||
summary = await handleNoteQuickAdd(rawInput, task);
|
||||
} else if (inputType === 'chat') {
|
||||
summary = await handleChatTranscriptQuickAdd(rawInput, task);
|
||||
} else {
|
||||
summary = await handleExtractionQuickAdd(inputType as ExtractionQuickAddType, rawInput, task);
|
||||
}
|
||||
|
||||
AgentDelegationService.completeDelegation(delegation.sessionId, summary, 'completed');
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'unknown error';
|
||||
AgentDelegationService.completeDelegation(
|
||||
delegation.sessionId,
|
||||
`Quick Add failed: ${message}`,
|
||||
'failed'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return delegation;
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { getDefaultToolNamesForRole } from '@/tools/infrastructure/registry';
|
||||
import { RAH_MAIN_SYSTEM_PROMPT } from '@/config/prompts/rah-main';
|
||||
import { RAH_EASY_SYSTEM_PROMPT } from '@/config/prompts/rah-easy';
|
||||
import { MINI_RAH_SYSTEM_PROMPT } from '@/config/prompts/rah-mini';
|
||||
import { WISE_RAH_SYSTEM_PROMPT } from '@/config/prompts/wise-rah';
|
||||
import type { AgentDefinition } from './types';
|
||||
|
||||
/**
|
||||
* Code-first agent registry (opinionated, not database-driven)
|
||||
* Agents are defined in code and cannot be modified by users
|
||||
*/
|
||||
export class AgentRegistry {
|
||||
// Deterministic agent definitions baked into code
|
||||
private static readonly AGENTS: Record<string, AgentDefinition> = {
|
||||
'ra-h': {
|
||||
id: 1,
|
||||
key: 'ra-h',
|
||||
displayName: 'ra-h (hard)',
|
||||
description: 'Opinionated orchestrator agent',
|
||||
model: 'anthropic/claude-sonnet-4.5',
|
||||
role: 'orchestrator',
|
||||
systemPrompt: RAH_MAIN_SYSTEM_PROMPT,
|
||||
availableTools: getDefaultToolNamesForRole('orchestrator'),
|
||||
enabled: true,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
memory: null,
|
||||
prompts: undefined
|
||||
},
|
||||
'ra-h-easy': {
|
||||
id: 4,
|
||||
key: 'ra-h-easy',
|
||||
displayName: 'ra-h (easy)',
|
||||
description: 'Fast, low-latency orchestrator',
|
||||
model: 'openai/gpt-5-mini',
|
||||
role: 'orchestrator',
|
||||
systemPrompt: RAH_EASY_SYSTEM_PROMPT,
|
||||
availableTools: getDefaultToolNamesForRole('orchestrator'),
|
||||
enabled: true,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
memory: null,
|
||||
prompts: undefined
|
||||
},
|
||||
'mini-rah': {
|
||||
id: 2,
|
||||
key: 'mini-rah',
|
||||
displayName: 'mini ra-h',
|
||||
description: 'Executor agent for delegated tasks',
|
||||
model: 'openai/gpt-4o-mini',
|
||||
role: 'executor',
|
||||
systemPrompt: MINI_RAH_SYSTEM_PROMPT,
|
||||
availableTools: getDefaultToolNamesForRole('executor'),
|
||||
enabled: true,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
memory: null,
|
||||
prompts: undefined
|
||||
},
|
||||
'wise-rah': {
|
||||
id: 3,
|
||||
key: 'wise-rah',
|
||||
displayName: 'wise ra-h',
|
||||
description: 'Complex workflow planner and orchestrator',
|
||||
model: 'openai/gpt-5',
|
||||
role: 'planner',
|
||||
systemPrompt: WISE_RAH_SYSTEM_PROMPT,
|
||||
availableTools: getDefaultToolNamesForRole('planner'),
|
||||
enabled: true,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
memory: null,
|
||||
prompts: undefined
|
||||
}
|
||||
};
|
||||
|
||||
static async getAgentByKey(key: string): Promise<AgentDefinition | null> {
|
||||
return this.AGENTS[key] || null;
|
||||
}
|
||||
|
||||
static async getAgentById(id: number): Promise<AgentDefinition | null> {
|
||||
return Object.values(this.AGENTS).find(a => a.id === id) || null;
|
||||
}
|
||||
|
||||
static async getEnabledAgents(): Promise<AgentDefinition[]> {
|
||||
return Object.values(this.AGENTS).filter(a => a.enabled);
|
||||
}
|
||||
|
||||
static async orchestrator(): Promise<AgentDefinition> {
|
||||
return this.AGENTS['ra-h'];
|
||||
}
|
||||
|
||||
static async orchestratorForMode(mode: 'easy' | 'hard' = 'easy'): Promise<AgentDefinition> {
|
||||
if (mode === 'hard') {
|
||||
return this.AGENTS['ra-h'];
|
||||
}
|
||||
return this.AGENTS['ra-h-easy'];
|
||||
}
|
||||
|
||||
static async executor(): Promise<AgentDefinition> {
|
||||
return this.AGENTS['mini-rah'];
|
||||
}
|
||||
|
||||
static async planner(): Promise<AgentDefinition> {
|
||||
return this.AGENTS['wise-rah'];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
function ensureString(value: unknown): string {
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
function ensureNumber(value: unknown): number | undefined {
|
||||
return typeof value === 'number' ? value : undefined;
|
||||
}
|
||||
|
||||
function truncateOrDefault(value: string, limit = 180, fallback = ''): string {
|
||||
if (!value) return fallback;
|
||||
if (value.length <= limit) return value;
|
||||
return `${value.slice(0, limit - 1)}…`;
|
||||
}
|
||||
|
||||
export function summarizeToolExecution(toolName: string, args: any, result: any): string {
|
||||
const fallback = `${toolName} completed.`;
|
||||
|
||||
if (typeof result === 'string') {
|
||||
const trimmed = result.trim();
|
||||
return trimmed || fallback;
|
||||
}
|
||||
|
||||
if (!result || typeof result !== 'object') {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
if (result.success === false) {
|
||||
const error = ensureString(result.error) || 'unknown error';
|
||||
return `${toolName} failed: ${error}`;
|
||||
}
|
||||
|
||||
const message = ensureString((result as any).message);
|
||||
if (message) {
|
||||
return message;
|
||||
}
|
||||
|
||||
if (toolName === 'think') {
|
||||
const trace = (result as any).data?.trace ?? args ?? {};
|
||||
const step = ensureNumber(trace.step ?? args?.step);
|
||||
const purpose = ensureString(trace.purpose ?? args?.purpose) || 'planning';
|
||||
const thoughts = ensureString(trace.thoughts ?? args?.thoughts);
|
||||
const next = ensureString(trace.next_action ?? args?.next_action);
|
||||
|
||||
let summary = `Plan${step ? ` step ${step}` : ''}: ${truncateOrDefault(purpose, 120, purpose)}`;
|
||||
if (thoughts) {
|
||||
summary += ` — ${truncateOrDefault(thoughts, 160, thoughts)}`;
|
||||
}
|
||||
if (next) {
|
||||
summary += `. Next: ${truncateOrDefault(next, 80, next)}`;
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
if (toolName === 'webSearch') {
|
||||
const query = ensureString(args?.query) || ensureString(result.data?.query);
|
||||
const results = Array.isArray(result.data?.results) ? result.data.results : [];
|
||||
if (results.length > 0) {
|
||||
const items = results.slice(0, 3).map((entry: any) => {
|
||||
const title = ensureString(entry.title) || ensureString(entry.url) || 'Result';
|
||||
const url = ensureString(entry.url);
|
||||
return url ? `${truncateOrDefault(title, 80, title)} (${url})` : truncateOrDefault(title, 80, title);
|
||||
});
|
||||
return `Web search${query ? ` for "${truncateOrDefault(query, 60, query)}"` : ''}: ${items.join('; ')}`;
|
||||
}
|
||||
return `Web search${query ? ` for "${truncateOrDefault(query, 60, query)}"` : ''}: no results.`;
|
||||
}
|
||||
|
||||
if (toolName === 'searchContentEmbeddings') {
|
||||
const query = ensureString(args?.query) || ensureString(result.data?.query);
|
||||
const chunks = Array.isArray(result.data?.chunks) ? result.data.chunks : [];
|
||||
if (chunks.length > 0) {
|
||||
const top = chunks[0];
|
||||
const snippet = ensureString(top.text);
|
||||
const nodeId = ensureNumber(top.node_id);
|
||||
const preview = truncateOrDefault(snippet, 160, snippet);
|
||||
return `Embedding search${query ? ` for "${truncateOrDefault(query, 60, query)}"` : ''} found ${chunks.length} chunk(s). Top${nodeId ? ` [NODE:${nodeId}]` : ''}: ${preview}`;
|
||||
}
|
||||
return `Embedding search${query ? ` for "${truncateOrDefault(query, 60, query)}"` : ''}: no matches.`;
|
||||
}
|
||||
|
||||
if (toolName === 'youtubeExtract') {
|
||||
const title = ensureString(result.data?.title) || ensureString(args?.title);
|
||||
const formatted = ensureString(result.data?.formatted_display);
|
||||
if (formatted) {
|
||||
return `YouTube extract created ${formatted}.`;
|
||||
}
|
||||
if (title) {
|
||||
return `YouTube extract processed "${truncateOrDefault(title, 80, title)}".`;
|
||||
}
|
||||
return 'YouTube extract completed.';
|
||||
}
|
||||
|
||||
if (toolName === 'queryNodes') {
|
||||
const nodes = Array.isArray(result.data?.nodes) ? result.data.nodes : [];
|
||||
if (nodes.length > 0) {
|
||||
const labels = nodes
|
||||
.slice(0, 3)
|
||||
.map((node: any) => ensureString(node.formatted_display) || ensureString(node.title) || `[NODE:${node.id}]`)
|
||||
.join(', ');
|
||||
return `Found ${nodes.length} node(s): ${labels}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (toolName === 'queryEdge') {
|
||||
const edges = Array.isArray(result.data?.edges) ? result.data.edges : [];
|
||||
if (edges.length > 0) {
|
||||
const edge = edges[0];
|
||||
return `Found ${edges.length} edge(s), e.g., ${edge.from_node_id} → ${edge.to_node_id}.`;
|
||||
}
|
||||
return 'No edges found.';
|
||||
}
|
||||
|
||||
if (result.data?.formatted_display) {
|
||||
return ensureString(result.data.formatted_display) || fallback;
|
||||
}
|
||||
|
||||
if (result.data?.title) {
|
||||
return `Processed "${truncateOrDefault(ensureString(result.data.title), 80, result.data.title)}".`;
|
||||
}
|
||||
|
||||
if (result.data?.count !== undefined) {
|
||||
const count = result.data.count;
|
||||
return `${toolName} returned ${count} item(s).`;
|
||||
}
|
||||
|
||||
try {
|
||||
const preview = JSON.stringify(result.data ?? result);
|
||||
return truncateOrDefault(preview, 200, fallback);
|
||||
} catch (error) {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { generateText } from 'ai';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
|
||||
export interface TranscriptSummaryResult {
|
||||
subject?: string;
|
||||
summary?: string;
|
||||
intent?: string;
|
||||
progress?: string;
|
||||
highlights?: string[];
|
||||
openQuestions?: string[];
|
||||
participants?: string[];
|
||||
stickingPoints?: string[];
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
const MAX_TRANSCRIPT_CHARS = 15000;
|
||||
|
||||
function buildPrompt(snippet: string) {
|
||||
return `You will receive a raw conversation transcript copy/pasted from various chat apps. It may contain timestamps, "You said:", "ChatGPT said:", missing speaker labels, or UI noise like "Thought for 11s".
|
||||
|
||||
Return ONLY valid JSON with this exact shape (no markdown, no commentary):
|
||||
{
|
||||
"subject": "Short descriptive thread title",
|
||||
"summary": "2-3 sentence narrative describing what was discussed and why it matters",
|
||||
"intent": "What the human seemed to be trying to accomplish (1 sentence, second-person)",
|
||||
"progress": "Where the conversation actually moved forward (1-2 sentences, second-person)",
|
||||
"stickingPoints": ["Concise phrases describing where you got stuck or needed help"],
|
||||
"highlights": ["Key takeaway 1", "Key takeaway 2"],
|
||||
"openQuestions": ["Follow-up 1"],
|
||||
"participants": ["user", "assistant"]
|
||||
}
|
||||
|
||||
Guidelines:
|
||||
- Never mention the formatting noise; describe the substance.
|
||||
- Keep highlights concise bullet phrases (<=12 words).
|
||||
- Sticking points should be authentic blockers, not restatements of highlights.
|
||||
- Open questions are the clearest next steps; omit the array when nothing actionable exists.
|
||||
- Participants should be lowercase role labels (user, assistant, reviewer, etc.).
|
||||
- Prefer phrasing that sounds like you're talking directly to the user ("you were trying to...").
|
||||
|
||||
Transcript:
|
||||
${snippet}`;
|
||||
}
|
||||
|
||||
export async function summarizeTranscript(transcript: string): Promise<TranscriptSummaryResult> {
|
||||
const limited = transcript.length > MAX_TRANSCRIPT_CHARS
|
||||
? transcript.slice(0, MAX_TRANSCRIPT_CHARS)
|
||||
: transcript;
|
||||
|
||||
try {
|
||||
const response = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
prompt: buildPrompt(limited),
|
||||
maxOutputTokens: 600,
|
||||
});
|
||||
|
||||
let content = response.text || '';
|
||||
content = content.replace(/```json/gi, '').replace(/```/g, '').trim();
|
||||
const parsed = JSON.parse(content);
|
||||
|
||||
return {
|
||||
subject: typeof parsed.subject === 'string' ? parsed.subject : undefined,
|
||||
summary: typeof parsed.summary === 'string' ? parsed.summary : undefined,
|
||||
intent: typeof parsed.intent === 'string' ? parsed.intent : undefined,
|
||||
progress: typeof parsed.progress === 'string' ? parsed.progress : undefined,
|
||||
highlights: Array.isArray(parsed.highlights)
|
||||
? parsed.highlights.filter((h: unknown) => typeof h === 'string' && h.trim().length > 0)
|
||||
: [],
|
||||
openQuestions: Array.isArray(parsed.openQuestions)
|
||||
? parsed.openQuestions.filter((q: unknown) => typeof q === 'string' && q.trim().length > 0)
|
||||
: [],
|
||||
participants: Array.isArray(parsed.participants)
|
||||
? parsed.participants.filter((p: unknown) => typeof p === 'string' && p.trim().length > 0)
|
||||
: [],
|
||||
stickingPoints: Array.isArray(parsed.stickingPoints)
|
||||
? parsed.stickingPoints.filter((s: unknown) => typeof s === 'string' && s.trim().length > 0)
|
||||
: [],
|
||||
truncated: transcript.length > MAX_TRANSCRIPT_CHARS,
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn('[TranscriptSummarizer] Failed to summarize transcript, falling back', error);
|
||||
return {
|
||||
summary: '',
|
||||
intent: '',
|
||||
progress: '',
|
||||
highlights: [],
|
||||
openQuestions: [],
|
||||
participants: [],
|
||||
stickingPoints: [],
|
||||
truncated: transcript.length > MAX_TRANSCRIPT_CHARS,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
export type AgentRole = 'orchestrator' | 'executor' | 'planner';
|
||||
|
||||
export interface AgentDefinition {
|
||||
id: number;
|
||||
key: string;
|
||||
displayName: string;
|
||||
description: string | null;
|
||||
model: string;
|
||||
role: AgentRole;
|
||||
systemPrompt: string;
|
||||
availableTools: string[];
|
||||
enabled: boolean;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
memory?: Record<string, unknown> | null;
|
||||
prompts?: Array<{ id: string; name: string; content: string }>;
|
||||
}
|
||||
|
||||
export interface AgentSummary {
|
||||
key: string;
|
||||
displayName: string;
|
||||
role: AgentRole;
|
||||
model: string;
|
||||
enabled: boolean;
|
||||
}
|
||||
@@ -0,0 +1,709 @@
|
||||
import { streamText, CoreMessage } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import type { LanguageModelV2ToolResultOutput } from '@ai-sdk/provider';
|
||||
import { AgentDelegationService } from '@/services/agents/delegation';
|
||||
import { WISE_RAH_SYSTEM_PROMPT } from '@/config/prompts/wise-rah';
|
||||
import { getToolsForRole } from '@/tools/infrastructure/registry';
|
||||
import { ChatLoggingMiddleware } from '@/services/chat/middleware';
|
||||
import { calculateCost } from '@/services/analytics/pricing';
|
||||
import { UsageData } from '@/types/analytics';
|
||||
import { summarizeToolExecution } from '@/services/agents/toolResultUtils';
|
||||
import { edgeService } from '@/services/database/edges';
|
||||
import { eventBroadcaster } from '@/services/events';
|
||||
import { delegationStreamBroadcaster } from '@/app/api/rah/delegations/stream/route';
|
||||
import { nodeService } from '@/services/database/nodes';
|
||||
import { RequestContext } from '@/services/context/requestContext';
|
||||
import { isLocalMode } from '@/config/runtime';
|
||||
|
||||
export interface WiseRAHExecutionInput {
|
||||
sessionId: string;
|
||||
task: string;
|
||||
context: string[];
|
||||
expectedOutcome?: string | null;
|
||||
traceId?: string;
|
||||
parentChatId?: number;
|
||||
workflowKey?: string;
|
||||
workflowNodeId?: number;
|
||||
}
|
||||
|
||||
export class WiseRAHExecutor {
|
||||
static async execute({ sessionId, task, context, expectedOutcome, traceId, parentChatId, workflowKey, workflowNodeId }: WiseRAHExecutionInput) {
|
||||
console.log('🧙 [WiseRAHExecutor] Starting execution', { sessionId, task: task.substring(0, 100) });
|
||||
try {
|
||||
const requestContext = RequestContext.get();
|
||||
const wiseRahKey =
|
||||
requestContext.apiKeys?.openai ||
|
||||
process.env.RAH_WISE_RAH_OPENAI_API_KEY ||
|
||||
process.env.OPENAI_API_KEY;
|
||||
if (!wiseRahKey) {
|
||||
throw new Error('RAH_WISE_RAH_OPENAI_API_KEY (or OPENAI_API_KEY) is not set.');
|
||||
}
|
||||
|
||||
AgentDelegationService.markInProgress(sessionId);
|
||||
console.log('✅ [WiseRAHExecutor] Delegation marked in progress');
|
||||
|
||||
const normalizedTask = task.toLowerCase();
|
||||
const normalizedOutcome = (expectedOutcome || '').toLowerCase();
|
||||
const isWorkflow = Boolean(workflowKey) || normalizedTask.startsWith('execute workflow');
|
||||
const explicitWriteRequest = /\b(create|update|edge|append|workflow|integrate|summarize into node|link node|embed|ingest|synchronize)\b/.test(normalizedTask) ||
|
||||
/\b(create|update|edge|append|workflow|integrate|summarize into node|link node|embed|ingest|synchronize)\b/.test(normalizedOutcome);
|
||||
const allowWrites = isWorkflow || explicitWriteRequest;
|
||||
const analysisOnly = !allowWrites;
|
||||
const maxIterationsLimit = isWorkflow ? 20 : 5;
|
||||
const maxDelegationsAllowed = allowWrites ? (isWorkflow ? 12 : 2) : 0;
|
||||
const maxDistinctWebSearches = isWorkflow ? 6 : 4;
|
||||
const maxDistinctEmbeddingSearches = isWorkflow ? 5 : 3;
|
||||
|
||||
const promptSections = [
|
||||
`Task: ${task}`,
|
||||
context.length ? `Context:\n- ${context.join('\n- ')}` : undefined,
|
||||
expectedOutcome ? `Expected outcome: ${expectedOutcome}` : undefined,
|
||||
analysisOnly ? 'Constraint: This is an analysis-only request. Stay strictly read-only: do not call delegateToMiniRAH and do not request new extractions. Work only with existing knowledge.' : undefined,
|
||||
'Return a structured summary following the format in your system prompt (Task/Actions/Result/Nodes/Follow-up).'
|
||||
].filter(Boolean);
|
||||
|
||||
const openaiProvider = createOpenAI({ apiKey: wiseRahKey });
|
||||
console.log('🔧 [WiseRAHExecutor] OpenAI provider created');
|
||||
|
||||
const plannerTools = getToolsForRole('planner');
|
||||
|
||||
// Remove delegateToMiniRAH for integrate workflow (wise-rah does updates directly)
|
||||
if (workflowKey === 'integrate' && plannerTools.delegateToMiniRAH) {
|
||||
delete plannerTools.delegateToMiniRAH;
|
||||
console.log('🚫 [WiseRAHExecutor] Removed delegateToMiniRAH for integrate workflow (direct updates only)');
|
||||
}
|
||||
|
||||
// For analysis-only tasks, also remove delegation
|
||||
if (analysisOnly && plannerTools.delegateToMiniRAH) {
|
||||
delete plannerTools.delegateToMiniRAH;
|
||||
}
|
||||
|
||||
console.log('🛠️ [WiseRAHExecutor] Planner tools retrieved:', Object.keys(plannerTools));
|
||||
|
||||
const toolsUsedInSession: string[] = [];
|
||||
const delegatedEdgeKeys = new Set<string>();
|
||||
|
||||
// Workflow progress is now streamed directly to delegation tabs via delegationStreamBroadcaster
|
||||
// No need to broadcast WORKFLOW_PROGRESS events to main chat anymore
|
||||
const wrappedTools = Object.fromEntries(
|
||||
Object.entries(plannerTools).map(([name, tool]) => {
|
||||
const wrapped = {
|
||||
...tool,
|
||||
async execute(params: any, context: any) {
|
||||
if (!toolsUsedInSession.includes(name)) {
|
||||
toolsUsedInSession.push(name);
|
||||
}
|
||||
if (name === 'delegateToMiniRAH') {
|
||||
const extractEdgeKey = () => {
|
||||
if (!params) return null;
|
||||
const tryFromTask = () => {
|
||||
if (typeof params.task !== 'string') return null;
|
||||
const matches = [...params.task.matchAll(/\[NODE:(\d+)/g)];
|
||||
if (matches.length >= 2) {
|
||||
const fromId = Number(matches[0][1]);
|
||||
const toId = Number(matches[1][1]);
|
||||
if (Number.isFinite(fromId) && Number.isFinite(toId)) {
|
||||
return `${fromId}->${toId}`;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const tryFromContext = () => {
|
||||
if (!Array.isArray(params.context)) return null;
|
||||
let fromId: number | null = null;
|
||||
let toId: number | null = null;
|
||||
for (const entry of params.context) {
|
||||
if (typeof entry === 'string') {
|
||||
const fromMatch = entry.match(/from_node_id\D+(\d+)/i);
|
||||
const toMatch = entry.match(/to_node_id\D+(\d+)/i);
|
||||
if (fromMatch && Number.isFinite(Number(fromMatch[1]))) {
|
||||
fromId = Number(fromMatch[1]);
|
||||
}
|
||||
if (toMatch && Number.isFinite(Number(toMatch[1]))) {
|
||||
toId = Number(toMatch[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Number.isFinite(fromId as number) && Number.isFinite(toId as number)) {
|
||||
return `${fromId}->${toId}`;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
return tryFromTask() || tryFromContext();
|
||||
};
|
||||
|
||||
const edgeKey = extractEdgeKey();
|
||||
if (edgeKey) {
|
||||
if (delegatedEdgeKeys.has(edgeKey)) {
|
||||
const [from, to] = edgeKey.split('->');
|
||||
const message = `Skipped duplicate edge delegation for nodes ${from}→${to}.`;
|
||||
workerSummaries.push(message);
|
||||
return message;
|
||||
}
|
||||
delegatedEdgeKeys.add(edgeKey);
|
||||
const [from, to] = edgeKey.split('->').map(Number);
|
||||
if (Number.isFinite(from) && Number.isFinite(to)) {
|
||||
const exists = await edgeService.edgeExists(from, to);
|
||||
if (exists) {
|
||||
const message = `Edge ${from}→${to} already exists; delegation skipped.`;
|
||||
workerSummaries.push(message);
|
||||
return message;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return await tool.execute(params, context);
|
||||
}
|
||||
};
|
||||
return [name, wrapped];
|
||||
})
|
||||
);
|
||||
|
||||
// Enforce read-only constraint - remove write tools EXCEPT for workflows
|
||||
// Workflows (like integrate) need updateNode for direct content updates
|
||||
const writeToolsToRemove = isWorkflow
|
||||
? ['createNode', 'createEdge', 'updateEdge', 'embedContent', 'youtubeExtract', 'websiteExtract', 'paperExtract', 'delegateToWiseRAH']
|
||||
: ['createNode', 'updateNode', 'createEdge', 'updateEdge', 'embedContent', 'youtubeExtract', 'websiteExtract', 'paperExtract', 'delegateToWiseRAH'];
|
||||
|
||||
writeToolsToRemove.forEach(toolName => {
|
||||
if (toolName in wrappedTools) {
|
||||
console.warn(`WiseRAHExecutor: ${toolName} detected in planner toolset. Removing to enforce read-only constraint.`);
|
||||
delete wrappedTools[toolName];
|
||||
}
|
||||
});
|
||||
|
||||
if (isWorkflow && 'updateNode' in wrappedTools) {
|
||||
console.log('✅ [WiseRAHExecutor] updateNode preserved for workflow execution');
|
||||
}
|
||||
console.log('🔒 [WiseRAHExecutor] Final tools after read-only enforcement:', Object.keys(wrappedTools));
|
||||
|
||||
console.log('📝 [WiseRAHExecutor] Starting manual agentic loop...');
|
||||
|
||||
const messages: CoreMessage[] = [
|
||||
{ role: 'system', content: WISE_RAH_SYSTEM_PROMPT },
|
||||
{ role: 'user', content: promptSections.join('\n\n') }
|
||||
];
|
||||
|
||||
let finalText = '';
|
||||
const totalUsage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
|
||||
const maxIterations = maxIterationsLimit;
|
||||
|
||||
const seenToolResults = new Map<string, { output: LanguageModelV2ToolResultOutput; summary: string }>();
|
||||
const workerSummaries: string[] = [];
|
||||
const workerSummarySet = new Set<string>();
|
||||
let hasPlan = false;
|
||||
let planReminderAdded = false;
|
||||
let planIncludesDelegation = false;
|
||||
let planRevisionNoticeSent = false;
|
||||
let delegationNudgeSent = false;
|
||||
let iterationsSincePlan = 0;
|
||||
let lastPlanSummary = '';
|
||||
let totalDelegations = 0;
|
||||
let didCreateEdge = false;
|
||||
let didUpdateNode = false;
|
||||
const uniqueWebQueries = new Set<string>();
|
||||
const uniqueEmbeddingQueries = new Set<string>();
|
||||
let finalSummaryRequested = false;
|
||||
let iterationsWithoutDelegation = 0;
|
||||
|
||||
const ensureString = (value: unknown) => (typeof value === 'string' ? value.trim() : '');
|
||||
|
||||
const sanitizeForBroadcast = (value: unknown) => {
|
||||
if (value === undefined) return undefined;
|
||||
try {
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
} catch (error) {
|
||||
console.warn('[WiseRAHExecutor] Failed to serialize delegation payload', error);
|
||||
if (typeof value === 'string') return value;
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const emitDelegationEvent = (payload: Record<string, unknown>) => {
|
||||
delegationStreamBroadcaster.broadcast(sessionId, payload);
|
||||
};
|
||||
|
||||
const emitToolStart = (toolCallId: string, toolName: string, input: unknown) => {
|
||||
emitDelegationEvent({
|
||||
type: 'tool-input-start',
|
||||
toolCallId,
|
||||
toolName,
|
||||
input: sanitizeForBroadcast(input),
|
||||
});
|
||||
};
|
||||
|
||||
const emitToolCompletion = (
|
||||
toolCallId: string,
|
||||
toolName: string,
|
||||
rawResult: unknown,
|
||||
summary: string,
|
||||
status: 'complete' | 'error' = 'complete',
|
||||
errorMessage?: string
|
||||
) => {
|
||||
emitDelegationEvent({
|
||||
type: 'tool-output-available',
|
||||
toolCallId,
|
||||
toolName,
|
||||
result: sanitizeForBroadcast(rawResult),
|
||||
summary,
|
||||
status,
|
||||
error: errorMessage,
|
||||
});
|
||||
};
|
||||
|
||||
const buildToolOutput = (toolName: string, summary: string, rawResult: any): LanguageModelV2ToolResultOutput => {
|
||||
const trimmedSummary = summary.trim();
|
||||
|
||||
if (rawResult && typeof rawResult === 'object' && rawResult.success === false) {
|
||||
const message = trimmedSummary || ensureString(rawResult.error) || `${toolName} failed.`;
|
||||
return { type: 'error-text', value: message };
|
||||
}
|
||||
|
||||
if (typeof rawResult === 'string') {
|
||||
const value = rawResult.trim() || trimmedSummary || `${toolName} completed.`;
|
||||
return { type: 'text', value };
|
||||
}
|
||||
|
||||
if (trimmedSummary) {
|
||||
return { type: 'text', value: trimmedSummary };
|
||||
}
|
||||
|
||||
return { type: 'text', value: `${toolName} completed.` };
|
||||
};
|
||||
|
||||
const requestFinalSummary = async (instruction: string) => {
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: instruction,
|
||||
});
|
||||
|
||||
const finalStreamResult = await streamText({
|
||||
model: openaiProvider('gpt-5'),
|
||||
messages,
|
||||
tools: {},
|
||||
maxOutputTokens: 500,
|
||||
});
|
||||
|
||||
// Collect the complete response
|
||||
const finalChunks: string[] = [];
|
||||
for await (const chunk of finalStreamResult.textStream) {
|
||||
finalChunks.push(chunk);
|
||||
}
|
||||
|
||||
const finalResponse = {
|
||||
text: finalChunks.join(''),
|
||||
usage: await finalStreamResult.usage,
|
||||
};
|
||||
|
||||
totalUsage.inputTokens += finalResponse.usage?.inputTokens || 0;
|
||||
totalUsage.outputTokens += finalResponse.usage?.outputTokens || 0;
|
||||
totalUsage.totalTokens += finalResponse.usage?.totalTokens || 0;
|
||||
|
||||
return finalResponse.text ?? '';
|
||||
};
|
||||
|
||||
const normaliseForSignature = (toolName: string, input: any) => {
|
||||
if (!input || typeof input !== 'object') {
|
||||
return input;
|
||||
}
|
||||
|
||||
if (toolName === 'webSearch' && 'query' in input) {
|
||||
const query = ensureString(input.query).toLowerCase().replace(/\s+/g, ' ').trim();
|
||||
return { ...input, query };
|
||||
}
|
||||
|
||||
if (toolName === 'searchContentEmbeddings' && 'query' in input) {
|
||||
const query = ensureString(input.query).toLowerCase().replace(/\s+/g, ' ').trim();
|
||||
return { ...input, query };
|
||||
}
|
||||
|
||||
return input;
|
||||
};
|
||||
|
||||
for (let i = 0; i < maxIterations; i++) {
|
||||
console.log(`🔄 [WiseRAHExecutor] Iteration ${i + 1}/${maxIterations}`);
|
||||
|
||||
// Touch delegation every iteration to prevent cleanup from killing it
|
||||
AgentDelegationService.touchDelegation(sessionId);
|
||||
|
||||
const streamResult = await streamText({
|
||||
model: openaiProvider('gpt-5'),
|
||||
messages,
|
||||
tools: wrappedTools,
|
||||
});
|
||||
|
||||
// Collect the complete response
|
||||
const chunks: string[] = [];
|
||||
for await (const chunk of streamResult.textStream) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
|
||||
const response = {
|
||||
text: chunks.join(''),
|
||||
finishReason: await streamResult.finishReason,
|
||||
usage: await streamResult.usage,
|
||||
toolCalls: await streamResult.toolCalls,
|
||||
};
|
||||
|
||||
totalUsage.inputTokens += response.usage?.inputTokens || 0;
|
||||
totalUsage.outputTokens += response.usage?.outputTokens || 0;
|
||||
totalUsage.totalTokens += response.usage?.totalTokens || 0;
|
||||
|
||||
console.log(`📊 [WiseRAHExecutor] Step ${i + 1} finishReason:`, response.finishReason);
|
||||
|
||||
// Stream text response to delegation chat
|
||||
if (response.text && response.text.trim()) {
|
||||
emitDelegationEvent({
|
||||
type: 'text-delta',
|
||||
delta: response.text,
|
||||
});
|
||||
}
|
||||
|
||||
if (response.finishReason !== 'tool-calls') {
|
||||
finalText = response.text;
|
||||
console.log('✅ [WiseRAHExecutor] Got final text');
|
||||
break;
|
||||
}
|
||||
|
||||
const toolCalls = response.toolCalls || [];
|
||||
console.log(`🔧 [WiseRAHExecutor] Executing ${toolCalls.length} tool calls`);
|
||||
|
||||
// Broadcast new assistant message for next iteration
|
||||
if (toolCalls.length > 0) {
|
||||
emitDelegationEvent({ type: 'assistant-message' });
|
||||
}
|
||||
|
||||
messages.push({
|
||||
role: 'assistant',
|
||||
content: toolCalls.map(call => ({
|
||||
type: 'tool-call' as const,
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
input: (call as any).input ?? (call as any).args,
|
||||
})),
|
||||
});
|
||||
|
||||
const toolResults: Array<{
|
||||
type: 'tool-result';
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
output: LanguageModelV2ToolResultOutput;
|
||||
}> = [];
|
||||
|
||||
let executedTool = false;
|
||||
|
||||
for (const call of toolCalls) {
|
||||
let callInputRaw = (call as any).input ?? (call as any).args;
|
||||
|
||||
// Append logic now handled in updateNode tool itself (lines 27-44)
|
||||
|
||||
const signatureInput = normaliseForSignature(call.toolName, callInputRaw);
|
||||
const signature = JSON.stringify({ tool: call.toolName, input: signatureInput });
|
||||
|
||||
// Broadcast tool call to delegation stream
|
||||
emitToolStart(call.toolCallId, call.toolName, callInputRaw);
|
||||
|
||||
if (!hasPlan && call.toolName !== 'think') {
|
||||
const warning = 'Planning required: use the think tool to outline a numbered plan (purpose, thoughts, next action) before other tools.';
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output: { type: 'error-text', value: warning },
|
||||
});
|
||||
emitToolCompletion(call.toolCallId, call.toolName, { success: false }, warning, 'error', warning);
|
||||
|
||||
if (!planReminderAdded) {
|
||||
planReminderAdded = true;
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: 'Before calling other tools, use the think tool to draft a numbered plan and specify your next action.',
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (call.toolName !== 'think' && seenToolResults.has(signature)) {
|
||||
const cached = seenToolResults.get(signature)!;
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output: cached.output,
|
||||
});
|
||||
|
||||
// Broadcast cached result
|
||||
emitToolCompletion(call.toolCallId, call.toolName, cached.summary || 'Cached result', cached.summary || 'Cached result');
|
||||
|
||||
if (call.toolName === 'delegateToMiniRAH' && cached.summary) {
|
||||
if (!workerSummarySet.has(cached.summary)) {
|
||||
workerSummarySet.add(cached.summary);
|
||||
workerSummaries.push(`[reused] ${cached.summary}`);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const tool = wrappedTools[call.toolName];
|
||||
if (!tool) {
|
||||
const warning = `Tool ${call.toolName} is not available to wise ra-h.`;
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output: { type: 'error-text', value: warning },
|
||||
});
|
||||
emitToolCompletion(call.toolCallId, call.toolName, { success: false }, warning, 'error', warning);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const rawResult = await tool.execute(callInputRaw, {});
|
||||
const summary = summarizeToolExecution(call.toolName, callInputRaw, rawResult);
|
||||
const output = buildToolOutput(call.toolName, summary, rawResult);
|
||||
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output,
|
||||
});
|
||||
emitToolCompletion(call.toolCallId, call.toolName, rawResult, summary, 'complete');
|
||||
|
||||
if (call.toolName === 'think') {
|
||||
hasPlan = true;
|
||||
lastPlanSummary = summary;
|
||||
if (allowWrites) {
|
||||
planIncludesDelegation = /delegate\b|delegateToMiniRAH|mini ra-h/i.test(summary);
|
||||
} else {
|
||||
planIncludesDelegation = false;
|
||||
}
|
||||
planRevisionNoticeSent = false;
|
||||
delegationNudgeSent = false;
|
||||
iterationsSincePlan = 0;
|
||||
} else {
|
||||
seenToolResults.set(signature, { output, summary });
|
||||
if (call.toolName === 'delegateToMiniRAH' && summary && !workerSummarySet.has(summary)) {
|
||||
workerSummarySet.add(summary);
|
||||
workerSummaries.push(summary);
|
||||
totalDelegations += 1;
|
||||
|
||||
if (/Created edge/i.test(summary) || /Edge created/i.test(summary)) {
|
||||
didCreateEdge = true;
|
||||
}
|
||||
if (/Updated node/i.test(summary) || /Appended/i.test(summary) || /content updated/i.test(summary)) {
|
||||
didUpdateNode = true;
|
||||
}
|
||||
}
|
||||
if (call.toolName === 'webSearch') {
|
||||
const query = ensureString(signatureInput?.query ?? callInputRaw?.query);
|
||||
if (query) {
|
||||
uniqueWebQueries.add(query);
|
||||
}
|
||||
}
|
||||
if (call.toolName === 'searchContentEmbeddings') {
|
||||
const query = ensureString(signatureInput?.query ?? callInputRaw?.query);
|
||||
if (query) {
|
||||
uniqueEmbeddingQueries.add(query);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
executedTool = true;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Tool execution failed';
|
||||
toolResults.push({
|
||||
type: 'tool-result',
|
||||
toolCallId: call.toolCallId,
|
||||
toolName: call.toolName,
|
||||
output: { type: 'error-text', value: message },
|
||||
});
|
||||
emitToolCompletion(call.toolCallId, call.toolName, { success: false }, message, 'error', message);
|
||||
}
|
||||
}
|
||||
|
||||
messages.push({
|
||||
role: 'tool',
|
||||
content: toolResults,
|
||||
});
|
||||
|
||||
if (hasPlan) {
|
||||
iterationsSincePlan += 1;
|
||||
// Legacy delegation nudges removed - wise-rah completes workflows independently
|
||||
}
|
||||
|
||||
const enoughMaterial = hasPlan && (allowWrites ? workerSummaries.length >= 2 : executedTool);
|
||||
const tooManyDelegations = allowWrites && totalDelegations >= maxDelegationsAllowed && maxDelegationsAllowed > 0;
|
||||
const webSearchCapReached = uniqueWebQueries.size >= maxDistinctWebSearches;
|
||||
const embeddingCapReached = uniqueEmbeddingQueries.size >= maxDistinctEmbeddingSearches;
|
||||
|
||||
if (allowWrites && workflowKey === 'integrate' && totalDelegations === 0) {
|
||||
iterationsWithoutDelegation += 1;
|
||||
if (!delegationNudgeSent && iterationsWithoutDelegation >= 4) {
|
||||
delegationNudgeSent = true;
|
||||
const targetInstruction = workflowNodeId
|
||||
? `Focus on node [NODE:${workflowNodeId}]. Delegate to mini ra-h now to (a) append integration insights to its content and (b) create edges to the highest-value related nodes you identified.`
|
||||
: 'Delegate to mini ra-h now to (a) append integration insights to the focused node and (b) create edges to the highest-value related nodes you identified.';
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: `${targetInstruction} Do not continue researching until those delegations are complete.`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
iterationsWithoutDelegation = 0;
|
||||
}
|
||||
|
||||
if (allowWrites && workflowKey === 'integrate' && didCreateEdge && !didUpdateNode) {
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: 'Edges are in place. Delegate to mini ra-h to append the Integration Insights section to the focused node before moving forward.',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const minIterationsBeforeSummary = allowWrites ? 2 : 1;
|
||||
if (!finalSummaryRequested && (tooManyDelegations || webSearchCapReached || embeddingCapReached || (enoughMaterial && executedTool && i >= minIterationsBeforeSummary))) {
|
||||
if (allowWrites && workflowKey === 'integrate' && !didUpdateNode) {
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: 'Before summarizing, delegate to mini ra-h to append the Integration Insights content to the focused node.',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (allowWrites && totalDelegations === 0) {
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: 'You have not delegated any execution yet. Identify the concrete write actions required and call delegateToMiniRAH to perform them before summarising.',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let instruction = 'You have enough evidence from the workers. Provide the final Task/Actions/Result/Nodes/Follow-up summary now without further tool calls.';
|
||||
if (tooManyDelegations) {
|
||||
instruction = `You have already delegated the maximum allowed (${maxDelegationsAllowed}). Synthesize the findings now using the Task/Actions/Result/Nodes/Follow-up format. Do not call additional tools.`;
|
||||
} else if (webSearchCapReached) {
|
||||
instruction = `You have already issued about ${maxDistinctWebSearches} distinct web searches. Consolidate what you found into the final Task/Actions/Result/Nodes/Follow-up summary now—no additional tool calls.`;
|
||||
} else if (embeddingCapReached) {
|
||||
instruction = `Embedding searches have covered the knowledge base (limit ${maxDistinctEmbeddingSearches}). Switch to producing the Task/Actions/Result/Nodes/Follow-up summary—do not call further tools.`;
|
||||
}
|
||||
|
||||
finalText = await requestFinalSummary(instruction);
|
||||
finalSummaryRequested = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!finalText) {
|
||||
if (allowWrites && totalDelegations === 0) {
|
||||
throw new Error('Wise ra-h attempted to summarize without delegating any execution to mini ra-h.');
|
||||
}
|
||||
console.warn('⚠️ [WiseRAHExecutor] Max iterations hit with no summary. Requesting final response without tools.');
|
||||
finalText = await requestFinalSummary('You have gathered everything needed. Provide the final Task/Actions/Result/Nodes/Follow-up summary now. Do not call any tools.');
|
||||
console.log('✅ [WiseRAHExecutor] Final summary obtained after tool cutoff.');
|
||||
}
|
||||
|
||||
const usage = totalUsage;
|
||||
let summary = typeof finalText === 'string' ? finalText.trim() : '';
|
||||
|
||||
if (summary.length > 2000) {
|
||||
console.log('⚠️ [WiseRAHExecutor] Summary too long, requesting concise version.');
|
||||
summary = (await requestFinalSummary('Condense the findings into ≤300 tokens using the Task/Actions/Result/Nodes/Follow-up format. Focus on the most salient insights and reference key nodes. Do not call any tools.')).trim();
|
||||
}
|
||||
if (summary.length > 1000) {
|
||||
summary = `${summary.slice(0, 997)}…`;
|
||||
}
|
||||
console.log('📄 [WiseRAHExecutor] Summary after trim:', summary);
|
||||
console.log('📏 [WiseRAHExecutor] Summary length:', summary.length);
|
||||
|
||||
if (!summary) {
|
||||
emitDelegationEvent({
|
||||
type: 'assistant-message',
|
||||
});
|
||||
emitDelegationEvent({
|
||||
type: 'text-delta',
|
||||
delta: 'Wise ra-h attempted to summarise but the response was empty. Check tool logs above for context.',
|
||||
});
|
||||
throw new Error('Wise ra-h returned empty summary');
|
||||
}
|
||||
|
||||
console.log('[WiseRAHExecutor] summary:', summary);
|
||||
|
||||
// Calculate cost and log to chats table
|
||||
if (usage) {
|
||||
const inputTokens = (usage as any).promptTokens || usage.inputTokens || 0;
|
||||
const outputTokens = (usage as any).completionTokens || usage.outputTokens || 0;
|
||||
const totalTokens = inputTokens + outputTokens;
|
||||
|
||||
const costResult = calculateCost({
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
modelId: 'gpt-5',
|
||||
});
|
||||
|
||||
const usageData: UsageData = {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens,
|
||||
estimatedCostUsd: costResult.totalCostUsd,
|
||||
modelUsed: 'gpt-5',
|
||||
provider: 'openai',
|
||||
toolsUsed: toolsUsedInSession.length > 0 ? toolsUsedInSession : undefined,
|
||||
toolCallsCount: toolsUsedInSession.length > 0 ? toolsUsedInSession.length : undefined,
|
||||
traceId,
|
||||
parentChatId,
|
||||
workflowKey,
|
||||
workflowNodeId,
|
||||
};
|
||||
|
||||
const delegation = AgentDelegationService.getDelegation(sessionId);
|
||||
const delegationId = delegation?.id;
|
||||
|
||||
await ChatLoggingMiddleware.logChatInteraction(
|
||||
task,
|
||||
summary,
|
||||
{
|
||||
helperName: 'wise-rah',
|
||||
agentType: 'planner',
|
||||
delegationId: delegationId ?? null,
|
||||
sessionId,
|
||||
usageData,
|
||||
traceId,
|
||||
parentChatId,
|
||||
workflowKey,
|
||||
workflowNodeId,
|
||||
systemMessage: WISE_RAH_SYSTEM_PROMPT,
|
||||
backendUsage: [],
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
console.log(`💰 [WiseRAHExecutor] Cost: $${costResult.totalCostUsd.toFixed(6)} (${totalTokens} tokens)`);
|
||||
}
|
||||
|
||||
console.log('✅ [WiseRAHExecutor] Completing delegation with summary');
|
||||
return AgentDelegationService.completeDelegation(sessionId, summary);
|
||||
} catch (error) {
|
||||
console.error('❌ [WiseRAHExecutor] Error during execution:', error);
|
||||
console.error('❌ [WiseRAHExecutor] Error stack:', error instanceof Error ? error.stack : 'No stack');
|
||||
const message = error instanceof Error ? error.message : 'Unknown delegation error';
|
||||
|
||||
// Broadcast error to delegation stream
|
||||
delegationStreamBroadcaster.broadcast(sessionId, {
|
||||
type: 'assistant-message',
|
||||
});
|
||||
delegationStreamBroadcaster.broadcast(sessionId, {
|
||||
type: 'text-delta',
|
||||
delta: `Wise ra-h failed: ${message}`,
|
||||
});
|
||||
|
||||
AgentDelegationService.completeDelegation(sessionId, `Wise ra-h failed: ${message}`, 'failed');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
import OpenAI from 'openai';
|
||||
import type { AgentDefinition } from './agents/types';
|
||||
import { Node } from '@/types/database';
|
||||
import { getToolSchemas, executeTool } from '../tools/infrastructure/registry';
|
||||
import { getSQLiteClient } from './database/sqlite-client';
|
||||
import { apiKeyService } from './storage/apiKeys';
|
||||
|
||||
// Initialize OpenAI client with dynamic API key support
|
||||
function getOpenAiClient(): OpenAI {
|
||||
const apiKey = apiKeyService.getOpenAiKey();
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API key required. Please:\n1. Click the Settings icon (⚙️) in the bottom left\n2. Go to API Keys tab\n3. Add your OpenAI API key\n\nGet your key at: https://platform.openai.com/api-keys');
|
||||
}
|
||||
return new OpenAI({ apiKey });
|
||||
}
|
||||
|
||||
export interface ChatMessage {
|
||||
role: 'system' | 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ToolCall {
|
||||
id: string;
|
||||
name: string;
|
||||
params: any;
|
||||
result: any;
|
||||
}
|
||||
|
||||
export interface ChatResponse {
|
||||
response: string;
|
||||
toolCalls?: ToolCall[];
|
||||
}
|
||||
|
||||
export class AIService {
|
||||
/**
|
||||
* Chat with a helper using GPT-4o-mini with function calling
|
||||
*/
|
||||
static async chatWithHelper(
|
||||
helper: AgentDefinition,
|
||||
message: string,
|
||||
selectedNodeIds: number[] = [],
|
||||
messageHistory: ChatMessage[] = []
|
||||
): Promise<ChatResponse> {
|
||||
try {
|
||||
// Get selected nodes details
|
||||
const selectedNodes = await this.getSelectedNodes(selectedNodeIds);
|
||||
|
||||
// Build context for tools
|
||||
const toolContext = {
|
||||
selectedNodes,
|
||||
database: getSQLiteClient()
|
||||
};
|
||||
|
||||
// Build messages array
|
||||
const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [
|
||||
{
|
||||
role: 'system',
|
||||
content: this.buildSystemPrompt(helper, selectedNodes)
|
||||
},
|
||||
// Add message history
|
||||
...messageHistory.map(msg => ({
|
||||
role: msg.role as 'user' | 'assistant',
|
||||
content: msg.content
|
||||
})),
|
||||
{
|
||||
role: 'user',
|
||||
content: message
|
||||
}
|
||||
];
|
||||
|
||||
// Get tool schemas for this helper
|
||||
const toolSchemas = getToolSchemas(helper.availableTools);
|
||||
|
||||
// Make OpenAI API call
|
||||
const openai = getOpenAiClient();
|
||||
const completion = await openai.chat.completions.create({
|
||||
model: helper.model,
|
||||
messages,
|
||||
tools: toolSchemas.length > 0 ? toolSchemas : undefined,
|
||||
tool_choice: toolSchemas.length > 0 ? 'auto' : undefined,
|
||||
temperature: 0.7,
|
||||
max_tokens: 2000,
|
||||
});
|
||||
|
||||
const assistantMessage = completion.choices[0]?.message;
|
||||
|
||||
if (!assistantMessage) {
|
||||
throw new Error('No response from OpenAI');
|
||||
}
|
||||
|
||||
let response = assistantMessage.content || '';
|
||||
const toolCalls: ToolCall[] = [];
|
||||
|
||||
// Handle tool calls if present
|
||||
if (assistantMessage.tool_calls && assistantMessage.tool_calls.length > 0) {
|
||||
console.log(`Agent ${helper.key} is calling ${assistantMessage.tool_calls.length} tools`);
|
||||
|
||||
for (const toolCall of assistantMessage.tool_calls) {
|
||||
try {
|
||||
const toolName = toolCall.function.name;
|
||||
const toolParams = JSON.parse(toolCall.function.arguments);
|
||||
|
||||
console.log(`Executing tool: ${toolName}`, toolParams);
|
||||
|
||||
// Execute the tool
|
||||
const result = await executeTool(toolName, toolParams, toolContext);
|
||||
|
||||
toolCalls.push({
|
||||
id: toolCall.id,
|
||||
name: toolName,
|
||||
params: toolParams,
|
||||
result
|
||||
});
|
||||
|
||||
console.log(`Tool ${toolName} completed:`, result.success ? 'success' : 'failed');
|
||||
} catch (error) {
|
||||
console.error(`Error executing tool ${toolCall.function.name}:`, error);
|
||||
toolCalls.push({
|
||||
id: toolCall.id,
|
||||
name: toolCall.function.name,
|
||||
params: {},
|
||||
result: {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Tool execution failed'
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// If we have tool results, make another call to get the final response
|
||||
if (toolCalls.length > 0) {
|
||||
const toolMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
|
||||
...messages,
|
||||
{
|
||||
role: 'assistant',
|
||||
content: assistantMessage.content,
|
||||
tool_calls: assistantMessage.tool_calls
|
||||
},
|
||||
...toolCalls.map(toolCall => ({
|
||||
role: 'tool' as const,
|
||||
tool_call_id: toolCall.id,
|
||||
content: JSON.stringify(toolCall.result)
|
||||
}))
|
||||
];
|
||||
|
||||
const finalCompletion = await getOpenAiClient().chat.completions.create({
|
||||
model: 'gpt-5-mini',
|
||||
messages: toolMessages,
|
||||
temperature: 0.7,
|
||||
max_tokens: 2000,
|
||||
});
|
||||
|
||||
response = finalCompletion.choices[0]?.message?.content || response;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
response,
|
||||
toolCalls: toolCalls.length > 0 ? toolCalls : undefined
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in chatWithHelper:', error);
|
||||
throw new Error(
|
||||
error instanceof Error ? error.message : 'Failed to process chat message'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get selected nodes with their details
|
||||
*/
|
||||
private static async getSelectedNodes(nodeIds: number[]): Promise<Node[]> {
|
||||
if (nodeIds.length === 0) return [];
|
||||
|
||||
try {
|
||||
const sqlite = getSQLiteClient();
|
||||
const placeholders = nodeIds.map(() => '?').join(', ');
|
||||
const result = sqlite.query<any>(
|
||||
`SELECT n.id, n.title, n.content, n.link, n.metadata, n.chunk,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
FROM nodes n
|
||||
WHERE n.id IN (${placeholders})
|
||||
ORDER BY n.created_at DESC`,
|
||||
nodeIds
|
||||
);
|
||||
return result.rows.map((row: any) => ({
|
||||
...row,
|
||||
dimensions: JSON.parse(row.dimensions_json || '[]')
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error('Error fetching selected nodes:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build system prompt with context
|
||||
*/
|
||||
private static buildSystemPrompt(helper: AgentDefinition, selectedNodes: Node[]): string {
|
||||
let systemPrompt = helper.systemPrompt;
|
||||
|
||||
// Universal rule: ensure clickable node labels in UI
|
||||
systemPrompt += '\n\nNode references: Always format nodes as [NODE:id:"title"] so the UI renders clickable labels.';
|
||||
|
||||
// Add context about selected nodes
|
||||
if (selectedNodes.length > 0) {
|
||||
systemPrompt += '\n\n## Selected Nodes Context\n';
|
||||
systemPrompt += `You have ${selectedNodes.length} node(s) selected:\n\n`;
|
||||
|
||||
selectedNodes.forEach((node, index) => {
|
||||
systemPrompt += `${index + 1}. **${node.title || 'Untitled'}**\n`;
|
||||
if (node.dimensions?.length > 0) {
|
||||
systemPrompt += ` - Dimensions: ${node.dimensions.join(', ')}\n`;
|
||||
}
|
||||
if (node.link) {
|
||||
systemPrompt += ` - URL: ${node.link}\n`;
|
||||
}
|
||||
systemPrompt += '\n';
|
||||
});
|
||||
} else {
|
||||
systemPrompt += '\n\n## Context\nNo nodes are currently selected. You can still help with general queries and web searches.';
|
||||
}
|
||||
|
||||
// Add available tools information
|
||||
if (helper.availableTools.length > 0) {
|
||||
systemPrompt += '\n\n## Available Tools\n';
|
||||
systemPrompt += `You have access to the following tools: ${helper.availableTools.join(', ')}\n`;
|
||||
systemPrompt += 'Use these tools when they would be helpful to answer the user\'s question.';
|
||||
}
|
||||
|
||||
// Subtle, hard-coded intent capture across all helpers (non-intrusive)
|
||||
systemPrompt += '\n\n## Subtle Intent Capture\n';
|
||||
systemPrompt += [
|
||||
'When it would clearly help you serve better, briefly ask for the user\'s intent (their "why").',
|
||||
'Keep it lightweight and optional (one short sentence, once per topic).',
|
||||
'Examples: "What are you hoping to do with this?", "What\'s the goal behind adding this?"',
|
||||
'If the user answers, incorporate the rationale into your next steps and outputs.'
|
||||
].join(' ');
|
||||
|
||||
return systemPrompt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
import { CostReport, TraceCostSummary, CacheEffectiveness } from '@/types/analytics';
|
||||
|
||||
export class CostAnalytics {
|
||||
static getCostsByDateRange(startDate: string, endDate: string): CostReport {
|
||||
const db = getSQLiteClient();
|
||||
|
||||
const chats = db.prepare(`
|
||||
SELECT
|
||||
id,
|
||||
helper_name,
|
||||
metadata,
|
||||
created_at
|
||||
FROM chats
|
||||
WHERE created_at >= ? AND created_at <= ?
|
||||
`).all(startDate, endDate) as Array<{
|
||||
id: number;
|
||||
helper_name: string;
|
||||
metadata: string;
|
||||
created_at: string;
|
||||
}>;
|
||||
|
||||
let totalCostUsd = 0;
|
||||
let totalTokens = 0;
|
||||
let inputTokens = 0;
|
||||
let outputTokens = 0;
|
||||
let cacheReadTokens = 0;
|
||||
let cacheWriteTokens = 0;
|
||||
let cacheHitCount = 0;
|
||||
let cacheSavingsUsd = 0;
|
||||
|
||||
const costByAgent: Record<string, { costUsd: number; chats: number; tokens: number }> = {};
|
||||
const costByModel: Record<string, { costUsd: number; chats: number; tokens: number }> = {};
|
||||
|
||||
for (const chat of chats) {
|
||||
let metadata: any = {};
|
||||
try {
|
||||
metadata = JSON.parse(chat.metadata || '{}');
|
||||
} catch (e) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const costUsd = metadata.estimated_cost_usd || 0;
|
||||
const tokens = metadata.total_tokens || 0;
|
||||
const inputToks = metadata.input_tokens || 0;
|
||||
const outputToks = metadata.output_tokens || 0;
|
||||
const cacheRead = metadata.cache_read_tokens || 0;
|
||||
const cacheWrite = metadata.cache_write_tokens || 0;
|
||||
const cacheHit = metadata.cache_hit || false;
|
||||
const model = metadata.model_used || 'unknown';
|
||||
|
||||
totalCostUsd += costUsd;
|
||||
totalTokens += tokens;
|
||||
inputTokens += inputToks;
|
||||
outputTokens += outputToks;
|
||||
cacheReadTokens += cacheRead;
|
||||
cacheWriteTokens += cacheWrite;
|
||||
if (cacheHit) cacheHitCount++;
|
||||
|
||||
if (metadata.cache_savings_pct && cacheHit) {
|
||||
const fullCost = costUsd / (1 - (metadata.cache_savings_pct / 100));
|
||||
cacheSavingsUsd += (fullCost - costUsd);
|
||||
}
|
||||
|
||||
if (!costByAgent[chat.helper_name]) {
|
||||
costByAgent[chat.helper_name] = { costUsd: 0, chats: 0, tokens: 0 };
|
||||
}
|
||||
costByAgent[chat.helper_name].costUsd += costUsd;
|
||||
costByAgent[chat.helper_name].chats += 1;
|
||||
costByAgent[chat.helper_name].tokens += tokens;
|
||||
|
||||
if (!costByModel[model]) {
|
||||
costByModel[model] = { costUsd: 0, chats: 0, tokens: 0 };
|
||||
}
|
||||
costByModel[model].costUsd += costUsd;
|
||||
costByModel[model].chats += 1;
|
||||
costByModel[model].tokens += tokens;
|
||||
}
|
||||
|
||||
const cacheRequests = chats.filter(c => {
|
||||
try {
|
||||
const meta = JSON.parse(c.metadata || '{}');
|
||||
return meta.provider === 'anthropic';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}).length;
|
||||
|
||||
return {
|
||||
periodStart: startDate,
|
||||
periodEnd: endDate,
|
||||
totalCostUsd: parseFloat(totalCostUsd.toFixed(6)),
|
||||
totalChats: chats.length,
|
||||
totalTokens,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheReadTokens,
|
||||
cacheWriteTokens,
|
||||
cacheHitRate: cacheRequests > 0 ? cacheHitCount / cacheRequests : 0,
|
||||
cacheSavingsUsd: parseFloat(cacheSavingsUsd.toFixed(6)),
|
||||
avgCostPerChat: chats.length > 0 ? parseFloat((totalCostUsd / chats.length).toFixed(6)) : 0,
|
||||
avgTokensPerChat: chats.length > 0 ? Math.round(totalTokens / chats.length) : 0,
|
||||
costByAgent,
|
||||
costByModel,
|
||||
};
|
||||
}
|
||||
|
||||
static getCostsByAgent(agentName: string, startDate?: string, endDate?: string): CostReport {
|
||||
const db = getSQLiteClient();
|
||||
|
||||
let query = `
|
||||
SELECT
|
||||
id,
|
||||
helper_name,
|
||||
metadata,
|
||||
created_at
|
||||
FROM chats
|
||||
WHERE helper_name = ?
|
||||
`;
|
||||
const params: any[] = [agentName];
|
||||
|
||||
if (startDate && endDate) {
|
||||
query += ` AND created_at >= ? AND created_at <= ?`;
|
||||
params.push(startDate, endDate);
|
||||
}
|
||||
|
||||
const chats = db.prepare(query).all(...params) as Array<{
|
||||
id: number;
|
||||
helper_name: string;
|
||||
metadata: string;
|
||||
created_at: string;
|
||||
}>;
|
||||
|
||||
if (chats.length === 0) {
|
||||
return {
|
||||
periodStart: startDate || '',
|
||||
periodEnd: endDate || '',
|
||||
totalCostUsd: 0,
|
||||
totalChats: 0,
|
||||
totalTokens: 0,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cacheReadTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
cacheHitRate: 0,
|
||||
cacheSavingsUsd: 0,
|
||||
avgCostPerChat: 0,
|
||||
avgTokensPerChat: 0,
|
||||
costByAgent: {},
|
||||
costByModel: {},
|
||||
};
|
||||
}
|
||||
|
||||
const start = startDate || chats[chats.length - 1].created_at;
|
||||
const end = endDate || chats[0].created_at;
|
||||
|
||||
return this.getCostsByDateRange(start, end);
|
||||
}
|
||||
|
||||
static getCostsByTrace(traceId: string): TraceCostSummary {
|
||||
const db = getSQLiteClient();
|
||||
|
||||
const chats = db.prepare(`
|
||||
SELECT
|
||||
id,
|
||||
helper_name,
|
||||
agent_type,
|
||||
metadata,
|
||||
created_at
|
||||
FROM chats
|
||||
WHERE json_extract(metadata, '$.trace_id') = ?
|
||||
ORDER BY created_at ASC
|
||||
`).all(traceId) as Array<{
|
||||
id: number;
|
||||
helper_name: string;
|
||||
agent_type: string;
|
||||
metadata: string;
|
||||
created_at: string;
|
||||
}>;
|
||||
|
||||
let totalCostUsd = 0;
|
||||
let orchestratorCost = 0;
|
||||
let executorCost = 0;
|
||||
let plannerCost = 0;
|
||||
let totalTokens = 0;
|
||||
|
||||
const interactions: TraceCostSummary['interactions'] = [];
|
||||
|
||||
for (const chat of chats) {
|
||||
let metadata: any = {};
|
||||
try {
|
||||
metadata = JSON.parse(chat.metadata || '{}');
|
||||
} catch (e) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const costUsd = metadata.estimated_cost_usd || 0;
|
||||
const tokens = metadata.total_tokens || 0;
|
||||
|
||||
totalCostUsd += costUsd;
|
||||
totalTokens += tokens;
|
||||
|
||||
if (chat.agent_type === 'orchestrator') {
|
||||
orchestratorCost += costUsd;
|
||||
} else if (chat.agent_type === 'executor') {
|
||||
executorCost += costUsd;
|
||||
} else if (chat.agent_type === 'planner') {
|
||||
plannerCost += costUsd;
|
||||
}
|
||||
|
||||
interactions.push({
|
||||
chatId: chat.id,
|
||||
agentName: chat.helper_name,
|
||||
costUsd: parseFloat(costUsd.toFixed(6)),
|
||||
tokens,
|
||||
createdAt: chat.created_at,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
traceId,
|
||||
totalCostUsd: parseFloat(totalCostUsd.toFixed(6)),
|
||||
chatCount: chats.length,
|
||||
orchestratorCost: parseFloat(orchestratorCost.toFixed(6)),
|
||||
executorCost: parseFloat(executorCost.toFixed(6)),
|
||||
plannerCost: parseFloat(plannerCost.toFixed(6)),
|
||||
totalTokens,
|
||||
interactions,
|
||||
};
|
||||
}
|
||||
|
||||
static getCacheEffectiveness(startDate?: string, endDate?: string): CacheEffectiveness {
|
||||
const db = getSQLiteClient();
|
||||
|
||||
let query = `
|
||||
SELECT
|
||||
metadata
|
||||
FROM chats
|
||||
WHERE json_extract(metadata, '$.provider') = 'anthropic'
|
||||
`;
|
||||
const params: any[] = [];
|
||||
|
||||
if (startDate && endDate) {
|
||||
query += ` AND created_at >= ? AND created_at <= ?`;
|
||||
params.push(startDate, endDate);
|
||||
}
|
||||
|
||||
const chats = db.prepare(query).all(...params) as Array<{ metadata: string }>;
|
||||
|
||||
let totalRequests = 0;
|
||||
let cacheHits = 0;
|
||||
let cacheMisses = 0;
|
||||
let totalCacheSavingsUsd = 0;
|
||||
let totalTokensSaved = 0;
|
||||
|
||||
for (const chat of chats) {
|
||||
let metadata: any = {};
|
||||
try {
|
||||
metadata = JSON.parse(chat.metadata || '{}');
|
||||
} catch (e) {
|
||||
continue;
|
||||
}
|
||||
|
||||
totalRequests++;
|
||||
|
||||
if (metadata.cache_hit) {
|
||||
cacheHits++;
|
||||
const cacheReadTokens = metadata.cache_read_tokens || 0;
|
||||
totalTokensSaved += cacheReadTokens;
|
||||
|
||||
if (metadata.cache_savings_pct && metadata.estimated_cost_usd) {
|
||||
const fullCost = metadata.estimated_cost_usd / (1 - (metadata.cache_savings_pct / 100));
|
||||
totalCacheSavingsUsd += (fullCost - metadata.estimated_cost_usd);
|
||||
}
|
||||
} else {
|
||||
cacheMisses++;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
totalRequests,
|
||||
cacheHits,
|
||||
cacheMisses,
|
||||
hitRate: totalRequests > 0 ? parseFloat((cacheHits / totalRequests).toFixed(4)) : 0,
|
||||
totalCacheSavingsUsd: parseFloat(totalCacheSavingsUsd.toFixed(6)),
|
||||
avgSavingsPerHit: cacheHits > 0 ? parseFloat((totalCacheSavingsUsd / cacheHits).toFixed(6)) : 0,
|
||||
totalTokensSaved,
|
||||
};
|
||||
}
|
||||
|
||||
static getDailyBreakdown(days: number = 7): Array<{ date: string; cost: number; chats: number; tokens: number }> {
|
||||
const db = getSQLiteClient();
|
||||
|
||||
const result = db.prepare(`
|
||||
SELECT
|
||||
DATE(created_at) as date,
|
||||
SUM(COALESCE(json_extract(metadata, '$.estimated_cost_usd'), 0)) as cost,
|
||||
COUNT(*) as chats,
|
||||
SUM(COALESCE(json_extract(metadata, '$.total_tokens'), 0)) as tokens
|
||||
FROM chats
|
||||
WHERE created_at >= DATE('now', '-' || ? || ' days')
|
||||
GROUP BY DATE(created_at)
|
||||
ORDER BY date DESC
|
||||
`).all(days) as Array<{ date: string; cost: number; chats: number; tokens: number }>;
|
||||
|
||||
return result.map(row => ({
|
||||
date: row.date,
|
||||
cost: parseFloat(Number(row.cost).toFixed(6)),
|
||||
chats: Number(row.chats),
|
||||
tokens: Number(row.tokens),
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { ModelPricing } from '@/types/analytics';
|
||||
|
||||
export const MODEL_PRICING: Record<string, ModelPricing> = {
|
||||
'claude-sonnet-4-5-20250929': {
|
||||
provider: 'anthropic',
|
||||
inputPer1M: 3.00,
|
||||
outputPer1M: 15.00,
|
||||
cacheWritePer1M: 3.75,
|
||||
cacheReadPer1M: 0.30,
|
||||
},
|
||||
'claude-3-5-sonnet-20241022': {
|
||||
provider: 'anthropic',
|
||||
inputPer1M: 3.00,
|
||||
outputPer1M: 15.00,
|
||||
cacheWritePer1M: 3.75,
|
||||
cacheReadPer1M: 0.30,
|
||||
},
|
||||
'gpt-4o-mini': {
|
||||
provider: 'openai',
|
||||
inputPer1M: 0.15,
|
||||
outputPer1M: 0.60,
|
||||
},
|
||||
'gpt-4o-mini-2024-07-18': {
|
||||
provider: 'openai',
|
||||
inputPer1M: 0.15,
|
||||
outputPer1M: 0.60,
|
||||
},
|
||||
'gpt-5o-mini': {
|
||||
provider: 'openai',
|
||||
inputPer1M: 0.25,
|
||||
outputPer1M: 2.00,
|
||||
},
|
||||
'gpt-5-mini': {
|
||||
provider: 'openai',
|
||||
inputPer1M: 0.25,
|
||||
outputPer1M: 2.00,
|
||||
},
|
||||
'gpt-5': {
|
||||
provider: 'openai',
|
||||
inputPer1M: 1.25,
|
||||
outputPer1M: 10.00,
|
||||
},
|
||||
};
|
||||
|
||||
export interface CostCalculationInput {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
cacheWriteTokens?: number;
|
||||
cacheReadTokens?: number;
|
||||
modelId: string;
|
||||
}
|
||||
|
||||
export interface CostCalculationResult {
|
||||
totalCostUsd: number;
|
||||
inputCostUsd: number;
|
||||
outputCostUsd: number;
|
||||
cacheWriteCostUsd: number;
|
||||
cacheReadCostUsd: number;
|
||||
cacheSavingsUsd: number;
|
||||
totalTokens: number;
|
||||
}
|
||||
|
||||
export function calculateCost(input: CostCalculationInput): CostCalculationResult {
|
||||
const pricing = MODEL_PRICING[input.modelId];
|
||||
|
||||
if (!pricing) {
|
||||
console.warn(`[Pricing] Unknown model: ${input.modelId}, using default pricing`);
|
||||
return {
|
||||
totalCostUsd: 0,
|
||||
inputCostUsd: 0,
|
||||
outputCostUsd: 0,
|
||||
cacheWriteCostUsd: 0,
|
||||
cacheReadCostUsd: 0,
|
||||
cacheSavingsUsd: 0,
|
||||
totalTokens: input.inputTokens + input.outputTokens + (input.cacheWriteTokens || 0) + (input.cacheReadTokens || 0),
|
||||
};
|
||||
}
|
||||
|
||||
const inputCostUsd = (input.inputTokens / 1_000_000) * pricing.inputPer1M;
|
||||
const outputCostUsd = (input.outputTokens / 1_000_000) * pricing.outputPer1M;
|
||||
|
||||
let cacheWriteCostUsd = 0;
|
||||
let cacheReadCostUsd = 0;
|
||||
let cacheSavingsUsd = 0;
|
||||
|
||||
if (pricing.cacheWritePer1M && input.cacheWriteTokens) {
|
||||
cacheWriteCostUsd = (input.cacheWriteTokens / 1_000_000) * pricing.cacheWritePer1M;
|
||||
}
|
||||
|
||||
if (pricing.cacheReadPer1M && input.cacheReadTokens) {
|
||||
cacheReadCostUsd = (input.cacheReadTokens / 1_000_000) * pricing.cacheReadPer1M;
|
||||
const regularCostForCachedTokens = (input.cacheReadTokens / 1_000_000) * pricing.inputPer1M;
|
||||
cacheSavingsUsd = regularCostForCachedTokens - cacheReadCostUsd;
|
||||
}
|
||||
|
||||
const totalCostUsd = inputCostUsd + outputCostUsd + cacheWriteCostUsd + cacheReadCostUsd;
|
||||
const totalTokens = input.inputTokens + input.outputTokens + (input.cacheWriteTokens || 0) + (input.cacheReadTokens || 0);
|
||||
|
||||
return {
|
||||
totalCostUsd: parseFloat(totalCostUsd.toFixed(6)),
|
||||
inputCostUsd: parseFloat(inputCostUsd.toFixed(6)),
|
||||
outputCostUsd: parseFloat(outputCostUsd.toFixed(6)),
|
||||
cacheWriteCostUsd: parseFloat(cacheWriteCostUsd.toFixed(6)),
|
||||
cacheReadCostUsd: parseFloat(cacheReadCostUsd.toFixed(6)),
|
||||
cacheSavingsUsd: parseFloat(cacheSavingsUsd.toFixed(6)),
|
||||
totalTokens,
|
||||
};
|
||||
}
|
||||
|
||||
export function getModelPricing(modelId: string): ModelPricing | null {
|
||||
return MODEL_PRICING[modelId] || null;
|
||||
}
|
||||
|
||||
export function getSupportedModels(): string[] {
|
||||
return Object.keys(MODEL_PRICING);
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
import { UsageData } from '@/types/analytics';
|
||||
import { RequestContext } from '@/services/context/requestContext';
|
||||
|
||||
interface ChatLogEntry {
|
||||
chat_type: string;
|
||||
user_message: string;
|
||||
assistant_message: string;
|
||||
thread_id: string;
|
||||
focused_node_id: number | null;
|
||||
helper_name: string;
|
||||
agent_type: 'orchestrator' | 'executor' | 'planner';
|
||||
delegation_id: number | null;
|
||||
metadata: any;
|
||||
}
|
||||
|
||||
interface StreamMetadata {
|
||||
helperName: string;
|
||||
openTabs?: any[];
|
||||
activeTabId?: number | null;
|
||||
currentView?: 'nodes' | 'memory';
|
||||
sessionId?: string;
|
||||
agentType?: 'orchestrator' | 'executor' | 'planner';
|
||||
delegationId?: number | null;
|
||||
usageData?: UsageData;
|
||||
traceId?: string;
|
||||
parentChatId?: number;
|
||||
systemMessage?: string;
|
||||
workflowKey?: string;
|
||||
workflowNodeId?: number;
|
||||
mode?: 'easy' | 'hard';
|
||||
toolCallsData?: any[];
|
||||
backendUsage?: Array<{
|
||||
provider: string;
|
||||
headers: Record<string, string>;
|
||||
}>;
|
||||
}
|
||||
|
||||
export class ChatLoggingMiddleware {
|
||||
private static generateThreadId(helperName: string, metadata: StreamMetadata): string {
|
||||
const { activeTabId = null, currentView, sessionId } = metadata;
|
||||
const timestamp = Date.now();
|
||||
const session = sessionId || `session_${timestamp}`;
|
||||
|
||||
if (activeTabId) {
|
||||
return `${helperName}-node-${activeTabId}-${session}`;
|
||||
}
|
||||
return `${helperName}-general-${session}`;
|
||||
}
|
||||
|
||||
private static extractUserMessage(messages: any[]): string | null {
|
||||
const userMessages = messages.filter(m => m.role === 'user');
|
||||
const lastUserMessage = userMessages[userMessages.length - 1];
|
||||
|
||||
if (!lastUserMessage) return null;
|
||||
|
||||
// Handle different message formats (AI SDK v5)
|
||||
if (typeof lastUserMessage.content === 'string') {
|
||||
return lastUserMessage.content;
|
||||
}
|
||||
|
||||
// Handle parts-based messages (from frontend)
|
||||
if (Array.isArray(lastUserMessage.parts)) {
|
||||
const textParts = lastUserMessage.parts
|
||||
.filter((part: any) => part.type === 'text')
|
||||
.map((part: any) => part.text);
|
||||
return textParts.join(' ');
|
||||
}
|
||||
|
||||
// Handle content as object or other formats
|
||||
if (lastUserMessage.content && typeof lastUserMessage.content === 'object') {
|
||||
return JSON.stringify(lastUserMessage.content);
|
||||
}
|
||||
|
||||
return lastUserMessage.content || null;
|
||||
}
|
||||
|
||||
static async logChatInteraction(
|
||||
userMessage: string,
|
||||
assistantMessage: string,
|
||||
metadata: StreamMetadata,
|
||||
messages: any[] = []
|
||||
): Promise<void> {
|
||||
try {
|
||||
const threadId = this.generateThreadId(metadata.helperName, metadata);
|
||||
|
||||
const createdAt = new Date().toISOString();
|
||||
|
||||
const chatEntry: ChatLogEntry = {
|
||||
chat_type: 'helper',
|
||||
user_message: userMessage,
|
||||
assistant_message: assistantMessage,
|
||||
thread_id: threadId,
|
||||
focused_node_id: metadata.activeTabId ?? null,
|
||||
helper_name: metadata.helperName,
|
||||
agent_type: metadata.agentType || 'orchestrator',
|
||||
delegation_id: metadata.delegationId ?? null,
|
||||
metadata: {
|
||||
timestamp: new Date().toISOString(),
|
||||
session_id: metadata.sessionId,
|
||||
current_view: metadata.currentView || 'nodes',
|
||||
open_tab_count: metadata.openTabs?.length || 0,
|
||||
has_focused_node: !!metadata.activeTabId,
|
||||
message_count: messages.length,
|
||||
...(metadata.mode && { mode: metadata.mode }),
|
||||
// System message
|
||||
...(metadata.systemMessage && { system_message: metadata.systemMessage }),
|
||||
// Enhanced usage data
|
||||
...(metadata.usageData && {
|
||||
input_tokens: metadata.usageData.inputTokens,
|
||||
output_tokens: metadata.usageData.outputTokens,
|
||||
total_tokens: metadata.usageData.totalTokens,
|
||||
cache_write_tokens: metadata.usageData.cacheWriteTokens,
|
||||
cache_read_tokens: metadata.usageData.cacheReadTokens,
|
||||
cache_hit: metadata.usageData.cacheHit,
|
||||
cache_savings_pct: metadata.usageData.cacheSavingsPct,
|
||||
estimated_cost_usd: metadata.usageData.estimatedCostUsd,
|
||||
model_used: metadata.usageData.modelUsed,
|
||||
provider: metadata.usageData.provider,
|
||||
tools_used: metadata.usageData.toolsUsed,
|
||||
tool_calls_count: metadata.usageData.toolCallsCount,
|
||||
capsule_version: metadata.usageData.capsuleVersion,
|
||||
context_sources_used: metadata.usageData.contextSourcesUsed,
|
||||
validation_status: metadata.usageData.validationStatus,
|
||||
validation_message: metadata.usageData.validationMessage,
|
||||
fallback_action: metadata.usageData.fallbackAction,
|
||||
}),
|
||||
// Tool calls data
|
||||
...(metadata.toolCallsData && metadata.toolCallsData.length > 0 && {
|
||||
tool_calls: metadata.toolCallsData
|
||||
}),
|
||||
// Trace grouping
|
||||
...(metadata.traceId && { trace_id: metadata.traceId }),
|
||||
...(metadata.parentChatId && { parent_chat_id: metadata.parentChatId }),
|
||||
// Workflow metadata
|
||||
...(metadata.workflowKey && {
|
||||
workflow_key: metadata.workflowKey,
|
||||
workflow_node_id: metadata.workflowNodeId,
|
||||
is_workflow: true,
|
||||
}),
|
||||
// Backend usage (for Supabase sync correlation)
|
||||
...(metadata.backendUsage && metadata.backendUsage.length > 0 && {
|
||||
backend_usage: metadata.backendUsage,
|
||||
}),
|
||||
}
|
||||
};
|
||||
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.prepare(`
|
||||
INSERT INTO chats (chat_type, user_message, assistant_message, thread_id, focused_node_id, helper_name, agent_type, delegation_id, created_at, metadata)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
chatEntry.chat_type,
|
||||
chatEntry.user_message,
|
||||
chatEntry.assistant_message,
|
||||
chatEntry.thread_id,
|
||||
chatEntry.focused_node_id,
|
||||
chatEntry.helper_name,
|
||||
chatEntry.agent_type,
|
||||
chatEntry.delegation_id,
|
||||
createdAt,
|
||||
JSON.stringify(chatEntry.metadata)
|
||||
);
|
||||
console.log(`✅ Chat logged for ${metadata.helperName}, ID: ${result.lastInsertRowid}`);
|
||||
|
||||
const lastInsertedChatId = Number(result.lastInsertRowid);
|
||||
|
||||
if (metadata.agentType === 'orchestrator' && (metadata.helperName === 'ra-h' || metadata.helperName === 'ra-h-easy')) {
|
||||
RequestContext.set({
|
||||
traceId: metadata.traceId,
|
||||
parentChatId: lastInsertedChatId
|
||||
});
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Chat logging error:', error);
|
||||
}
|
||||
}
|
||||
|
||||
static createLoggingHandlers(metadata: StreamMetadata, messages: any[]) {
|
||||
let assistantResponse = '';
|
||||
const userMessage = this.extractUserMessage(messages);
|
||||
|
||||
return {
|
||||
onFinish: async (result: any) => {
|
||||
const { text, toolCalls, steps } = result;
|
||||
// Log if we have a user message and either text OR tool activity
|
||||
const hasActivity = text || toolCalls?.length > 0 || steps?.length > 0;
|
||||
|
||||
if (userMessage && hasActivity) {
|
||||
// Capture tool calls if present
|
||||
const toolCallsData = toolCalls && toolCalls.length > 0 ? toolCalls.map((tc: any) => ({
|
||||
toolName: tc.toolName,
|
||||
args: tc.args,
|
||||
result: typeof tc.result === 'object' ? tc.result : { value: tc.result }
|
||||
})) : undefined;
|
||||
|
||||
if (toolCallsData) {
|
||||
console.log(`🔧 Captured ${toolCallsData.length} tool calls for logging`);
|
||||
}
|
||||
|
||||
const enhancedMetadata = {
|
||||
...metadata,
|
||||
toolCallsData
|
||||
};
|
||||
|
||||
await this.logChatInteraction(
|
||||
userMessage,
|
||||
text || '[Tool calls only - no text response]',
|
||||
enhancedMetadata,
|
||||
messages
|
||||
);
|
||||
} else if (userMessage && !hasActivity) {
|
||||
console.warn(`⚠️ Skipping chat log - no text or tool activity for user message: ${userMessage.substring(0, 50)}...`);
|
||||
}
|
||||
},
|
||||
onChunk: ({ chunk }: { chunk: any }) => {
|
||||
if (chunk.type === 'text-delta' && chunk.textDelta) {
|
||||
assistantResponse += chunk.textDelta;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function withChatLogging(
|
||||
streamConfig: any,
|
||||
metadata: StreamMetadata,
|
||||
messages: any[]
|
||||
) {
|
||||
const handlers = ChatLoggingMiddleware.createLoggingHandlers(metadata, messages);
|
||||
const originalOnFinish = streamConfig.onFinish;
|
||||
|
||||
return {
|
||||
...streamConfig,
|
||||
onFinish: async (result: any) => {
|
||||
// Call original onFinish first (for cache stats)
|
||||
if (originalOnFinish) {
|
||||
await originalOnFinish(result);
|
||||
}
|
||||
// Then call logging handler
|
||||
await handlers.onFinish(result);
|
||||
},
|
||||
onChunk: handlers.onChunk
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
import { getAutoContextSettings } from '@/services/settings/autoContextSettings';
|
||||
|
||||
interface AutoContextRow {
|
||||
id: number;
|
||||
title: string | null;
|
||||
updated_at: string;
|
||||
edge_count: number | null;
|
||||
}
|
||||
|
||||
export interface AutoContextSummary {
|
||||
id: number;
|
||||
title: string;
|
||||
edgeCount: number;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
function fetchAutoContextRows(limit: number): AutoContextSummary[] {
|
||||
const db = getSQLiteClient();
|
||||
const rows = db
|
||||
.query<AutoContextRow>(
|
||||
`
|
||||
SELECT n.id,
|
||||
n.title,
|
||||
n.updated_at,
|
||||
COUNT(DISTINCT e.id) AS edge_count
|
||||
FROM nodes n
|
||||
LEFT JOIN edges e
|
||||
ON (e.from_node_id = n.id OR e.to_node_id = n.id)
|
||||
WHERE n.type IS NULL OR n.type != 'memory'
|
||||
GROUP BY n.id
|
||||
ORDER BY edge_count DESC, n.updated_at DESC
|
||||
LIMIT ?
|
||||
`,
|
||||
[limit]
|
||||
)
|
||||
.rows;
|
||||
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
title: row.title || 'Untitled node',
|
||||
updatedAt: row.updated_at,
|
||||
edgeCount: Number(row.edge_count ?? 0),
|
||||
}));
|
||||
}
|
||||
|
||||
export function getAutoContextSummaries(limit = 10): AutoContextSummary[] {
|
||||
const settings = getAutoContextSettings();
|
||||
if (!settings.autoContextEnabled) {
|
||||
return [];
|
||||
}
|
||||
return fetchAutoContextRows(limit);
|
||||
}
|
||||
|
||||
export function buildAutoContextBlock(limit = 10): string | null {
|
||||
const summaries = getAutoContextSummaries(limit);
|
||||
if (summaries.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const lines: string[] = [
|
||||
'=== BACKGROUND CONTEXT ===',
|
||||
'Top 10 most-connected nodes (important knowledge hubs). Use queryNodes/getNodesById if relevant.',
|
||||
'',
|
||||
];
|
||||
|
||||
for (const summary of summaries) {
|
||||
lines.push(`[NODE:${summary.id}:"${summary.title}"] (edges: ${summary.edgeCount})`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { Node } from '@/types/database';
|
||||
|
||||
interface RequestContext {
|
||||
traceId?: string;
|
||||
parentChatId?: number;
|
||||
workflowKey?: string;
|
||||
workflowNodeId?: number;
|
||||
openTabs?: Node[];
|
||||
activeTabId?: number | null;
|
||||
mode?: 'easy' | 'hard';
|
||||
apiKeys?: {
|
||||
openai?: string;
|
||||
anthropic?: string;
|
||||
};
|
||||
}
|
||||
|
||||
let currentContext: RequestContext = {};
|
||||
|
||||
export const RequestContext = {
|
||||
set(context: Partial<RequestContext>) {
|
||||
Object.entries(context).forEach(([key, value]) => {
|
||||
if (value === undefined) {
|
||||
delete (currentContext as Record<string, unknown>)[key];
|
||||
} else {
|
||||
(currentContext as Record<string, unknown>)[key] = value;
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
get(): RequestContext {
|
||||
return currentContext;
|
||||
},
|
||||
|
||||
clear() {
|
||||
currentContext = {};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,331 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { Chunk, ChunkData } from '@/types/database';
|
||||
|
||||
export class ChunkService {
|
||||
async getChunksByNodeId(nodeId: number): Promise<Chunk[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Chunk>('SELECT * FROM chunks WHERE node_id = ? ORDER BY chunk_idx ASC', [nodeId]);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async getChunkById(id: number): Promise<Chunk | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Chunk>('SELECT * FROM chunks WHERE id = ?', [id]);
|
||||
return result.rows[0] || null;
|
||||
}
|
||||
|
||||
async createChunk(chunkData: ChunkData): Promise<Chunk> {
|
||||
return this.createChunkSQLite(chunkData);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async createChunkSQLite(chunkData: ChunkData): Promise<Chunk> {
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.prepare(`
|
||||
INSERT INTO chunks (node_id, chunk_idx, text, embedding, embedding_type, metadata, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
chunkData.node_id,
|
||||
chunkData.chunk_idx || null,
|
||||
chunkData.text,
|
||||
chunkData.embedding || null,
|
||||
chunkData.embedding_type,
|
||||
chunkData.metadata ? JSON.stringify(chunkData.metadata) : null,
|
||||
now
|
||||
);
|
||||
|
||||
const chunkId = Number(result.lastInsertRowid);
|
||||
const createdChunk = await this.getChunkById(chunkId);
|
||||
|
||||
if (!createdChunk) {
|
||||
throw new Error('Failed to create chunk');
|
||||
}
|
||||
|
||||
return createdChunk;
|
||||
}
|
||||
|
||||
async createChunks(chunksData: ChunkData[]): Promise<Chunk[]> {
|
||||
if (chunksData.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.createChunksSQLite(chunksData);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async createChunksSQLite(chunksData: ChunkData[]): Promise<Chunk[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const now = new Date().toISOString();
|
||||
const createdChunks: Chunk[] = [];
|
||||
|
||||
// Use transaction for bulk insert
|
||||
sqlite.transaction(() => {
|
||||
const stmt = sqlite.prepare(`
|
||||
INSERT INTO chunks (node_id, chunk_idx, text, embedding, embedding_type, metadata, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
for (const chunk of chunksData) {
|
||||
stmt.run(
|
||||
chunk.node_id,
|
||||
chunk.chunk_idx || null,
|
||||
chunk.text,
|
||||
chunk.embedding || null,
|
||||
chunk.embedding_type,
|
||||
chunk.metadata ? JSON.stringify(chunk.metadata) : null,
|
||||
now
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Get all created chunks by node_id (since we know they were just created)
|
||||
const nodeIds = [...new Set(chunksData.map(c => c.node_id))];
|
||||
for (const nodeId of nodeIds) {
|
||||
const chunks = await this.getChunksByNodeId(nodeId);
|
||||
createdChunks.push(...chunks.filter(c => c.created_at === now));
|
||||
}
|
||||
|
||||
return createdChunks;
|
||||
}
|
||||
|
||||
async updateChunk(id: number, updates: Partial<Chunk>): Promise<Chunk> {
|
||||
return this.updateChunkSQLite(id, updates);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async updateChunkSQLite(id: number, updates: Partial<Chunk>): Promise<Chunk> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const updateFields: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
// Build dynamic update query
|
||||
Object.entries(updates).forEach(([key, value]) => {
|
||||
if (key !== 'id' && key !== 'created_at' && value !== undefined) {
|
||||
updateFields.push(`${key} = ?`);
|
||||
if (key === 'metadata') {
|
||||
params.push(typeof value === 'object' ? JSON.stringify(value) : value);
|
||||
} else {
|
||||
params.push(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (updateFields.length === 0) {
|
||||
throw new Error('No valid fields to update');
|
||||
}
|
||||
|
||||
params.push(id); // Add ID for WHERE clause
|
||||
|
||||
const query = `UPDATE chunks SET ${updateFields.join(', ')} WHERE id = ?`;
|
||||
const result = sqlite.query(query, params);
|
||||
|
||||
if (result.changes === 0) {
|
||||
throw new Error(`Chunk with ID ${id} not found`);
|
||||
}
|
||||
|
||||
const updatedChunk = await this.getChunkById(id);
|
||||
if (!updatedChunk) {
|
||||
throw new Error(`Failed to retrieve updated chunk with ID ${id}`);
|
||||
}
|
||||
|
||||
return updatedChunk;
|
||||
}
|
||||
|
||||
async deleteChunk(id: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('DELETE FROM chunks WHERE id = ?', [id]);
|
||||
if ((result.changes || 0) === 0) {
|
||||
throw new Error(`Chunk with ID ${id} not found`);
|
||||
}
|
||||
}
|
||||
|
||||
async deleteChunksByNodeId(nodeId: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
sqlite.query('DELETE FROM chunks WHERE node_id = ?', [nodeId]);
|
||||
}
|
||||
|
||||
async searchChunks(
|
||||
queryEmbedding: number[],
|
||||
similarityThreshold = 0.5,
|
||||
matchCount = 5,
|
||||
nodeIds?: number[],
|
||||
fallbackQuery?: string
|
||||
): Promise<Array<Chunk & { similarity: number }>> {
|
||||
try {
|
||||
return await this.searchChunksSQLite(queryEmbedding, similarityThreshold, matchCount, nodeIds);
|
||||
} catch (error) {
|
||||
console.warn('Vector search failed, attempting text fallback:', error);
|
||||
if (fallbackQuery) {
|
||||
return await this.textSearchFallback(fallbackQuery, matchCount, nodeIds);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async searchChunksSQLite(
|
||||
queryEmbedding: number[],
|
||||
similarityThreshold = 0.5,
|
||||
matchCount = 5,
|
||||
nodeIds?: number[]
|
||||
): Promise<Array<Chunk & { similarity: number }>> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
// Step 1: Determine vector search limit - more conservative approach
|
||||
let vectorLimit = 50; // Start smaller for efficiency
|
||||
|
||||
if (nodeIds && nodeIds.length > 0) {
|
||||
// When searching specific nodes, get exact count and use reasonable multiplier
|
||||
const candidateCountQuery = `SELECT COUNT(*) as count FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
|
||||
const candidateResult = sqlite.query<{count: number}>(candidateCountQuery, nodeIds);
|
||||
const candidateCount = Number(candidateResult.rows[0].count);
|
||||
|
||||
// Use 2x the candidate count but cap at reasonable limits
|
||||
vectorLimit = Math.min(Math.max(candidateCount * 2, 50), 1000);
|
||||
console.log(`🔍 Node-scoped search: ${candidateCount} candidates, using vector limit ${vectorLimit}`);
|
||||
} else {
|
||||
// For global search, use adaptive limit based on expected results
|
||||
vectorLimit = Math.max(matchCount * 10, 50);
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
// Step 2: Use CTE-based query to avoid UNION ALL explosion
|
||||
const vectorString = `[${queryEmbedding.join(',')}]`;
|
||||
let query = `
|
||||
WITH vector_results AS (
|
||||
SELECT chunk_id, distance
|
||||
FROM vec_chunks
|
||||
WHERE embedding MATCH ?
|
||||
ORDER BY distance
|
||||
LIMIT ?
|
||||
)
|
||||
SELECT c.*, (1.0 / (1.0 + vr.distance)) AS similarity
|
||||
FROM vector_results vr
|
||||
JOIN chunks c ON c.id = vr.chunk_id
|
||||
`;
|
||||
|
||||
const params: any[] = [vectorString, vectorLimit];
|
||||
const conditions = [];
|
||||
|
||||
// Node ID filter if provided
|
||||
if (nodeIds && nodeIds.length > 0) {
|
||||
conditions.push(`c.node_id IN (${nodeIds.map(() => '?').join(',')})`);
|
||||
params.push(...nodeIds);
|
||||
}
|
||||
|
||||
// Similarity threshold filter
|
||||
conditions.push(`(1.0 / (1.0 + vr.distance)) >= ?`);
|
||||
params.push(similarityThreshold);
|
||||
|
||||
if (conditions.length > 0) {
|
||||
query += ` WHERE ${conditions.join(' AND ')}`;
|
||||
}
|
||||
|
||||
query += ` ORDER BY similarity DESC LIMIT ?`;
|
||||
params.push(matchCount);
|
||||
|
||||
const result = sqlite.query<Chunk & { similarity: number }>(query, params);
|
||||
const searchTime = Date.now() - startTime;
|
||||
|
||||
console.log(`📊 Vector search: ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
|
||||
if (result.rows.length > 0) {
|
||||
console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
|
||||
}
|
||||
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async textSearchFallback(
|
||||
query: string,
|
||||
matchCount = 5,
|
||||
nodeIds?: number[]
|
||||
): Promise<Array<Chunk & { similarity: number }>> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const startTime = Date.now();
|
||||
|
||||
// Clean query for LIKE search
|
||||
const cleanQuery = query.trim().toLowerCase();
|
||||
const searchTerms = cleanQuery.split(/\s+/).filter(term => term.length > 2);
|
||||
|
||||
if (searchTerms.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Build LIKE conditions for each term
|
||||
const likeConditions = searchTerms.map(() => 'LOWER(text) LIKE ?').join(' AND ');
|
||||
const likeParams = searchTerms.map(term => `%${term}%`);
|
||||
|
||||
let textQuery = `
|
||||
SELECT *, 0.8 as similarity
|
||||
FROM chunks
|
||||
WHERE ${likeConditions}
|
||||
`;
|
||||
|
||||
const params = [...likeParams];
|
||||
|
||||
// Add node filter if provided
|
||||
if (nodeIds && nodeIds.length > 0) {
|
||||
textQuery += ` AND node_id IN (${nodeIds.map(() => '?').join(',')})`;
|
||||
params.push(...nodeIds.map(String));
|
||||
}
|
||||
|
||||
textQuery += ` ORDER BY LENGTH(text) ASC LIMIT ?`;
|
||||
params.push(String(matchCount));
|
||||
|
||||
const result = sqlite.query<Chunk & { similarity: number }>(textQuery, params);
|
||||
const searchTime = Date.now() - startTime;
|
||||
|
||||
console.log(`📝 Text fallback: ${result.rows.length} chunks found, time=${searchTime}ms`);
|
||||
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async getChunkCount(): Promise<number> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT COUNT(*) as count FROM chunks');
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async getChunkCountByNodeId(nodeId: number): Promise<number> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT COUNT(*) as count FROM chunks WHERE node_id = ?', [nodeId]);
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async getNodesWithChunks(): Promise<Array<{ node_id: number; chunk_count: number }>> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query(`
|
||||
SELECT node_id, COUNT(*) as chunk_count
|
||||
FROM chunks
|
||||
GROUP BY node_id
|
||||
ORDER BY chunk_count DESC
|
||||
`);
|
||||
return result.rows.map((row: any) => ({
|
||||
node_id: Number(row.node_id),
|
||||
chunk_count: Number(row.chunk_count)
|
||||
}));
|
||||
}
|
||||
|
||||
async getChunksWithoutEmbeddings(): Promise<Chunk[]> {
|
||||
// In SQLite, chunk vectors live in vec_chunks; report chunks without corresponding vector rows
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Chunk>(`
|
||||
SELECT c.*
|
||||
FROM chunks c
|
||||
LEFT JOIN vec_chunks v ON v.chunk_id = c.id
|
||||
WHERE v.chunk_id IS NULL
|
||||
ORDER BY c.created_at ASC
|
||||
`);
|
||||
return result.rows;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const chunkService = new ChunkService();
|
||||
@@ -0,0 +1,202 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { openai as openaiProvider } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
|
||||
export interface Dimension {
|
||||
name: string;
|
||||
description: string | null;
|
||||
is_priority: boolean;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface LockedDimension {
|
||||
name: string;
|
||||
description: string | null;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export class DimensionService {
|
||||
/**
|
||||
* Get all locked (priority) dimensions with their descriptions
|
||||
*/
|
||||
static async getLockedDimensions(): Promise<LockedDimension[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.query(`
|
||||
WITH dimension_counts AS (
|
||||
SELECT nd.dimension, COUNT(*) AS count
|
||||
FROM node_dimensions nd
|
||||
GROUP BY nd.dimension
|
||||
)
|
||||
SELECT
|
||||
d.name,
|
||||
d.description,
|
||||
COALESCE(dc.count, 0) AS count
|
||||
FROM dimensions d
|
||||
LEFT JOIN dimension_counts dc ON dc.dimension = d.name
|
||||
WHERE d.is_priority = 1
|
||||
ORDER BY d.name ASC
|
||||
`);
|
||||
|
||||
return result.rows.map((row: any) => ({
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
count: Number(row.count)
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatically assign locked dimensions to a node based on its content
|
||||
*/
|
||||
static async assignLockedDimensions(nodeData: {
|
||||
title: string;
|
||||
content?: string;
|
||||
link?: string;
|
||||
}): Promise<string[]> {
|
||||
try {
|
||||
const lockedDimensions = await this.getLockedDimensions();
|
||||
|
||||
if (lockedDimensions.length === 0) {
|
||||
console.log('No locked dimensions available for assignment');
|
||||
return [];
|
||||
}
|
||||
|
||||
const prompt = this.buildAssignmentPrompt(nodeData, lockedDimensions);
|
||||
|
||||
const response = await generateText({
|
||||
model: openaiProvider('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 100,
|
||||
temperature: 0.1, // Low temperature for consistent results
|
||||
});
|
||||
|
||||
const assignedDimensions = this.parseAssignmentResponse(response.text, lockedDimensions);
|
||||
|
||||
console.log(`Assigned dimensions for "${nodeData.title}": ${assignedDimensions.join(', ')}`);
|
||||
return assignedDimensions;
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error assigning dimensions:', error);
|
||||
return []; // Graceful fallback
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update dimension description
|
||||
*/
|
||||
static async updateDimensionDescription(name: string, description: string): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
sqlite.query(`
|
||||
INSERT INTO dimensions(name, description, is_priority, updated_at)
|
||||
VALUES (?, ?, 0, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
description = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`, [name, description, description]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get dimension by name with description
|
||||
*/
|
||||
static async getDimensionByName(name: string): Promise<Dimension | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.query(`
|
||||
SELECT name, description, is_priority, updated_at
|
||||
FROM dimensions
|
||||
WHERE name = ?
|
||||
`, [name]);
|
||||
|
||||
if (result.rows.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const row = result.rows[0] as any;
|
||||
return {
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
is_priority: Boolean(row.is_priority),
|
||||
updated_at: row.updated_at
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build AI prompt for dimension assignment
|
||||
*/
|
||||
private static buildAssignmentPrompt(
|
||||
nodeData: { title: string; content?: string; link?: string },
|
||||
lockedDimensions: LockedDimension[]
|
||||
): string {
|
||||
const contentPreview = nodeData.content?.slice(0, 500) || '';
|
||||
const dimensionsList = lockedDimensions
|
||||
.map(d => `- ${d.name}: ${d.description || `Infer purpose from name: ${d.name}`}`)
|
||||
.join('\n');
|
||||
|
||||
return `Analyze this node content and assign appropriate dimensions from the available locked dimensions.
|
||||
|
||||
Node:
|
||||
Title: ${nodeData.title}
|
||||
Content: ${contentPreview}${nodeData.content && nodeData.content.length > 500 ? '...' : ''}
|
||||
URL: ${nodeData.link || 'none'}
|
||||
|
||||
Available Locked Dimensions:
|
||||
${dimensionsList}
|
||||
|
||||
Return only the dimension names that best match this content, maximum 3 dimensions.
|
||||
Respond with just the dimension names, one per line.
|
||||
If no dimensions are appropriate, respond with "none".
|
||||
|
||||
Examples of good responses:
|
||||
Work
|
||||
Learning
|
||||
|
||||
or
|
||||
|
||||
Research
|
||||
Ideas
|
||||
Tech
|
||||
|
||||
or
|
||||
|
||||
none`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse AI response and validate dimension names
|
||||
*/
|
||||
private static parseAssignmentResponse(
|
||||
response: string,
|
||||
availableDimensions: LockedDimension[]
|
||||
): string[] {
|
||||
const lines = response.trim().toLowerCase().split('\n');
|
||||
const availableNames = availableDimensions.map(d => d.name.toLowerCase());
|
||||
const validDimensions: string[] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
const dimensionName = line.trim();
|
||||
|
||||
if (dimensionName === 'none') {
|
||||
break;
|
||||
}
|
||||
|
||||
// Find matching dimension (case-insensitive)
|
||||
const matchedDimension = availableDimensions.find(
|
||||
d => d.name.toLowerCase() === dimensionName
|
||||
);
|
||||
|
||||
if (matchedDimension && !validDimensions.includes(matchedDimension.name)) {
|
||||
validDimensions.push(matchedDimension.name);
|
||||
|
||||
// Limit to 3 dimensions max
|
||||
if (validDimensions.length >= 3) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return validDimensions;
|
||||
}
|
||||
}
|
||||
|
||||
export const dimensionService = new DimensionService();
|
||||
@@ -0,0 +1,340 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { Edge, EdgeData, NodeConnection, Node } from '@/types/database';
|
||||
import { eventBroadcaster } from '../events';
|
||||
|
||||
export class EdgeService {
|
||||
async getEdges(): Promise<Edge[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Edge>('SELECT * FROM edges ORDER BY created_at DESC');
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async getEdgeById(id: number): Promise<Edge | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query<Edge>('SELECT * FROM edges WHERE id = ?', [id]);
|
||||
return result.rows[0] || null;
|
||||
}
|
||||
|
||||
async createEdge(edgeData: EdgeData): Promise<Edge> {
|
||||
return this.createEdgeSQLite(edgeData);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async createEdgeSQLite(edgeData: EdgeData): Promise<Edge> {
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.prepare(`
|
||||
INSERT INTO edges (from_node_id, to_node_id, context, source, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
edgeData.from_node_id,
|
||||
edgeData.to_node_id,
|
||||
JSON.stringify(edgeData.context || {}),
|
||||
edgeData.source,
|
||||
now
|
||||
);
|
||||
|
||||
const edgeId = Number(result.lastInsertRowid);
|
||||
const newEdge = await this.getEdgeById(edgeId);
|
||||
|
||||
if (!newEdge) {
|
||||
throw new Error('Failed to create edge');
|
||||
}
|
||||
|
||||
// Broadcast edge creation event
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'EDGE_CREATED',
|
||||
data: {
|
||||
fromNodeId: newEdge.from_node_id,
|
||||
toNodeId: newEdge.to_node_id,
|
||||
edge: newEdge
|
||||
}
|
||||
});
|
||||
|
||||
return newEdge;
|
||||
}
|
||||
|
||||
async updateEdge(id: number, updates: Partial<Edge>): Promise<Edge> {
|
||||
return this.updateEdgeSQLite(id, updates);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async updateEdgeSQLite(id: number, updates: Partial<Edge>): Promise<Edge> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const updateFields: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
// Build dynamic update query
|
||||
Object.entries(updates).forEach(([key, value]) => {
|
||||
if (key !== 'id' && key !== 'created_at' && value !== undefined) {
|
||||
updateFields.push(`${key} = ?`);
|
||||
if (key === 'context') {
|
||||
params.push(typeof value === 'object' ? JSON.stringify(value) : value);
|
||||
} else {
|
||||
params.push(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (updateFields.length === 0) {
|
||||
throw new Error('No valid fields to update');
|
||||
}
|
||||
|
||||
params.push(id); // Add ID for WHERE clause
|
||||
|
||||
const query = `UPDATE edges SET ${updateFields.join(', ')} WHERE id = ?`;
|
||||
const result = sqlite.query(query, params);
|
||||
|
||||
if (result.changes === 0) {
|
||||
throw new Error(`Edge with ID ${id} not found`);
|
||||
}
|
||||
|
||||
const updatedEdge = await this.getEdgeById(id);
|
||||
if (!updatedEdge) {
|
||||
throw new Error(`Failed to retrieve updated edge with ID ${id}`);
|
||||
}
|
||||
|
||||
return updatedEdge;
|
||||
}
|
||||
|
||||
async deleteEdge(id: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('DELETE FROM edges WHERE id = ?', [id]);
|
||||
if ((result.changes || 0) === 0) {
|
||||
throw new Error(`Edge with ID ${id} not found`);
|
||||
}
|
||||
// Broadcast edge deletion event
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'EDGE_DELETED',
|
||||
data: { edgeId: id }
|
||||
});
|
||||
}
|
||||
|
||||
async deleteEdgesByNodeId(nodeId: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
sqlite.query(
|
||||
'DELETE FROM edges WHERE from_node_id = ? OR to_node_id = ?',
|
||||
[nodeId, nodeId]
|
||||
);
|
||||
}
|
||||
|
||||
async getNodeConnections(nodeId: number): Promise<NodeConnection[]> {
|
||||
return this.getNodeConnectionsSQLite(nodeId);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async getNodeConnectionsSQLite(nodeId: number): Promise<NodeConnection[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query(`
|
||||
SELECT
|
||||
e.*,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.id
|
||||
ELSE n_from.id
|
||||
END as connected_node_id,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.title
|
||||
ELSE n_from.title
|
||||
END as connected_node_title,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.content
|
||||
ELSE n_from.content
|
||||
END as connected_node_content,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.link
|
||||
ELSE n_from.link
|
||||
END as connected_node_link,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.chunk
|
||||
ELSE n_from.chunk
|
||||
END as connected_node_chunk,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.metadata
|
||||
ELSE n_from.metadata
|
||||
END as connected_node_metadata,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.created_at
|
||||
ELSE n_from.created_at
|
||||
END as connected_node_created_at,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN n_to.updated_at
|
||||
ELSE n_from.updated_at
|
||||
END as connected_node_updated_at,
|
||||
CASE
|
||||
WHEN e.from_node_id = ? THEN (
|
||||
SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n_to.id
|
||||
)
|
||||
ELSE (
|
||||
SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n_from.id
|
||||
)
|
||||
END as connected_node_dimensions_json
|
||||
FROM edges e
|
||||
LEFT JOIN nodes n_from ON e.from_node_id = n_from.id
|
||||
LEFT JOIN nodes n_to ON e.to_node_id = n_to.id
|
||||
WHERE e.from_node_id = ? OR e.to_node_id = ?
|
||||
ORDER BY e.created_at DESC
|
||||
`, [
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId,
|
||||
nodeId
|
||||
]);
|
||||
|
||||
return this.mapNodeConnectionsSQLite(result.rows);
|
||||
}
|
||||
|
||||
private mapNodeConnections(rows: any[]): NodeConnection[] {
|
||||
return rows.map(row => {
|
||||
const edge: Edge = {
|
||||
id: row.id,
|
||||
from_node_id: row.from_node_id,
|
||||
to_node_id: row.to_node_id,
|
||||
context: row.context,
|
||||
source: row.source,
|
||||
created_at: row.created_at
|
||||
};
|
||||
|
||||
const connected_node: Node = {
|
||||
id: row.connected_node_id,
|
||||
title: row.connected_node_title,
|
||||
content: row.connected_node_content,
|
||||
link: row.connected_node_link,
|
||||
dimensions: row.connected_node_dimensions,
|
||||
embedding: undefined, // Not needed for display
|
||||
chunk: row.connected_node_chunk,
|
||||
metadata: row.connected_node_metadata,
|
||||
created_at: row.connected_node_created_at,
|
||||
updated_at: row.connected_node_updated_at
|
||||
};
|
||||
|
||||
return {
|
||||
id: edge.id,
|
||||
connected_node,
|
||||
edge
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private mapNodeConnectionsSQLite(rows: any[]): NodeConnection[] {
|
||||
return rows.map(row => {
|
||||
let context: any = row.context;
|
||||
if (typeof row.context === 'string') {
|
||||
const trimmed = row.context.trim();
|
||||
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
||||
try {
|
||||
context = JSON.parse(trimmed);
|
||||
} catch (error) {
|
||||
console.warn('[edges] Failed to parse JSON context for edge', row.id, error);
|
||||
context = row.context;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const edge: Edge = {
|
||||
id: row.id,
|
||||
from_node_id: row.from_node_id,
|
||||
to_node_id: row.to_node_id,
|
||||
context,
|
||||
source: row.source,
|
||||
created_at: row.created_at
|
||||
};
|
||||
|
||||
const connected_node: Node = {
|
||||
id: row.connected_node_id,
|
||||
title: row.connected_node_title,
|
||||
content: row.connected_node_content,
|
||||
link: row.connected_node_link,
|
||||
dimensions: JSON.parse(row.connected_node_dimensions_json || '[]'),
|
||||
embedding: undefined, // Not needed for display
|
||||
chunk: row.connected_node_chunk,
|
||||
metadata: typeof row.connected_node_metadata === 'string' ? JSON.parse(row.connected_node_metadata) : row.connected_node_metadata,
|
||||
created_at: row.connected_node_created_at,
|
||||
updated_at: row.connected_node_updated_at
|
||||
};
|
||||
|
||||
return {
|
||||
id: edge.id,
|
||||
connected_node,
|
||||
edge
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async edgeExists(fromId: number, toId: number): Promise<boolean> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT 1 FROM edges WHERE from_node_id = ? AND to_node_id = ?', [fromId, toId]);
|
||||
return result.rows.length > 0;
|
||||
}
|
||||
|
||||
async getEdgeCount(): Promise<number> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT COUNT(*) as count FROM edges');
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
|
||||
async getMostConnectedNodes(limit = 10): Promise<Array<{ node_id: number; connection_count: number }>> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query(`
|
||||
SELECT
|
||||
node_id,
|
||||
COUNT(*) as connection_count
|
||||
FROM (
|
||||
SELECT from_node_id as node_id FROM edges
|
||||
UNION ALL
|
||||
SELECT to_node_id as node_id FROM edges
|
||||
) combined
|
||||
GROUP BY node_id
|
||||
ORDER BY connection_count DESC
|
||||
LIMIT ?
|
||||
`, [limit]);
|
||||
|
||||
return result.rows.map((row: any) => ({
|
||||
node_id: Number(row.node_id),
|
||||
connection_count: Number(row.connection_count)
|
||||
}));
|
||||
}
|
||||
|
||||
async createBidirectionalEdge(fromId: number, toId: number, options?: {
|
||||
context?: any;
|
||||
source?: 'user' | 'ai_similarity' | 'helper_name';
|
||||
}): Promise<Edge[]> {
|
||||
const edges: Edge[] = [];
|
||||
|
||||
// Create edge from A to B
|
||||
const forwardEdge = await this.createEdge({
|
||||
from_node_id: fromId,
|
||||
to_node_id: toId,
|
||||
context: options?.context,
|
||||
source: options?.source || 'ai_similarity'
|
||||
});
|
||||
edges.push(forwardEdge);
|
||||
|
||||
// Create edge from B to A
|
||||
const backwardEdge = await this.createEdge({
|
||||
from_node_id: toId,
|
||||
to_node_id: fromId,
|
||||
context: options?.context,
|
||||
source: options?.source || 'ai_similarity'
|
||||
});
|
||||
edges.push(backwardEdge);
|
||||
|
||||
return edges;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const edgeService = new EdgeService();
|
||||
@@ -0,0 +1,70 @@
|
||||
// Service instances
|
||||
export { nodeService, NodeService } from './nodes';
|
||||
export { chunkService, ChunkService } from './chunks';
|
||||
export { edgeService, EdgeService } from './edges';
|
||||
export { dimensionService, DimensionService } from './dimensionService';
|
||||
// export { HelperService } from './helpers'; // Removed - migrated to JSON-based service
|
||||
|
||||
// Types
|
||||
export * from '@/types/database';
|
||||
|
||||
// Health check utility
|
||||
export async function checkDatabaseHealth(): Promise<{
|
||||
connected: boolean;
|
||||
vectorExtension: boolean;
|
||||
tablesExist: boolean;
|
||||
error?: string;
|
||||
}> {
|
||||
try {
|
||||
return checkSQLiteDatabaseHealth();
|
||||
} catch (error) {
|
||||
return {
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function checkSQLiteDatabaseHealth(): Promise<{
|
||||
connected: boolean;
|
||||
vectorExtension: boolean;
|
||||
tablesExist: boolean;
|
||||
error?: string;
|
||||
}> {
|
||||
try {
|
||||
const { getSQLiteClient } = await import('./sqlite-client');
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const connected = await sqlite.testConnection();
|
||||
if (!connected) {
|
||||
return {
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
error: 'SQLite connection failed'
|
||||
};
|
||||
}
|
||||
|
||||
const vectorExtension = await sqlite.checkVectorExtension();
|
||||
|
||||
// Check if main tables exist
|
||||
const tables = await sqlite.checkTables();
|
||||
const requiredTables = ['nodes', 'chunks', 'edges'];
|
||||
const tablesExist = requiredTables.every(table => tables.includes(table));
|
||||
|
||||
return {
|
||||
connected,
|
||||
vectorExtension,
|
||||
tablesExist
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
connected: false,
|
||||
vectorExtension: false,
|
||||
tablesExist: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
import { getSQLiteClient } from './sqlite-client';
|
||||
import { Node, NodeFilters } from '@/types/database';
|
||||
import { eventBroadcaster } from '../events';
|
||||
|
||||
export class NodeService {
|
||||
async getNodes(filters: NodeFilters = {}): Promise<Node[]> {
|
||||
return this.getNodesSQLite(filters);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async getNodesSQLite(filters: NodeFilters = {}): Promise<Node[]> {
|
||||
const { dimensions, search, limit = 100, offset = 0, sortBy } = filters;
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
// Use nodes_v view for array-like dimensions behavior (exclude embedding BLOB for performance)
|
||||
let query = `
|
||||
SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
(SELECT COUNT(*) FROM edges WHERE from_node_id = n.id OR to_node_id = n.id) as edge_count
|
||||
FROM nodes n
|
||||
WHERE 1=1
|
||||
`;
|
||||
const params: any[] = [];
|
||||
|
||||
// Filter by dimensions (SQLite JOIN with node_dimensions)
|
||||
if (dimensions && dimensions.length > 0) {
|
||||
query += ` AND EXISTS (
|
||||
SELECT 1 FROM node_dimensions nd
|
||||
WHERE nd.node_id = n.id
|
||||
AND nd.dimension IN (${dimensions.map(() => '?').join(',')})
|
||||
)`;
|
||||
params.push(...dimensions);
|
||||
}
|
||||
|
||||
// Text search in title and content (SQLite LIKE with COLLATE NOCASE)
|
||||
if (search) {
|
||||
query += ` AND (n.title LIKE ? COLLATE NOCASE OR n.content LIKE ? COLLATE NOCASE)`;
|
||||
params.push(`%${search}%`, `%${search}%`);
|
||||
}
|
||||
|
||||
// Sorting logic
|
||||
if (search) {
|
||||
// For search queries, prioritize by relevance: exact title → starts with → contains in title → content
|
||||
query += ` ORDER BY
|
||||
CASE WHEN LOWER(n.title) = LOWER(?) THEN 1 ELSE 5 END,
|
||||
CASE WHEN LOWER(n.title) LIKE LOWER(?) THEN 2 ELSE 5 END,
|
||||
CASE WHEN n.title LIKE ? COLLATE NOCASE THEN 3 ELSE 5 END,
|
||||
CASE WHEN n.content LIKE ? COLLATE NOCASE THEN 4 ELSE 5 END,
|
||||
n.updated_at DESC`;
|
||||
params.push(
|
||||
search, // Exact match (case-insensitive)
|
||||
`${search}%`, // Starts with search term
|
||||
`%${search}%`, // Contains in title
|
||||
`%${search}%` // Contains in content
|
||||
);
|
||||
} else if (sortBy === 'edges') {
|
||||
// Sort by edge count (most connected first)
|
||||
query += ' ORDER BY edge_count DESC, n.updated_at DESC';
|
||||
} else {
|
||||
query += ' ORDER BY n.updated_at DESC';
|
||||
}
|
||||
|
||||
if (limit) {
|
||||
query += ` LIMIT ?`;
|
||||
params.push(limit);
|
||||
}
|
||||
|
||||
if (offset > 0) {
|
||||
query += ` OFFSET ?`;
|
||||
params.push(offset);
|
||||
}
|
||||
|
||||
const result = sqlite.query<Node & { dimensions_json: string }>(query, params);
|
||||
|
||||
// Parse dimensions_json back to array for compatibility
|
||||
return result.rows.map(row => ({
|
||||
...row,
|
||||
dimensions: JSON.parse(row.dimensions_json || '[]')
|
||||
}));
|
||||
}
|
||||
|
||||
async getNodeById(id: number): Promise<Node | null> {
|
||||
return this.getNodeByIdSQLite(id);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async getNodeByIdSQLite(id: number): Promise<Node | null> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT n.id, n.title, n.description, n.content, n.link, n.type, n.metadata, n.chunk,
|
||||
n.chunk_status, n.embedding_updated_at, n.embedding_text,
|
||||
n.created_at, n.updated_at,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json
|
||||
FROM nodes n
|
||||
WHERE n.id = ?
|
||||
`;
|
||||
const result = sqlite.query<Node & { dimensions_json: string }>(query, [id]);
|
||||
|
||||
if (result.rows.length === 0) return null;
|
||||
|
||||
const row = result.rows[0];
|
||||
return {
|
||||
...row,
|
||||
dimensions: JSON.parse(row.dimensions_json || '[]')
|
||||
};
|
||||
}
|
||||
|
||||
async createNode(nodeData: Partial<Node>): Promise<Node> {
|
||||
return this.createNodeSQLite(nodeData);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async createNodeSQLite(nodeData: Partial<Node>): Promise<Node> {
|
||||
const {
|
||||
title,
|
||||
description,
|
||||
content,
|
||||
link,
|
||||
type,
|
||||
dimensions = [],
|
||||
chunk,
|
||||
chunk_status,
|
||||
metadata = {}
|
||||
} = nodeData;
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const nodeId = sqlite.transaction(() => {
|
||||
// Insert node using prepare/run for lastInsertRowid access
|
||||
const nodeResult = sqlite.prepare(`
|
||||
INSERT INTO nodes (title, description, content, link, type, metadata, chunk, chunk_status, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
title,
|
||||
description ?? null,
|
||||
content ?? null,
|
||||
link ?? null,
|
||||
type ?? null,
|
||||
JSON.stringify(metadata),
|
||||
chunk ?? null,
|
||||
chunk_status ?? null,
|
||||
now,
|
||||
now
|
||||
);
|
||||
|
||||
const id = Number(nodeResult.lastInsertRowid);
|
||||
|
||||
// Insert dimensions separately with INSERT OR IGNORE for safety
|
||||
if (dimensions.length > 0) {
|
||||
const stmt = sqlite.prepare(
|
||||
"INSERT OR IGNORE INTO node_dimensions (node_id, dimension) VALUES (?, ?)"
|
||||
);
|
||||
for (const dimension of dimensions) {
|
||||
stmt.run(id, dimension);
|
||||
}
|
||||
}
|
||||
|
||||
return id; // Returns number directly
|
||||
});
|
||||
|
||||
// Get the created node with dimensions (outside transaction)
|
||||
const createdNode = await this.getNodeByIdSQLite(nodeId);
|
||||
if (!createdNode) {
|
||||
throw new Error('Failed to create node');
|
||||
}
|
||||
|
||||
// Broadcast node creation event
|
||||
console.log('🚀 Broadcasting NODE_CREATED event for:', createdNode.title);
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'NODE_CREATED',
|
||||
data: { node: createdNode }
|
||||
});
|
||||
|
||||
return createdNode;
|
||||
}
|
||||
|
||||
async updateNode(id: number, updates: Partial<Node>): Promise<Node> {
|
||||
return this.updateNodeSQLite(id, updates);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async updateNodeSQLite(id: number, updates: Partial<Node>): Promise<Node> {
|
||||
const { title, description, content, link, type, dimensions, chunk, metadata } = updates;
|
||||
const now = new Date().toISOString();
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const existingRow = sqlite
|
||||
.query<{ id: number }>('SELECT id FROM nodes WHERE id = ?', [id])
|
||||
.rows[0];
|
||||
|
||||
if (!existingRow) {
|
||||
throw new Error(`Node with ID ${id} not found`);
|
||||
}
|
||||
|
||||
sqlite.transaction(() => {
|
||||
// Update node columns (only update provided fields)
|
||||
const setFields: string[] = [];
|
||||
const params: any[] = [];
|
||||
|
||||
if (title !== undefined) { setFields.push('title = ?'); params.push(title); }
|
||||
if (description !== undefined) { setFields.push('description = ?'); params.push(description); }
|
||||
if (content !== undefined) { setFields.push('content = ?'); params.push(content); }
|
||||
if (link !== undefined) { setFields.push('link = ?'); params.push(link); }
|
||||
if (type !== undefined) { setFields.push('type = ?'); params.push(type); }
|
||||
if (chunk !== undefined) { setFields.push('chunk = ?'); params.push(chunk); }
|
||||
if (Object.prototype.hasOwnProperty.call(updates, 'chunk_status')) {
|
||||
setFields.push('chunk_status = ?');
|
||||
params.push(updates.chunk_status ?? null);
|
||||
}
|
||||
if (metadata !== undefined) {
|
||||
setFields.push('metadata = ?');
|
||||
params.push(JSON.stringify(metadata));
|
||||
}
|
||||
|
||||
// Always update timestamp
|
||||
setFields.push('updated_at = ?');
|
||||
params.push(now, id); // id for WHERE clause
|
||||
|
||||
if (setFields.length > 1) { // More than just updated_at
|
||||
const stmt = sqlite.prepare(`UPDATE nodes SET ${setFields.join(', ')} WHERE id = ?`);
|
||||
stmt.run(...params);
|
||||
}
|
||||
|
||||
// Handle dimensions separately
|
||||
if (Array.isArray(dimensions)) {
|
||||
sqlite.prepare('DELETE FROM node_dimensions WHERE node_id = ?').run(id);
|
||||
const dimStmt = sqlite.prepare('INSERT OR IGNORE INTO node_dimensions (node_id, dimension) VALUES (?, ?)');
|
||||
for (const dim of dimensions) {
|
||||
dimStmt.run(id, dim);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Get updated node
|
||||
const updatedNode = await this.getNodeByIdSQLite(id);
|
||||
if (!updatedNode) {
|
||||
throw new Error(`Node with ID ${id} not found`);
|
||||
}
|
||||
|
||||
// Broadcast node update event
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'NODE_UPDATED',
|
||||
data: { nodeId: id, node: updatedNode }
|
||||
});
|
||||
|
||||
return updatedNode;
|
||||
}
|
||||
|
||||
async deleteNode(id: number): Promise<void> {
|
||||
return this.deleteNodeSQLite(id);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async deleteNodeSQLite(id: number): Promise<void> {
|
||||
const sqlite = getSQLiteClient();
|
||||
|
||||
const result = sqlite.query('DELETE FROM nodes WHERE id = ?', [id]);
|
||||
|
||||
if (result.changes === 0) {
|
||||
throw new Error(`Node with ID ${id} not found`);
|
||||
}
|
||||
|
||||
// Broadcast node deletion event
|
||||
eventBroadcaster.broadcast({
|
||||
type: 'NODE_DELETED',
|
||||
data: { nodeId: id }
|
||||
});
|
||||
}
|
||||
|
||||
// Dimension-based filtering methods
|
||||
async getNodesByDimension(dimension: string): Promise<Node[]> {
|
||||
return this.getNodes({ dimensions: [dimension] });
|
||||
}
|
||||
|
||||
async searchNodes(searchTerm: string, limit = 50): Promise<Node[]> {
|
||||
return this.getNodes({ search: searchTerm, limit });
|
||||
}
|
||||
|
||||
async getNodeCount(): Promise<number> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const result = sqlite.query('SELECT COUNT(*) as count FROM nodes');
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async bulkUpdateNodes(ids: number[], updates: Partial<Node>): Promise<Node[]> {
|
||||
if (ids.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.bulkUpdateNodesSQLite(ids, updates);
|
||||
}
|
||||
|
||||
// PostgreSQL path removed in SQLite-only consolidation
|
||||
|
||||
private async bulkUpdateNodesSQLite(ids: number[], updates: Partial<Node>): Promise<Node[]> {
|
||||
// For SQLite, use IN (SELECT value FROM json_each(?)) for safety
|
||||
const sqlite = getSQLiteClient();
|
||||
const idsJson = JSON.stringify(ids);
|
||||
|
||||
// For now, just update one by one - could optimize later
|
||||
const updatedNodes: Node[] = [];
|
||||
for (const id of ids) {
|
||||
const updated = await this.updateNodeSQLite(id, updates);
|
||||
updatedNodes.push(updated);
|
||||
}
|
||||
return updatedNodes;
|
||||
}
|
||||
|
||||
// Get all unique dimensions for UI filtering
|
||||
async getAllDimensions(): Promise<string[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT DISTINCT dimension
|
||||
FROM node_dimensions
|
||||
ORDER BY dimension
|
||||
`;
|
||||
const result = sqlite.query<{dimension: string}>(query);
|
||||
return result.rows.map(row => row.dimension);
|
||||
}
|
||||
|
||||
// Get dimension usage statistics
|
||||
async getDimensionStats(): Promise<{dimension: string, count: number}[]> {
|
||||
const sqlite = getSQLiteClient();
|
||||
const query = `
|
||||
SELECT dimension, COUNT(*) as count
|
||||
FROM node_dimensions
|
||||
GROUP BY dimension
|
||||
ORDER BY count DESC
|
||||
`;
|
||||
const result = sqlite.query<{dimension: string, count: number}>(query);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const nodeService = new NodeService();
|
||||
|
||||
// Legacy export for backwards compatibility during migration
|
||||
export const itemService = nodeService;
|
||||
export const ItemService = NodeService;
|
||||
@@ -0,0 +1,686 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { DatabaseError } from '@/types/database';
|
||||
|
||||
export interface SQLiteConfig {
|
||||
dbPath: string;
|
||||
vecExtensionPath: string;
|
||||
}
|
||||
|
||||
export interface SQLiteQueryResult<T = any> {
|
||||
rows: T[];
|
||||
changes?: number;
|
||||
lastInsertRowid?: number;
|
||||
}
|
||||
|
||||
class SQLiteClient {
|
||||
private static instance: SQLiteClient;
|
||||
private db: Database.Database;
|
||||
private config: SQLiteConfig;
|
||||
private readonly readOnly: boolean;
|
||||
|
||||
private constructor() {
|
||||
this.config = this.getSQLiteConfig();
|
||||
this.readOnly = process.env.SQLITE_READONLY === 'true';
|
||||
|
||||
// Initialize database connection
|
||||
const dbDirectory = path.dirname(this.config.dbPath);
|
||||
if (!this.readOnly && !fs.existsSync(dbDirectory)) {
|
||||
fs.mkdirSync(dbDirectory, { recursive: true });
|
||||
}
|
||||
this.db = this.readOnly
|
||||
? new Database(this.config.dbPath, { readonly: true, fileMustExist: true })
|
||||
: new Database(this.config.dbPath);
|
||||
|
||||
// Load sqlite-vec extension
|
||||
try {
|
||||
this.db.loadExtension(this.config.vecExtensionPath);
|
||||
console.log('SQLite vector extension loaded successfully');
|
||||
} catch (error) {
|
||||
// Do not fail hard — allow the app to run without vector features
|
||||
console.error('Warning: Failed to load vector extension:', error);
|
||||
}
|
||||
|
||||
// Configure SQLite settings
|
||||
if (this.readOnly) {
|
||||
try {
|
||||
this.db.pragma('query_only = ON');
|
||||
} catch (error) {
|
||||
console.warn('Failed to enable query_only pragma in read-only mode:', error);
|
||||
}
|
||||
} else {
|
||||
this.db.pragma('foreign_keys = ON');
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.pragma('synchronous = NORMAL');
|
||||
this.db.pragma('cache_size = 10000');
|
||||
this.db.pragma('temp_store = memory');
|
||||
this.db.pragma('busy_timeout = 5000');
|
||||
|
||||
// Ensure vector virtual tables are present and healthy
|
||||
this.ensureVectorTables();
|
||||
this.healVectorTablesIfCorrupt();
|
||||
|
||||
// Ensure logging schema (rename memory->logs if needed, create triggers/views)
|
||||
this.ensureLoggingAndMemorySchema();
|
||||
}
|
||||
|
||||
console.log('SQLite client initialized successfully');
|
||||
}
|
||||
|
||||
private getSQLiteConfig(): SQLiteConfig {
|
||||
const dbPath = process.env.SQLITE_DB_PATH || path.join(
|
||||
process.env.HOME || '~',
|
||||
'Library/Application Support/RA-H/db/rah.sqlite'
|
||||
);
|
||||
|
||||
const vecExtensionPath = process.env.SQLITE_VEC_EXTENSION_PATH ||
|
||||
'./vendor/sqlite-extensions/vec0.dylib';
|
||||
|
||||
return {
|
||||
dbPath,
|
||||
vecExtensionPath
|
||||
};
|
||||
}
|
||||
|
||||
public static getInstance(): SQLiteClient {
|
||||
if (!SQLiteClient.instance) {
|
||||
SQLiteClient.instance = new SQLiteClient();
|
||||
}
|
||||
return SQLiteClient.instance;
|
||||
}
|
||||
|
||||
public query<T extends Record<string, any> = any>(
|
||||
sql: string,
|
||||
params?: any[]
|
||||
): SQLiteQueryResult<T> {
|
||||
try {
|
||||
const sqlLower = sql.trim().toLowerCase();
|
||||
|
||||
// Handle different query types
|
||||
if (sqlLower.startsWith('select') ||
|
||||
sqlLower.startsWith('with') ||
|
||||
sqlLower.includes('returning')) {
|
||||
// SELECT queries and queries with RETURNING clause
|
||||
const stmt = this.db.prepare(sql);
|
||||
const rows = params ? stmt.all(...params) : stmt.all();
|
||||
return { rows: rows as T[] };
|
||||
} else {
|
||||
// INSERT/UPDATE/DELETE queries without RETURNING
|
||||
const stmt = this.db.prepare(sql);
|
||||
const result = params ? stmt.run(...params) : stmt.run();
|
||||
return {
|
||||
rows: [],
|
||||
changes: result.changes,
|
||||
lastInsertRowid: Number(result.lastInsertRowid)
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('SQLite query error:', error);
|
||||
throw this.handleError(error);
|
||||
}
|
||||
}
|
||||
|
||||
public prepare(sql: string) {
|
||||
return this.db.prepare(sql);
|
||||
}
|
||||
|
||||
public transaction<T>(callback: () => T): T {
|
||||
if (this.readOnly) {
|
||||
throw {
|
||||
message: 'SQLite client is read-only',
|
||||
code: 'SQLITE_READONLY',
|
||||
details: 'Transactions are not allowed in read-only mode'
|
||||
} as DatabaseError;
|
||||
}
|
||||
// Proactively validate/repair vec vtables before any write transaction
|
||||
this.healVectorTablesIfCorrupt();
|
||||
const txn = this.db.transaction(callback);
|
||||
try {
|
||||
return txn();
|
||||
} catch (error) {
|
||||
throw this.handleError(error);
|
||||
}
|
||||
}
|
||||
|
||||
public async testConnection(): Promise<boolean> {
|
||||
try {
|
||||
const result = this.query('SELECT datetime() as current_time');
|
||||
return result.rows.length > 0;
|
||||
} catch (error) {
|
||||
console.error('SQLite connection test failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async checkVectorExtension(): Promise<boolean> {
|
||||
try {
|
||||
const result = this.query('SELECT vec_version() as version');
|
||||
return result.rows.length > 0;
|
||||
} catch (error) {
|
||||
console.error('Vector extension check failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async checkTables(): Promise<string[]> {
|
||||
try {
|
||||
const result = this.query(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
|
||||
);
|
||||
return result.rows.map(row => row.name);
|
||||
} catch (error) {
|
||||
console.error('Table check failed:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
public ensureVectorExtensions(): void {
|
||||
try {
|
||||
// Test for vec_nodes and vec_chunks; create them if missing
|
||||
const hasVecNodes = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_nodes');
|
||||
if (!hasVecNodes) {
|
||||
this.db.exec(`
|
||||
CREATE VIRTUAL TABLE vec_nodes USING vec0(
|
||||
node_id INTEGER PRIMARY KEY,
|
||||
embedding FLOAT[1536]
|
||||
);
|
||||
`);
|
||||
console.log('Created vec_nodes virtual table');
|
||||
}
|
||||
|
||||
const hasVecChunks = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get('vec_chunks');
|
||||
if (!hasVecChunks) {
|
||||
this.db.exec(`
|
||||
CREATE VIRTUAL TABLE vec_chunks USING vec0(
|
||||
chunk_id INTEGER PRIMARY KEY,
|
||||
embedding FLOAT[1536]
|
||||
);
|
||||
`);
|
||||
console.log('Created vec_chunks virtual table');
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Vector extension not available:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private ensureVectorTables(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
// Wrapper to keep existing public API stable
|
||||
this.ensureVectorExtensions();
|
||||
}
|
||||
|
||||
private ensureLoggingAndMemorySchema(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// 1) If logs table missing but legacy memory table exists, migrate
|
||||
const hasLogs = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
|
||||
const hasLegacyMemory = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memory'").get();
|
||||
if (!hasLogs && hasLegacyMemory) {
|
||||
// Drop old view to release dependency
|
||||
this.db.exec(`DROP VIEW IF EXISTS memory_v;`);
|
||||
this.db.exec(`ALTER TABLE memory RENAME TO logs;`);
|
||||
}
|
||||
|
||||
// 2) Ensure logs table exists (fresh install)
|
||||
const hasLogsNow = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='logs'").get();
|
||||
if (!hasLogsNow) {
|
||||
this.db.exec(`
|
||||
CREATE TABLE logs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
ts TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP),
|
||||
table_name TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
row_id INTEGER NOT NULL,
|
||||
summary TEXT,
|
||||
snapshot_json TEXT
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
// Ensure nodes table has expected columns for memory nodes
|
||||
try {
|
||||
const nodeCols = this.db.prepare('PRAGMA table_info(nodes)').all() as Array<{ name: string }>;
|
||||
const ensureNodeCol = (name: string, ddl: string) => {
|
||||
if (!nodeCols.some(col => col.name === name)) {
|
||||
try {
|
||||
this.db.exec(ddl);
|
||||
} catch (colErr) {
|
||||
console.warn(`Failed to add nodes.${name}`, colErr);
|
||||
}
|
||||
}
|
||||
};
|
||||
ensureNodeCol('description', "ALTER TABLE nodes ADD COLUMN description TEXT;");
|
||||
ensureNodeCol('type', "ALTER TABLE nodes ADD COLUMN type TEXT;");
|
||||
} catch (nodeErr) {
|
||||
console.warn('Failed to ensure nodes columns:', nodeErr);
|
||||
}
|
||||
|
||||
// Ensure chats table tracks creation timestamp for ordering
|
||||
try {
|
||||
const chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as Array<{ name: string }>;
|
||||
if (chatCols.some(col => col.name === 'created_at')) {
|
||||
// no-op, column exists
|
||||
} else if (chatCols.length > 0) {
|
||||
this.db.exec("ALTER TABLE chats ADD COLUMN created_at TEXT DEFAULT (CURRENT_TIMESTAMP);");
|
||||
}
|
||||
} catch (chatErr) {
|
||||
console.warn('Failed to ensure chats.created_at column:', chatErr);
|
||||
}
|
||||
|
||||
// 3) Helpful indexes on logs (clean up old names first)
|
||||
this.db.exec(`
|
||||
DROP INDEX IF EXISTS idx_memory_ts;
|
||||
DROP INDEX IF EXISTS idx_memory_table_ts;
|
||||
DROP INDEX IF EXISTS idx_memory_table_row;
|
||||
CREATE INDEX IF NOT EXISTS idx_logs_ts ON logs(ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_logs_table_ts ON logs(table_name, ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_logs_table_row ON logs(table_name, row_id);
|
||||
`);
|
||||
|
||||
// 4) Recreate triggers to write to logs (use CREATE IF NOT EXISTS)
|
||||
const hasChats = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='chats'").get();
|
||||
this.db.exec(`
|
||||
DROP TRIGGER IF EXISTS trg_nodes_ai;
|
||||
DROP TRIGGER IF EXISTS trg_nodes_au;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_nodes_ai AFTER INSERT ON nodes BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('nodes', 'insert', NEW.id,
|
||||
printf('node created: %s', COALESCE(NEW.title,'')),
|
||||
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_nodes_au AFTER UPDATE ON nodes BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('nodes', 'update', NEW.id,
|
||||
printf('node updated: %s', COALESCE(NEW.title,'')),
|
||||
json_object('id', NEW.id, 'title', NEW.title, 'link', NEW.link));
|
||||
END;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_edges_ai;
|
||||
DROP TRIGGER IF EXISTS trg_edges_au;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_edges_ai AFTER INSERT ON edges BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('edges', 'insert', NEW.id,
|
||||
printf('edge %d→%d (%s)', NEW.from_node_id, NEW.to_node_id, COALESCE(NEW.source,'')),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'from', NEW.from_node_id,
|
||||
'to', NEW.to_node_id,
|
||||
'source', NEW.source,
|
||||
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
|
||||
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
|
||||
));
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_edges_au AFTER UPDATE ON edges BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('edges', 'update', NEW.id,
|
||||
printf('edge updated %d→%d', NEW.from_node_id, NEW.to_node_id),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'from', NEW.from_node_id,
|
||||
'to', NEW.to_node_id,
|
||||
'source', NEW.source,
|
||||
'from_title', substr((SELECT title FROM nodes WHERE id = NEW.from_node_id), 1, 120),
|
||||
'to_title', substr((SELECT title FROM nodes WHERE id = NEW.to_node_id), 1, 120)
|
||||
));
|
||||
END;
|
||||
`);
|
||||
|
||||
if (hasChats) {
|
||||
this.db.exec(`
|
||||
DROP TRIGGER IF EXISTS trg_chats_ai;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_chats_ai AFTER INSERT ON chats BEGIN
|
||||
INSERT INTO logs(table_name, action, row_id, summary, snapshot_json)
|
||||
VALUES('chats', 'insert', NEW.id,
|
||||
printf('chat: %s (%s)', COALESCE(NEW.helper_name,''), COALESCE(NEW.thread_id,'')),
|
||||
json_object(
|
||||
'id', NEW.id,
|
||||
'helper', NEW.helper_name,
|
||||
'thread', NEW.thread_id,
|
||||
'user_message', COALESCE(NEW.user_message,''),
|
||||
'assistant_message', COALESCE(NEW.assistant_message,''),
|
||||
'user_preview', substr(COALESCE(NEW.user_message,''), 1, 120),
|
||||
'assistant_preview', substr(COALESCE(NEW.assistant_message,''), 1, 120),
|
||||
'system_message', COALESCE(json_extract(NEW.metadata, '$.system_message'), ''),
|
||||
'input_tokens', COALESCE(json_extract(NEW.metadata, '$.input_tokens'), 0),
|
||||
'output_tokens', COALESCE(json_extract(NEW.metadata, '$.output_tokens'), 0),
|
||||
'cost_usd', COALESCE(json_extract(NEW.metadata, '$.estimated_cost_usd'), 0.0),
|
||||
'cache_hit', COALESCE(json_extract(NEW.metadata, '$.cache_hit'), 0),
|
||||
'model', COALESCE(json_extract(NEW.metadata, '$.model_used'), ''),
|
||||
'tools_count', COALESCE(json_extract(NEW.metadata, '$.tool_calls_count'), 0),
|
||||
'trace_id', COALESCE(json_extract(NEW.metadata, '$.trace_id'), ''),
|
||||
'voice_tts_chars', COALESCE(json_extract(NEW.metadata, '$.voice_tts_chars'), 0),
|
||||
'voice_tts_cost_usd', COALESCE(json_extract(NEW.metadata, '$.voice_tts_cost_usd'), 0),
|
||||
'voice_tts_chars_total', COALESCE(json_extract(NEW.metadata, '$.voice_tts_chars_total'), 0),
|
||||
'voice_tts_cost_usd_total', COALESCE(json_extract(NEW.metadata, '$.voice_tts_cost_usd_total'), 0),
|
||||
'voice_request_id', COALESCE(json_extract(NEW.metadata, '$.voice_request_id'), ''),
|
||||
'voice_tts_request_count', COALESCE(json_extract(NEW.metadata, '$.voice_tts_request_count'), 0)
|
||||
));
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS voice_usage (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
chat_id INTEGER,
|
||||
session_id TEXT,
|
||||
helper_name TEXT,
|
||||
request_id TEXT,
|
||||
message_id TEXT,
|
||||
voice TEXT,
|
||||
model TEXT,
|
||||
chars INTEGER,
|
||||
cost_usd REAL,
|
||||
duration_ms INTEGER,
|
||||
text_preview TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE SET NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_voice_usage_session ON voice_usage(session_id, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_voice_usage_chat ON voice_usage(chat_id);
|
||||
`);
|
||||
|
||||
// 5) Views: logs_v (drop any legacy memory_v alias)
|
||||
this.db.exec(`DROP VIEW IF EXISTS logs_v; DROP VIEW IF EXISTS memory_v;`);
|
||||
try {
|
||||
this.db.exec(`
|
||||
CREATE VIEW logs_v AS
|
||||
SELECT
|
||||
m.id,
|
||||
m.ts,
|
||||
m.table_name,
|
||||
m.action,
|
||||
m.row_id,
|
||||
m.summary,
|
||||
m.enriched_summary,
|
||||
m.snapshot_json,
|
||||
CASE WHEN m.table_name='nodes' THEN n.title END AS node_title,
|
||||
CASE WHEN m.table_name='edges' THEN nf.title END AS edge_from_title,
|
||||
CASE WHEN m.table_name='edges' THEN nt.title END AS edge_to_title,
|
||||
CASE WHEN m.table_name='chats' THEN c.helper_name END AS chat_helper,
|
||||
CASE WHEN m.table_name='chats' THEN substr(c.user_message,1,120) END AS chat_user_preview,
|
||||
CASE WHEN m.table_name='chats' THEN substr(c.assistant_message,1,120) END AS chat_assistant_preview,
|
||||
CASE WHEN m.table_name='chats' THEN c.user_message END AS chat_user_full,
|
||||
CASE WHEN m.table_name='chats' THEN c.assistant_message END AS chat_assistant_full
|
||||
FROM logs m
|
||||
LEFT JOIN nodes n ON (m.table_name='nodes' AND m.row_id = n.id)
|
||||
LEFT JOIN edges e ON (m.table_name='edges' AND m.row_id = e.id)
|
||||
LEFT JOIN nodes nf ON e.from_node_id = nf.id
|
||||
LEFT JOIN nodes nt ON e.to_node_id = nt.id
|
||||
LEFT JOIN chats c ON (m.table_name='chats' AND m.row_id = c.id);
|
||||
`);
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof Error) ||
|
||||
!/already exists/i.test(error.message || '')
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
// Do not recreate memory_v; alias has been removed.
|
||||
|
||||
// 6) Chat memory state tracking for chat-triggered memories
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS chat_memory_state (
|
||||
thread_id TEXT PRIMARY KEY,
|
||||
helper_name TEXT,
|
||||
last_processed_chat_id INTEGER DEFAULT 0,
|
||||
last_processed_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_memory_thread ON chat_memory_state(thread_id);
|
||||
`);
|
||||
|
||||
// Agent delegation table for orchestrator/worker coordination
|
||||
try {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS agent_delegations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT UNIQUE NOT NULL,
|
||||
task TEXT NOT NULL,
|
||||
context TEXT,
|
||||
expected_outcome TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
summary TEXT,
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
} catch (e) {
|
||||
console.warn('Failed to ensure agent_delegations table:', e);
|
||||
}
|
||||
|
||||
// 8) Logs retention trigger (~10k most recent rows)
|
||||
try {
|
||||
this.db.exec(`
|
||||
DROP TRIGGER IF EXISTS trg_logs_prune;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_logs_prune AFTER INSERT ON logs BEGIN
|
||||
DELETE FROM logs WHERE id < NEW.id - 10000;
|
||||
END;
|
||||
`);
|
||||
} catch {}
|
||||
|
||||
// 7) Ensure agents table schema (backward compatibility)
|
||||
try {
|
||||
const agentCols = this.db.prepare('PRAGMA table_info(agents)').all() as any[];
|
||||
if (agentCols.length) {
|
||||
const hasKey = agentCols.some(col => col.name === 'key');
|
||||
const hasComponentKey = agentCols.some(col => col.name === 'component_key');
|
||||
if (!hasKey && hasComponentKey) {
|
||||
try { this.db.exec('ALTER TABLE agents RENAME COLUMN component_key TO key;'); } catch {}
|
||||
}
|
||||
|
||||
if (!agentCols.some(col => col.name === 'role')) {
|
||||
try { this.db.exec("ALTER TABLE agents ADD COLUMN role TEXT NOT NULL DEFAULT 'executor';"); } catch {}
|
||||
}
|
||||
if (!agentCols.some(col => col.name === 'memory')) {
|
||||
try { this.db.exec('ALTER TABLE agents ADD COLUMN memory TEXT;'); } catch {}
|
||||
}
|
||||
if (!agentCols.some(col => col.name === 'prompts')) {
|
||||
try { this.db.exec("ALTER TABLE agents ADD COLUMN prompts TEXT DEFAULT '[]';"); } catch {}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Agent schema ensure failed:', e);
|
||||
}
|
||||
|
||||
// 8) Ensure chats schema (remove legacy focused_memory_id, ensure agent columns)
|
||||
if (hasChats) {
|
||||
try {
|
||||
let chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as any[];
|
||||
const hasFocusedMemoryId = chatCols.some((c: any) => c.name === 'focused_memory_id');
|
||||
if (hasFocusedMemoryId) {
|
||||
console.log('Removing legacy chats.focused_memory_id column');
|
||||
let flippedForeignKeys = false;
|
||||
try {
|
||||
this.db.exec('PRAGMA foreign_keys=OFF;');
|
||||
flippedForeignKeys = true;
|
||||
this.db.exec(`
|
||||
BEGIN TRANSACTION;
|
||||
ALTER TABLE chats RENAME TO chats_legacy_cleanup;
|
||||
CREATE TABLE chats (
|
||||
id INTEGER PRIMARY KEY,
|
||||
chat_type TEXT,
|
||||
helper_name TEXT,
|
||||
agent_type TEXT DEFAULT 'orchestrator',
|
||||
delegation_id INTEGER,
|
||||
user_message TEXT,
|
||||
assistant_message TEXT,
|
||||
thread_id TEXT,
|
||||
focused_node_id INTEGER,
|
||||
created_at TEXT DEFAULT (CURRENT_TIMESTAMP),
|
||||
metadata TEXT,
|
||||
FOREIGN KEY (focused_node_id) REFERENCES nodes(id) ON DELETE SET NULL
|
||||
);
|
||||
INSERT INTO chats (
|
||||
id, chat_type, helper_name, agent_type, delegation_id,
|
||||
user_message, assistant_message, thread_id, focused_node_id,
|
||||
created_at, metadata
|
||||
)
|
||||
SELECT id, chat_type, helper_name, agent_type, delegation_id,
|
||||
user_message, assistant_message, thread_id, focused_node_id,
|
||||
created_at, metadata
|
||||
FROM chats_legacy_cleanup;
|
||||
DROP TABLE chats_legacy_cleanup;
|
||||
CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);
|
||||
COMMIT;
|
||||
`);
|
||||
} catch (migrationErr) {
|
||||
console.warn('Failed to migrate chats table (focused_memory_id removal):', migrationErr);
|
||||
try { this.db.exec('ROLLBACK;'); } catch {}
|
||||
} finally {
|
||||
if (flippedForeignKeys) {
|
||||
try { this.db.exec('PRAGMA foreign_keys=ON;'); } catch {}
|
||||
}
|
||||
}
|
||||
chatCols = this.db.prepare('PRAGMA table_info(chats)').all() as any[];
|
||||
}
|
||||
|
||||
this.db.exec("CREATE INDEX IF NOT EXISTS idx_chats_thread ON chats(thread_id);");
|
||||
|
||||
const ensureCol = (name: string, ddl: string) => {
|
||||
if (!chatCols.some((c: any) => c.name === name)) {
|
||||
try { this.db.exec(ddl); } catch (colErr) { console.warn(`Failed to add chats.${name}`, colErr); }
|
||||
}
|
||||
};
|
||||
ensureCol('agent_type', "ALTER TABLE chats ADD COLUMN agent_type TEXT DEFAULT 'orchestrator';");
|
||||
ensureCol('delegation_id', 'ALTER TABLE chats ADD COLUMN delegation_id INTEGER;');
|
||||
} catch (e) {
|
||||
console.warn('Failed to update chats schema:', e);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const chatColsPost = hasChats
|
||||
? this.db.prepare('PRAGMA table_info(chats)').all() as any[]
|
||||
: [];
|
||||
const stillHasFocusedMemoryId = chatColsPost.some((c: any) => c.name === 'focused_memory_id');
|
||||
if (stillHasFocusedMemoryId) {
|
||||
console.warn('Skipping legacy memory table drop because chats.focused_memory_id is still present.');
|
||||
} else {
|
||||
this.db.exec(`
|
||||
DROP TRIGGER IF EXISTS trg_episodic_prune;
|
||||
DROP TABLE IF EXISTS episodic_memory;
|
||||
DROP TABLE IF EXISTS episodic_pipeline_state;
|
||||
DROP TABLE IF EXISTS semantic_memory;
|
||||
DROP TABLE IF EXISTS semantic_pipeline_state;
|
||||
DROP TABLE IF EXISTS memory_pipeline_state;
|
||||
DROP TABLE IF EXISTS memory;
|
||||
`);
|
||||
}
|
||||
} catch (dropLegacyErr) {
|
||||
console.warn('Failed to drop legacy memory pipeline tables:', dropLegacyErr);
|
||||
}
|
||||
|
||||
// 9) Ensure dimensions table exists (v0.1.16+ schema migration)
|
||||
const hasDimensions = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='dimensions'").get();
|
||||
if (!hasDimensions) {
|
||||
console.log('Creating dimensions table for v0.1.16+ features...');
|
||||
this.db.exec(`
|
||||
CREATE TABLE dimensions (
|
||||
name TEXT PRIMARY KEY,
|
||||
description TEXT,
|
||||
is_priority INTEGER DEFAULT 0,
|
||||
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
// Seed default locked dimensions
|
||||
const defaultDimensions = ['research', 'ideas', 'projects', 'memory', 'preferences'];
|
||||
const insertDimension = this.db.prepare(`
|
||||
INSERT INTO dimensions (name, is_priority, updated_at)
|
||||
VALUES (?, 1, datetime('now'))
|
||||
ON CONFLICT(name) DO UPDATE SET is_priority = 1, updated_at = datetime('now')
|
||||
`);
|
||||
|
||||
for (const dimension of defaultDimensions) {
|
||||
try {
|
||||
insertDimension.run(dimension);
|
||||
} catch (e) {
|
||||
console.warn(`Failed to seed dimension '${dimension}':`, e);
|
||||
}
|
||||
}
|
||||
console.log('Dimensions table created and seeded with default locked dimensions');
|
||||
} else {
|
||||
// Check if existing dimensions table has description column
|
||||
const dimensionCols = this.db.prepare('PRAGMA table_info(dimensions)').all() as Array<{ name: string }>;
|
||||
const hasDescription = dimensionCols.some(col => col.name === 'description');
|
||||
if (!hasDescription) {
|
||||
console.log('Adding description column to existing dimensions table...');
|
||||
try {
|
||||
this.db.exec('ALTER TABLE dimensions ADD COLUMN description TEXT;');
|
||||
console.log('Description column added to dimensions table');
|
||||
} catch (e) {
|
||||
console.warn('Failed to add description column to dimensions table:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Logging + memory schema ensured');
|
||||
} catch (error) {
|
||||
console.error('Failed to ensure logging/memory schema:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private healVectorTablesIfCorrupt(): void {
|
||||
if (this.readOnly) {
|
||||
return;
|
||||
}
|
||||
// Attempt lightweight reads to detect CORRUPT_VTAB; if detected, drop/recreate vtables
|
||||
const tryRead = (table: string) => {
|
||||
try {
|
||||
this.db.prepare(`SELECT COUNT(*) as c FROM ${table}`).get();
|
||||
} catch (e: any) {
|
||||
const msg = String(e?.message || '');
|
||||
const code = (e && e.code) ? String(e.code) : '';
|
||||
if (code === 'SQLITE_CORRUPT_VTAB' || msg.includes('database disk image is malformed') || msg.includes('CORRUPT_VTAB')) {
|
||||
console.warn(`Detected corrupted virtual table ${table} (${code || 'error'}). Recreating...`);
|
||||
try {
|
||||
this.db.exec(`DROP TABLE IF EXISTS ${table};`);
|
||||
} catch {}
|
||||
const ddl = table === 'vec_nodes'
|
||||
? `CREATE VIRTUAL TABLE vec_nodes USING vec0(node_id INTEGER PRIMARY KEY, embedding FLOAT[1536]);`
|
||||
: `CREATE VIRTUAL TABLE vec_chunks USING vec0(chunk_id INTEGER PRIMARY KEY, embedding FLOAT[1536]);`;
|
||||
try {
|
||||
this.db.exec(ddl);
|
||||
console.log(`Recreated ${table} virtual table`);
|
||||
} catch (re) {
|
||||
console.error(`Failed to recreate ${table}:`, re);
|
||||
}
|
||||
} else {
|
||||
// Other errors should bubble up normally
|
||||
// eslint-disable-next-line no-unsafe-finally
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
tryRead('vec_nodes');
|
||||
tryRead('vec_chunks');
|
||||
}
|
||||
|
||||
private handleError(error: any): DatabaseError {
|
||||
return {
|
||||
message: error.message || 'SQLite operation failed',
|
||||
code: error.code || 'SQLITE_ERROR',
|
||||
details: error
|
||||
};
|
||||
}
|
||||
|
||||
public close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance (similar to PostgreSQL client interface)
|
||||
export const sqliteDb = SQLiteClient.getInstance();
|
||||
|
||||
// Export function to get client instance
|
||||
export const getSQLiteClient = () => sqliteDb;
|
||||
|
||||
// Export class for testing
|
||||
export { SQLiteClient };
|
||||
@@ -0,0 +1,113 @@
|
||||
import { embedNodeContent } from '@/services/embedding/ingestion';
|
||||
import { nodeService } from '@/services/database';
|
||||
|
||||
interface AutoEmbedTask {
|
||||
nodeId: number;
|
||||
force?: boolean;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_COOLDOWN_MS = 2 * 60 * 1000; // 2 minutes between automatic runs per node
|
||||
|
||||
export class AutoEmbedQueue {
|
||||
private readonly queue: number[] = [];
|
||||
private readonly pendingTasks = new Map<number, AutoEmbedTask>();
|
||||
private readonly running = new Set<number>();
|
||||
private readonly lastRunAt = new Map<number, number>();
|
||||
private readonly maxConcurrent = 1;
|
||||
private readonly cooldownMs = DEFAULT_COOLDOWN_MS;
|
||||
|
||||
enqueue(nodeId: number, task: Omit<AutoEmbedTask, 'nodeId'> = {}): boolean {
|
||||
const existing = this.pendingTasks.get(nodeId);
|
||||
if (!existing) {
|
||||
this.pendingTasks.set(nodeId, { nodeId, ...task });
|
||||
this.queue.push(nodeId);
|
||||
} else {
|
||||
existing.force = existing.force || task.force;
|
||||
existing.reason = existing.reason || task.reason;
|
||||
}
|
||||
|
||||
this.processQueue();
|
||||
return true;
|
||||
}
|
||||
|
||||
private processQueue() {
|
||||
if (this.running.size >= this.maxConcurrent) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextId = this.queue.shift();
|
||||
if (typeof nextId !== 'number') {
|
||||
return;
|
||||
}
|
||||
|
||||
const task = this.pendingTasks.get(nextId);
|
||||
if (!task) {
|
||||
// Task was removed; try next
|
||||
this.processQueue();
|
||||
return;
|
||||
}
|
||||
this.pendingTasks.delete(nextId);
|
||||
|
||||
const now = Date.now();
|
||||
const lastRun = this.lastRunAt.get(task.nodeId);
|
||||
if (!task.force && lastRun && now - lastRun < this.cooldownMs) {
|
||||
const delay = this.cooldownMs - (now - lastRun);
|
||||
setTimeout(() => this.enqueue(task.nodeId, task), delay);
|
||||
this.processQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
this.running.add(task.nodeId);
|
||||
this.executeTask(task)
|
||||
.catch(error => {
|
||||
console.error('[AutoEmbedQueue] Task failed', task.nodeId, error);
|
||||
})
|
||||
.finally(() => {
|
||||
this.running.delete(task.nodeId);
|
||||
this.lastRunAt.set(task.nodeId, Date.now());
|
||||
if (this.queue.length > 0) {
|
||||
setTimeout(() => this.processQueue(), 10);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async executeTask(task: AutoEmbedTask) {
|
||||
const node = await nodeService.getNodeById(task.nodeId);
|
||||
if (!node) {
|
||||
console.warn('[AutoEmbedQueue] Node missing, skipping', task.nodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
const chunkText = node.chunk?.trim();
|
||||
if (!chunkText) {
|
||||
console.warn('[AutoEmbedQueue] Node has no chunk content, skipping', task.nodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!task.force && node.chunk_status === 'chunked') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.chunk_status === 'chunking' && !task.force) {
|
||||
console.log('[AutoEmbedQueue] Node already chunking, skipping duplicate run', task.nodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`🔄 [AutoEmbedQueue] Embedding node ${task.nodeId}${task.reason ? ` (${task.reason})` : ''}`);
|
||||
const result = await embedNodeContent(task.nodeId);
|
||||
if (!result.success) {
|
||||
console.error('[AutoEmbedQueue] Embedding failed', task.nodeId, result.error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var autoEmbedQueue: AutoEmbedQueue | undefined;
|
||||
}
|
||||
|
||||
export const autoEmbedQueue = globalThis.autoEmbedQueue ?? new AutoEmbedQueue();
|
||||
if (!globalThis.autoEmbedQueue) {
|
||||
globalThis.autoEmbedQueue = autoEmbedQueue;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export const AUTO_EMBED_MIN_CHARS = 200;
|
||||
|
||||
export function hasSufficientContent(text?: string | null): boolean {
|
||||
if (typeof text !== 'string') {
|
||||
return false;
|
||||
}
|
||||
return text.trim().length >= AUTO_EMBED_MIN_CHARS;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { nodeService } from '@/services/database';
|
||||
import { NodeEmbedder } from '@/services/typescript/embed-nodes';
|
||||
import { UniversalEmbedder } from '@/services/typescript/embed-universal';
|
||||
import type { Node } from '@/types/database';
|
||||
|
||||
export interface EmbeddingStageStatus {
|
||||
status: 'pending' | 'completed' | 'failed' | 'skipped';
|
||||
message: string;
|
||||
chunks_created?: number;
|
||||
}
|
||||
|
||||
export interface EmbeddingPipelineResult {
|
||||
success: boolean;
|
||||
error?: string;
|
||||
node_embedding: EmbeddingStageStatus;
|
||||
chunk_embeddings: EmbeddingStageStatus;
|
||||
overall_status: 'pending' | 'fully_embedded' | 'partially_embedded' | 'no_content' | 'failed';
|
||||
}
|
||||
|
||||
interface EmbeddingResult {
|
||||
success: boolean;
|
||||
error?: string;
|
||||
output?: string;
|
||||
}
|
||||
|
||||
async function runNodeEmbedding(nodeId: number): Promise<EmbeddingResult> {
|
||||
const embedder = new NodeEmbedder();
|
||||
try {
|
||||
const result = await embedder.embedNodes({ nodeId });
|
||||
if (result.processed > 0) {
|
||||
return { success: true, output: `Embedded ${result.processed} nodes` };
|
||||
}
|
||||
if (result.failed > 0) {
|
||||
return { success: false, error: 'Failed to embed node' };
|
||||
}
|
||||
return { success: true, output: 'Node already has embedding' };
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Embedding failed'
|
||||
};
|
||||
} finally {
|
||||
embedder.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function updateChunkStatus(nodeId: number, status: Node['chunk_status']) {
|
||||
try {
|
||||
await nodeService.updateNode(nodeId, { chunk_status: status });
|
||||
} catch (error) {
|
||||
console.warn('Failed to update chunk_status for node', nodeId, status, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function runChunkEmbedding(nodeId: number): Promise<EmbeddingResult> {
|
||||
const embedder = new UniversalEmbedder();
|
||||
try {
|
||||
await updateChunkStatus(nodeId, 'chunking');
|
||||
const result = await embedder.processNode({ nodeId });
|
||||
return {
|
||||
success: true,
|
||||
output: `Stored ${result.chunks} chunks`
|
||||
};
|
||||
} catch (error) {
|
||||
await updateChunkStatus(nodeId, 'error');
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Chunking failed'
|
||||
};
|
||||
} finally {
|
||||
embedder.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function embedNodeContent(nodeId: number): Promise<EmbeddingPipelineResult> {
|
||||
const node = await nodeService.getNodeById(nodeId);
|
||||
if (!node) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Node not found',
|
||||
node_embedding: { status: 'failed', message: 'Node not found' },
|
||||
chunk_embeddings: { status: 'failed', message: 'Node not found' },
|
||||
overall_status: 'failed'
|
||||
};
|
||||
}
|
||||
|
||||
const results = {
|
||||
node_embedding: { status: 'pending', message: '' } as EmbeddingStageStatus,
|
||||
chunk_embeddings: { status: 'pending', message: '', chunks_created: 0 } as EmbeddingStageStatus,
|
||||
overall_status: 'pending' as EmbeddingPipelineResult['overall_status']
|
||||
};
|
||||
|
||||
const nodeResult = await runNodeEmbedding(nodeId);
|
||||
if (nodeResult.success) {
|
||||
results.node_embedding = {
|
||||
status: 'completed',
|
||||
message: nodeResult.output || 'Node metadata embedded successfully'
|
||||
};
|
||||
} else {
|
||||
results.node_embedding = {
|
||||
status: 'failed',
|
||||
message: nodeResult.error || 'Failed to embed node metadata'
|
||||
};
|
||||
}
|
||||
|
||||
if (!node.chunk || !node.chunk.trim()) {
|
||||
results.chunk_embeddings = {
|
||||
status: 'skipped',
|
||||
message: 'No chunk content to embed',
|
||||
chunks_created: 0
|
||||
};
|
||||
} else {
|
||||
const chunkResult = await runChunkEmbedding(nodeId);
|
||||
if (chunkResult.success) {
|
||||
const chunkMatch = chunkResult.output?.match(/Stored (\d+) chunks/);
|
||||
const chunksCreated = chunkMatch ? parseInt(chunkMatch[1], 10) : 0;
|
||||
results.chunk_embeddings = {
|
||||
status: 'completed',
|
||||
message: chunkResult.output || 'Chunk content embedded successfully',
|
||||
chunks_created: chunksCreated
|
||||
};
|
||||
} else {
|
||||
results.chunk_embeddings = {
|
||||
status: 'failed',
|
||||
message: chunkResult.error || 'Failed to embed chunk content'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const nodeSuccess = results.node_embedding.status === 'completed';
|
||||
const chunkSuccess = results.chunk_embeddings.status === 'completed';
|
||||
const chunkSkipped = results.chunk_embeddings.status === 'skipped';
|
||||
|
||||
if (nodeSuccess && (chunkSuccess || chunkSkipped)) {
|
||||
results.overall_status = chunkSkipped ? 'no_content' : 'fully_embedded';
|
||||
if (chunkSuccess) {
|
||||
await updateChunkStatus(nodeId, 'chunked');
|
||||
}
|
||||
} else if (nodeSuccess || chunkSuccess) {
|
||||
results.overall_status = 'partially_embedded';
|
||||
} else {
|
||||
results.overall_status = 'failed';
|
||||
}
|
||||
|
||||
const errorParts: string[] = [];
|
||||
if (results.node_embedding.status === 'failed') {
|
||||
errorParts.push(`node: ${results.node_embedding.message}`);
|
||||
}
|
||||
if (results.chunk_embeddings.status === 'failed') {
|
||||
errorParts.push(`chunks: ${results.chunk_embeddings.message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: results.overall_status !== 'failed',
|
||||
error: errorParts.length ? errorParts.join('; ') : undefined,
|
||||
...results
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import OpenAI from 'openai';
|
||||
import { apiKeyService } from './storage/apiKeys';
|
||||
|
||||
// Initialize OpenAI client with dynamic API key support
|
||||
function getOpenAiClient(): OpenAI {
|
||||
const apiKey = apiKeyService.getOpenAiKey();
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API key required. Please:\n1. Click the Settings icon (⚙️) in the bottom left\n2. Go to API Keys tab\n3. Add your OpenAI API key\n\nGet your key at: https://platform.openai.com/api-keys');
|
||||
}
|
||||
return new OpenAI({ apiKey });
|
||||
}
|
||||
|
||||
export class EmbeddingService {
|
||||
/**
|
||||
* Generate embedding for a search query using OpenAI's text-embedding-3-small model
|
||||
* This matches the same model used in embed_universal.py for consistency
|
||||
*/
|
||||
static async generateQueryEmbedding(query: string): Promise<number[]> {
|
||||
try {
|
||||
const openai = getOpenAiClient();
|
||||
const response = await openai.embeddings.create({
|
||||
model: "text-embedding-3-small",
|
||||
input: query.trim(),
|
||||
encoding_format: "float"
|
||||
});
|
||||
|
||||
if (!response.data?.[0]?.embedding) {
|
||||
throw new Error('No embedding returned from OpenAI API');
|
||||
}
|
||||
|
||||
return response.data[0].embedding;
|
||||
} catch (error) {
|
||||
console.error('Failed to generate query embedding:', error);
|
||||
throw new Error(`Embedding generation failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate embedding dimensions match expected size (1536 for text-embedding-3-small)
|
||||
*/
|
||||
static validateEmbedding(embedding: number[]): boolean {
|
||||
return Array.isArray(embedding) && embedding.length === 1536;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Event Broadcasting Service for Real-time UI Updates
|
||||
* Manages SSE connections and broadcasts database change events
|
||||
*/
|
||||
|
||||
export interface DatabaseEvent {
|
||||
type:
|
||||
| 'NODE_CREATED'
|
||||
| 'NODE_UPDATED'
|
||||
| 'NODE_DELETED'
|
||||
| 'EDGE_CREATED'
|
||||
| 'EDGE_DELETED'
|
||||
| 'DIMENSION_UPDATED'
|
||||
| 'HELPER_UPDATED'
|
||||
| 'AGENT_UPDATED'
|
||||
| 'AGENT_DELEGATION_CREATED'
|
||||
| 'AGENT_DELEGATION_UPDATED'
|
||||
| 'WORKFLOW_PROGRESS'
|
||||
| 'CONNECTION_ESTABLISHED';
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data: any;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
class EventBroadcaster {
|
||||
private connections = new Set<ReadableStreamDefaultController>();
|
||||
|
||||
/**
|
||||
* Add a new SSE connection
|
||||
*/
|
||||
addConnection(controller: ReadableStreamDefaultController) {
|
||||
this.connections.add(controller);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove an SSE connection
|
||||
*/
|
||||
removeConnection(controller: ReadableStreamDefaultController) {
|
||||
this.connections.delete(controller);
|
||||
}
|
||||
|
||||
/**
|
||||
* Broadcast an event to all connected clients
|
||||
*/
|
||||
broadcast(event: Omit<DatabaseEvent, 'timestamp'>) {
|
||||
console.log(`📡 Broadcasting ${event.type} to ${this.connections.size} connections`);
|
||||
|
||||
const eventWithTimestamp: DatabaseEvent = {
|
||||
...event,
|
||||
timestamp: Date.now()
|
||||
};
|
||||
|
||||
const message = `data: ${JSON.stringify(eventWithTimestamp)}\n\n`;
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(message);
|
||||
|
||||
// Send to all connected clients
|
||||
let successCount = 0;
|
||||
for (const controller of this.connections) {
|
||||
try {
|
||||
controller.enqueue(data);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
// Connection is closed, remove it
|
||||
console.log('🔌 Removing dead SSE connection');
|
||||
this.connections.delete(controller);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Broadcasted to ${successCount} active connections`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send keep-alive ping to maintain connections
|
||||
*/
|
||||
sendKeepAlive() {
|
||||
const ping = `: keep-alive\n\n`;
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(ping);
|
||||
|
||||
for (const controller of this.connections) {
|
||||
try {
|
||||
controller.enqueue(data);
|
||||
} catch (error) {
|
||||
this.connections.delete(controller);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get connection count for debugging
|
||||
*/
|
||||
getConnectionCount(): number {
|
||||
return this.connections.size;
|
||||
}
|
||||
}
|
||||
|
||||
// Global singleton instance with proper Next.js dev mode handling
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var eventBroadcaster: EventBroadcaster | undefined;
|
||||
// eslint-disable-next-line no-var
|
||||
var keepAliveInterval: NodeJS.Timeout | undefined;
|
||||
}
|
||||
|
||||
export const eventBroadcaster = globalThis.eventBroadcaster ?? new EventBroadcaster();
|
||||
|
||||
if (typeof window === 'undefined') {
|
||||
globalThis.eventBroadcaster = eventBroadcaster;
|
||||
|
||||
// Keep-alive interval (every 30 seconds) - only create once
|
||||
if (!globalThis.keepAliveInterval) {
|
||||
globalThis.keepAliveInterval = setInterval(() => {
|
||||
eventBroadcaster.sendKeepAlive();
|
||||
}, 30000);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
import { Node } from '@/types/database';
|
||||
import { AgentRegistry } from '@/services/agents/registry';
|
||||
import { WorkflowRegistry } from '@/services/workflows/registry';
|
||||
import { getHelperTools, getDefaultToolNamesForRole } from '@/tools/infrastructure/registry';
|
||||
import type { CacheableBlock, SystemPromptResult } from '@/types/prompts';
|
||||
import { buildAutoContextBlock } from '@/services/context/autoContext';
|
||||
|
||||
export interface NodeContext {
|
||||
nodes: Node[];
|
||||
activeNodeId: number | null;
|
||||
}
|
||||
|
||||
export interface ContextBuilderOptions {
|
||||
maxPrimaryContent?: number; // Default: 500 chars
|
||||
maxSecondaryContent?: number; // Default: 200 chars
|
||||
includeMetadata?: boolean; // Include created/updated timestamps
|
||||
}
|
||||
|
||||
const BASE_CONTEXT = `=== RA-H BASE CONTEXT ===
|
||||
- Nodes store content (title, content, dimensions, metadata, link, chunk)
|
||||
- Edges capture directed relationships between nodes
|
||||
- When auto-context is enabled you'll see BACKGROUND CONTEXT with the 10 most-connected nodes (ID + title only)
|
||||
- Focused nodes show truncated content; use queryNodes, searchContentEmbeddings, or queryEdge when you need full detail
|
||||
- Node references must use [NODE:id:"title"] so the UI renders clickable labels
|
||||
- Pronouns or phrases like "this conversation/paper/video" refer to the primary focused node unless clarified
|
||||
`;
|
||||
|
||||
function buildStaticBaseContext(): string {
|
||||
return BASE_CONTEXT;
|
||||
}
|
||||
|
||||
function buildAgentInstructionsBlock(helperKey: string, systemPrompt: string): string {
|
||||
return `=== AGENT INSTRUCTIONS (${helperKey}) ===\n${systemPrompt}\n`;
|
||||
}
|
||||
|
||||
function isPrimaryOrchestrator(helperKey: string): boolean {
|
||||
return helperKey === 'ra-h' || helperKey === 'ra-h-easy';
|
||||
}
|
||||
|
||||
function buildToolDefinitionsBlock(toolNames: string[]): string {
|
||||
if (!Array.isArray(toolNames) || toolNames.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const tools = getHelperTools(toolNames);
|
||||
const lines: string[] = ['=== AVAILABLE TOOLS ==='];
|
||||
|
||||
Object.entries(tools).forEach(([name, tool]) => {
|
||||
if (tool?.description) {
|
||||
lines.push(`\n## ${name}\n${tool.description.trim()}`);
|
||||
}
|
||||
});
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function buildWorkflowDefinitionsBlock(helperKey: string): Promise<string | null> {
|
||||
// Only include for primary orchestrator variants
|
||||
if (!isPrimaryOrchestrator(helperKey)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const workflows = await WorkflowRegistry.getEnabledWorkflows();
|
||||
if (workflows.length === 0) return null;
|
||||
|
||||
const lines: string[] = ['=== AVAILABLE WORKFLOWS ==='];
|
||||
|
||||
for (const workflow of workflows) {
|
||||
lines.push(`\n## ${workflow.displayName} (${workflow.key})`);
|
||||
lines.push(workflow.description);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
} catch (error) {
|
||||
console.warn('Workflow definitions load failed (contextBuilder):', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function truncateToWords(text: string, maxWords: number): string {
|
||||
const words = text.trim().split(/\s+/);
|
||||
if (words.length <= maxWords) return text;
|
||||
return words.slice(0, maxWords).join(' ') + '…';
|
||||
}
|
||||
|
||||
function parseMetadata(metadata: unknown): Record<string, any> {
|
||||
if (!metadata) return {};
|
||||
if (typeof metadata === 'string') {
|
||||
try {
|
||||
return JSON.parse(metadata);
|
||||
} catch (error) {
|
||||
console.warn('Failed to parse node metadata JSON:', error);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
if (typeof metadata === 'object') {
|
||||
return metadata as Record<string, any>;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function describeChunkStatus(node: Node): string {
|
||||
const status = node.chunk_status || 'unknown';
|
||||
const chunkLength = typeof node.chunk === 'string' ? node.chunk.length : 0;
|
||||
const approxChars = chunkLength > 0 ? ` (~${Math.max(1, Math.round(chunkLength / 1000))}k chars)` : '';
|
||||
const metadata = parseMetadata(node.metadata);
|
||||
const transcriptLength = typeof metadata.transcript_length === 'number'
|
||||
? metadata.transcript_length
|
||||
: undefined;
|
||||
let transcriptLabel = '';
|
||||
if (transcriptLength && transcriptLength > 0) {
|
||||
transcriptLabel = `, transcript ≈${Math.max(1, Math.round(transcriptLength / 1000))}k chars`;
|
||||
}
|
||||
const embeddingsAvailable = status === 'chunked' || chunkLength > 0;
|
||||
return `Chunks: ${status}${approxChars}${transcriptLabel}; Embeddings: ${embeddingsAvailable ? 'available' : 'missing'}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the dynamic focused nodes context section
|
||||
*/
|
||||
export function buildFocusedNodesBlock(
|
||||
context: NodeContext,
|
||||
options: ContextBuilderOptions = {}
|
||||
): string {
|
||||
if (!context.nodes || context.nodes.length === 0) {
|
||||
return '\n=== CURRENT FOCUS ===\nNo nodes currently in focus.';
|
||||
}
|
||||
|
||||
let contextString = '=== CURRENT FOCUS ===';
|
||||
contextString += '\n25-word previews; use queryNodes/searchContentEmbeddings for full detail\n';
|
||||
|
||||
const validNodes = context.nodes.filter(n => n != null);
|
||||
const activeNode = validNodes.find(n => n.id === context.activeNodeId);
|
||||
const otherNodes = validNodes.filter(n => n.id !== context.activeNodeId);
|
||||
|
||||
if (activeNode) {
|
||||
contextString += `\n[PRIMARY - Tab 1]\nID: ${activeNode.id} | "${activeNode.title || 'Untitled'}"\n${truncateToWords(activeNode.content || 'No content', 25)}\nLink: ${activeNode.link || 'No link'}`;
|
||||
contextString += `\n${describeChunkStatus(activeNode)}`;
|
||||
}
|
||||
|
||||
if (otherNodes.length > 0) {
|
||||
otherNodes.forEach((node, index) => {
|
||||
contextString += `\n\n[Tab ${index + 2}]\nID: ${node.id} | "${node.title || 'Untitled'}"\n${truncateToWords(node.content || 'No content', 25)}\nLink: ${node.link || 'No link'}\n${describeChunkStatus(node)}`;
|
||||
});
|
||||
}
|
||||
|
||||
contextString += '\n===================================';
|
||||
return contextString;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds system prompt as cacheable blocks (Anthropic prompt caching)
|
||||
*/
|
||||
export async function buildSystemPromptBlocks(
|
||||
nodeContext: NodeContext,
|
||||
helperComponentKey: string,
|
||||
options?: ContextBuilderOptions
|
||||
): Promise<SystemPromptResult> {
|
||||
const helper = await AgentRegistry.getAgentByKey(helperComponentKey);
|
||||
const isAnthropic = helper?.model?.startsWith('anthropic/');
|
||||
const cacheControl = isAnthropic ? { type: 'ephemeral' as const } : undefined;
|
||||
const blocks: CacheableBlock[] = [];
|
||||
|
||||
const baseContext = buildStaticBaseContext().trim();
|
||||
const helperPrompt = helper?.systemPrompt || 'No instructions provided.';
|
||||
const instructionsBlock = buildAgentInstructionsBlock(helperComponentKey, helperPrompt).trim();
|
||||
|
||||
const combinedInstructions = [baseContext, instructionsBlock]
|
||||
.filter(section => section.length > 0)
|
||||
.join('\n\n');
|
||||
|
||||
if (combinedInstructions.length > 0) {
|
||||
blocks.push({
|
||||
type: 'text',
|
||||
text: combinedInstructions,
|
||||
...(cacheControl ? { cache_control: cacheControl } : {})
|
||||
});
|
||||
}
|
||||
|
||||
const availableToolNames = helper?.availableTools?.length
|
||||
? helper.availableTools
|
||||
: getDefaultToolNamesForRole(helper?.role === 'executor' ? 'executor' : 'orchestrator');
|
||||
const toolBlock = buildToolDefinitionsBlock(availableToolNames);
|
||||
if (toolBlock.trim().length > 0) {
|
||||
blocks.push({
|
||||
type: 'text',
|
||||
text: toolBlock,
|
||||
...(cacheControl ? { cache_control: cacheControl } : {})
|
||||
});
|
||||
}
|
||||
|
||||
const workflowBlock = await buildWorkflowDefinitionsBlock(helperComponentKey);
|
||||
if (workflowBlock && workflowBlock.trim().length > 0) {
|
||||
blocks.push({
|
||||
type: 'text',
|
||||
text: workflowBlock,
|
||||
...(cacheControl ? { cache_control: cacheControl } : {})
|
||||
});
|
||||
}
|
||||
|
||||
const autoContextBlock = isPrimaryOrchestrator(helperComponentKey)
|
||||
? buildAutoContextBlock()
|
||||
: null;
|
||||
if (autoContextBlock && autoContextBlock.trim().length > 0) {
|
||||
blocks.push({
|
||||
type: 'text',
|
||||
text: autoContextBlock,
|
||||
...(cacheControl ? { cache_control: cacheControl } : {})
|
||||
});
|
||||
}
|
||||
|
||||
const focusBlock = buildFocusedNodesBlock(nodeContext, options);
|
||||
blocks.push({ type: 'text', text: focusBlock });
|
||||
|
||||
return { blocks, cacheHit: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy string-based system prompt (for backward compatibility)
|
||||
* @deprecated Use buildSystemPromptBlocks for Anthropic caching support
|
||||
*/
|
||||
export async function buildSystemPrompt(
|
||||
nodeContext: NodeContext,
|
||||
helperComponentKey: string,
|
||||
options?: ContextBuilderOptions
|
||||
): Promise<{ systemPrompt: string; cacheHit: boolean }> {
|
||||
// Convert blocks to string for legacy callers
|
||||
const result = await buildSystemPromptBlocks(nodeContext, helperComponentKey, options);
|
||||
const systemPrompt = result.blocks.map(b => b.text).join('');
|
||||
return { systemPrompt, cacheHit: result.cacheHit };
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads helper instructions from JSON file
|
||||
*/
|
||||
// DB-backed; no-op placeholder kept for API stability if imported elsewhere
|
||||
@@ -0,0 +1,126 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
interface LogEntry {
|
||||
timestamp: string;
|
||||
helper: string;
|
||||
type: 'USER_MESSAGE' | 'SYSTEM_PROMPT' | 'TOOL_CALL' | 'TOOL_RESULT' | 'ASSISTANT_RESPONSE' | 'ERROR';
|
||||
content: any;
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
const LOG_DIR = path.join(process.cwd(), 'logs');
|
||||
const LOG_FILE = path.join(LOG_DIR, 'helper-interactions.log');
|
||||
|
||||
class HelperLogger {
|
||||
private sessionId: string;
|
||||
private logDirEnsured = false;
|
||||
|
||||
constructor() {
|
||||
this.sessionId = Date.now().toString();
|
||||
}
|
||||
|
||||
private async ensureLogDir() {
|
||||
if (this.logDirEnsured) return;
|
||||
try {
|
||||
await fs.mkdir(LOG_DIR, { recursive: true });
|
||||
this.logDirEnsured = true;
|
||||
} catch (error) {
|
||||
console.error('Failed to create log directory:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private async writeLog(entry: LogEntry) {
|
||||
try {
|
||||
await this.ensureLogDir();
|
||||
const logLine = JSON.stringify(entry) + '\n';
|
||||
await fs.appendFile(LOG_FILE, logLine);
|
||||
} catch (error) {
|
||||
console.error('Failed to write log:', error);
|
||||
}
|
||||
}
|
||||
|
||||
logUserMessage(helper: string, messages: any[], openTabs: any[], activeTabId: any) {
|
||||
const entry: LogEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
helper,
|
||||
type: 'USER_MESSAGE',
|
||||
content: {
|
||||
lastMessage: messages[messages.length - 1],
|
||||
messageCount: messages.length,
|
||||
openTabIds: openTabs.map(t => t.id),
|
||||
activeTabId
|
||||
},
|
||||
sessionId: this.sessionId
|
||||
};
|
||||
this.writeLog(entry);
|
||||
console.log(`✓ [${helper}] Request received`);
|
||||
}
|
||||
|
||||
logSystemPrompt(helper: string, systemPrompt: string, cacheHit: boolean) {
|
||||
const entry: LogEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
helper,
|
||||
type: 'SYSTEM_PROMPT',
|
||||
content: {
|
||||
systemPrompt,
|
||||
cacheHit
|
||||
},
|
||||
sessionId: this.sessionId
|
||||
};
|
||||
this.writeLog(entry);
|
||||
}
|
||||
|
||||
logToolCall(helper: string, toolName: string, parameters: any) {
|
||||
const entry: LogEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
helper,
|
||||
type: 'TOOL_CALL',
|
||||
content: { toolName, parameters },
|
||||
sessionId: this.sessionId
|
||||
};
|
||||
this.writeLog(entry);
|
||||
}
|
||||
|
||||
logToolResult(helper: string, toolName: string, result: any) {
|
||||
const entry: LogEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
helper,
|
||||
type: 'TOOL_RESULT',
|
||||
content: {
|
||||
toolName,
|
||||
result
|
||||
},
|
||||
sessionId: this.sessionId
|
||||
};
|
||||
this.writeLog(entry);
|
||||
}
|
||||
|
||||
logAssistantResponse(helper: string, response: string) {
|
||||
const entry: LogEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
helper,
|
||||
type: 'ASSISTANT_RESPONSE',
|
||||
content: { response },
|
||||
sessionId: this.sessionId
|
||||
};
|
||||
this.writeLog(entry);
|
||||
}
|
||||
|
||||
logError(helper: string, error: Error | string) {
|
||||
const entry: LogEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
helper,
|
||||
type: 'ERROR',
|
||||
content: error instanceof Error ? {
|
||||
message: error.message,
|
||||
stack: error.stack
|
||||
} : { message: error },
|
||||
sessionId: this.sessionId
|
||||
};
|
||||
this.writeLog(entry);
|
||||
console.error(`❌ [${helper}] Error occurred`);
|
||||
}
|
||||
}
|
||||
|
||||
export const helperLogger = new HelperLogger();
|
||||
@@ -0,0 +1,124 @@
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
|
||||
export interface AutoContextSettings {
|
||||
autoContextEnabled: boolean;
|
||||
lastPinnedMigration?: string;
|
||||
}
|
||||
|
||||
const SETTINGS_FILE = 'settings.json';
|
||||
const DEFAULT_SETTINGS: AutoContextSettings = {
|
||||
autoContextEnabled: false,
|
||||
};
|
||||
|
||||
let bootstrapAttempted = false;
|
||||
|
||||
function resolveBaseConfigDir(): string {
|
||||
const override = process.env.RAH_CONFIG_DIR;
|
||||
if (override && override.trim().length > 0) {
|
||||
return override;
|
||||
}
|
||||
|
||||
const home = os.homedir();
|
||||
if (process.platform === 'darwin') {
|
||||
return path.join(home, 'Library', 'Application Support', 'RA-H');
|
||||
}
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
const roaming = process.env.APPDATA || path.join(home, 'AppData', 'Roaming');
|
||||
return path.join(roaming, 'RA-H');
|
||||
}
|
||||
|
||||
const xdgConfig = process.env.XDG_CONFIG_HOME || path.join(home, '.config');
|
||||
return path.join(xdgConfig, 'ra-h');
|
||||
}
|
||||
|
||||
function getSettingsDir(): string {
|
||||
return path.join(resolveBaseConfigDir(), 'config');
|
||||
}
|
||||
|
||||
function getSettingsPath(): string {
|
||||
return path.join(getSettingsDir(), SETTINGS_FILE);
|
||||
}
|
||||
|
||||
function ensureSettingsDirExists(): void {
|
||||
const dir = getSettingsDir();
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
function writeSettingsFile(settings: AutoContextSettings): AutoContextSettings {
|
||||
ensureSettingsDirExists();
|
||||
fs.writeFileSync(getSettingsPath(), JSON.stringify(settings, null, 2), 'utf-8');
|
||||
return settings;
|
||||
}
|
||||
|
||||
function bootstrapFromLegacyPins(): void {
|
||||
if (bootstrapAttempted) return;
|
||||
bootstrapAttempted = true;
|
||||
|
||||
const settingsPath = getSettingsPath();
|
||||
if (fs.existsSync(settingsPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const db = getSQLiteClient();
|
||||
const countRow = db
|
||||
.query<{ count: number }>('SELECT COUNT(*) as count FROM nodes WHERE is_pinned = 1')
|
||||
.rows[0];
|
||||
const pinnedCount = Number(countRow?.count ?? 0);
|
||||
if (pinnedCount > 0) {
|
||||
writeSettingsFile({
|
||||
autoContextEnabled: true,
|
||||
lastPinnedMigration: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Auto-context pin bootstrap failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
export function getAutoContextSettings(): AutoContextSettings {
|
||||
bootstrapFromLegacyPins();
|
||||
const settingsPath = getSettingsPath();
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(settingsPath)) {
|
||||
return { ...DEFAULT_SETTINGS };
|
||||
}
|
||||
|
||||
const raw = fs.readFileSync(settingsPath, 'utf-8');
|
||||
const parsed = JSON.parse(raw);
|
||||
return {
|
||||
...DEFAULT_SETTINGS,
|
||||
...parsed,
|
||||
autoContextEnabled: Boolean(parsed?.autoContextEnabled),
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn('Failed to read auto-context settings, using defaults:', error);
|
||||
return { ...DEFAULT_SETTINGS };
|
||||
}
|
||||
}
|
||||
|
||||
export function updateAutoContextSettings(
|
||||
partial: Partial<AutoContextSettings>
|
||||
): AutoContextSettings {
|
||||
const current = getAutoContextSettings();
|
||||
const next: AutoContextSettings = {
|
||||
...current,
|
||||
...partial,
|
||||
autoContextEnabled:
|
||||
typeof partial.autoContextEnabled === 'boolean'
|
||||
? partial.autoContextEnabled
|
||||
: current.autoContextEnabled,
|
||||
};
|
||||
return writeSettingsFile(next);
|
||||
}
|
||||
|
||||
export function setAutoContextEnabled(enabled: boolean): AutoContextSettings {
|
||||
return updateAutoContextSettings({ autoContextEnabled: enabled });
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// API Key Storage Service
|
||||
// Handles secure storage and retrieval of user-provided API keys
|
||||
|
||||
export interface ApiKeys {
|
||||
openai?: string;
|
||||
anthropic?: string;
|
||||
}
|
||||
|
||||
export interface ApiKeyStatus {
|
||||
openai: 'connected' | 'failed' | 'testing' | 'not-set';
|
||||
anthropic: 'connected' | 'failed' | 'testing' | 'not-set';
|
||||
}
|
||||
|
||||
const STORAGE_KEY = 'ra-h-api-keys';
|
||||
|
||||
export class ApiKeyService {
|
||||
private static instance: ApiKeyService;
|
||||
private keys: ApiKeys = {};
|
||||
private status: ApiKeyStatus = {
|
||||
openai: 'not-set',
|
||||
anthropic: 'not-set'
|
||||
};
|
||||
|
||||
static getInstance(): ApiKeyService {
|
||||
if (!ApiKeyService.instance) {
|
||||
ApiKeyService.instance = new ApiKeyService();
|
||||
}
|
||||
return ApiKeyService.instance;
|
||||
}
|
||||
|
||||
constructor() {
|
||||
this.loadKeys();
|
||||
}
|
||||
|
||||
private notifyUpdate(): void {
|
||||
if (typeof window !== 'undefined') {
|
||||
window.dispatchEvent(new CustomEvent('api-keys:updated'));
|
||||
}
|
||||
}
|
||||
|
||||
// Load keys from localStorage
|
||||
private loadKeys(): void {
|
||||
try {
|
||||
if (typeof window !== 'undefined') {
|
||||
const stored = localStorage.getItem(STORAGE_KEY);
|
||||
if (stored) {
|
||||
this.keys = JSON.parse(stored);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Failed to load API keys from storage:', error);
|
||||
this.keys = {};
|
||||
}
|
||||
}
|
||||
|
||||
// Save keys to localStorage
|
||||
private saveKeys(): void {
|
||||
try {
|
||||
if (typeof window !== 'undefined') {
|
||||
localStorage.setItem(STORAGE_KEY, JSON.stringify(this.keys));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to save API keys to storage:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Get OpenAI API key (user key or fallback to env)
|
||||
getOpenAiKey(): string | undefined {
|
||||
// Priority: User key > Environment key
|
||||
return this.keys.openai || process.env.OPENAI_API_KEY;
|
||||
}
|
||||
|
||||
// Get Anthropic API key (user key or fallback to env)
|
||||
getAnthropicKey(): string | undefined {
|
||||
// Priority: User key > Environment key
|
||||
return this.keys.anthropic || process.env.ANTHROPIC_API_KEY;
|
||||
}
|
||||
|
||||
// Set OpenAI API key
|
||||
setOpenAiKey(key: string): void {
|
||||
if (this.validateOpenAiKey(key)) {
|
||||
this.keys.openai = key;
|
||||
this.saveKeys();
|
||||
this.notifyUpdate();
|
||||
} else {
|
||||
throw new Error('Invalid OpenAI API key format');
|
||||
}
|
||||
}
|
||||
|
||||
// Set Anthropic API key
|
||||
setAnthropicKey(key: string): void {
|
||||
if (this.validateAnthropicKey(key)) {
|
||||
this.keys.anthropic = key;
|
||||
this.saveKeys();
|
||||
this.notifyUpdate();
|
||||
} else {
|
||||
throw new Error('Invalid Anthropic API key format');
|
||||
}
|
||||
}
|
||||
|
||||
// Clear specific key
|
||||
clearOpenAiKey(): void {
|
||||
delete this.keys.openai;
|
||||
this.saveKeys();
|
||||
this.status.openai = 'not-set';
|
||||
this.notifyUpdate();
|
||||
}
|
||||
|
||||
clearAnthropicKey(): void {
|
||||
delete this.keys.anthropic;
|
||||
this.saveKeys();
|
||||
this.status.anthropic = 'not-set';
|
||||
this.notifyUpdate();
|
||||
}
|
||||
|
||||
// Clear all keys
|
||||
clearAllKeys(): void {
|
||||
this.keys = {};
|
||||
this.saveKeys();
|
||||
this.status = {
|
||||
openai: 'not-set',
|
||||
anthropic: 'not-set'
|
||||
};
|
||||
this.notifyUpdate();
|
||||
}
|
||||
|
||||
// Get masked key for display (show only last 4 characters)
|
||||
getMaskedKey(provider: 'openai' | 'anthropic'): string {
|
||||
const key = provider === 'openai' ? this.keys.openai : this.keys.anthropic;
|
||||
if (!key) return '';
|
||||
return '••••••••••••••••••••••••••••••••••••••••••••••••••••••••••••' + key.slice(-4);
|
||||
}
|
||||
|
||||
// Check if user has provided custom keys
|
||||
hasUserKeys(): boolean {
|
||||
return !!(this.keys.openai || this.keys.anthropic);
|
||||
}
|
||||
|
||||
// Get current keys (for internal use)
|
||||
getStoredKeys(): ApiKeys {
|
||||
return { ...this.keys };
|
||||
}
|
||||
|
||||
// Validate OpenAI key format
|
||||
private validateOpenAiKey(key: string): boolean {
|
||||
return typeof key === 'string' &&
|
||||
key.length > 20 &&
|
||||
(key.startsWith('sk-') || key.startsWith('sk-proj-'));
|
||||
}
|
||||
|
||||
// Validate Anthropic key format
|
||||
private validateAnthropicKey(key: string): boolean {
|
||||
return typeof key === 'string' &&
|
||||
key.length > 20 &&
|
||||
key.startsWith('sk-ant-');
|
||||
}
|
||||
|
||||
// Test connection to OpenAI
|
||||
async testOpenAiConnection(key?: string): Promise<boolean> {
|
||||
const testKey = key || this.getOpenAiKey();
|
||||
if (!testKey) return false;
|
||||
|
||||
this.status.openai = 'testing';
|
||||
|
||||
try {
|
||||
const response = await fetch('https://api.openai.com/v1/models', {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${testKey}`,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
});
|
||||
|
||||
const isConnected = response.ok;
|
||||
this.status.openai = isConnected ? 'connected' : 'failed';
|
||||
return isConnected;
|
||||
} catch (error) {
|
||||
console.error('OpenAI connection test failed:', error);
|
||||
this.status.openai = 'failed';
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Test connection to Anthropic
|
||||
async testAnthropicConnection(key?: string): Promise<boolean> {
|
||||
const testKey = key || this.getAnthropicKey();
|
||||
if (!testKey) return false;
|
||||
this.status.anthropic = 'testing';
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/local/test-anthropic', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ apiKey: testKey }),
|
||||
});
|
||||
const data = await response.json();
|
||||
const isConnected = Boolean(data?.ok);
|
||||
this.status.anthropic = isConnected ? 'connected' : 'failed';
|
||||
return isConnected;
|
||||
} catch (error) {
|
||||
console.error('Anthropic connection test failed:', error);
|
||||
this.status.anthropic = 'failed';
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Get connection status
|
||||
getStatus(): ApiKeyStatus {
|
||||
return { ...this.status };
|
||||
}
|
||||
|
||||
// Update status
|
||||
updateStatus(provider: 'openai' | 'anthropic', status: ApiKeyStatus['openai']): void {
|
||||
this.status[provider] = status;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const apiKeyService = ApiKeyService.getInstance();
|
||||
@@ -0,0 +1,32 @@
|
||||
type Entry = { data: any; ts: number };
|
||||
|
||||
class ResultCache {
|
||||
private store = new Map<string, Entry>();
|
||||
private ttlMs = 10 * 60 * 1000; // 10 minutes
|
||||
|
||||
set(id: string, data: any) {
|
||||
if (!id) return;
|
||||
this.store.set(id, { data, ts: Date.now() });
|
||||
this.gc();
|
||||
}
|
||||
|
||||
get(id: string): any | null {
|
||||
const e = this.store.get(id);
|
||||
if (!e) return null;
|
||||
if (Date.now() - e.ts > this.ttlMs) {
|
||||
this.store.delete(id);
|
||||
return null;
|
||||
}
|
||||
return e.data;
|
||||
}
|
||||
|
||||
private gc() {
|
||||
const now = Date.now();
|
||||
for (const [k, v] of this.store.entries()) {
|
||||
if (now - v.ts > this.ttlMs) this.store.delete(k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const resultCache = new ResultCache();
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* Node metadata embedding service for RA-H knowledge management system
|
||||
* Embeds node metadata (title, content, dimensions, AI analysis) into nodes.embedding field
|
||||
*/
|
||||
|
||||
import OpenAI from 'openai';
|
||||
import { generateText } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import {
|
||||
createDatabaseConnection,
|
||||
serializeFloat32Vector,
|
||||
formatEmbeddingText,
|
||||
batchProcess
|
||||
} from './sqlite-vec';
|
||||
|
||||
interface NodeRecord {
|
||||
id: number;
|
||||
title: string;
|
||||
content: string | null;
|
||||
dimensions_json: string;
|
||||
embedding?: Buffer | null;
|
||||
embedding_updated_at?: string | null;
|
||||
embedding_text?: string | null;
|
||||
}
|
||||
|
||||
interface EmbedNodeOptions {
|
||||
nodeId?: number;
|
||||
forceReEmbed?: boolean;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
export class NodeEmbedder {
|
||||
private openaiClient: OpenAI;
|
||||
private openaiProvider: ReturnType<typeof createOpenAI>;
|
||||
private db: ReturnType<typeof createDatabaseConnection>;
|
||||
private processedCount: number = 0;
|
||||
private failedCount: number = 0;
|
||||
|
||||
constructor() {
|
||||
const apiKey = process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error('OPENAI_API_KEY environment variable is not set');
|
||||
}
|
||||
|
||||
this.openaiClient = new OpenAI({ apiKey });
|
||||
this.openaiProvider = createOpenAI({ apiKey });
|
||||
this.db = createDatabaseConnection();
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze node content with AI to extract insights
|
||||
*/
|
||||
private async analyzeNodeWithAI(node: NodeRecord): Promise<string> {
|
||||
const dimensions = node.dimensions_json ? JSON.parse(node.dimensions_json) : [];
|
||||
const dimensionsText = Array.isArray(dimensions) && dimensions.length ? dimensions.join(', ') : 'none';
|
||||
|
||||
const prompt = `Analyze this content and provide 2-3 key insights or themes in a concise paragraph (max 100 words):
|
||||
|
||||
Title: ${node.title}
|
||||
Content: ${node.content || 'No content'}
|
||||
Dimensions: ${dimensionsText}
|
||||
|
||||
Focus on the main concepts, key relationships, and practical implications.`;
|
||||
|
||||
try {
|
||||
const { text } = await generateText({
|
||||
model: this.openaiProvider('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 150,
|
||||
temperature: 0.3,
|
||||
});
|
||||
|
||||
return text;
|
||||
} catch (error) {
|
||||
console.error(`AI analysis failed for node ${node.id}:`, error);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding for text using OpenAI
|
||||
*/
|
||||
private async generateEmbedding(text: string): Promise<number[]> {
|
||||
const response = await this.openaiClient.embeddings.create({
|
||||
model: 'text-embedding-3-small',
|
||||
input: text,
|
||||
});
|
||||
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed a single node
|
||||
*/
|
||||
private async embedNode(node: NodeRecord, forceReEmbed: boolean = false): Promise<void> {
|
||||
// Skip if already embedded and not forcing
|
||||
if (node.embedding && !forceReEmbed) {
|
||||
console.log(`Skipping node ${node.id} - already has embedding`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse dimensions from JSON string
|
||||
const dimensions = node.dimensions_json ? JSON.parse(node.dimensions_json) : [];
|
||||
|
||||
// Create base embedding text
|
||||
let embeddingText = formatEmbeddingText(
|
||||
node.title,
|
||||
node.content || '',
|
||||
dimensions
|
||||
);
|
||||
|
||||
// Add AI analysis if content exists
|
||||
if (node.content && node.content.trim().length > 0) {
|
||||
const analysis = await this.analyzeNodeWithAI(node);
|
||||
if (analysis) {
|
||||
embeddingText += `\n\nAI Analysis: ${analysis}`;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Generate embedding
|
||||
const embedding = await this.generateEmbedding(embeddingText);
|
||||
const embeddingBlob = serializeFloat32Vector(embedding);
|
||||
|
||||
// Update database
|
||||
const updateStmt = this.db.prepare(`
|
||||
UPDATE nodes
|
||||
SET embedding = ?,
|
||||
embedding_updated_at = ?,
|
||||
embedding_text = ?
|
||||
WHERE id = ?
|
||||
`);
|
||||
|
||||
const now = new Date().toISOString();
|
||||
updateStmt.run(embeddingBlob, now, embeddingText, node.id);
|
||||
|
||||
// Update vec_nodes virtual table
|
||||
try {
|
||||
// Determine correct column name for primary key (node_id vs id)
|
||||
// Use declared PK column from your DB schema (confirmed: node_id)
|
||||
const pkCol = 'node_id';
|
||||
|
||||
// Delete existing entry if any
|
||||
const deleteStmt = this.db.prepare(`DELETE FROM vec_nodes WHERE ${pkCol} = ?`);
|
||||
deleteStmt.run(BigInt(node.id));
|
||||
|
||||
// Insert new entry (use bracketed string format compatible with sqlite-vec)
|
||||
const vectorString = `[${embedding.join(',')}]`;
|
||||
const insertStmt = this.db.prepare(`INSERT INTO vec_nodes (${pkCol}, embedding) VALUES (?, ?)`);
|
||||
insertStmt.run(BigInt(node.id), vectorString);
|
||||
} catch (vecError) {
|
||||
console.warn(`Could not update vec_nodes for node ${node.id}:`, vecError);
|
||||
// Continue - main embedding is still saved
|
||||
}
|
||||
|
||||
this.processedCount++;
|
||||
console.log(`✓ Embedded node ${node.id}: "${node.title}"`);
|
||||
|
||||
} catch (error) {
|
||||
this.failedCount++;
|
||||
console.error(`✗ Failed to embed node ${node.id}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed nodes based on options
|
||||
*/
|
||||
async embedNodes(options: EmbedNodeOptions = {}): Promise<{ processed: number; failed: number }> {
|
||||
const { nodeId, forceReEmbed = false, verbose = false } = options;
|
||||
|
||||
let query: string;
|
||||
let params: any[] = [];
|
||||
|
||||
if (nodeId) {
|
||||
// Single node
|
||||
query = `
|
||||
SELECT n.id, n.title, n.content,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
n.embedding, n.embedding_updated_at
|
||||
FROM nodes n
|
||||
WHERE n.id = ?
|
||||
`;
|
||||
params = [nodeId];
|
||||
} else if (forceReEmbed) {
|
||||
// All nodes
|
||||
query = `
|
||||
SELECT n.id, n.title, n.content,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
n.embedding, n.embedding_updated_at
|
||||
FROM nodes n
|
||||
ORDER BY n.id
|
||||
`;
|
||||
} else {
|
||||
// Only nodes without embeddings
|
||||
query = `
|
||||
SELECT n.id, n.title, n.content,
|
||||
COALESCE((SELECT JSON_GROUP_ARRAY(d.dimension)
|
||||
FROM node_dimensions d WHERE d.node_id = n.id), '[]') as dimensions_json,
|
||||
n.embedding, n.embedding_updated_at
|
||||
FROM nodes n
|
||||
WHERE n.embedding IS NULL OR n.embedding_updated_at IS NULL
|
||||
ORDER BY n.id
|
||||
`;
|
||||
}
|
||||
|
||||
const stmt = this.db.prepare(query);
|
||||
const nodes = stmt.all(...params) as NodeRecord[];
|
||||
|
||||
if (nodes.length === 0) {
|
||||
console.log('No nodes to process');
|
||||
return { processed: 0, failed: 0 };
|
||||
}
|
||||
|
||||
console.log(`Processing ${nodes.length} nodes...`);
|
||||
|
||||
// Process in batches
|
||||
await batchProcess(
|
||||
nodes,
|
||||
async (node) => {
|
||||
try {
|
||||
await this.embedNode(node, forceReEmbed);
|
||||
} catch (error) {
|
||||
// Error already logged in embedNode
|
||||
}
|
||||
},
|
||||
5, // Batch size
|
||||
verbose ? (processed, total) => {
|
||||
console.log(`Progress: ${processed}/${total} nodes`);
|
||||
} : undefined
|
||||
);
|
||||
|
||||
console.log(`\nComplete! Processed: ${this.processedCount}, Failed: ${this.failedCount}`);
|
||||
|
||||
return {
|
||||
processed: this.processedCount,
|
||||
failed: this.failedCount
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Close database connection
|
||||
*/
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI interface for direct execution
|
||||
*/
|
||||
export async function runCLI(args: string[]): Promise<void> {
|
||||
const nodeId = args.includes('--node-id')
|
||||
? parseInt(args[args.indexOf('--node-id') + 1])
|
||||
: undefined;
|
||||
|
||||
const forceReEmbed = args.includes('--force');
|
||||
const verbose = args.includes('--verbose');
|
||||
|
||||
const embedder = new NodeEmbedder();
|
||||
|
||||
try {
|
||||
await embedder.embedNodes({ nodeId, forceReEmbed, verbose });
|
||||
} finally {
|
||||
embedder.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly (for testing)
|
||||
if (require.main === module) {
|
||||
runCLI(process.argv.slice(2)).catch(error => {
|
||||
console.error('Error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
/**
|
||||
* Universal chunking and embedding service for RA-H knowledge management system
|
||||
* Takes a node_id, reads chunk content from nodes table, chunks it, and stores in chunks table
|
||||
*/
|
||||
|
||||
import OpenAI from 'openai';
|
||||
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
|
||||
import {
|
||||
createDatabaseConnection,
|
||||
serializeFloat32Vector,
|
||||
batchProcess
|
||||
} from './sqlite-vec';
|
||||
|
||||
interface Node {
|
||||
id: number;
|
||||
title: string;
|
||||
chunk: string | null;
|
||||
chunk_status?: string | null;
|
||||
}
|
||||
|
||||
interface ChunkData {
|
||||
content: string;
|
||||
metadata: {
|
||||
node_id: number;
|
||||
chunk_index: number;
|
||||
start_char: number;
|
||||
end_char: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface EmbedUniversalOptions {
|
||||
nodeId: number;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
export class UniversalEmbedder {
|
||||
private openaiClient: OpenAI;
|
||||
private db: ReturnType<typeof createDatabaseConnection>;
|
||||
private textSplitter: RecursiveCharacterTextSplitter;
|
||||
private vecChunksInsertSQL: string | null = null;
|
||||
|
||||
constructor() {
|
||||
const apiKey = process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error('OPENAI_API_KEY environment variable is not set');
|
||||
}
|
||||
|
||||
this.openaiClient = new OpenAI({ apiKey });
|
||||
this.db = createDatabaseConnection();
|
||||
|
||||
// Configure text splitter (same as old KMS system)
|
||||
this.textSplitter = new RecursiveCharacterTextSplitter({
|
||||
chunkSize: 1000,
|
||||
chunkOverlap: 200,
|
||||
separators: ["\n\n", "\n", ". ", " ", ""],
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine correct insert SQL for vec_chunks based on actual schema
|
||||
*/
|
||||
private resolveVecChunksInsertSQL(): string {
|
||||
// Use declared PK column from your DB schema (confirmed: chunk_id)
|
||||
if (!this.vecChunksInsertSQL) {
|
||||
this.vecChunksInsertSQL = 'INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)';
|
||||
}
|
||||
return this.vecChunksInsertSQL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding for text using OpenAI
|
||||
*/
|
||||
private async generateEmbedding(text: string): Promise<number[]> {
|
||||
const response = await this.openaiClient.embeddings.create({
|
||||
model: 'text-embedding-3-small',
|
||||
input: text,
|
||||
});
|
||||
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete existing chunks for a node
|
||||
*/
|
||||
private deleteExistingChunks(nodeId: number): void {
|
||||
// First, get all chunk IDs for this node
|
||||
const chunkIds = this.db.prepare('SELECT id FROM chunks WHERE node_id = ?').all(nodeId) as Array<{ id: number }>;
|
||||
|
||||
// Delete from vec_chunks first, one by one to ensure they're removed
|
||||
for (const chunk of chunkIds) {
|
||||
try {
|
||||
const deleteVecStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
|
||||
deleteVecStmt.run(BigInt(chunk.id));
|
||||
} catch (error) {
|
||||
console.warn(`Could not delete vec_chunk ${chunk.id}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// Then delete from chunks table
|
||||
const deleteChunksStmt = this.db.prepare('DELETE FROM chunks WHERE node_id = ?');
|
||||
deleteChunksStmt.run(nodeId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a chunk with its embedding
|
||||
*/
|
||||
private async storeChunk(
|
||||
nodeId: number,
|
||||
chunkContent: string,
|
||||
chunkIndex: number,
|
||||
metadata: any
|
||||
): Promise<void> {
|
||||
// Generate embedding
|
||||
const embedding = await this.generateEmbedding(chunkContent);
|
||||
|
||||
// Insert into chunks table (align with existing schema)
|
||||
const insertStmt = this.db.prepare(`
|
||||
INSERT INTO chunks (node_id, chunk_idx, text, embedding_type, metadata, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const result = insertStmt.run(
|
||||
nodeId,
|
||||
chunkIndex,
|
||||
chunkContent,
|
||||
'text-embedding-3-small',
|
||||
JSON.stringify(metadata),
|
||||
now
|
||||
);
|
||||
|
||||
const chunkId = Number(result.lastInsertRowid);
|
||||
|
||||
// Insert into vec_chunks virtual table (use bracketed string format)
|
||||
try {
|
||||
const vectorString = `[${embedding.join(',')}]`;
|
||||
// First try to delete any existing vec_chunk with this ID
|
||||
try {
|
||||
const deleteStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
|
||||
deleteStmt.run(BigInt(chunkId));
|
||||
} catch {}
|
||||
|
||||
// Now insert the new embedding
|
||||
const sql = 'INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)';
|
||||
const vecInsertStmt = this.db.prepare(sql);
|
||||
vecInsertStmt.run(BigInt(chunkId), vectorString);
|
||||
} catch (error) {
|
||||
console.warn(`Could not insert into vec_chunks for chunk ${chunkId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single node for chunking and embedding
|
||||
*/
|
||||
async processNode(options: EmbedUniversalOptions): Promise<{ chunks: number }> {
|
||||
const { nodeId, verbose = false } = options;
|
||||
|
||||
// Get node data
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT id, title, chunk, chunk_status
|
||||
FROM nodes
|
||||
WHERE id = ?
|
||||
`);
|
||||
|
||||
const node = stmt.get(nodeId) as Node | undefined;
|
||||
|
||||
if (!node) {
|
||||
throw new Error(`Node ${nodeId} not found`);
|
||||
}
|
||||
|
||||
if (!node.chunk || node.chunk.trim().length === 0) {
|
||||
console.log(`Node ${nodeId} has no chunk content to process`);
|
||||
return { chunks: 0 };
|
||||
}
|
||||
|
||||
console.log(`Processing node ${nodeId}: "${node.title}"`);
|
||||
|
||||
// Delete existing chunks
|
||||
this.deleteExistingChunks(nodeId);
|
||||
|
||||
// Split text into chunks
|
||||
const chunks = await this.textSplitter.splitText(node.chunk);
|
||||
|
||||
if (verbose) {
|
||||
console.log(`Split into ${chunks.length} chunks`);
|
||||
}
|
||||
|
||||
// Process each chunk
|
||||
let startChar = 0;
|
||||
await batchProcess(
|
||||
chunks.map((chunkContent, index) => ({ chunkContent, index })),
|
||||
async ({ chunkContent, index }) => {
|
||||
const endChar = startChar + chunkContent.length;
|
||||
|
||||
const metadata = {
|
||||
node_id: nodeId,
|
||||
chunk_index: index,
|
||||
start_char: startChar,
|
||||
end_char: endChar,
|
||||
title: node.title,
|
||||
};
|
||||
|
||||
await this.storeChunk(nodeId, chunkContent, index, metadata);
|
||||
|
||||
if (verbose) {
|
||||
console.log(` Chunk ${index + 1}/${chunks.length}: ${chunkContent.substring(0, 50)}...`);
|
||||
}
|
||||
|
||||
startChar = endChar;
|
||||
},
|
||||
5, // Batch size
|
||||
verbose ? (processed, total) => {
|
||||
console.log(`Embedding progress: ${processed}/${total} chunks`);
|
||||
} : undefined
|
||||
);
|
||||
|
||||
// Update node chunk_status
|
||||
const updateStmt = this.db.prepare(`
|
||||
UPDATE nodes
|
||||
SET chunk_status = 'chunked'
|
||||
WHERE id = ?
|
||||
`);
|
||||
updateStmt.run(nodeId);
|
||||
|
||||
console.log(`✓ Created ${chunks.length} chunks for node ${nodeId}`);
|
||||
|
||||
return { chunks: chunks.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics about chunks in the database
|
||||
*/
|
||||
getStats(): { totalChunks: number; totalNodes: number; avgChunksPerNode: number } {
|
||||
const statsStmt = this.db.prepare(`
|
||||
SELECT
|
||||
COUNT(DISTINCT node_id) as total_nodes,
|
||||
COUNT(*) as total_chunks
|
||||
FROM chunks
|
||||
`);
|
||||
|
||||
const stats = statsStmt.get() as any;
|
||||
|
||||
return {
|
||||
totalChunks: stats.total_chunks || 0,
|
||||
totalNodes: stats.total_nodes || 0,
|
||||
avgChunksPerNode: stats.total_nodes > 0
|
||||
? Math.round(stats.total_chunks / stats.total_nodes * 10) / 10
|
||||
: 0
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Close database connection
|
||||
*/
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI interface for direct execution
|
||||
*/
|
||||
export async function runCLI(args: string[]): Promise<void> {
|
||||
if (!args.includes('--node-id')) {
|
||||
console.error('Error: --node-id is required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const nodeId = parseInt(args[args.indexOf('--node-id') + 1]);
|
||||
const verbose = args.includes('--verbose');
|
||||
|
||||
if (isNaN(nodeId)) {
|
||||
console.error('Error: Invalid node ID');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const embedder = new UniversalEmbedder();
|
||||
|
||||
try {
|
||||
const result = await embedder.processNode({ nodeId, verbose });
|
||||
|
||||
if (verbose) {
|
||||
const stats = embedder.getStats();
|
||||
console.log('\nDatabase statistics:');
|
||||
console.log(` Total chunks: ${stats.totalChunks}`);
|
||||
console.log(` Total nodes with chunks: ${stats.totalNodes}`);
|
||||
console.log(` Average chunks per node: ${stats.avgChunksPerNode}`);
|
||||
}
|
||||
} finally {
|
||||
embedder.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly (for testing)
|
||||
if (require.main === module) {
|
||||
runCLI(process.argv.slice(2)).catch(error => {
|
||||
console.error('Error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
/**
|
||||
* PDF/Paper content extraction for RA-H knowledge management system
|
||||
* Extracts text content from PDF files and returns formatted content
|
||||
*/
|
||||
|
||||
// Import pdf-parse directly from lib to avoid index.js debug side effects
|
||||
// (pdf-parse/index.js conditionally reads ./test/data/05-versions-space.pdf when module.parent is falsy in some bundles)
|
||||
// See: node_modules/pdf-parse/index.js
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
|
||||
const pdf = require('pdf-parse/lib/pdf-parse.js');
|
||||
import * as fs from 'fs/promises';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
interface PaperMetadata {
|
||||
title?: string;
|
||||
pages: number;
|
||||
info?: any;
|
||||
text_length: number;
|
||||
filename?: string;
|
||||
extraction_method?: string;
|
||||
}
|
||||
|
||||
interface ExtractionResult {
|
||||
content: string;
|
||||
chunk: string;
|
||||
metadata: PaperMetadata;
|
||||
url: string;
|
||||
}
|
||||
|
||||
type PdfJsTextItem = {
|
||||
str?: string;
|
||||
};
|
||||
|
||||
type PdfMetadataInfo = Record<string, unknown>;
|
||||
|
||||
export class PaperExtractor {
|
||||
private headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
};
|
||||
|
||||
private isLikelyPdf(buffer: Buffer): boolean {
|
||||
const header = buffer.slice(0, 8).toString('utf8');
|
||||
return header.includes('%PDF');
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean academic PDF content to reduce citation fragments and corrupted text
|
||||
*/
|
||||
private cleanAcademicContent(content: string): string {
|
||||
const lines = content.split('\n');
|
||||
const cleanedLines: string[] = [];
|
||||
|
||||
for (let line of lines) {
|
||||
line = line.trim();
|
||||
|
||||
// Skip empty lines for now
|
||||
if (line.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip lines that look like headers/footers (page numbers, dates)
|
||||
if (/^[\d\s\-\/]+$/.test(line)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip lines that are just citations like "[1]" or "(Smith, 2020)"
|
||||
if (/^\[\d+\]$/.test(line) || /^\([^)]+,\s*\d{4}\)$/.test(line)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip very short lines that are likely artifacts
|
||||
if (line.length < 3) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip lines that are mostly special characters (likely corrupted)
|
||||
const specialCharCount = (line.match(/[^\w\s]/g) || []).length;
|
||||
if (specialCharCount > line.length * 0.5) {
|
||||
continue;
|
||||
}
|
||||
|
||||
cleanedLines.push(line);
|
||||
}
|
||||
|
||||
// Combine lines into paragraphs
|
||||
const paragraphs: string[] = [];
|
||||
let currentParagraph = '';
|
||||
|
||||
for (const line of cleanedLines) {
|
||||
// Check if line looks like it ends a sentence
|
||||
const endsWithPunctuation = /[.!?]$/.test(line);
|
||||
|
||||
// Check if next line starts with capital or is a heading
|
||||
const looksLikeNewParagraph = /^[A-Z0-9]/.test(line) && currentParagraph.endsWith('.');
|
||||
|
||||
if (currentParagraph.length === 0) {
|
||||
currentParagraph = line;
|
||||
} else if (looksLikeNewParagraph) {
|
||||
paragraphs.push(currentParagraph);
|
||||
currentParagraph = line;
|
||||
} else if (endsWithPunctuation) {
|
||||
currentParagraph += ' ' + line;
|
||||
} else {
|
||||
currentParagraph += ' ' + line;
|
||||
}
|
||||
}
|
||||
|
||||
if (currentParagraph.length > 0) {
|
||||
paragraphs.push(currentParagraph);
|
||||
}
|
||||
|
||||
// Filter out very short paragraphs (likely noise)
|
||||
return paragraphs.filter(p => p.length > 30).join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Download PDF from URL to temporary file
|
||||
*/
|
||||
private async downloadPDF(url: string): Promise<string> {
|
||||
const response = await fetch(url, {
|
||||
headers: this.headers,
|
||||
signal: AbortSignal.timeout(60000), // 60 second timeout for PDFs
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
// Create temp file
|
||||
const tempDir = os.tmpdir();
|
||||
const tempFile = path.join(tempDir, `pdf_${Date.now()}.pdf`);
|
||||
|
||||
// Get PDF data as buffer
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
|
||||
// Write to temp file
|
||||
await fs.writeFile(tempFile, buffer);
|
||||
|
||||
return tempFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract text from PDF buffer
|
||||
*/
|
||||
private async extractFromPDF(buffer: Buffer): Promise<{ text: string; metadata: PaperMetadata }> {
|
||||
try {
|
||||
const data = await pdf(buffer);
|
||||
|
||||
const metadata: PaperMetadata = {
|
||||
pages: data.numpages,
|
||||
info: data.info,
|
||||
text_length: data.text.length,
|
||||
};
|
||||
|
||||
// Try to extract title from metadata or first lines
|
||||
if (data.info && data.info.Title) {
|
||||
metadata.title = data.info.Title;
|
||||
} else {
|
||||
// Try to get title from first few lines
|
||||
const firstLines = data.text.split('\n').slice(0, 5);
|
||||
const possibleTitle = firstLines.find((line: string) =>
|
||||
line.length > 10 && line.length < 200 && /[A-Z]/.test(line)
|
||||
);
|
||||
if (possibleTitle) {
|
||||
metadata.title = possibleTitle.trim();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
text: data.text,
|
||||
metadata,
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
const primaryMessage = this.formatErrorMessage(error);
|
||||
console.warn('Primary pdf-parse extraction failed, attempting pdfjs-dist fallback:', primaryMessage);
|
||||
try {
|
||||
return await this.extractWithPdfJs(buffer);
|
||||
} catch (fallbackError: unknown) {
|
||||
const fallbackMessage = this.formatErrorMessage(fallbackError);
|
||||
throw new Error(`PDF parsing failed: ${primaryMessage}; fallback error: ${fallbackMessage}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async extractWithPdfJs(buffer: Buffer): Promise<{ text: string; metadata: PaperMetadata }> {
|
||||
const pdfjsLib = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
||||
const loadingTask = pdfjsLib.getDocument({ data: buffer, useSystemFonts: true });
|
||||
const doc = await loadingTask.promise;
|
||||
|
||||
let aggregatedText = '';
|
||||
for (let pageIndex = 1; pageIndex <= doc.numPages; pageIndex++) {
|
||||
const page = await doc.getPage(pageIndex);
|
||||
const textContent = await page.getTextContent();
|
||||
const pageText = (textContent.items as PdfJsTextItem[])
|
||||
.map((item) => (typeof item?.str === 'string' ? item.str : ''))
|
||||
.join(' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
if (pageText) {
|
||||
aggregatedText += pageText + '\n\n';
|
||||
}
|
||||
}
|
||||
|
||||
let info: PdfMetadataInfo = {};
|
||||
try {
|
||||
const metadata = await doc.getMetadata();
|
||||
info = (metadata?.info as PdfMetadataInfo) || {};
|
||||
} catch (metadataError: unknown) {
|
||||
console.warn('pdfjs-dist metadata extraction failed:', metadataError);
|
||||
}
|
||||
|
||||
const metadata: PaperMetadata = {
|
||||
pages: doc.numPages,
|
||||
info,
|
||||
text_length: aggregatedText.length,
|
||||
extraction_method: 'typescript_pdfjs_dist'
|
||||
};
|
||||
|
||||
const title = typeof info['Title'] === 'string' ? (info['Title'] as string) : undefined;
|
||||
if (!metadata.title && title) {
|
||||
metadata.title = title;
|
||||
}
|
||||
|
||||
return {
|
||||
text: aggregatedText,
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format content for node creation
|
||||
*/
|
||||
private formatContent(metadata: PaperMetadata, text: string): string {
|
||||
const sections: string[] = [];
|
||||
|
||||
// Add metadata section
|
||||
sections.push('## Document Information');
|
||||
|
||||
if (metadata.title) {
|
||||
sections.push(`**Title:** ${metadata.title}`);
|
||||
}
|
||||
|
||||
sections.push(`**Pages:** ${metadata.pages}`);
|
||||
sections.push(`**Text Length:** ${metadata.text_length} characters`);
|
||||
|
||||
if (metadata.info) {
|
||||
if (metadata.info.Author) {
|
||||
sections.push(`**Author:** ${metadata.info.Author}`);
|
||||
}
|
||||
if (metadata.info.Creator) {
|
||||
sections.push(`**Creator:** ${metadata.info.Creator}`);
|
||||
}
|
||||
if (metadata.info.Producer) {
|
||||
sections.push(`**Producer:** ${metadata.info.Producer}`);
|
||||
}
|
||||
if (metadata.info.CreationDate) {
|
||||
sections.push(`**Creation Date:** ${metadata.info.CreationDate}`);
|
||||
}
|
||||
}
|
||||
|
||||
sections.push('');
|
||||
|
||||
// Add content
|
||||
sections.push('## Content');
|
||||
sections.push(text);
|
||||
|
||||
return sections.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Main extraction method
|
||||
*/
|
||||
async extract(url: string): Promise<ExtractionResult> {
|
||||
let tempFile: string | null = null;
|
||||
|
||||
try {
|
||||
// Validate URL
|
||||
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
||||
throw new Error('Invalid URL format - must start with http:// or https://');
|
||||
}
|
||||
|
||||
// Check if URL looks like it points to a PDF
|
||||
const urlLower = url.toLowerCase();
|
||||
if (!urlLower.includes('.pdf') && !urlLower.includes('arxiv.org')) {
|
||||
console.warn('Warning: URL does not appear to point to a PDF file');
|
||||
}
|
||||
|
||||
// Download PDF
|
||||
tempFile = await this.downloadPDF(url);
|
||||
|
||||
// Read PDF buffer
|
||||
const buffer = await fs.readFile(tempFile);
|
||||
|
||||
if (!this.isLikelyPdf(buffer)) {
|
||||
const preview = buffer.slice(0, 64).toString('utf8');
|
||||
throw new Error(`Downloaded file does not appear to be a PDF (header: ${preview})`);
|
||||
}
|
||||
|
||||
// Extract text and metadata
|
||||
const { text, metadata } = await this.extractFromPDF(buffer);
|
||||
|
||||
// Clean the text
|
||||
const cleanedText = this.cleanAcademicContent(text);
|
||||
|
||||
// Add filename to metadata
|
||||
metadata.filename = path.basename(url);
|
||||
// Mark extraction method for downstream metadata
|
||||
if (!metadata.extraction_method) {
|
||||
metadata.extraction_method = 'typescript_pdf-parse';
|
||||
}
|
||||
|
||||
// Format content for display
|
||||
const content = this.formatContent(metadata, cleanedText);
|
||||
|
||||
// Chunk is the cleaned text
|
||||
const chunk = cleanedText;
|
||||
|
||||
return {
|
||||
content,
|
||||
chunk,
|
||||
metadata,
|
||||
url,
|
||||
};
|
||||
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
throw new Error('Request timeout - PDF download took too long');
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
// Clean up temp file
|
||||
if (tempFile) {
|
||||
try {
|
||||
await fs.unlink(tempFile);
|
||||
} catch (cleanupError: unknown) {
|
||||
console.warn('Could not delete temp file:', tempFile, cleanupError);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private formatErrorMessage(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
if (typeof error === 'string') {
|
||||
return error;
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(error);
|
||||
} catch {
|
||||
return 'Unknown error';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone extraction function for direct use
|
||||
*/
|
||||
export async function extractPaper(url: string): Promise<ExtractionResult> {
|
||||
const extractor = new PaperExtractor();
|
||||
return extractor.extract(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI interface for direct execution
|
||||
*/
|
||||
export async function runCLI(args: string[]): Promise<void> {
|
||||
if (args.length === 0) {
|
||||
console.error('Usage: paper-extract <url>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const url = args[0];
|
||||
|
||||
try {
|
||||
const result = await extractPaper(url);
|
||||
// Output as JSON for compatibility with existing tools
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error: any) {
|
||||
console.error('Error:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly (for testing)
|
||||
if (require.main === module) {
|
||||
runCLI(process.argv.slice(2)).catch(error => {
|
||||
console.error('Error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* Website content extraction for RA-H knowledge management system
|
||||
* Extracts text content from web pages and returns formatted content
|
||||
*/
|
||||
|
||||
import * as cheerio from 'cheerio';
|
||||
|
||||
interface WebsiteMetadata {
|
||||
title: string;
|
||||
author?: string;
|
||||
date?: string;
|
||||
description?: string;
|
||||
og_image?: string;
|
||||
site_name?: string;
|
||||
extraction_method?: string;
|
||||
}
|
||||
|
||||
interface ExtractionResult {
|
||||
content: string;
|
||||
chunk: string;
|
||||
metadata: WebsiteMetadata;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export class WebsiteExtractor {
|
||||
private headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
};
|
||||
|
||||
/**
|
||||
* Clean extracted content for better readability
|
||||
*/
|
||||
private cleanContent(content: string): string {
|
||||
// Remove excessive whitespace
|
||||
content = content.replace(/\s+/g, ' ').trim();
|
||||
|
||||
// Remove cookie/privacy policy mentions
|
||||
content = content.replace(/cookie\s+policy|privacy\s+policy|terms\s+of\s+service/gi, '');
|
||||
|
||||
// Split into paragraphs and clean
|
||||
const paragraphs = content
|
||||
.split('\n')
|
||||
.map(p => p.trim())
|
||||
.filter(p => p.length > 20); // Remove very short paragraphs (likely nav/UI)
|
||||
|
||||
return paragraphs.join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract metadata from HTML
|
||||
*/
|
||||
private extractMetadata($: cheerio.CheerioAPI): WebsiteMetadata {
|
||||
const metadata: WebsiteMetadata = {
|
||||
title: '',
|
||||
};
|
||||
|
||||
// Title extraction (priority order)
|
||||
metadata.title =
|
||||
$('meta[property="og:title"]').attr('content') ||
|
||||
$('meta[name="twitter:title"]').attr('content') ||
|
||||
$('title').text() ||
|
||||
$('h1').first().text() ||
|
||||
'Untitled';
|
||||
|
||||
// Author extraction
|
||||
metadata.author =
|
||||
$('meta[name="author"]').attr('content') ||
|
||||
$('meta[property="article:author"]').attr('content') ||
|
||||
$('.author').first().text() ||
|
||||
$('[rel="author"]').first().text() ||
|
||||
undefined;
|
||||
|
||||
// Date extraction
|
||||
metadata.date =
|
||||
$('meta[property="article:published_time"]').attr('content') ||
|
||||
$('meta[name="publish_date"]').attr('content') ||
|
||||
$('time').first().attr('datetime') ||
|
||||
$('.date').first().text() ||
|
||||
undefined;
|
||||
|
||||
// Description
|
||||
metadata.description =
|
||||
$('meta[property="og:description"]').attr('content') ||
|
||||
$('meta[name="description"]').attr('content') ||
|
||||
undefined;
|
||||
|
||||
// Image
|
||||
metadata.og_image =
|
||||
$('meta[property="og:image"]').attr('content') ||
|
||||
undefined;
|
||||
|
||||
// Site name
|
||||
metadata.site_name =
|
||||
$('meta[property="og:site_name"]').attr('content') ||
|
||||
undefined;
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract main content from HTML
|
||||
*/
|
||||
private extractMainContent($: cheerio.CheerioAPI): string {
|
||||
// Remove script and style elements
|
||||
$('script, style, noscript').remove();
|
||||
|
||||
// Remove common navigation and footer elements
|
||||
$('nav, header, footer, aside, .nav, .header, .footer, .sidebar, .menu, .advertisement').remove();
|
||||
|
||||
// Try to find main content areas (in priority order)
|
||||
const contentSelectors = [
|
||||
'main',
|
||||
'article',
|
||||
'[role="main"]',
|
||||
'.content',
|
||||
'.post',
|
||||
'.article-body',
|
||||
'.entry-content',
|
||||
'#content',
|
||||
'.container',
|
||||
'body',
|
||||
];
|
||||
|
||||
let mainContent = '';
|
||||
|
||||
for (const selector of contentSelectors) {
|
||||
const element = $(selector).first();
|
||||
if (element.length > 0) {
|
||||
// Extract text from paragraphs, headings, lists
|
||||
const textElements = element.find('p, h1, h2, h3, h4, h5, h6, li, blockquote, td, th');
|
||||
|
||||
if (textElements.length > 0) {
|
||||
const texts: string[] = [];
|
||||
textElements.each((_, el) => {
|
||||
const text = $(el).text().trim();
|
||||
if (text.length > 0) {
|
||||
texts.push(text);
|
||||
}
|
||||
});
|
||||
|
||||
if (texts.length > 0) {
|
||||
mainContent = texts.join('\n\n');
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to all text if no main content found
|
||||
if (!mainContent) {
|
||||
mainContent = $('body').text();
|
||||
}
|
||||
|
||||
return this.cleanContent(mainContent);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format content for node creation
|
||||
*/
|
||||
private formatContent(metadata: WebsiteMetadata, mainContent: string): string {
|
||||
const sections: string[] = [];
|
||||
|
||||
// Add metadata section
|
||||
sections.push('## Article Information');
|
||||
sections.push(`**Title:** ${metadata.title}`);
|
||||
|
||||
if (metadata.author) {
|
||||
sections.push(`**Author:** ${metadata.author}`);
|
||||
}
|
||||
|
||||
if (metadata.date) {
|
||||
sections.push(`**Date:** ${metadata.date}`);
|
||||
}
|
||||
|
||||
if (metadata.site_name) {
|
||||
sections.push(`**Source:** ${metadata.site_name}`);
|
||||
}
|
||||
|
||||
if (metadata.description) {
|
||||
sections.push(`**Description:** ${metadata.description}`);
|
||||
}
|
||||
|
||||
sections.push('');
|
||||
|
||||
// Add main content
|
||||
sections.push('## Content');
|
||||
sections.push(mainContent);
|
||||
|
||||
return sections.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Main extraction method
|
||||
*/
|
||||
async extract(url: string): Promise<ExtractionResult> {
|
||||
// Validate URL
|
||||
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
||||
throw new Error('Invalid URL format - must start with http:// or https://');
|
||||
}
|
||||
|
||||
try {
|
||||
// Fetch the webpage
|
||||
const response = await fetch(url, {
|
||||
headers: this.headers,
|
||||
signal: AbortSignal.timeout(30000), // 30 second timeout
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
|
||||
// Parse HTML with cheerio
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
// Extract metadata and content
|
||||
const metadata = this.extractMetadata($);
|
||||
// Mark extraction method for downstream metadata
|
||||
metadata.extraction_method = 'typescript_cheerio';
|
||||
const mainContent = this.extractMainContent($);
|
||||
|
||||
// Format content for display
|
||||
const content = this.formatContent(metadata, mainContent);
|
||||
|
||||
// Chunk is the main content text
|
||||
const chunk = mainContent;
|
||||
|
||||
return {
|
||||
content,
|
||||
chunk,
|
||||
metadata,
|
||||
url,
|
||||
};
|
||||
|
||||
} catch (error: any) {
|
||||
if (error.name === 'AbortError') {
|
||||
throw new Error('Request timeout - website took too long to respond');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone extraction function for direct use
|
||||
*/
|
||||
export async function extractWebsite(url: string): Promise<ExtractionResult> {
|
||||
const extractor = new WebsiteExtractor();
|
||||
return extractor.extract(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI interface for direct execution
|
||||
*/
|
||||
export async function runCLI(args: string[]): Promise<void> {
|
||||
if (args.length === 0) {
|
||||
console.error('Usage: website-extract <url>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const url = args[0];
|
||||
|
||||
try {
|
||||
const result = await extractWebsite(url);
|
||||
// Output as JSON for compatibility with existing tools
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error: any) {
|
||||
console.error('Error:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly (for testing)
|
||||
if (require.main === module) {
|
||||
runCLI(process.argv.slice(2)).catch(error => {
|
||||
console.error('Error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* YouTube content extraction for RA-H knowledge management system
|
||||
* Uses youtube-transcript npm package - more similar to Python youtube-transcript-api
|
||||
*/
|
||||
|
||||
import { YoutubeTranscript } from 'youtube-transcript';
|
||||
|
||||
interface TranscriptSegment {
|
||||
text: string;
|
||||
start: number;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
interface YouTubeMetadata {
|
||||
video_id: string;
|
||||
video_url: string;
|
||||
video_title: string;
|
||||
channel_name: string;
|
||||
channel_url: string;
|
||||
thumbnail_url: string;
|
||||
source_type: string;
|
||||
transcript_length: number;
|
||||
total_segments: number;
|
||||
content_format: string;
|
||||
language?: string;
|
||||
provider: string;
|
||||
extraction_method: string;
|
||||
}
|
||||
|
||||
interface ExtractionResult {
|
||||
success: boolean;
|
||||
content: string;
|
||||
chunk: string; // Same as content, but tool expects this field name
|
||||
metadata: YouTubeMetadata;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class YouTubeExtractor {
|
||||
/**
|
||||
* Extract video ID from YouTube URL
|
||||
*/
|
||||
private extractVideoId(url: string): string | null {
|
||||
if (!url) return null;
|
||||
|
||||
if (url.includes('youtu.be')) {
|
||||
return url.split('/').pop()?.split('?')[0] || null;
|
||||
} else if (url.includes('youtube.com/watch')) {
|
||||
const urlParams = new URLSearchParams(url.split('?')[1]);
|
||||
return urlParams.get('v');
|
||||
} else if (url.includes('youtube.com/live')) {
|
||||
return url.split('/live/')[1]?.split('?')[0] || null;
|
||||
} else if (url.includes('youtube.com/embed')) {
|
||||
return url.split('/embed/')[1]?.split('?')[0] || null;
|
||||
} else if (url.includes('youtube.com/v')) {
|
||||
return url.split('/v/')[1]?.split('?')[0] || null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get video metadata from YouTube oEmbed API
|
||||
*/
|
||||
private async getVideoMetadata(url: string): Promise<{ title: string; author_name: string; author_url: string; thumbnail_url: string }> {
|
||||
try {
|
||||
const oembedUrl = `https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`;
|
||||
const response = await fetch(oembedUrl, {
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
return {
|
||||
title: data.title || 'YouTube Video',
|
||||
author_name: data.author_name || 'Unknown Channel',
|
||||
author_url: data.author_url || '',
|
||||
thumbnail_url: data.thumbnail_url || ''
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('oEmbed extraction failed:', error);
|
||||
}
|
||||
|
||||
// Fallback metadata
|
||||
const videoId = this.extractVideoId(url);
|
||||
return {
|
||||
title: `YouTube Video ${videoId || 'Unknown'}`,
|
||||
author_name: 'Unknown Channel',
|
||||
author_url: '',
|
||||
thumbnail_url: ''
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get transcript using youtube-transcript npm package (like Python approach)
|
||||
*/
|
||||
private async getTranscript(url: string): Promise<{ transcript: string; segments: TranscriptSegment[]; language?: string }> {
|
||||
try {
|
||||
// Get transcript using npm package (more similar to Python approach)
|
||||
const transcriptData = await YoutubeTranscript.fetchTranscript(url);
|
||||
|
||||
if (!transcriptData || transcriptData.length === 0) {
|
||||
throw new Error('No transcript segments found');
|
||||
}
|
||||
|
||||
// Convert to our format
|
||||
const segments: TranscriptSegment[] = transcriptData.map(item => ({
|
||||
text: item.text,
|
||||
start: item.offset / 1000, // Convert ms to seconds
|
||||
duration: item.duration ? item.duration / 1000 : 0 // Convert ms to seconds
|
||||
}));
|
||||
|
||||
// Format with timestamps like Python version
|
||||
const formattedSegments: string[] = [];
|
||||
for (const segment of segments) {
|
||||
formattedSegments.push(`[${segment.start.toFixed(1)}s] ${segment.text}`);
|
||||
}
|
||||
|
||||
const fullTranscript = formattedSegments.join('\n');
|
||||
|
||||
return {
|
||||
transcript: fullTranscript,
|
||||
segments,
|
||||
language: 'en'
|
||||
};
|
||||
|
||||
} catch (error: any) {
|
||||
throw new Error(`Failed to extract YouTube transcript: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main extraction method - uses youtube-transcript npm package like Python script uses youtube-transcript-api
|
||||
*/
|
||||
async extract(url: string): Promise<ExtractionResult> {
|
||||
try {
|
||||
// Validate URL
|
||||
if (!url.includes('youtube.com') && !url.includes('youtu.be')) {
|
||||
throw new Error('Invalid YouTube URL');
|
||||
}
|
||||
|
||||
const videoId = this.extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Could not extract video ID from URL');
|
||||
}
|
||||
|
||||
// Get video metadata
|
||||
const videoMetadata = await this.getVideoMetadata(url);
|
||||
|
||||
// Get transcript using npm package (similar to Python approach)
|
||||
const { transcript, segments, language } = await this.getTranscript(url);
|
||||
|
||||
// Create metadata matching Python version exactly
|
||||
const metadata: YouTubeMetadata = {
|
||||
video_id: videoId,
|
||||
video_url: url,
|
||||
video_title: videoMetadata.title,
|
||||
channel_name: videoMetadata.author_name,
|
||||
channel_url: videoMetadata.author_url,
|
||||
thumbnail_url: videoMetadata.thumbnail_url,
|
||||
source_type: 'youtube_transcript',
|
||||
transcript_length: transcript.length,
|
||||
total_segments: segments.length,
|
||||
content_format: 'timestamped_transcript',
|
||||
language: language || 'unknown',
|
||||
provider: 'YouTube',
|
||||
extraction_method: 'typescript_npm_youtube_transcript'
|
||||
};
|
||||
|
||||
return {
|
||||
success: true,
|
||||
content: transcript,
|
||||
chunk: transcript, // Tool expects this field
|
||||
metadata
|
||||
};
|
||||
|
||||
} catch (error: any) {
|
||||
return {
|
||||
success: false,
|
||||
content: '',
|
||||
chunk: '', // Tool expects this field
|
||||
metadata: {} as YouTubeMetadata,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main function for command line usage (matching Python interface)
|
||||
*/
|
||||
export async function main(url: string): Promise<ExtractionResult> {
|
||||
const extractor = new YouTubeExtractor();
|
||||
return extractor.extract(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone extraction function for direct use
|
||||
*/
|
||||
export async function extractYouTube(url: string): Promise<ExtractionResult> {
|
||||
const extractor = new YouTubeExtractor();
|
||||
return extractor.extract(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI interface for direct execution (matching Python interface)
|
||||
*/
|
||||
export async function runCLI(): Promise<void> {
|
||||
if (process.argv.length !== 3) {
|
||||
console.log(JSON.stringify({
|
||||
success: false,
|
||||
error: "Usage: node youtube.js <youtube_url>"
|
||||
}));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const url = process.argv[2];
|
||||
const result = await main(url);
|
||||
console.log(JSON.stringify(result));
|
||||
|
||||
if (!result.success) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly (for testing)
|
||||
if (require.main === module) {
|
||||
runCLI().catch(error => {
|
||||
console.log(JSON.stringify({
|
||||
success: false,
|
||||
error: error.message
|
||||
}));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* YouTube content extraction for RA-H knowledge management system
|
||||
* Uses innertube-based transcript extraction via youtube-transcript-plus
|
||||
* Falls back to the legacy npm extractor if captions cannot be retrieved
|
||||
*/
|
||||
import {
|
||||
fetchTranscript as fetchTranscriptPlus,
|
||||
YoutubeTranscriptNotAvailableLanguageError,
|
||||
} from 'youtube-transcript-plus';
|
||||
import { extractYouTube as extractYouTubeNpm } from './youtube-npm';
|
||||
|
||||
interface TranscriptSegment {
|
||||
text: string;
|
||||
start: number;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
interface YouTubeMetadata {
|
||||
video_id: string;
|
||||
video_url: string;
|
||||
video_title: string;
|
||||
channel_name: string;
|
||||
channel_url: string;
|
||||
thumbnail_url: string;
|
||||
source_type: string;
|
||||
transcript_length: number;
|
||||
total_segments: number;
|
||||
content_format: string;
|
||||
language?: string;
|
||||
provider: string;
|
||||
extraction_method: string;
|
||||
}
|
||||
|
||||
interface ExtractionResult {
|
||||
success: boolean;
|
||||
content: string;
|
||||
chunk: string;
|
||||
metadata: YouTubeMetadata;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class YouTubeExtractor {
|
||||
private decodeHtmlEntities(input: string): string {
|
||||
if (!input) {
|
||||
return '';
|
||||
}
|
||||
|
||||
let result = input.replace(/&/g, '&');
|
||||
result = result.replace(/&#(\d+);/g, (_match, dec) => {
|
||||
const code = Number.parseInt(dec, 10);
|
||||
return Number.isNaN(code) ? _match : String.fromCharCode(code);
|
||||
});
|
||||
result = result
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/‘/g, "'")
|
||||
.replace(/’/g, "'")
|
||||
.replace(/“/g, '"')
|
||||
.replace(/”/g, '"')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private formatSegments(segments: TranscriptSegment[]): string {
|
||||
return segments
|
||||
.map((segment) => {
|
||||
const startTime = Number.isFinite(segment.start) ? segment.start : 0;
|
||||
return `[${startTime.toFixed(1)}s] ${segment.text}`;
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
private extractVideoId(url: string): string | null {
|
||||
if (!url) return null;
|
||||
|
||||
if (url.includes('youtu.be')) {
|
||||
return url.split('/').pop()?.split('?')[0] || null;
|
||||
} else if (url.includes('youtube.com/watch')) {
|
||||
const urlParams = new URLSearchParams(url.split('?')[1]);
|
||||
return urlParams.get('v');
|
||||
} else if (url.includes('youtube.com/live')) {
|
||||
return url.split('/live/')[1]?.split('?')[0] || null;
|
||||
} else if (url.includes('youtube.com/embed')) {
|
||||
return url.split('/embed/')[1]?.split('?')[0] || null;
|
||||
} else if (url.includes('youtube.com/v')) {
|
||||
return url.split('/v/')[1]?.split('?')[0] || null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async getVideoMetadata(url: string): Promise<{ title: string; author_name: string; author_url: string; thumbnail_url: string }> {
|
||||
try {
|
||||
const oembedUrl = `https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`;
|
||||
const response = await fetch(oembedUrl, {
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
return {
|
||||
title: data.title || 'YouTube Video',
|
||||
author_name: data.author_name || 'Unknown Channel',
|
||||
author_url: data.author_url || '',
|
||||
thumbnail_url: data.thumbnail_url || ''
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('oEmbed extraction failed:', error);
|
||||
}
|
||||
|
||||
const videoId = this.extractVideoId(url);
|
||||
return {
|
||||
title: `YouTube Video ${videoId || 'Unknown'}`,
|
||||
author_name: 'Unknown Channel',
|
||||
author_url: '',
|
||||
thumbnail_url: ''
|
||||
};
|
||||
}
|
||||
|
||||
private async fetchPrimaryTranscript(url: string): Promise<{
|
||||
transcript: string;
|
||||
segments: TranscriptSegment[];
|
||||
language?: string;
|
||||
extractionMethod: string;
|
||||
}> {
|
||||
const attempts: Array<{ lang?: string; label: string }> = [
|
||||
{ lang: 'en', label: 'typescript_youtube_transcript_plus_en' },
|
||||
{ label: 'typescript_youtube_transcript_plus' },
|
||||
];
|
||||
|
||||
let lastError: unknown = null;
|
||||
|
||||
for (const attempt of attempts) {
|
||||
try {
|
||||
const entries = await fetchTranscriptPlus(url, attempt.lang ? { lang: attempt.lang } : undefined);
|
||||
const language = entries.find((entry) => entry.lang)?.lang;
|
||||
|
||||
const segments = entries
|
||||
.map((entry) => ({
|
||||
text: this.decodeHtmlEntities(entry.text ?? '').trim(),
|
||||
start: Number(entry.offset ?? 0),
|
||||
duration: Number(entry.duration ?? 0),
|
||||
}))
|
||||
.filter((segment) => segment.text.length > 0);
|
||||
|
||||
if (segments.length === 0) {
|
||||
throw new Error('Transcript returned no segments');
|
||||
}
|
||||
|
||||
const transcript = this.formatSegments(segments);
|
||||
|
||||
return {
|
||||
transcript,
|
||||
segments,
|
||||
language,
|
||||
extractionMethod: attempt.label,
|
||||
};
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt.lang && error instanceof YoutubeTranscriptNotAvailableLanguageError) {
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error('Unable to fetch transcript');
|
||||
}
|
||||
|
||||
private formatErrorMessage(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
if (typeof error === 'string') {
|
||||
return error;
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(error);
|
||||
} catch {
|
||||
return 'Unknown error';
|
||||
}
|
||||
}
|
||||
|
||||
async extract(url: string): Promise<ExtractionResult> {
|
||||
try {
|
||||
if (!url.includes('youtube.com') && !url.includes('youtu.be')) {
|
||||
throw new Error('Invalid YouTube URL');
|
||||
}
|
||||
|
||||
const videoId = this.extractVideoId(url);
|
||||
if (!videoId) {
|
||||
throw new Error('Could not extract video ID from URL');
|
||||
}
|
||||
|
||||
const { transcript, segments, language, extractionMethod } = await this.fetchPrimaryTranscript(url);
|
||||
const videoMetadata = await this.getVideoMetadata(url);
|
||||
|
||||
const metadata: YouTubeMetadata = {
|
||||
video_id: videoId,
|
||||
video_url: url,
|
||||
video_title: videoMetadata.title,
|
||||
channel_name: videoMetadata.author_name,
|
||||
channel_url: videoMetadata.author_url,
|
||||
thumbnail_url: videoMetadata.thumbnail_url,
|
||||
source_type: 'youtube_transcript',
|
||||
transcript_length: transcript.length,
|
||||
total_segments: segments.length,
|
||||
content_format: 'timestamped_transcript',
|
||||
language: language || 'unknown',
|
||||
provider: 'YouTube',
|
||||
extraction_method: extractionMethod,
|
||||
};
|
||||
|
||||
return {
|
||||
success: true,
|
||||
content: transcript,
|
||||
chunk: transcript,
|
||||
metadata,
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
try {
|
||||
return await this.runNpmFallback(url);
|
||||
} catch (fallbackError: unknown) {
|
||||
return {
|
||||
success: false,
|
||||
content: '',
|
||||
chunk: '',
|
||||
metadata: {} as YouTubeMetadata,
|
||||
error: `${this.formatErrorMessage(error)}; fallback error: ${this.formatErrorMessage(fallbackError)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runNpmFallback(url: string): Promise<ExtractionResult> {
|
||||
console.warn('Primary transcript extraction failed; falling back to legacy youtube-transcript package');
|
||||
const fallback = await extractYouTubeNpm(url);
|
||||
return {
|
||||
success: fallback.success,
|
||||
content: fallback.content,
|
||||
chunk: fallback.chunk,
|
||||
metadata: fallback.metadata as YouTubeMetadata,
|
||||
error: fallback.error,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(url: string): Promise<ExtractionResult> {
|
||||
const extractor = new YouTubeExtractor();
|
||||
return extractor.extract(url);
|
||||
}
|
||||
|
||||
export async function extractYouTube(url: string): Promise<ExtractionResult> {
|
||||
const extractor = new YouTubeExtractor();
|
||||
return extractor.extract(url);
|
||||
}
|
||||
|
||||
export async function runCLI(): Promise<void> {
|
||||
if (process.argv.length !== 3) {
|
||||
console.log(JSON.stringify({
|
||||
success: false,
|
||||
error: "Usage: node youtube.js <youtube_url>"
|
||||
}));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const url = process.argv[2];
|
||||
const result = await main(url);
|
||||
console.log(JSON.stringify(result));
|
||||
|
||||
if (!result.success) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
runCLI().catch(error => {
|
||||
console.log(JSON.stringify({
|
||||
success: false,
|
||||
error: error.message
|
||||
}));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* SQLite vec0 utilities for TypeScript
|
||||
* Handles binary serialization for vec0 BLOB storage
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
|
||||
/**
|
||||
* Serialize a float array to binary format for vec0 storage
|
||||
* Equivalent to Python's struct.pack(f'{len(vector)}f', *vector)
|
||||
*/
|
||||
export function serializeFloat32Vector(vector: number[]): Buffer {
|
||||
const buffer = Buffer.allocUnsafe(vector.length * 4);
|
||||
for (let i = 0; i < vector.length; i++) {
|
||||
buffer.writeFloatLE(vector[i], i * 4);
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deserialize a vec0 BLOB back to float array
|
||||
*/
|
||||
export function deserializeFloat32Vector(blob: Buffer): number[] {
|
||||
const vector: number[] = [];
|
||||
for (let i = 0; i < blob.length; i += 4) {
|
||||
vector.push(blob.readFloatLE(i));
|
||||
}
|
||||
return vector;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get SQLite database path from environment or default location
|
||||
*/
|
||||
export function getDatabasePath(): string {
|
||||
const envPath = process.env.SQLITE_DB_PATH;
|
||||
if (envPath) {
|
||||
return envPath;
|
||||
}
|
||||
|
||||
// Default path: ~/Library/Application Support/RA-H/db/rah.sqlite
|
||||
const homeDir = os.homedir();
|
||||
return path.join(homeDir, 'Library', 'Application Support', 'RA-H', 'db', 'rah.sqlite');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get vec extension path from environment or default location
|
||||
*/
|
||||
export function getVecExtensionPath(): string {
|
||||
const envPath = process.env.SQLITE_VEC_EXTENSION_PATH;
|
||||
if (envPath) {
|
||||
return envPath;
|
||||
}
|
||||
|
||||
// Default path relative to project root
|
||||
return path.join(process.cwd(), 'vendor', 'sqlite-extensions', 'vec0.dylib');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create database connection with vec0 extension loaded
|
||||
*/
|
||||
export function createDatabaseConnection(): Database.Database {
|
||||
const dbPath = getDatabasePath();
|
||||
const vecPath = getVecExtensionPath();
|
||||
|
||||
const db = new Database(dbPath);
|
||||
|
||||
// Load vec0 extension
|
||||
try {
|
||||
db.loadExtension(vecPath);
|
||||
} catch (error) {
|
||||
console.error('Warning: Could not load vec0 extension:', error);
|
||||
// Continue without vector support for non-vector operations
|
||||
}
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format embedding text for node metadata
|
||||
*/
|
||||
export function formatEmbeddingText(
|
||||
title: string,
|
||||
content: string,
|
||||
dimensions: string[]
|
||||
): string {
|
||||
const dimensionsText = dimensions.length > 0 ? dimensions.join(', ') : 'none';
|
||||
return `Title: ${title}\n\nContent: ${content}\n\nDimensions: ${dimensionsText}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch process items with progress logging
|
||||
*/
|
||||
export async function batchProcess<T, R>(
|
||||
items: T[],
|
||||
processor: (item: T) => Promise<R>,
|
||||
batchSize: number = 10,
|
||||
onProgress?: (processed: number, total: number) => void
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
|
||||
for (let i = 0; i < items.length; i += batchSize) {
|
||||
const batch = items.slice(i, Math.min(i + batchSize, items.length));
|
||||
const batchResults = await Promise.all(batch.map(processor));
|
||||
results.push(...batchResults);
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(Math.min(i + batchSize, items.length), items.length);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import { getSQLiteClient } from '@/services/database/sqlite-client';
|
||||
|
||||
export interface VoiceUsageLogEntry {
|
||||
sessionId?: string | null;
|
||||
helperName?: string | null;
|
||||
requestId: string;
|
||||
messageId?: string | null;
|
||||
voice: string;
|
||||
model: string;
|
||||
charCount: number;
|
||||
costUsd: number;
|
||||
durationMs?: number | null;
|
||||
textPreview?: string | null;
|
||||
}
|
||||
|
||||
type ChatRow = {
|
||||
id: number;
|
||||
metadata: string | null;
|
||||
};
|
||||
|
||||
function parseMetadata(raw: unknown): Record<string, any> {
|
||||
if (!raw) return {};
|
||||
if (typeof raw === 'string') {
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch (error) {
|
||||
console.warn('[VoiceUsage] Failed to parse chat metadata JSON', error);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
if (typeof raw === 'object') {
|
||||
return (raw as Record<string, any>) || {};
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function applyVoiceUsageMetadata(
|
||||
metadata: Record<string, any>,
|
||||
entry: VoiceUsageLogEntry,
|
||||
loggedAt: string
|
||||
): Record<string, any> {
|
||||
const usageList = Array.isArray(metadata.voice_usage) ? metadata.voice_usage : [];
|
||||
|
||||
const usageEntry = {
|
||||
request_id: entry.requestId,
|
||||
message_id: entry.messageId ?? null,
|
||||
chars: entry.charCount,
|
||||
cost_usd: entry.costUsd,
|
||||
voice: entry.voice,
|
||||
model: entry.model,
|
||||
duration_ms: entry.durationMs ?? null,
|
||||
logged_at: loggedAt,
|
||||
};
|
||||
|
||||
usageList.push(usageEntry);
|
||||
const MAX_USAGE_HISTORY = 20;
|
||||
metadata.voice_usage = usageList.slice(-MAX_USAGE_HISTORY);
|
||||
|
||||
const currentCharsTotal = Number(metadata.voice_tts_chars_total) || 0;
|
||||
const currentCostTotal = Number(metadata.voice_tts_cost_usd_total) || Number(metadata.voice_tts_cost_total_usd) || 0;
|
||||
const currentRequestCount = Number(metadata.voice_tts_request_count) || 0;
|
||||
|
||||
metadata.voice_tts_chars_total = currentCharsTotal + entry.charCount;
|
||||
metadata.voice_tts_cost_usd_total = parseFloat((currentCostTotal + entry.costUsd).toFixed(6));
|
||||
metadata.voice_tts_request_count = currentRequestCount + 1;
|
||||
|
||||
metadata.voice_tts_chars = entry.charCount;
|
||||
metadata.voice_tts_cost_usd = entry.costUsd;
|
||||
metadata.voice_request_id = entry.requestId;
|
||||
metadata.voice_tts_voice = entry.voice;
|
||||
metadata.voice_tts_model = entry.model;
|
||||
metadata.voice_tts_duration_ms = entry.durationMs ?? null;
|
||||
metadata.voice_tts_last_logged_at = loggedAt;
|
||||
metadata.voice_message_id = entry.messageId ?? null;
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
export function recordVoiceUsage(entry: VoiceUsageLogEntry): void {
|
||||
try {
|
||||
const sqlite = getSQLiteClient();
|
||||
const loggedAt = new Date().toISOString();
|
||||
let chatId: number | null = null;
|
||||
|
||||
if (entry.sessionId) {
|
||||
try {
|
||||
const row = sqlite
|
||||
.prepare(
|
||||
`
|
||||
SELECT id, metadata
|
||||
FROM chats
|
||||
WHERE json_extract(metadata, '$.session_id') = ?
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
`
|
||||
)
|
||||
.get(entry.sessionId) as ChatRow | undefined;
|
||||
|
||||
if (row) {
|
||||
chatId = row.id;
|
||||
const parsedMetadata = applyVoiceUsageMetadata(parseMetadata(row.metadata), entry, loggedAt);
|
||||
sqlite
|
||||
.prepare(`UPDATE chats SET metadata = ? WHERE id = ?`)
|
||||
.run(JSON.stringify(parsedMetadata), row.id);
|
||||
} else {
|
||||
console.warn(`[VoiceUsage] No chat row found for session ${entry.sessionId}, logging standalone entry.`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[VoiceUsage] Failed to attach usage to chat metadata', error);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
sqlite
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO voice_usage (
|
||||
chat_id,
|
||||
session_id,
|
||||
helper_name,
|
||||
request_id,
|
||||
message_id,
|
||||
voice,
|
||||
model,
|
||||
chars,
|
||||
cost_usd,
|
||||
duration_ms,
|
||||
text_preview,
|
||||
created_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
)
|
||||
.run(
|
||||
chatId,
|
||||
entry.sessionId ?? null,
|
||||
entry.helperName ?? null,
|
||||
entry.requestId,
|
||||
entry.messageId ?? null,
|
||||
entry.voice,
|
||||
entry.model,
|
||||
entry.charCount,
|
||||
entry.costUsd,
|
||||
entry.durationMs ?? null,
|
||||
entry.textPreview ?? null,
|
||||
loggedAt
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('[VoiceUsage] Failed to insert voice usage row', error);
|
||||
}
|
||||
} catch (outerError) {
|
||||
console.error('[VoiceUsage] Unexpected error while recording usage', outerError);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { INTEGRATE_WORKFLOW_INSTRUCTIONS } from '@/config/workflows/integrate';
|
||||
import type { WorkflowDefinition } from './types';
|
||||
|
||||
export class WorkflowRegistry {
|
||||
private static readonly WORKFLOWS: Record<string, WorkflowDefinition> = {
|
||||
'integrate': {
|
||||
id: 1,
|
||||
key: 'integrate',
|
||||
displayName: 'Integrate',
|
||||
description: 'Deep analysis and connection-building for focused node',
|
||||
instructions: INTEGRATE_WORKFLOW_INSTRUCTIONS,
|
||||
enabled: true,
|
||||
requiresFocusedNode: true,
|
||||
primaryActor: 'oracle',
|
||||
expectedOutcome: 'Focused node updated with insights; 3-5 high-value edges created',
|
||||
}
|
||||
};
|
||||
|
||||
static async getWorkflowByKey(key: string): Promise<WorkflowDefinition | null> {
|
||||
return this.WORKFLOWS[key] || null;
|
||||
}
|
||||
|
||||
static async getEnabledWorkflows(): Promise<WorkflowDefinition[]> {
|
||||
return Object.values(this.WORKFLOWS).filter(w => w.enabled);
|
||||
}
|
||||
|
||||
static async getAllWorkflows(): Promise<WorkflowDefinition[]> {
|
||||
return Object.values(this.WORKFLOWS);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
export interface WorkflowDefinition {
|
||||
id: number;
|
||||
key: string;
|
||||
displayName: string;
|
||||
description: string;
|
||||
instructions: string;
|
||||
enabled: boolean;
|
||||
requiresFocusedNode: boolean;
|
||||
primaryActor: 'oracle' | 'main';
|
||||
expectedOutcome?: string;
|
||||
}
|
||||
Reference in New Issue
Block a user