sync: bug fixes + eval system from private repo
- Fix chunk_status: pass chunk_status/chunk/metadata to context builder - Fix vector search: scope by node_id BEFORE similarity search - Add eval logging system (RAH_EVALS_LOG=1) - Add eval dashboard at /evals - Add vitest for testing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
“BeeRad”
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
e15f223ed8
commit
2f2ef10ec9
@@ -0,0 +1,24 @@
|
|||||||
|
import { NextRequest } from 'next/server';
|
||||||
|
import { upsertEvalComment } from '@/services/evals/evalsStore';
|
||||||
|
|
||||||
|
export async function POST(request: NextRequest) {
|
||||||
|
const evalsEnabled = process.env.RAH_EVALS_LOG === '1' || process.env.RAH_EVALS_LOG === 'true';
|
||||||
|
if (process.env.NODE_ENV === 'production' || !evalsEnabled) {
|
||||||
|
return new Response('Not Found', { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = await request.json();
|
||||||
|
const traceId = typeof body?.traceId === 'string' ? body.traceId : null;
|
||||||
|
const scenarioId = typeof body?.scenarioId === 'string' ? body.scenarioId : null;
|
||||||
|
const comment = typeof body?.comment === 'string' ? body.comment : null;
|
||||||
|
|
||||||
|
if (!traceId || comment === null) {
|
||||||
|
return new Response(JSON.stringify({ error: 'Invalid payload' }), { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
upsertEvalComment(traceId, scenarioId, comment);
|
||||||
|
return new Response(JSON.stringify({ ok: true }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,376 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useMemo, useState } from 'react';
|
||||||
|
import type { EvalTrace } from '@/services/evals/evalsStore';
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
traces: EvalTrace[];
|
||||||
|
scenarioList: Array<{
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
tools?: string[];
|
||||||
|
enabled?: boolean;
|
||||||
|
notes?: string;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
|
function formatPreview(text: string | null, max = 80) {
|
||||||
|
if (!text) return '';
|
||||||
|
const compact = text.replace(/\s+/g, ' ').trim();
|
||||||
|
return compact.length > max ? `${compact.slice(0, max)}...` : compact;
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusLabel(success: number | null) {
|
||||||
|
if (success === null) return 'n/a';
|
||||||
|
return success ? 'success' : 'fail';
|
||||||
|
}
|
||||||
|
|
||||||
|
function badgeStyle(kind: 'success' | 'fail' | 'neutral') {
|
||||||
|
const base = { display: 'inline-block', padding: '2px 8px', borderRadius: 999, fontSize: 12 };
|
||||||
|
if (kind === 'success') return { ...base, background: '#e6f4ea', color: '#146c2e' };
|
||||||
|
if (kind === 'fail') return { ...base, background: '#fdecea', color: '#b42318' };
|
||||||
|
return { ...base, background: '#f2f2f2', color: '#333' };
|
||||||
|
}
|
||||||
|
|
||||||
|
function prettyJson(value: string | null) {
|
||||||
|
if (!value) return '';
|
||||||
|
try {
|
||||||
|
return JSON.stringify(JSON.parse(value), null, 2);
|
||||||
|
} catch {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function EvalsClient({ traces, scenarioList }: Props) {
|
||||||
|
const [openTraceId, setOpenTraceId] = useState<string | null>(traces[0]?.chat.trace_id || null);
|
||||||
|
const [comments, setComments] = useState<Record<string, string>>(() => {
|
||||||
|
const initial: Record<string, string> = {};
|
||||||
|
traces.forEach((trace) => {
|
||||||
|
if (trace.comment) {
|
||||||
|
initial[trace.chat.trace_id] = trace.comment;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return initial;
|
||||||
|
});
|
||||||
|
const [sourceFilter, setSourceFilter] = useState<string>('all'); // 'all' | 'live' | 'scenario'
|
||||||
|
const [scenarioFilter, setScenarioFilter] = useState<string>('all');
|
||||||
|
const [statusFilter, setStatusFilter] = useState<string>('all');
|
||||||
|
const [searchFilter, setSearchFilter] = useState<string>('');
|
||||||
|
|
||||||
|
const rows = useMemo(() => {
|
||||||
|
return traces.map((trace) => {
|
||||||
|
const { chat, toolCalls } = trace;
|
||||||
|
const status = statusLabel(chat.success);
|
||||||
|
const isLive = !chat.scenario_id;
|
||||||
|
return {
|
||||||
|
trace,
|
||||||
|
id: chat.trace_id,
|
||||||
|
source: isLive ? 'live' : 'scenario',
|
||||||
|
scenario: chat.scenario_id || '—',
|
||||||
|
model: chat.model || 'n/a',
|
||||||
|
latency: chat.latency_ms ?? 'n/a',
|
||||||
|
tokens: `${chat.input_tokens ?? 0}/${chat.output_tokens ?? 0} (${chat.total_tokens ?? 0})`,
|
||||||
|
cost: chat.estimated_cost_usd ?? null,
|
||||||
|
cache: chat.cache_hit == null ? 'n/a' : chat.cache_hit ? 'hit' : 'miss',
|
||||||
|
cacheTokens: `${chat.cache_read_tokens ?? 0}/${chat.cache_write_tokens ?? 0}`,
|
||||||
|
toolCount: toolCalls.length,
|
||||||
|
status,
|
||||||
|
userPreview: formatPreview(chat.user_message),
|
||||||
|
timestamp: chat.ts,
|
||||||
|
mode: chat.mode || 'n/a',
|
||||||
|
workflow: chat.workflow_key || '—',
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}, [traces]);
|
||||||
|
|
||||||
|
const filteredRows = useMemo(() => {
|
||||||
|
return rows.filter((row) => {
|
||||||
|
if (sourceFilter !== 'all' && row.source !== sourceFilter) return false;
|
||||||
|
if (scenarioFilter !== 'all' && row.scenario !== scenarioFilter) return false;
|
||||||
|
if (statusFilter !== 'all' && row.status !== statusFilter) return false;
|
||||||
|
if (searchFilter.trim()) {
|
||||||
|
const needle = searchFilter.toLowerCase();
|
||||||
|
if (!row.userPreview.toLowerCase().includes(needle)) return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}, [rows, sourceFilter, scenarioFilter, statusFilter, searchFilter]);
|
||||||
|
|
||||||
|
const openTrace = traces.find((trace) => trace.chat.trace_id === openTraceId) || traces[0];
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<div style={{ marginBottom: 24, border: '1px solid #ddd', borderRadius: 8, background: '#fff', padding: 16 }}>
|
||||||
|
<div style={{ fontWeight: 600, marginBottom: 8 }}>Scenario Set</div>
|
||||||
|
<div style={{ overflowX: 'auto' }}>
|
||||||
|
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}>
|
||||||
|
<thead>
|
||||||
|
<tr style={{ textAlign: 'left', borderBottom: '1px solid #ddd' }}>
|
||||||
|
<th style={{ padding: '6px 6px' }}>ID</th>
|
||||||
|
<th style={{ padding: '6px 6px' }}>Name</th>
|
||||||
|
<th style={{ padding: '6px 6px' }}>Description</th>
|
||||||
|
<th style={{ padding: '6px 6px' }}>Tools</th>
|
||||||
|
<th style={{ padding: '6px 6px' }}>Enabled</th>
|
||||||
|
<th style={{ padding: '6px 6px' }}>Notes</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{scenarioList.map((scenario) => (
|
||||||
|
<tr key={scenario.id} style={{ borderBottom: '1px solid #eee' }}>
|
||||||
|
<td style={{ padding: '6px 6px' }}>{scenario.id}</td>
|
||||||
|
<td style={{ padding: '6px 6px' }}>{scenario.name}</td>
|
||||||
|
<td style={{ padding: '6px 6px', color: '#555' }}>{scenario.description || '—'}</td>
|
||||||
|
<td style={{ padding: '6px 6px', color: '#555' }}>{scenario.tools?.join(', ') || '—'}</td>
|
||||||
|
<td style={{ padding: '6px 6px' }}>{scenario.enabled === false ? 'no' : 'yes'}</td>
|
||||||
|
<td style={{ padding: '6px 6px', color: '#555' }}>{scenario.notes || '—'}</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style={{ overflowX: 'auto' }}>
|
||||||
|
<div style={{ display: 'flex', gap: 12, alignItems: 'center', flexWrap: 'wrap', marginBottom: 12 }}>
|
||||||
|
<label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||||
|
<span>Source</span>
|
||||||
|
<select
|
||||||
|
value={sourceFilter}
|
||||||
|
onChange={(event) => setSourceFilter(event.target.value)}
|
||||||
|
style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd', fontWeight: 500 }}
|
||||||
|
>
|
||||||
|
<option value="all">All</option>
|
||||||
|
<option value="live">🟢 Live Runs</option>
|
||||||
|
<option value="scenario">🔬 Scenarios</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||||
|
<span>Scenario</span>
|
||||||
|
<select
|
||||||
|
value={scenarioFilter}
|
||||||
|
onChange={(event) => setScenarioFilter(event.target.value)}
|
||||||
|
style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd' }}
|
||||||
|
>
|
||||||
|
<option value="all">All</option>
|
||||||
|
{Array.from(new Set(rows.filter(row => row.scenario !== '—').map(row => row.scenario))).map((scenario) => (
|
||||||
|
<option key={scenario} value={scenario}>{scenario}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||||
|
<span>Status</span>
|
||||||
|
<select
|
||||||
|
value={statusFilter}
|
||||||
|
onChange={(event) => setStatusFilter(event.target.value)}
|
||||||
|
style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd' }}
|
||||||
|
>
|
||||||
|
<option value="all">All</option>
|
||||||
|
<option value="success">Success</option>
|
||||||
|
<option value="fail">Fail</option>
|
||||||
|
<option value="n/a">N/A</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||||
|
<span>Search</span>
|
||||||
|
<input
|
||||||
|
value={searchFilter}
|
||||||
|
onChange={(event) => setSearchFilter(event.target.value)}
|
||||||
|
placeholder="User input contains..."
|
||||||
|
style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd', minWidth: 240 }}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}>
|
||||||
|
<thead>
|
||||||
|
<tr style={{ textAlign: 'left', borderBottom: '1px solid #ddd' }}>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Source</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Scenario</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Status</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Latency</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Tokens</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Cost</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Cache</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Tools</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Model</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Mode</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Workflow</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>User Input</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Time</th>
|
||||||
|
<th style={{ padding: '8px 6px' }}>Comment</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{filteredRows.map((row) => {
|
||||||
|
const isOpen = row.id === openTraceId;
|
||||||
|
const statusKind = row.status === 'success' ? 'success' : row.status === 'fail' ? 'fail' : 'neutral';
|
||||||
|
const commentValue = comments[row.id] || '';
|
||||||
|
return (
|
||||||
|
<tr
|
||||||
|
key={row.id}
|
||||||
|
onClick={() => setOpenTraceId(row.id)}
|
||||||
|
style={{
|
||||||
|
cursor: 'pointer',
|
||||||
|
background: isOpen ? '#eef4ff' : '#fff',
|
||||||
|
borderBottom: '1px solid #eee',
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<td style={{ padding: '8px 6px' }}>
|
||||||
|
<span style={{
|
||||||
|
...badgeStyle(row.source === 'live' ? 'success' : 'neutral'),
|
||||||
|
fontSize: 11,
|
||||||
|
}}>
|
||||||
|
{row.source === 'live' ? '🟢 Live' : '🔬 Scenario'}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.scenario}</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}><span style={badgeStyle(statusKind)}>{row.status}</span></td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.latency} ms</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.tokens}</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.cost == null ? 'n/a' : `$${row.cost.toFixed(4)}`}</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.cache} ({row.cacheTokens})</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.toolCount}</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.model}</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.mode}</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>{row.workflow}</td>
|
||||||
|
<td style={{ padding: '8px 6px', color: '#555' }}>{row.userPreview}</td>
|
||||||
|
<td style={{ padding: '8px 6px', color: '#666' }}>{row.timestamp}</td>
|
||||||
|
<td style={{ padding: '8px 6px' }}>
|
||||||
|
<textarea
|
||||||
|
value={commentValue}
|
||||||
|
onChange={(event) => {
|
||||||
|
const nextValue = event.target.value;
|
||||||
|
setComments((prev) => ({ ...prev, [row.id]: nextValue }));
|
||||||
|
}}
|
||||||
|
onBlur={async (event) => {
|
||||||
|
const nextValue = event.target.value;
|
||||||
|
try {
|
||||||
|
await fetch('/api/evals/comment', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
traceId: row.id,
|
||||||
|
scenarioId: row.scenario,
|
||||||
|
comment: nextValue,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Ignore failures in the UI, keep local state.
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
onClick={(event) => event.stopPropagation()}
|
||||||
|
rows={2}
|
||||||
|
style={{
|
||||||
|
width: 220,
|
||||||
|
resize: 'vertical',
|
||||||
|
fontFamily: 'inherit',
|
||||||
|
fontSize: 12,
|
||||||
|
padding: 6,
|
||||||
|
borderRadius: 6,
|
||||||
|
border: '1px solid #ddd',
|
||||||
|
}}
|
||||||
|
placeholder="Add notes..."
|
||||||
|
/>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{openTrace ? (
|
||||||
|
<div style={{ marginTop: 24, border: '1px solid #ddd', borderRadius: 8, background: '#fff', padding: 16 }}>
|
||||||
|
<div style={{ display: 'flex', gap: 16, flexWrap: 'wrap', marginBottom: 12 }}>
|
||||||
|
<div><strong>Source:</strong> {openTrace.chat.scenario_id ? '🔬 Scenario' : '🟢 Live Run'}</div>
|
||||||
|
<div><strong>Scenario:</strong> {openTrace.chat.scenario_id || '—'}</div>
|
||||||
|
<div><strong>Trace:</strong> {openTrace.chat.trace_id}</div>
|
||||||
|
<div><strong>Helper:</strong> {openTrace.chat.helper_name || 'n/a'}</div>
|
||||||
|
<div><strong>Model:</strong> {openTrace.chat.model || 'n/a'}</div>
|
||||||
|
<div><strong>Mode:</strong> {openTrace.chat.mode || 'n/a'}</div>
|
||||||
|
<div><strong>Workflow:</strong> {openTrace.chat.workflow_key || '—'}</div>
|
||||||
|
<div><strong>Latency:</strong> {openTrace.chat.latency_ms ?? 'n/a'} ms</div>
|
||||||
|
<div><strong>Tokens:</strong> {openTrace.chat.input_tokens ?? 0}/{openTrace.chat.output_tokens ?? 0} (total {openTrace.chat.total_tokens ?? 0})</div>
|
||||||
|
<div><strong>Cost:</strong> {openTrace.chat.estimated_cost_usd == null ? 'n/a' : `$${openTrace.chat.estimated_cost_usd.toFixed(4)}`}</div>
|
||||||
|
<div><strong>Cache:</strong> {openTrace.chat.cache_hit == null ? 'n/a' : openTrace.chat.cache_hit ? 'hit' : 'miss'} (read {openTrace.chat.cache_read_tokens ?? 0} / write {openTrace.chat.cache_write_tokens ?? 0})</div>
|
||||||
|
<div><strong>Tools:</strong> {openTrace.toolCalls.length}</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style={{ marginBottom: 12 }}>
|
||||||
|
<div><strong>Comment:</strong></div>
|
||||||
|
<div style={{ color: '#555' }}>{comments[openTrace.chat.trace_id] || '—'}</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<details open>
|
||||||
|
<summary style={{ cursor: 'pointer', marginBottom: 8 }}><strong>System Message</strong></summary>
|
||||||
|
<pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.system_message || 'n/a'}</pre>
|
||||||
|
</details>
|
||||||
|
|
||||||
|
<div style={{ marginTop: 16 }}>
|
||||||
|
<div style={{ fontWeight: 600, marginBottom: 8 }}>Trace Steps</div>
|
||||||
|
{[
|
||||||
|
{
|
||||||
|
type: 'chat',
|
||||||
|
ts: openTrace.chat.ts,
|
||||||
|
title: 'LLM Chat',
|
||||||
|
spanId: openTrace.chat.span_id,
|
||||||
|
parentSpanId: null,
|
||||||
|
latency: openTrace.chat.latency_ms,
|
||||||
|
success: openTrace.chat.success,
|
||||||
|
content: (
|
||||||
|
<>
|
||||||
|
<div><strong>User</strong></div>
|
||||||
|
<pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.user_message}</pre>
|
||||||
|
<div><strong>Assistant</strong></div>
|
||||||
|
<pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.assistant_message}</pre>
|
||||||
|
</>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
...openTrace.toolCalls.map((tool) => ({
|
||||||
|
type: 'tool',
|
||||||
|
ts: tool.ts,
|
||||||
|
title: `Tool: ${tool.tool_name}`,
|
||||||
|
spanId: tool.span_id,
|
||||||
|
parentSpanId: tool.parent_span_id,
|
||||||
|
latency: tool.latency_ms,
|
||||||
|
success: tool.success,
|
||||||
|
content: (
|
||||||
|
<>
|
||||||
|
<div><strong>Args</strong></div>
|
||||||
|
<pre style={{ whiteSpace: 'pre-wrap' }}>{prettyJson(tool.args_json)}</pre>
|
||||||
|
<div><strong>Result</strong></div>
|
||||||
|
<pre style={{ whiteSpace: 'pre-wrap' }}>{prettyJson(tool.result_json)}</pre>
|
||||||
|
{tool.error ? (
|
||||||
|
<>
|
||||||
|
<div><strong>Error</strong></div>
|
||||||
|
<pre style={{ whiteSpace: 'pre-wrap' }}>{tool.error}</pre>
|
||||||
|
</>
|
||||||
|
) : null}
|
||||||
|
</>
|
||||||
|
),
|
||||||
|
})),
|
||||||
|
].sort((a, b) => a.ts.localeCompare(b.ts)).map((step, index) => {
|
||||||
|
const stepStatus = statusLabel(step.success);
|
||||||
|
const stepKind = stepStatus === 'success' ? 'success' : stepStatus === 'fail' ? 'fail' : 'neutral';
|
||||||
|
return (
|
||||||
|
<div key={`${openTrace.chat.trace_id}-${step.type}-${index}`} style={{ borderTop: '1px solid #eee', paddingTop: 12, marginTop: 12 }}>
|
||||||
|
<div style={{ display: 'flex', gap: 12, alignItems: 'center', flexWrap: 'wrap' }}>
|
||||||
|
<div><strong>Step {index + 1}:</strong> {step.title}</div>
|
||||||
|
<span style={badgeStyle(stepKind)}>{stepStatus}</span>
|
||||||
|
<div style={{ color: '#666' }}>{step.ts}</div>
|
||||||
|
</div>
|
||||||
|
<div style={{ marginTop: 6, color: '#555' }}>
|
||||||
|
<div><strong>Span:</strong> {step.spanId || 'n/a'}</div>
|
||||||
|
{step.parentSpanId ? <div><strong>Parent Span:</strong> {step.parentSpanId}</div> : null}
|
||||||
|
<div><strong>Latency:</strong> {step.latency ?? 'n/a'} ms</div>
|
||||||
|
</div>
|
||||||
|
<div style={{ marginTop: 8 }}>{step.content}</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import { notFound } from 'next/navigation';
|
||||||
|
import { loadEvalTraces } from '@/services/evals/evalsStore';
|
||||||
|
import EvalsClient from './EvalsClient';
|
||||||
|
import { scenarios } from '../../tests/evals/scenarios';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
export default function EvalsPage() {
|
||||||
|
const evalsEnabled = process.env.RAH_EVALS_LOG === '1' || process.env.RAH_EVALS_LOG === 'true';
|
||||||
|
if (process.env.NODE_ENV === 'production' || !evalsEnabled) {
|
||||||
|
notFound();
|
||||||
|
}
|
||||||
|
|
||||||
|
const traces = loadEvalTraces(25);
|
||||||
|
const scenarioList = scenarios.map((scenario) => ({
|
||||||
|
id: scenario.id,
|
||||||
|
name: scenario.name,
|
||||||
|
description: scenario.description,
|
||||||
|
tools: scenario.tools,
|
||||||
|
enabled: scenario.enabled,
|
||||||
|
notes: scenario.notes,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ height: '100vh', overflowY: 'auto', background: '#f5f5f5', color: '#111' }}>
|
||||||
|
<div style={{ padding: 24, maxWidth: 1200, margin: '0 auto', fontFamily: 'ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace' }}>
|
||||||
|
<div style={{ position: 'sticky', top: 0, background: '#f5f5f5', paddingBottom: 12, marginBottom: 12, zIndex: 1 }}>
|
||||||
|
<h1 style={{ fontSize: 24, marginBottom: 6 }}>Eval Traces</h1>
|
||||||
|
<div style={{ color: '#444' }}>Trace table on top, full span detail below.</div>
|
||||||
|
</div>
|
||||||
|
{traces.length === 0 ? (
|
||||||
|
<p>No eval traces found. Run evals with RAH_EVALS_LOG=1.</p>
|
||||||
|
) : (
|
||||||
|
<EvalsClient traces={traces} scenarioList={scenarioList} />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
+7
-2
@@ -15,7 +15,10 @@
|
|||||||
"type-check": "tsc --noEmit",
|
"type-check": "tsc --noEmit",
|
||||||
"setup": "npm install",
|
"setup": "npm install",
|
||||||
"sqlite:backup": "bash scripts/database/sqlite-backup.sh",
|
"sqlite:backup": "bash scripts/database/sqlite-backup.sh",
|
||||||
"sqlite:restore": "bash scripts/database/sqlite-restore.sh"
|
"sqlite:restore": "bash scripts/database/sqlite-restore.sh",
|
||||||
|
"test": "vitest run",
|
||||||
|
"test:watch": "vitest",
|
||||||
|
"evals": "RAH_EVALS_LOG=1 tsx tests/evals/runner.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@ai-sdk/anthropic": "^2.0.27",
|
"@ai-sdk/anthropic": "^2.0.27",
|
||||||
@@ -62,6 +65,8 @@
|
|||||||
"prettier": "^3.3.3",
|
"prettier": "^3.3.3",
|
||||||
"tailwindcss": "^3.4.17",
|
"tailwindcss": "^3.4.17",
|
||||||
"tailwindcss-animate": "^1.0.7",
|
"tailwindcss-animate": "^1.0.7",
|
||||||
"typescript": "^5.7.2"
|
"tsx": "^4.19.4",
|
||||||
|
"typescript": "^5.7.2",
|
||||||
|
"vitest": "^3.2.4"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -75,7 +75,10 @@ export default function ThreePanelLayout() {
|
|||||||
content: node.content,
|
content: node.content,
|
||||||
dimensions: node.dimensions,
|
dimensions: node.dimensions,
|
||||||
created_at: node.created_at,
|
created_at: node.created_at,
|
||||||
updated_at: node.updated_at
|
updated_at: node.updated_at,
|
||||||
|
chunk_status: node.chunk_status,
|
||||||
|
chunk: node.chunk,
|
||||||
|
metadata: node.metadata,
|
||||||
}));
|
}));
|
||||||
setOpenTabsData(validNodes);
|
setOpenTabsData(validNodes);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ interface RequestContext {
|
|||||||
openai?: string;
|
openai?: string;
|
||||||
anthropic?: string;
|
anthropic?: string;
|
||||||
};
|
};
|
||||||
|
helperName?: string;
|
||||||
|
evalDatasetId?: string;
|
||||||
|
evalScenarioId?: string;
|
||||||
|
evalChatSpanId?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
let currentContext: RequestContext = {};
|
let currentContext: RequestContext = {};
|
||||||
@@ -26,11 +30,11 @@ export const RequestContext = {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|
||||||
get(): RequestContext {
|
get(): RequestContext {
|
||||||
return currentContext;
|
return currentContext;
|
||||||
},
|
},
|
||||||
|
|
||||||
clear() {
|
clear() {
|
||||||
currentContext = {};
|
currentContext = {};
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -170,35 +170,57 @@ export class ChunkService {
|
|||||||
// PostgreSQL path removed in SQLite-only consolidation
|
// PostgreSQL path removed in SQLite-only consolidation
|
||||||
|
|
||||||
private async searchChunksSQLite(
|
private async searchChunksSQLite(
|
||||||
queryEmbedding: number[],
|
queryEmbedding: number[],
|
||||||
similarityThreshold = 0.5,
|
similarityThreshold = 0.5,
|
||||||
matchCount = 5,
|
matchCount = 5,
|
||||||
nodeIds?: number[]
|
nodeIds?: number[]
|
||||||
): Promise<Array<Chunk & { similarity: number }>> {
|
): Promise<Array<Chunk & { similarity: number }>> {
|
||||||
const sqlite = getSQLiteClient();
|
const sqlite = getSQLiteClient();
|
||||||
|
|
||||||
// Step 1: Determine vector search limit - more conservative approach
|
|
||||||
let vectorLimit = 50; // Start smaller for efficiency
|
|
||||||
|
|
||||||
if (nodeIds && nodeIds.length > 0) {
|
|
||||||
// When searching specific nodes, get exact count and use reasonable multiplier
|
|
||||||
const candidateCountQuery = `SELECT COUNT(*) as count FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
|
|
||||||
const candidateResult = sqlite.query<{count: number}>(candidateCountQuery, nodeIds);
|
|
||||||
const candidateCount = Number(candidateResult.rows[0].count);
|
|
||||||
|
|
||||||
// Use 2x the candidate count but cap at reasonable limits
|
|
||||||
vectorLimit = Math.min(Math.max(candidateCount * 2, 50), 1000);
|
|
||||||
console.log(`🔍 Node-scoped search: ${candidateCount} candidates, using vector limit ${vectorLimit}`);
|
|
||||||
} else {
|
|
||||||
// For global search, use adaptive limit based on expected results
|
|
||||||
vectorLimit = Math.max(matchCount * 10, 50);
|
|
||||||
}
|
|
||||||
|
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
|
|
||||||
// Step 2: Use CTE-based query to avoid UNION ALL explosion
|
|
||||||
const vectorString = `[${queryEmbedding.join(',')}]`;
|
const vectorString = `[${queryEmbedding.join(',')}]`;
|
||||||
let query = `
|
|
||||||
|
// When searching specific nodes, first get their chunk_ids to scope the vector search
|
||||||
|
if (nodeIds && nodeIds.length > 0) {
|
||||||
|
// Get chunk IDs for the target nodes
|
||||||
|
const chunkIdsQuery = `SELECT id FROM chunks WHERE node_id IN (${nodeIds.map(() => '?').join(',')})`;
|
||||||
|
const chunkIdsResult = sqlite.query<{id: number}>(chunkIdsQuery, nodeIds);
|
||||||
|
const chunkIds = chunkIdsResult.rows.map(r => r.id);
|
||||||
|
|
||||||
|
if (chunkIds.length === 0) {
|
||||||
|
console.log(`🔍 Node-scoped search: no chunks found for nodes ${nodeIds.join(', ')}`);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`🔍 Node-scoped search: ${chunkIds.length} chunks in nodes ${nodeIds.join(', ')}`);
|
||||||
|
|
||||||
|
// Search ONLY within those chunks using rowid filter
|
||||||
|
const query = `
|
||||||
|
SELECT c.*, (1.0 / (1.0 + v.distance)) AS similarity
|
||||||
|
FROM vec_chunks v
|
||||||
|
JOIN chunks c ON c.id = v.chunk_id
|
||||||
|
WHERE v.embedding MATCH ?
|
||||||
|
AND v.chunk_id IN (${chunkIds.map(() => '?').join(',')})
|
||||||
|
AND (1.0 / (1.0 + v.distance)) >= ?
|
||||||
|
ORDER BY v.distance
|
||||||
|
LIMIT ?
|
||||||
|
`;
|
||||||
|
|
||||||
|
const params = [vectorString, ...chunkIds, similarityThreshold, matchCount];
|
||||||
|
const result = sqlite.query<Chunk & { similarity: number }>(query, params);
|
||||||
|
const searchTime = Date.now() - startTime;
|
||||||
|
|
||||||
|
console.log(`📊 Vector search (node-scoped): ${result.rows.length} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
|
||||||
|
if (result.rows.length > 0) {
|
||||||
|
console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Global search (no node filter)
|
||||||
|
const vectorLimit = Math.max(matchCount * 10, 50);
|
||||||
|
|
||||||
|
const query = `
|
||||||
WITH vector_results AS (
|
WITH vector_results AS (
|
||||||
SELECT chunk_id, distance
|
SELECT chunk_id, distance
|
||||||
FROM vec_chunks
|
FROM vec_chunks
|
||||||
@@ -209,36 +231,20 @@ export class ChunkService {
|
|||||||
SELECT c.*, (1.0 / (1.0 + vr.distance)) AS similarity
|
SELECT c.*, (1.0 / (1.0 + vr.distance)) AS similarity
|
||||||
FROM vector_results vr
|
FROM vector_results vr
|
||||||
JOIN chunks c ON c.id = vr.chunk_id
|
JOIN chunks c ON c.id = vr.chunk_id
|
||||||
|
WHERE (1.0 / (1.0 + vr.distance)) >= ?
|
||||||
|
ORDER BY similarity DESC
|
||||||
|
LIMIT ?
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const params: any[] = [vectorString, vectorLimit];
|
|
||||||
const conditions = [];
|
|
||||||
|
|
||||||
// Node ID filter if provided
|
|
||||||
if (nodeIds && nodeIds.length > 0) {
|
|
||||||
conditions.push(`c.node_id IN (${nodeIds.map(() => '?').join(',')})`);
|
|
||||||
params.push(...nodeIds);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Similarity threshold filter
|
|
||||||
conditions.push(`(1.0 / (1.0 + vr.distance)) >= ?`);
|
|
||||||
params.push(similarityThreshold);
|
|
||||||
|
|
||||||
if (conditions.length > 0) {
|
|
||||||
query += ` WHERE ${conditions.join(' AND ')}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
query += ` ORDER BY similarity DESC LIMIT ?`;
|
|
||||||
params.push(matchCount);
|
|
||||||
|
|
||||||
|
const params = [vectorString, vectorLimit, similarityThreshold, matchCount];
|
||||||
const result = sqlite.query<Chunk & { similarity: number }>(query, params);
|
const result = sqlite.query<Chunk & { similarity: number }>(query, params);
|
||||||
const searchTime = Date.now() - startTime;
|
const searchTime = Date.now() - startTime;
|
||||||
|
|
||||||
console.log(`📊 Vector search: ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
|
console.log(`📊 Vector search (global): ${result.rows.length}/${vectorLimit} chunks, threshold=${similarityThreshold}, time=${searchTime}ms`);
|
||||||
if (result.rows.length > 0) {
|
if (result.rows.length > 0) {
|
||||||
console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
|
console.log(`🎯 Top result: chunk ${result.rows[0].id} (similarity: ${result.rows[0].similarity.toFixed(3)})`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.rows;
|
return result.rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,269 @@
|
|||||||
|
import Database from 'better-sqlite3';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { RequestContext } from '@/services/context/requestContext';
|
||||||
|
|
||||||
|
type EvalToolCallLog = {
|
||||||
|
toolName: string;
|
||||||
|
args?: unknown;
|
||||||
|
result?: unknown;
|
||||||
|
error?: unknown;
|
||||||
|
latencyMs?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type EvalChatLog = {
|
||||||
|
traceId?: string;
|
||||||
|
spanId?: string;
|
||||||
|
helperName?: string;
|
||||||
|
model?: string;
|
||||||
|
promptVersion?: string;
|
||||||
|
systemMessage?: string | null;
|
||||||
|
userMessage?: string | null;
|
||||||
|
assistantMessage?: string | null;
|
||||||
|
inputTokens?: number;
|
||||||
|
outputTokens?: number;
|
||||||
|
totalTokens?: number;
|
||||||
|
cacheWriteTokens?: number;
|
||||||
|
cacheReadTokens?: number;
|
||||||
|
cacheHit?: boolean;
|
||||||
|
cacheSavingsPct?: number;
|
||||||
|
estimatedCostUsd?: number;
|
||||||
|
provider?: string | null;
|
||||||
|
mode?: string | null;
|
||||||
|
workflowKey?: string | null;
|
||||||
|
workflowNodeId?: number | null;
|
||||||
|
latencyMs?: number;
|
||||||
|
success?: boolean;
|
||||||
|
error?: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const EVALS_LOG_FLAG = process.env.RAH_EVALS_LOG;
|
||||||
|
const EVALS_LOG_ENABLED = EVALS_LOG_FLAG === '1' || EVALS_LOG_FLAG === 'true';
|
||||||
|
const LOG_DIR = path.join(process.cwd(), 'logs');
|
||||||
|
const DB_PATH = path.join(LOG_DIR, 'evals.sqlite');
|
||||||
|
|
||||||
|
let evalsDb: Database.Database | null = null;
|
||||||
|
|
||||||
|
function shouldLogEvals() {
|
||||||
|
// Log ALL interactions when RAH_EVALS_LOG=1
|
||||||
|
// - Real app interactions: scenario_id will be NULL
|
||||||
|
// - Synthetic scenarios: scenario_id will be set
|
||||||
|
return EVALS_LOG_ENABLED;
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureSchema(db: Database.Database) {
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS tool_calls (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
ts TEXT NOT NULL,
|
||||||
|
trace_id TEXT,
|
||||||
|
span_id TEXT,
|
||||||
|
parent_span_id TEXT,
|
||||||
|
helper_name TEXT,
|
||||||
|
tool_name TEXT NOT NULL,
|
||||||
|
args_json TEXT,
|
||||||
|
result_json TEXT,
|
||||||
|
success INTEGER,
|
||||||
|
latency_ms INTEGER,
|
||||||
|
error TEXT,
|
||||||
|
dataset_id TEXT,
|
||||||
|
scenario_id TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tool_calls_trace ON tool_calls(trace_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tool_calls_scenario ON tool_calls(scenario_id);
|
||||||
|
`);
|
||||||
|
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS llm_chats (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
ts TEXT NOT NULL,
|
||||||
|
trace_id TEXT,
|
||||||
|
span_id TEXT,
|
||||||
|
helper_name TEXT,
|
||||||
|
model TEXT,
|
||||||
|
prompt_version TEXT,
|
||||||
|
system_message TEXT,
|
||||||
|
user_message TEXT,
|
||||||
|
assistant_message TEXT,
|
||||||
|
input_tokens INTEGER,
|
||||||
|
output_tokens INTEGER,
|
||||||
|
total_tokens INTEGER,
|
||||||
|
cache_write_tokens INTEGER,
|
||||||
|
cache_read_tokens INTEGER,
|
||||||
|
cache_hit INTEGER,
|
||||||
|
cache_savings_pct INTEGER,
|
||||||
|
estimated_cost_usd REAL,
|
||||||
|
provider TEXT,
|
||||||
|
mode TEXT,
|
||||||
|
workflow_key TEXT,
|
||||||
|
workflow_node_id INTEGER,
|
||||||
|
latency_ms INTEGER,
|
||||||
|
success INTEGER,
|
||||||
|
error TEXT,
|
||||||
|
dataset_id TEXT,
|
||||||
|
scenario_id TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_llm_chats_trace ON llm_chats(trace_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_llm_chats_scenario ON llm_chats(scenario_id);
|
||||||
|
`);
|
||||||
|
|
||||||
|
const columns = db.prepare(`PRAGMA table_info(llm_chats);`).all() as { name: string }[];
|
||||||
|
const columnNames = new Set(columns.map(column => column.name));
|
||||||
|
if (!columnNames.has('system_message')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN system_message TEXT;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('cache_write_tokens')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_write_tokens INTEGER;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('cache_read_tokens')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_read_tokens INTEGER;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('cache_hit')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_hit INTEGER;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('cache_savings_pct')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN cache_savings_pct INTEGER;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('estimated_cost_usd')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN estimated_cost_usd REAL;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('provider')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN provider TEXT;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('mode')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN mode TEXT;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('workflow_key')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN workflow_key TEXT;`);
|
||||||
|
}
|
||||||
|
if (!columnNames.has('workflow_node_id')) {
|
||||||
|
db.exec(`ALTER TABLE llm_chats ADD COLUMN workflow_node_id INTEGER;`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDb() {
|
||||||
|
if (!EVALS_LOG_ENABLED) return null;
|
||||||
|
if (evalsDb) return evalsDb;
|
||||||
|
|
||||||
|
fs.mkdirSync(LOG_DIR, { recursive: true });
|
||||||
|
evalsDb = new Database(DB_PATH);
|
||||||
|
evalsDb.pragma('journal_mode = WAL');
|
||||||
|
evalsDb.pragma('synchronous = NORMAL');
|
||||||
|
evalsDb.pragma('busy_timeout = 3000');
|
||||||
|
ensureSchema(evalsDb);
|
||||||
|
return evalsDb;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringifySafe(value: unknown) {
|
||||||
|
if (value === undefined) return null;
|
||||||
|
try {
|
||||||
|
return JSON.stringify(value);
|
||||||
|
} catch (error) {
|
||||||
|
return JSON.stringify({
|
||||||
|
error: 'Failed to serialize payload',
|
||||||
|
message: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function logEvalToolCall(entry: EvalToolCallLog) {
|
||||||
|
if (!shouldLogEvals()) return;
|
||||||
|
const context = RequestContext.get();
|
||||||
|
const traceId = context.traceId;
|
||||||
|
if (!traceId) return;
|
||||||
|
|
||||||
|
const db = getDb();
|
||||||
|
if (!db) return;
|
||||||
|
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const spanId = randomUUID();
|
||||||
|
const parentSpanId = context.evalChatSpanId || null;
|
||||||
|
const helperName = context.helperName || null;
|
||||||
|
const datasetId = context.evalDatasetId || null;
|
||||||
|
const scenarioId = context.evalScenarioId || null;
|
||||||
|
const errorMessage = entry.error instanceof Error
|
||||||
|
? entry.error.message
|
||||||
|
: entry.error
|
||||||
|
? String(entry.error)
|
||||||
|
: null;
|
||||||
|
const success = typeof entry.error === 'undefined'
|
||||||
|
? entry.result && typeof (entry.result as any).success === 'boolean'
|
||||||
|
? ((entry.result as any).success ? 1 : 0)
|
||||||
|
: 1
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
db.prepare(`
|
||||||
|
INSERT INTO tool_calls (
|
||||||
|
ts, trace_id, span_id, parent_span_id, helper_name,
|
||||||
|
tool_name, args_json, result_json, success, latency_ms, error,
|
||||||
|
dataset_id, scenario_id
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
`).run(
|
||||||
|
now,
|
||||||
|
traceId,
|
||||||
|
spanId,
|
||||||
|
parentSpanId,
|
||||||
|
helperName,
|
||||||
|
entry.toolName,
|
||||||
|
stringifySafe(entry.args),
|
||||||
|
stringifySafe(entry.result),
|
||||||
|
success,
|
||||||
|
entry.latencyMs ?? null,
|
||||||
|
errorMessage,
|
||||||
|
datasetId,
|
||||||
|
scenarioId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function logEvalChat(entry: EvalChatLog) {
|
||||||
|
if (!shouldLogEvals()) return;
|
||||||
|
const context = RequestContext.get();
|
||||||
|
const traceId = entry.traceId || context.traceId;
|
||||||
|
if (!traceId) return;
|
||||||
|
|
||||||
|
const db = getDb();
|
||||||
|
if (!db) return;
|
||||||
|
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const spanId = entry.spanId || context.evalChatSpanId || randomUUID();
|
||||||
|
const datasetId = context.evalDatasetId || null;
|
||||||
|
const scenarioId = context.evalScenarioId || null;
|
||||||
|
|
||||||
|
db.prepare(`
|
||||||
|
INSERT INTO llm_chats (
|
||||||
|
ts, trace_id, span_id, helper_name, model, prompt_version, system_message,
|
||||||
|
user_message, assistant_message, input_tokens, output_tokens, total_tokens,
|
||||||
|
cache_write_tokens, cache_read_tokens, cache_hit, cache_savings_pct,
|
||||||
|
estimated_cost_usd, provider, mode, workflow_key, workflow_node_id,
|
||||||
|
latency_ms, success, error, dataset_id, scenario_id
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
`).run(
|
||||||
|
now,
|
||||||
|
traceId,
|
||||||
|
spanId,
|
||||||
|
entry.helperName ?? null,
|
||||||
|
entry.model ?? null,
|
||||||
|
entry.promptVersion ?? null,
|
||||||
|
entry.systemMessage ?? null,
|
||||||
|
entry.userMessage ?? null,
|
||||||
|
entry.assistantMessage ?? null,
|
||||||
|
entry.inputTokens ?? null,
|
||||||
|
entry.outputTokens ?? null,
|
||||||
|
entry.totalTokens ?? null,
|
||||||
|
entry.cacheWriteTokens ?? null,
|
||||||
|
entry.cacheReadTokens ?? null,
|
||||||
|
typeof entry.cacheHit === 'boolean' ? (entry.cacheHit ? 1 : 0) : null,
|
||||||
|
entry.cacheSavingsPct ?? null,
|
||||||
|
entry.estimatedCostUsd ?? null,
|
||||||
|
entry.provider ?? null,
|
||||||
|
entry.mode ?? null,
|
||||||
|
entry.workflowKey ?? null,
|
||||||
|
entry.workflowNodeId ?? null,
|
||||||
|
entry.latencyMs ?? null,
|
||||||
|
typeof entry.success === 'boolean' ? (entry.success ? 1 : 0) : null,
|
||||||
|
entry.error ?? null,
|
||||||
|
datasetId,
|
||||||
|
scenarioId
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
import Database from 'better-sqlite3';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
export type EvalChatRow = {
|
||||||
|
id: number;
|
||||||
|
ts: string;
|
||||||
|
trace_id: string;
|
||||||
|
span_id: string | null;
|
||||||
|
helper_name: string | null;
|
||||||
|
model: string | null;
|
||||||
|
prompt_version: string | null;
|
||||||
|
system_message: string | null;
|
||||||
|
user_message: string | null;
|
||||||
|
assistant_message: string | null;
|
||||||
|
input_tokens: number | null;
|
||||||
|
output_tokens: number | null;
|
||||||
|
total_tokens: number | null;
|
||||||
|
cache_write_tokens: number | null;
|
||||||
|
cache_read_tokens: number | null;
|
||||||
|
cache_hit: number | null;
|
||||||
|
cache_savings_pct: number | null;
|
||||||
|
estimated_cost_usd: number | null;
|
||||||
|
provider: string | null;
|
||||||
|
mode: string | null;
|
||||||
|
workflow_key: string | null;
|
||||||
|
workflow_node_id: number | null;
|
||||||
|
latency_ms: number | null;
|
||||||
|
success: number | null;
|
||||||
|
error: string | null;
|
||||||
|
dataset_id: string | null;
|
||||||
|
scenario_id: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type EvalToolCallRow = {
|
||||||
|
id: number;
|
||||||
|
ts: string;
|
||||||
|
trace_id: string;
|
||||||
|
span_id: string | null;
|
||||||
|
parent_span_id: string | null;
|
||||||
|
helper_name: string | null;
|
||||||
|
tool_name: string;
|
||||||
|
args_json: string | null;
|
||||||
|
result_json: string | null;
|
||||||
|
success: number | null;
|
||||||
|
latency_ms: number | null;
|
||||||
|
error: string | null;
|
||||||
|
dataset_id: string | null;
|
||||||
|
scenario_id: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type EvalTrace = {
|
||||||
|
chat: EvalChatRow;
|
||||||
|
toolCalls: EvalToolCallRow[];
|
||||||
|
comment: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
|
||||||
|
|
||||||
|
function ensureCommentSchema(db: Database.Database) {
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS eval_comments (
|
||||||
|
trace_id TEXT PRIMARY KEY,
|
||||||
|
scenario_id TEXT,
|
||||||
|
comment TEXT,
|
||||||
|
updated_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_eval_comments_scenario ON eval_comments(scenario_id);
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function upsertEvalComment(traceId: string, scenarioId: string | null, comment: string) {
|
||||||
|
if (!fs.existsSync(LOG_DB_PATH)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const db = new Database(LOG_DB_PATH);
|
||||||
|
ensureCommentSchema(db);
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
db.prepare(`
|
||||||
|
INSERT INTO eval_comments (trace_id, scenario_id, comment, updated_at)
|
||||||
|
VALUES (?, ?, ?, ?)
|
||||||
|
ON CONFLICT(trace_id) DO UPDATE SET
|
||||||
|
comment = excluded.comment,
|
||||||
|
updated_at = excluded.updated_at,
|
||||||
|
scenario_id = excluded.scenario_id
|
||||||
|
`).run(traceId, scenarioId, comment, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function loadEvalTraces(limit = 25): EvalTrace[] {
|
||||||
|
if (!fs.existsSync(LOG_DB_PATH)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const db = new Database(LOG_DB_PATH, { readonly: true, fileMustExist: true });
|
||||||
|
const chats = db.prepare(`
|
||||||
|
SELECT *
|
||||||
|
FROM llm_chats
|
||||||
|
ORDER BY ts DESC
|
||||||
|
LIMIT ?
|
||||||
|
`).all(limit) as EvalChatRow[];
|
||||||
|
|
||||||
|
if (chats.length === 0) return [];
|
||||||
|
|
||||||
|
const traceIds = Array.from(new Set(chats.map(chat => chat.trace_id)));
|
||||||
|
const placeholders = traceIds.map(() => '?').join(', ');
|
||||||
|
const toolCalls = db.prepare(`
|
||||||
|
SELECT *
|
||||||
|
FROM tool_calls
|
||||||
|
WHERE trace_id IN (${placeholders})
|
||||||
|
ORDER BY ts ASC
|
||||||
|
`).all(...traceIds) as EvalToolCallRow[];
|
||||||
|
|
||||||
|
const toolCallsByTrace = new Map<string, EvalToolCallRow[]>();
|
||||||
|
toolCalls.forEach(call => {
|
||||||
|
const list = toolCallsByTrace.get(call.trace_id) || [];
|
||||||
|
list.push(call);
|
||||||
|
toolCallsByTrace.set(call.trace_id, list);
|
||||||
|
});
|
||||||
|
|
||||||
|
const commentsByTrace = new Map<string, string | null>();
|
||||||
|
if (placeholders.length > 0) {
|
||||||
|
try {
|
||||||
|
const commentRows = db.prepare(`
|
||||||
|
SELECT trace_id, comment
|
||||||
|
FROM eval_comments
|
||||||
|
WHERE trace_id IN (${placeholders})
|
||||||
|
`).all(...traceIds) as Array<{ trace_id: string; comment: string | null }>;
|
||||||
|
commentRows.forEach(row => {
|
||||||
|
commentsByTrace.set(row.trace_id, row.comment ?? null);
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Comments table doesn't exist yet in readonly mode.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return chats.map(chat => ({
|
||||||
|
chat,
|
||||||
|
toolCalls: toolCallsByTrace.get(chat.trace_id) || [],
|
||||||
|
comment: commentsByTrace.get(chat.trace_id) ?? null,
|
||||||
|
}));
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"id": "golden-v1",
|
||||||
|
"name": "Golden Dataset v1",
|
||||||
|
"description": "Baseline eval scenarios for core RA-H helper behavior."
|
||||||
|
}
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
import Database from 'better-sqlite3';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { scenarios } from './scenarios';
|
||||||
|
import { Scenario } from './types';
|
||||||
|
|
||||||
|
type EvalChatRow = {
|
||||||
|
trace_id: string;
|
||||||
|
scenario_id: string;
|
||||||
|
assistant_message: string | null;
|
||||||
|
latency_ms: number | null;
|
||||||
|
input_tokens: number | null;
|
||||||
|
output_tokens: number | null;
|
||||||
|
total_tokens: number | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
type EvalToolCallRow = {
|
||||||
|
tool_name: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type EvalResult = {
|
||||||
|
scenario: string;
|
||||||
|
passed: boolean;
|
||||||
|
failures: string[];
|
||||||
|
warnings: string[];
|
||||||
|
latencyMs?: number | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const BASE_URL = process.env.RAH_EVALS_BASE_URL || 'http://localhost:3000';
|
||||||
|
const DATASET_ENV = process.env.RAH_EVALS_DATASET_ID;
|
||||||
|
const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
|
||||||
|
const RAH_DB_PATH = process.env.SQLITE_DB_PATH || path.join(
|
||||||
|
process.env.HOME || '~',
|
||||||
|
'Library/Application Support/RA-H/db/rah.sqlite'
|
||||||
|
);
|
||||||
|
|
||||||
|
function loadDatasetId() {
|
||||||
|
if (DATASET_ENV) return DATASET_ENV;
|
||||||
|
const datasetPath = path.join(process.cwd(), 'tests', 'evals', 'dataset.json');
|
||||||
|
const raw = fs.readFileSync(datasetPath, 'utf-8');
|
||||||
|
const parsed = JSON.parse(raw);
|
||||||
|
return parsed.id || 'default';
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): number | null {
|
||||||
|
if (!query) return null;
|
||||||
|
if (!fs.existsSync(RAH_DB_PATH)) return null;
|
||||||
|
const db = new Database(RAH_DB_PATH, { readonly: true, fileMustExist: true });
|
||||||
|
if (query.titleEquals) {
|
||||||
|
const row = db.prepare(`
|
||||||
|
SELECT id
|
||||||
|
FROM nodes
|
||||||
|
WHERE title = ?
|
||||||
|
ORDER BY datetime(created_at) DESC
|
||||||
|
LIMIT 1
|
||||||
|
`).get(query.titleEquals) as { id?: number } | undefined;
|
||||||
|
return row?.id ?? null;
|
||||||
|
}
|
||||||
|
if (query.titleContains) {
|
||||||
|
const row = db.prepare(`
|
||||||
|
SELECT id
|
||||||
|
FROM nodes
|
||||||
|
WHERE title LIKE ?
|
||||||
|
ORDER BY datetime(created_at) DESC
|
||||||
|
LIMIT 1
|
||||||
|
`).get(`%${query.titleContains}%`) as { id?: number } | undefined;
|
||||||
|
return row?.id ?? null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function drainResponse(response: Response) {
|
||||||
|
if (!response.body) return;
|
||||||
|
const reader = response.body.getReader();
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
while (true) {
|
||||||
|
const { value, done } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
decoder.decode(value, { stream: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function openEvalDb() {
|
||||||
|
if (!fs.existsSync(LOG_DB_PATH)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new Database(LOG_DB_PATH, { readonly: true, fileMustExist: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
function getEvalChatRow(db: Database.Database, traceId: string, scenarioId: string) {
|
||||||
|
return db.prepare(`
|
||||||
|
SELECT trace_id, scenario_id, assistant_message, latency_ms, input_tokens, output_tokens, total_tokens
|
||||||
|
FROM llm_chats
|
||||||
|
WHERE trace_id = ? AND scenario_id = ?
|
||||||
|
ORDER BY id DESC
|
||||||
|
LIMIT 1
|
||||||
|
`).get(traceId, scenarioId) as EvalChatRow | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getEvalToolCalls(db: Database.Database, traceId: string, scenarioId: string) {
|
||||||
|
return db.prepare(`
|
||||||
|
SELECT tool_name
|
||||||
|
FROM tool_calls
|
||||||
|
WHERE trace_id = ? AND scenario_id = ?
|
||||||
|
`).all(traceId, scenarioId) as EvalToolCallRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForEvalRow(traceId: string, scenarioId: string, timeoutMs = 10000) {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
while (Date.now() - startedAt < timeoutMs) {
|
||||||
|
const db = openEvalDb();
|
||||||
|
if (db) {
|
||||||
|
const row = getEvalChatRow(db, traceId, scenarioId);
|
||||||
|
if (row) return { row, db };
|
||||||
|
}
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 200));
|
||||||
|
}
|
||||||
|
return { row: undefined, db: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeContains(text: string) {
|
||||||
|
return text.toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkScenario(
|
||||||
|
scenario: Scenario,
|
||||||
|
chatRow: EvalChatRow,
|
||||||
|
toolCalls: EvalToolCallRow[]
|
||||||
|
): EvalResult {
|
||||||
|
const failures: string[] = [];
|
||||||
|
const warnings: string[] = [];
|
||||||
|
const expect = scenario.expect || {};
|
||||||
|
const toolNames = toolCalls.map(call => call.tool_name);
|
||||||
|
const responseText = chatRow.assistant_message || '';
|
||||||
|
|
||||||
|
(expect.toolsCalled || []).forEach(tool => {
|
||||||
|
if (!toolNames.includes(tool)) {
|
||||||
|
failures.push(`Expected tool "${tool}" not called`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
(expect.toolsCalledSoft || []).forEach(tool => {
|
||||||
|
if (!toolNames.includes(tool)) {
|
||||||
|
warnings.push(`(soft) Expected tool "${tool}" not called`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
(expect.responseContains || []).forEach(text => {
|
||||||
|
if (!normalizeContains(responseText).includes(normalizeContains(text))) {
|
||||||
|
failures.push(`Response missing "${text}"`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
(expect.responseContainsSoft || []).forEach(text => {
|
||||||
|
if (!normalizeContains(responseText).includes(normalizeContains(text))) {
|
||||||
|
warnings.push(`(soft) Response missing "${text}"`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
(expect.responseNotContains || []).forEach(text => {
|
||||||
|
if (normalizeContains(responseText).includes(normalizeContains(text))) {
|
||||||
|
failures.push(`Response should not contain "${text}"`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (typeof expect.maxLatencyMs === 'number' && chatRow.latency_ms !== null) {
|
||||||
|
if (chatRow.latency_ms > expect.maxLatencyMs) {
|
||||||
|
failures.push(`Latency ${chatRow.latency_ms}ms exceeded ${expect.maxLatencyMs}ms`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
scenario: scenario.name,
|
||||||
|
passed: failures.length === 0,
|
||||||
|
failures,
|
||||||
|
warnings,
|
||||||
|
latencyMs: chatRow.latency_ms,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runScenario(scenario: Scenario, datasetId: string): Promise<EvalResult> {
|
||||||
|
const traceId = `eval_${Date.now()}_${randomUUID().slice(0, 8)}`;
|
||||||
|
const resolvedFocusedNodeId =
|
||||||
|
scenario.input.focusedNodeId ?? resolveFocusedNodeId(scenario.input.focusedNodeQuery);
|
||||||
|
const response = await fetch(`${BASE_URL}/api/rah/chat`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
messages: [{ role: 'user', content: scenario.input.message }],
|
||||||
|
openTabs: [],
|
||||||
|
activeTabId: resolvedFocusedNodeId ?? null,
|
||||||
|
currentView: 'nodes',
|
||||||
|
mode: scenario.input.mode ?? 'easy',
|
||||||
|
traceId,
|
||||||
|
evals: {
|
||||||
|
datasetId,
|
||||||
|
scenarioId: scenario.id,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
return {
|
||||||
|
scenario: scenario.name,
|
||||||
|
passed: false,
|
||||||
|
failures: [`HTTP ${response.status} from /api/rah/chat`],
|
||||||
|
warnings: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
await drainResponse(response);
|
||||||
|
|
||||||
|
const { row, db } = await waitForEvalRow(traceId, scenario.id);
|
||||||
|
if (!row) {
|
||||||
|
return {
|
||||||
|
scenario: scenario.name,
|
||||||
|
passed: false,
|
||||||
|
failures: ['Timed out waiting for eval logs'],
|
||||||
|
warnings: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const toolCalls = db ? getEvalToolCalls(db, traceId, scenario.id) : [];
|
||||||
|
return checkScenario(scenario, row, toolCalls);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runAll() {
|
||||||
|
const datasetId = loadDatasetId();
|
||||||
|
console.log(`Running ${scenarios.length} scenarios (dataset: ${datasetId})...\n`);
|
||||||
|
|
||||||
|
const results: EvalResult[] = [];
|
||||||
|
for (const scenario of scenarios.filter(s => s.enabled !== false)) {
|
||||||
|
const result = await runScenario(scenario, datasetId);
|
||||||
|
results.push(result);
|
||||||
|
const icon = result.passed ? '✓' : '✗';
|
||||||
|
const latency = result.latencyMs ? ` (${result.latencyMs}ms)` : '';
|
||||||
|
console.log(`${icon} ${result.scenario}${latency}`);
|
||||||
|
result.failures.forEach(failure => console.log(` - ${failure}`));
|
||||||
|
result.warnings.forEach(warning => console.log(` - ${warning}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
const failed = results.filter(result => !result.passed);
|
||||||
|
const warnings = results.filter(result => result.warnings.length > 0);
|
||||||
|
console.log('\nSummary');
|
||||||
|
console.log(`- Passed: ${results.length - failed.length}`);
|
||||||
|
console.log(`- Failed: ${failed.length}`);
|
||||||
|
console.log(`- With warnings: ${warnings.length}`);
|
||||||
|
|
||||||
|
if (failed.length > 0) {
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
runAll().catch(error => {
|
||||||
|
console.error('Eval runner failed:', error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'create-edge',
|
||||||
|
name: 'Create edge between two new nodes',
|
||||||
|
description: 'Create two nodes and connect them with an edge.',
|
||||||
|
tools: ['createNode', 'createEdge'],
|
||||||
|
input: {
|
||||||
|
message: 'Create nodes titled "Eval Edge A" and "Eval Edge B" (one sentence each), then create an edge from A to B labeled "related".',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['createNode', 'createEdge'],
|
||||||
|
responseContainsSoft: ['edge', 'connected'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'create-node',
|
||||||
|
name: 'Create node request',
|
||||||
|
description: 'Create a new node via chat.',
|
||||||
|
tools: ['createNode'],
|
||||||
|
input: {
|
||||||
|
message: 'Create a node titled "Eval: Test Node" with a short summary about evals.',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['createNode'],
|
||||||
|
responseContainsSoft: ['node', 'created', 'added'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'delegate-mini',
|
||||||
|
name: 'Delegate to Mini RAH',
|
||||||
|
description: 'Ask the system to delegate a short task to mini helper.',
|
||||||
|
tools: ['delegateToMiniRAH'],
|
||||||
|
input: {
|
||||||
|
message: 'Delegate to mini RAH: summarize my notes on plaintext productivity in 3 bullets.',
|
||||||
|
mode: 'hard',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['delegateToMiniRAH'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'delegate-wise',
|
||||||
|
name: 'Delegate to Wise RAH',
|
||||||
|
description: 'Ask the system to delegate a research comparison task.',
|
||||||
|
tools: ['delegateToWiseRAH'],
|
||||||
|
input: {
|
||||||
|
message: 'Delegate to wise RAH: compare SQLite vs markdown storage for PKM in 5 bullets.',
|
||||||
|
mode: 'hard',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['delegateToWiseRAH'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'dimension-lifecycle',
|
||||||
|
name: 'Dimension lifecycle (create/update/lock/unlock)',
|
||||||
|
description: 'Create a dimension, update its description, lock it, then unlock it.',
|
||||||
|
tools: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
|
||||||
|
input: {
|
||||||
|
message: 'Create a dimension named "eval-dim" with description "temporary eval dimension", then update the description to "eval dimension updated", then lock it, then unlock it.',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['createDimension', 'updateDimension', 'lockDimension', 'unlockDimension'],
|
||||||
|
responseContainsSoft: ['eval-dim'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'get-dimension',
|
||||||
|
name: 'Get single dimension',
|
||||||
|
description: 'Fetch detail for a known dimension.',
|
||||||
|
tools: ['getDimension'],
|
||||||
|
input: {
|
||||||
|
message: 'Get details for the dimension "ai".',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['getDimension'],
|
||||||
|
responseContainsSoft: ['ai'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'hard-mode-query',
|
||||||
|
name: 'Hard mode retrieval query',
|
||||||
|
description: 'Run a baseline retrieval query in hard mode.',
|
||||||
|
tools: ['queryNodes', 'searchContentEmbeddings'],
|
||||||
|
input: {
|
||||||
|
message: 'What have I captured about plaintext productivity and tools?',
|
||||||
|
mode: 'hard',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['queryNodes'],
|
||||||
|
responseContainsSoft: ['plaintext', 'productivity'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { scenario as simpleQuery } from './simple-query';
|
||||||
|
import { scenario as searchEmbeddings } from './search-embeddings';
|
||||||
|
import { scenario as createNode } from './create-node';
|
||||||
|
import { scenario as hardModeQuery } from './hard-mode-query';
|
||||||
|
import { scenario as workflowIntegrate } from './workflow-integrate';
|
||||||
|
import { scenario as updateNode } from './update-node';
|
||||||
|
import { scenario as createEdge } from './create-edge';
|
||||||
|
import { scenario as queryDimensions } from './query-dimensions';
|
||||||
|
import { scenario as getDimension } from './get-dimension';
|
||||||
|
import { scenario as dimensionLifecycle } from './dimension-lifecycle';
|
||||||
|
import { scenario as delegateMini } from './delegate-mini';
|
||||||
|
import { scenario as delegateWise } from './delegate-wise';
|
||||||
|
import { scenario as youtubeExtract } from './youtube-extract';
|
||||||
|
import { scenario as websiteExtract } from './website-extract';
|
||||||
|
import { scenario as paperExtract } from './paper-extract';
|
||||||
|
|
||||||
|
export const scenarios = [
|
||||||
|
simpleQuery,
|
||||||
|
searchEmbeddings,
|
||||||
|
createNode,
|
||||||
|
updateNode,
|
||||||
|
createEdge,
|
||||||
|
queryDimensions,
|
||||||
|
getDimension,
|
||||||
|
dimensionLifecycle,
|
||||||
|
hardModeQuery,
|
||||||
|
workflowIntegrate,
|
||||||
|
delegateMini,
|
||||||
|
delegateWise,
|
||||||
|
youtubeExtract,
|
||||||
|
websiteExtract,
|
||||||
|
paperExtract,
|
||||||
|
];
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'paper-extract',
|
||||||
|
name: 'Paper extract (optional)',
|
||||||
|
description: 'Extract and ingest a PDF (requires network).',
|
||||||
|
tools: ['paperExtract'],
|
||||||
|
enabled: false,
|
||||||
|
notes: 'Enable when network is available; can be slow.',
|
||||||
|
input: {
|
||||||
|
message: 'Add this paper: https://arxiv.org/pdf/1706.03762.pdf',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['paperExtract'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'query-dimensions',
|
||||||
|
name: 'Query dimensions',
|
||||||
|
description: 'List available dimensions and basic info.',
|
||||||
|
tools: ['queryDimensions'],
|
||||||
|
input: {
|
||||||
|
message: 'List my top dimensions and briefly describe what they represent.',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['queryDimensions'],
|
||||||
|
responseContainsSoft: ['dimension'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'search-embeddings',
|
||||||
|
name: 'Embedding search triggers retrieval',
|
||||||
|
description: 'Semantic search over stored knowledge.',
|
||||||
|
tools: ['searchContentEmbeddings', 'queryNodes'],
|
||||||
|
input: {
|
||||||
|
message: 'Find my notes about deep learning architectures.',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['searchContentEmbeddings'],
|
||||||
|
responseContainsSoft: ['found', 'node'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'simple-query',
|
||||||
|
name: 'Simple query routes to helper',
|
||||||
|
description: 'Baseline retrieval query against existing notes.',
|
||||||
|
tools: ['queryNodes'],
|
||||||
|
input: {
|
||||||
|
message: 'What do I know about machine learning?',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['queryNodes'],
|
||||||
|
responseContainsSoft: ['node', 'found'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'update-node',
|
||||||
|
name: 'Update node content',
|
||||||
|
description: 'Create and then update a node in one request.',
|
||||||
|
tools: ['createNode', 'updateNode'],
|
||||||
|
input: {
|
||||||
|
message: 'Create a node titled "Eval: Update Node" with one sentence, then append one more sentence to it.',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['createNode', 'updateNode'],
|
||||||
|
responseContainsSoft: ['updated', 'appended'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'website-extract',
|
||||||
|
name: 'Website extract (optional)',
|
||||||
|
description: 'Extract and ingest a webpage (requires network).',
|
||||||
|
tools: ['websiteExtract'],
|
||||||
|
enabled: false,
|
||||||
|
notes: 'Enable when network is available; can be slow.',
|
||||||
|
input: {
|
||||||
|
message: 'Add this article: https://sive.rs/plaintext',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['websiteExtract'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'workflow-integrate',
|
||||||
|
name: 'Integrate workflow on focused node',
|
||||||
|
description: 'Execute integrate workflow on a real node.',
|
||||||
|
tools: ['executeWorkflow'],
|
||||||
|
input: {
|
||||||
|
message: 'Integrate this.',
|
||||||
|
focusedNodeQuery: { titleContains: 'Markdown vs database backends for PKM' },
|
||||||
|
mode: 'hard',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['executeWorkflow'],
|
||||||
|
responseContainsSoft: ['integrate', 'updated'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { Scenario } from '../types';
|
||||||
|
|
||||||
|
export const scenario: Scenario = {
|
||||||
|
id: 'youtube-extract',
|
||||||
|
name: 'YouTube extract (optional)',
|
||||||
|
description: 'Extract and ingest a YouTube video (requires network + API keys).',
|
||||||
|
tools: ['youtubeExtract'],
|
||||||
|
enabled: false,
|
||||||
|
notes: 'Enable when network + API keys are configured; can be flaky.',
|
||||||
|
input: {
|
||||||
|
message: 'Add this video: https://www.youtube.com/watch?v=dQw4w9WgXcQ',
|
||||||
|
},
|
||||||
|
expect: {
|
||||||
|
toolsCalledSoft: ['youtubeExtract'],
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
export type ScenarioExpectations = {
|
||||||
|
toolsCalled?: string[];
|
||||||
|
toolsCalledSoft?: string[];
|
||||||
|
responseContains?: string[];
|
||||||
|
responseContainsSoft?: string[];
|
||||||
|
responseNotContains?: string[];
|
||||||
|
maxLatencyMs?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ScenarioInput = {
|
||||||
|
message: string;
|
||||||
|
focusedNodeId?: number;
|
||||||
|
focusedNodeQuery?: {
|
||||||
|
titleContains?: string;
|
||||||
|
titleEquals?: string;
|
||||||
|
};
|
||||||
|
mode?: 'easy' | 'hard';
|
||||||
|
};
|
||||||
|
|
||||||
|
export type Scenario = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
input: ScenarioInput;
|
||||||
|
expect?: ScenarioExpectations;
|
||||||
|
description?: string;
|
||||||
|
tools?: string[];
|
||||||
|
enabled?: boolean;
|
||||||
|
notes?: string;
|
||||||
|
};
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { defineConfig } from 'vitest/config';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
test: {
|
||||||
|
environment: 'node',
|
||||||
|
include: ['tests/**/*.test.ts'],
|
||||||
|
globals: true,
|
||||||
|
setupFiles: ['tests/unit/setup.ts'],
|
||||||
|
},
|
||||||
|
resolve: {
|
||||||
|
alias: {
|
||||||
|
'@': path.resolve(__dirname, './src'),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user