sync: bug fixes + eval system from private repo
- Fix chunk_status: pass chunk_status/chunk/metadata to context builder - Fix vector search: scope by node_id BEFORE similarity search - Add eval logging system (RAH_EVALS_LOG=1) - Add eval dashboard at /evals - Add vitest for testing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
“BeeRad”
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
e15f223ed8
commit
2f2ef10ec9
@@ -0,0 +1,376 @@
|
||||
'use client';
|
||||
|
||||
import { useMemo, useState } from 'react';
|
||||
import type { EvalTrace } from '@/services/evals/evalsStore';
|
||||
|
||||
type Props = {
|
||||
traces: EvalTrace[];
|
||||
scenarioList: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
tools?: string[];
|
||||
enabled?: boolean;
|
||||
notes?: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
function formatPreview(text: string | null, max = 80) {
|
||||
if (!text) return '';
|
||||
const compact = text.replace(/\s+/g, ' ').trim();
|
||||
return compact.length > max ? `${compact.slice(0, max)}...` : compact;
|
||||
}
|
||||
|
||||
function statusLabel(success: number | null) {
|
||||
if (success === null) return 'n/a';
|
||||
return success ? 'success' : 'fail';
|
||||
}
|
||||
|
||||
function badgeStyle(kind: 'success' | 'fail' | 'neutral') {
|
||||
const base = { display: 'inline-block', padding: '2px 8px', borderRadius: 999, fontSize: 12 };
|
||||
if (kind === 'success') return { ...base, background: '#e6f4ea', color: '#146c2e' };
|
||||
if (kind === 'fail') return { ...base, background: '#fdecea', color: '#b42318' };
|
||||
return { ...base, background: '#f2f2f2', color: '#333' };
|
||||
}
|
||||
|
||||
function prettyJson(value: string | null) {
|
||||
if (!value) return '';
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(value), null, 2);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
export default function EvalsClient({ traces, scenarioList }: Props) {
|
||||
const [openTraceId, setOpenTraceId] = useState<string | null>(traces[0]?.chat.trace_id || null);
|
||||
const [comments, setComments] = useState<Record<string, string>>(() => {
|
||||
const initial: Record<string, string> = {};
|
||||
traces.forEach((trace) => {
|
||||
if (trace.comment) {
|
||||
initial[trace.chat.trace_id] = trace.comment;
|
||||
}
|
||||
});
|
||||
return initial;
|
||||
});
|
||||
const [sourceFilter, setSourceFilter] = useState<string>('all'); // 'all' | 'live' | 'scenario'
|
||||
const [scenarioFilter, setScenarioFilter] = useState<string>('all');
|
||||
const [statusFilter, setStatusFilter] = useState<string>('all');
|
||||
const [searchFilter, setSearchFilter] = useState<string>('');
|
||||
|
||||
const rows = useMemo(() => {
|
||||
return traces.map((trace) => {
|
||||
const { chat, toolCalls } = trace;
|
||||
const status = statusLabel(chat.success);
|
||||
const isLive = !chat.scenario_id;
|
||||
return {
|
||||
trace,
|
||||
id: chat.trace_id,
|
||||
source: isLive ? 'live' : 'scenario',
|
||||
scenario: chat.scenario_id || '—',
|
||||
model: chat.model || 'n/a',
|
||||
latency: chat.latency_ms ?? 'n/a',
|
||||
tokens: `${chat.input_tokens ?? 0}/${chat.output_tokens ?? 0} (${chat.total_tokens ?? 0})`,
|
||||
cost: chat.estimated_cost_usd ?? null,
|
||||
cache: chat.cache_hit == null ? 'n/a' : chat.cache_hit ? 'hit' : 'miss',
|
||||
cacheTokens: `${chat.cache_read_tokens ?? 0}/${chat.cache_write_tokens ?? 0}`,
|
||||
toolCount: toolCalls.length,
|
||||
status,
|
||||
userPreview: formatPreview(chat.user_message),
|
||||
timestamp: chat.ts,
|
||||
mode: chat.mode || 'n/a',
|
||||
workflow: chat.workflow_key || '—',
|
||||
};
|
||||
});
|
||||
}, [traces]);
|
||||
|
||||
const filteredRows = useMemo(() => {
|
||||
return rows.filter((row) => {
|
||||
if (sourceFilter !== 'all' && row.source !== sourceFilter) return false;
|
||||
if (scenarioFilter !== 'all' && row.scenario !== scenarioFilter) return false;
|
||||
if (statusFilter !== 'all' && row.status !== statusFilter) return false;
|
||||
if (searchFilter.trim()) {
|
||||
const needle = searchFilter.toLowerCase();
|
||||
if (!row.userPreview.toLowerCase().includes(needle)) return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}, [rows, sourceFilter, scenarioFilter, statusFilter, searchFilter]);
|
||||
|
||||
const openTrace = traces.find((trace) => trace.chat.trace_id === openTraceId) || traces[0];
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div style={{ marginBottom: 24, border: '1px solid #ddd', borderRadius: 8, background: '#fff', padding: 16 }}>
|
||||
<div style={{ fontWeight: 600, marginBottom: 8 }}>Scenario Set</div>
|
||||
<div style={{ overflowX: 'auto' }}>
|
||||
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}>
|
||||
<thead>
|
||||
<tr style={{ textAlign: 'left', borderBottom: '1px solid #ddd' }}>
|
||||
<th style={{ padding: '6px 6px' }}>ID</th>
|
||||
<th style={{ padding: '6px 6px' }}>Name</th>
|
||||
<th style={{ padding: '6px 6px' }}>Description</th>
|
||||
<th style={{ padding: '6px 6px' }}>Tools</th>
|
||||
<th style={{ padding: '6px 6px' }}>Enabled</th>
|
||||
<th style={{ padding: '6px 6px' }}>Notes</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{scenarioList.map((scenario) => (
|
||||
<tr key={scenario.id} style={{ borderBottom: '1px solid #eee' }}>
|
||||
<td style={{ padding: '6px 6px' }}>{scenario.id}</td>
|
||||
<td style={{ padding: '6px 6px' }}>{scenario.name}</td>
|
||||
<td style={{ padding: '6px 6px', color: '#555' }}>{scenario.description || '—'}</td>
|
||||
<td style={{ padding: '6px 6px', color: '#555' }}>{scenario.tools?.join(', ') || '—'}</td>
|
||||
<td style={{ padding: '6px 6px' }}>{scenario.enabled === false ? 'no' : 'yes'}</td>
|
||||
<td style={{ padding: '6px 6px', color: '#555' }}>{scenario.notes || '—'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ overflowX: 'auto' }}>
|
||||
<div style={{ display: 'flex', gap: 12, alignItems: 'center', flexWrap: 'wrap', marginBottom: 12 }}>
|
||||
<label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||
<span>Source</span>
|
||||
<select
|
||||
value={sourceFilter}
|
||||
onChange={(event) => setSourceFilter(event.target.value)}
|
||||
style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd', fontWeight: 500 }}
|
||||
>
|
||||
<option value="all">All</option>
|
||||
<option value="live">🟢 Live Runs</option>
|
||||
<option value="scenario">🔬 Scenarios</option>
|
||||
</select>
|
||||
</label>
|
||||
<label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||
<span>Scenario</span>
|
||||
<select
|
||||
value={scenarioFilter}
|
||||
onChange={(event) => setScenarioFilter(event.target.value)}
|
||||
style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd' }}
|
||||
>
|
||||
<option value="all">All</option>
|
||||
{Array.from(new Set(rows.filter(row => row.scenario !== '—').map(row => row.scenario))).map((scenario) => (
|
||||
<option key={scenario} value={scenario}>{scenario}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||
<span>Status</span>
|
||||
<select
|
||||
value={statusFilter}
|
||||
onChange={(event) => setStatusFilter(event.target.value)}
|
||||
style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd' }}
|
||||
>
|
||||
<option value="all">All</option>
|
||||
<option value="success">Success</option>
|
||||
<option value="fail">Fail</option>
|
||||
<option value="n/a">N/A</option>
|
||||
</select>
|
||||
</label>
|
||||
<label style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||
<span>Search</span>
|
||||
<input
|
||||
value={searchFilter}
|
||||
onChange={(event) => setSearchFilter(event.target.value)}
|
||||
placeholder="User input contains..."
|
||||
style={{ padding: '4px 6px', borderRadius: 6, border: '1px solid #ddd', minWidth: 240 }}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}>
|
||||
<thead>
|
||||
<tr style={{ textAlign: 'left', borderBottom: '1px solid #ddd' }}>
|
||||
<th style={{ padding: '8px 6px' }}>Source</th>
|
||||
<th style={{ padding: '8px 6px' }}>Scenario</th>
|
||||
<th style={{ padding: '8px 6px' }}>Status</th>
|
||||
<th style={{ padding: '8px 6px' }}>Latency</th>
|
||||
<th style={{ padding: '8px 6px' }}>Tokens</th>
|
||||
<th style={{ padding: '8px 6px' }}>Cost</th>
|
||||
<th style={{ padding: '8px 6px' }}>Cache</th>
|
||||
<th style={{ padding: '8px 6px' }}>Tools</th>
|
||||
<th style={{ padding: '8px 6px' }}>Model</th>
|
||||
<th style={{ padding: '8px 6px' }}>Mode</th>
|
||||
<th style={{ padding: '8px 6px' }}>Workflow</th>
|
||||
<th style={{ padding: '8px 6px' }}>User Input</th>
|
||||
<th style={{ padding: '8px 6px' }}>Time</th>
|
||||
<th style={{ padding: '8px 6px' }}>Comment</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filteredRows.map((row) => {
|
||||
const isOpen = row.id === openTraceId;
|
||||
const statusKind = row.status === 'success' ? 'success' : row.status === 'fail' ? 'fail' : 'neutral';
|
||||
const commentValue = comments[row.id] || '';
|
||||
return (
|
||||
<tr
|
||||
key={row.id}
|
||||
onClick={() => setOpenTraceId(row.id)}
|
||||
style={{
|
||||
cursor: 'pointer',
|
||||
background: isOpen ? '#eef4ff' : '#fff',
|
||||
borderBottom: '1px solid #eee',
|
||||
}}
|
||||
>
|
||||
<td style={{ padding: '8px 6px' }}>
|
||||
<span style={{
|
||||
...badgeStyle(row.source === 'live' ? 'success' : 'neutral'),
|
||||
fontSize: 11,
|
||||
}}>
|
||||
{row.source === 'live' ? '🟢 Live' : '🔬 Scenario'}
|
||||
</span>
|
||||
</td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.scenario}</td>
|
||||
<td style={{ padding: '8px 6px' }}><span style={badgeStyle(statusKind)}>{row.status}</span></td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.latency} ms</td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.tokens}</td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.cost == null ? 'n/a' : `$${row.cost.toFixed(4)}`}</td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.cache} ({row.cacheTokens})</td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.toolCount}</td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.model}</td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.mode}</td>
|
||||
<td style={{ padding: '8px 6px' }}>{row.workflow}</td>
|
||||
<td style={{ padding: '8px 6px', color: '#555' }}>{row.userPreview}</td>
|
||||
<td style={{ padding: '8px 6px', color: '#666' }}>{row.timestamp}</td>
|
||||
<td style={{ padding: '8px 6px' }}>
|
||||
<textarea
|
||||
value={commentValue}
|
||||
onChange={(event) => {
|
||||
const nextValue = event.target.value;
|
||||
setComments((prev) => ({ ...prev, [row.id]: nextValue }));
|
||||
}}
|
||||
onBlur={async (event) => {
|
||||
const nextValue = event.target.value;
|
||||
try {
|
||||
await fetch('/api/evals/comment', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
traceId: row.id,
|
||||
scenarioId: row.scenario,
|
||||
comment: nextValue,
|
||||
}),
|
||||
});
|
||||
} catch {
|
||||
// Ignore failures in the UI, keep local state.
|
||||
}
|
||||
}}
|
||||
onClick={(event) => event.stopPropagation()}
|
||||
rows={2}
|
||||
style={{
|
||||
width: 220,
|
||||
resize: 'vertical',
|
||||
fontFamily: 'inherit',
|
||||
fontSize: 12,
|
||||
padding: 6,
|
||||
borderRadius: 6,
|
||||
border: '1px solid #ddd',
|
||||
}}
|
||||
placeholder="Add notes..."
|
||||
/>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{openTrace ? (
|
||||
<div style={{ marginTop: 24, border: '1px solid #ddd', borderRadius: 8, background: '#fff', padding: 16 }}>
|
||||
<div style={{ display: 'flex', gap: 16, flexWrap: 'wrap', marginBottom: 12 }}>
|
||||
<div><strong>Source:</strong> {openTrace.chat.scenario_id ? '🔬 Scenario' : '🟢 Live Run'}</div>
|
||||
<div><strong>Scenario:</strong> {openTrace.chat.scenario_id || '—'}</div>
|
||||
<div><strong>Trace:</strong> {openTrace.chat.trace_id}</div>
|
||||
<div><strong>Helper:</strong> {openTrace.chat.helper_name || 'n/a'}</div>
|
||||
<div><strong>Model:</strong> {openTrace.chat.model || 'n/a'}</div>
|
||||
<div><strong>Mode:</strong> {openTrace.chat.mode || 'n/a'}</div>
|
||||
<div><strong>Workflow:</strong> {openTrace.chat.workflow_key || '—'}</div>
|
||||
<div><strong>Latency:</strong> {openTrace.chat.latency_ms ?? 'n/a'} ms</div>
|
||||
<div><strong>Tokens:</strong> {openTrace.chat.input_tokens ?? 0}/{openTrace.chat.output_tokens ?? 0} (total {openTrace.chat.total_tokens ?? 0})</div>
|
||||
<div><strong>Cost:</strong> {openTrace.chat.estimated_cost_usd == null ? 'n/a' : `$${openTrace.chat.estimated_cost_usd.toFixed(4)}`}</div>
|
||||
<div><strong>Cache:</strong> {openTrace.chat.cache_hit == null ? 'n/a' : openTrace.chat.cache_hit ? 'hit' : 'miss'} (read {openTrace.chat.cache_read_tokens ?? 0} / write {openTrace.chat.cache_write_tokens ?? 0})</div>
|
||||
<div><strong>Tools:</strong> {openTrace.toolCalls.length}</div>
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<div><strong>Comment:</strong></div>
|
||||
<div style={{ color: '#555' }}>{comments[openTrace.chat.trace_id] || '—'}</div>
|
||||
</div>
|
||||
|
||||
<details open>
|
||||
<summary style={{ cursor: 'pointer', marginBottom: 8 }}><strong>System Message</strong></summary>
|
||||
<pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.system_message || 'n/a'}</pre>
|
||||
</details>
|
||||
|
||||
<div style={{ marginTop: 16 }}>
|
||||
<div style={{ fontWeight: 600, marginBottom: 8 }}>Trace Steps</div>
|
||||
{[
|
||||
{
|
||||
type: 'chat',
|
||||
ts: openTrace.chat.ts,
|
||||
title: 'LLM Chat',
|
||||
spanId: openTrace.chat.span_id,
|
||||
parentSpanId: null,
|
||||
latency: openTrace.chat.latency_ms,
|
||||
success: openTrace.chat.success,
|
||||
content: (
|
||||
<>
|
||||
<div><strong>User</strong></div>
|
||||
<pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.user_message}</pre>
|
||||
<div><strong>Assistant</strong></div>
|
||||
<pre style={{ whiteSpace: 'pre-wrap' }}>{openTrace.chat.assistant_message}</pre>
|
||||
</>
|
||||
),
|
||||
},
|
||||
...openTrace.toolCalls.map((tool) => ({
|
||||
type: 'tool',
|
||||
ts: tool.ts,
|
||||
title: `Tool: ${tool.tool_name}`,
|
||||
spanId: tool.span_id,
|
||||
parentSpanId: tool.parent_span_id,
|
||||
latency: tool.latency_ms,
|
||||
success: tool.success,
|
||||
content: (
|
||||
<>
|
||||
<div><strong>Args</strong></div>
|
||||
<pre style={{ whiteSpace: 'pre-wrap' }}>{prettyJson(tool.args_json)}</pre>
|
||||
<div><strong>Result</strong></div>
|
||||
<pre style={{ whiteSpace: 'pre-wrap' }}>{prettyJson(tool.result_json)}</pre>
|
||||
{tool.error ? (
|
||||
<>
|
||||
<div><strong>Error</strong></div>
|
||||
<pre style={{ whiteSpace: 'pre-wrap' }}>{tool.error}</pre>
|
||||
</>
|
||||
) : null}
|
||||
</>
|
||||
),
|
||||
})),
|
||||
].sort((a, b) => a.ts.localeCompare(b.ts)).map((step, index) => {
|
||||
const stepStatus = statusLabel(step.success);
|
||||
const stepKind = stepStatus === 'success' ? 'success' : stepStatus === 'fail' ? 'fail' : 'neutral';
|
||||
return (
|
||||
<div key={`${openTrace.chat.trace_id}-${step.type}-${index}`} style={{ borderTop: '1px solid #eee', paddingTop: 12, marginTop: 12 }}>
|
||||
<div style={{ display: 'flex', gap: 12, alignItems: 'center', flexWrap: 'wrap' }}>
|
||||
<div><strong>Step {index + 1}:</strong> {step.title}</div>
|
||||
<span style={badgeStyle(stepKind)}>{stepStatus}</span>
|
||||
<div style={{ color: '#666' }}>{step.ts}</div>
|
||||
</div>
|
||||
<div style={{ marginTop: 6, color: '#555' }}>
|
||||
<div><strong>Span:</strong> {step.spanId || 'n/a'}</div>
|
||||
{step.parentSpanId ? <div><strong>Parent Span:</strong> {step.parentSpanId}</div> : null}
|
||||
<div><strong>Latency:</strong> {step.latency ?? 'n/a'} ms</div>
|
||||
</div>
|
||||
<div style={{ marginTop: 8 }}>{step.content}</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { notFound } from 'next/navigation';
|
||||
import { loadEvalTraces } from '@/services/evals/evalsStore';
|
||||
import EvalsClient from './EvalsClient';
|
||||
import { scenarios } from '../../tests/evals/scenarios';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export default function EvalsPage() {
|
||||
const evalsEnabled = process.env.RAH_EVALS_LOG === '1' || process.env.RAH_EVALS_LOG === 'true';
|
||||
if (process.env.NODE_ENV === 'production' || !evalsEnabled) {
|
||||
notFound();
|
||||
}
|
||||
|
||||
const traces = loadEvalTraces(25);
|
||||
const scenarioList = scenarios.map((scenario) => ({
|
||||
id: scenario.id,
|
||||
name: scenario.name,
|
||||
description: scenario.description,
|
||||
tools: scenario.tools,
|
||||
enabled: scenario.enabled,
|
||||
notes: scenario.notes,
|
||||
}));
|
||||
|
||||
return (
|
||||
<div style={{ height: '100vh', overflowY: 'auto', background: '#f5f5f5', color: '#111' }}>
|
||||
<div style={{ padding: 24, maxWidth: 1200, margin: '0 auto', fontFamily: 'ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace' }}>
|
||||
<div style={{ position: 'sticky', top: 0, background: '#f5f5f5', paddingBottom: 12, marginBottom: 12, zIndex: 1 }}>
|
||||
<h1 style={{ fontSize: 24, marginBottom: 6 }}>Eval Traces</h1>
|
||||
<div style={{ color: '#444' }}>Trace table on top, full span detail below.</div>
|
||||
</div>
|
||||
{traces.length === 0 ? (
|
||||
<p>No eval traces found. Run evals with RAH_EVALS_LOG=1.</p>
|
||||
) : (
|
||||
<EvalsClient traces={traces} scenarioList={scenarioList} />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user