From 2f2ef10ec9c404811c402b5fff82e71a814313f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CBeeRad=E2=80=9D?= Date: Sun, 28 Dec 2025 16:09:25 +1100 Subject: [PATCH] sync: bug fixes + eval system from private repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix chunk_status: pass chunk_status/chunk/metadata to context builder - Fix vector search: scope by node_id BEFORE similarity search - Add eval logging system (RAH_EVALS_LOG=1) - Add eval dashboard at /evals - Add vitest for testing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- app/api/evals/comment/route.ts | 24 ++ app/evals/EvalsClient.tsx | 376 +++++++++++++++++++ app/evals/page.tsx | 39 ++ package.json | 9 +- src/components/layout/ThreePanelLayout.tsx | 5 +- src/services/context/requestContext.ts | 8 +- src/services/database/chunks.ts | 96 ++--- src/services/evals/evalsLogger.ts | 269 +++++++++++++ src/services/evals/evalsStore.ts | 141 +++++++ tests/evals/dataset.json | 5 + tests/evals/runner.ts | 258 +++++++++++++ tests/evals/scenarios/create-edge.ts | 15 + tests/evals/scenarios/create-node.ts | 15 + tests/evals/scenarios/delegate-mini.ts | 15 + tests/evals/scenarios/delegate-wise.ts | 15 + tests/evals/scenarios/dimension-lifecycle.ts | 15 + tests/evals/scenarios/get-dimension.ts | 15 + tests/evals/scenarios/hard-mode-query.ts | 16 + tests/evals/scenarios/index.ts | 33 ++ tests/evals/scenarios/paper-extract.ts | 16 + tests/evals/scenarios/query-dimensions.ts | 15 + tests/evals/scenarios/search-embeddings.ts | 15 + tests/evals/scenarios/simple-query.ts | 15 + tests/evals/scenarios/update-node.ts | 15 + tests/evals/scenarios/website-extract.ts | 16 + tests/evals/scenarios/workflow-integrate.ts | 17 + tests/evals/scenarios/youtube-extract.ts | 16 + tests/evals/types.ts | 29 ++ vitest.config.ts | 16 + 29 files changed, 1489 insertions(+), 50 deletions(-) create mode 100644 app/api/evals/comment/route.ts create mode 100644 app/evals/EvalsClient.tsx create mode 100644 app/evals/page.tsx create mode 100644 src/services/evals/evalsLogger.ts create mode 100644 src/services/evals/evalsStore.ts create mode 100644 tests/evals/dataset.json create mode 100644 tests/evals/runner.ts create mode 100644 tests/evals/scenarios/create-edge.ts create mode 100644 tests/evals/scenarios/create-node.ts create mode 100644 tests/evals/scenarios/delegate-mini.ts create mode 100644 tests/evals/scenarios/delegate-wise.ts create mode 100644 tests/evals/scenarios/dimension-lifecycle.ts create mode 100644 tests/evals/scenarios/get-dimension.ts create mode 100644 tests/evals/scenarios/hard-mode-query.ts create mode 100644 tests/evals/scenarios/index.ts create mode 100644 tests/evals/scenarios/paper-extract.ts create mode 100644 tests/evals/scenarios/query-dimensions.ts create mode 100644 tests/evals/scenarios/search-embeddings.ts create mode 100644 tests/evals/scenarios/simple-query.ts create mode 100644 tests/evals/scenarios/update-node.ts create mode 100644 tests/evals/scenarios/website-extract.ts create mode 100644 tests/evals/scenarios/workflow-integrate.ts create mode 100644 tests/evals/scenarios/youtube-extract.ts create mode 100644 tests/evals/types.ts create mode 100644 vitest.config.ts diff --git a/app/api/evals/comment/route.ts b/app/api/evals/comment/route.ts new file mode 100644 index 0000000..2daec14 --- /dev/null +++ b/app/api/evals/comment/route.ts @@ -0,0 +1,24 @@ +import { NextRequest } from 'next/server'; +import { upsertEvalComment } from '@/services/evals/evalsStore'; + +export async function POST(request: NextRequest) { + const evalsEnabled = process.env.RAH_EVALS_LOG === '1' || process.env.RAH_EVALS_LOG === 'true'; + if (process.env.NODE_ENV === 'production' || !evalsEnabled) { + return new Response('Not Found', { status: 404 }); + } + + const body = await request.json(); + const traceId = typeof body?.traceId === 'string' ? body.traceId : null; + const scenarioId = typeof body?.scenarioId === 'string' ? body.scenarioId : null; + const comment = typeof body?.comment === 'string' ? body.comment : null; + + if (!traceId || comment === null) { + return new Response(JSON.stringify({ error: 'Invalid payload' }), { status: 400 }); + } + + upsertEvalComment(traceId, scenarioId, comment); + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); +} diff --git a/app/evals/EvalsClient.tsx b/app/evals/EvalsClient.tsx new file mode 100644 index 0000000..bae24db --- /dev/null +++ b/app/evals/EvalsClient.tsx @@ -0,0 +1,376 @@ +'use client'; + +import { useMemo, useState } from 'react'; +import type { EvalTrace } from '@/services/evals/evalsStore'; + +type Props = { + traces: EvalTrace[]; + scenarioList: Array<{ + id: string; + name: string; + description?: string; + tools?: string[]; + enabled?: boolean; + notes?: string; + }>; +}; + +function formatPreview(text: string | null, max = 80) { + if (!text) return ''; + const compact = text.replace(/\s+/g, ' ').trim(); + return compact.length > max ? `${compact.slice(0, max)}...` : compact; +} + +function statusLabel(success: number | null) { + if (success === null) return 'n/a'; + return success ? 'success' : 'fail'; +} + +function badgeStyle(kind: 'success' | 'fail' | 'neutral') { + const base = { display: 'inline-block', padding: '2px 8px', borderRadius: 999, fontSize: 12 }; + if (kind === 'success') return { ...base, background: '#e6f4ea', color: '#146c2e' }; + if (kind === 'fail') return { ...base, background: '#fdecea', color: '#b42318' }; + return { ...base, background: '#f2f2f2', color: '#333' }; +} + +function prettyJson(value: string | null) { + if (!value) return ''; + try { + return JSON.stringify(JSON.parse(value), null, 2); + } catch { + return value; + } +} + +export default function EvalsClient({ traces, scenarioList }: Props) { + const [openTraceId, setOpenTraceId] = useState(traces[0]?.chat.trace_id || null); + const [comments, setComments] = useState>(() => { + const initial: Record = {}; + traces.forEach((trace) => { + if (trace.comment) { + initial[trace.chat.trace_id] = trace.comment; + } + }); + return initial; + }); + const [sourceFilter, setSourceFilter] = useState('all'); // 'all' | 'live' | 'scenario' + const [scenarioFilter, setScenarioFilter] = useState('all'); + const [statusFilter, setStatusFilter] = useState('all'); + const [searchFilter, setSearchFilter] = useState(''); + + const rows = useMemo(() => { + return traces.map((trace) => { + const { chat, toolCalls } = trace; + const status = statusLabel(chat.success); + const isLive = !chat.scenario_id; + return { + trace, + id: chat.trace_id, + source: isLive ? 'live' : 'scenario', + scenario: chat.scenario_id || '—', + model: chat.model || 'n/a', + latency: chat.latency_ms ?? 'n/a', + tokens: `${chat.input_tokens ?? 0}/${chat.output_tokens ?? 0} (${chat.total_tokens ?? 0})`, + cost: chat.estimated_cost_usd ?? null, + cache: chat.cache_hit == null ? 'n/a' : chat.cache_hit ? 'hit' : 'miss', + cacheTokens: `${chat.cache_read_tokens ?? 0}/${chat.cache_write_tokens ?? 0}`, + toolCount: toolCalls.length, + status, + userPreview: formatPreview(chat.user_message), + timestamp: chat.ts, + mode: chat.mode || 'n/a', + workflow: chat.workflow_key || '—', + }; + }); + }, [traces]); + + const filteredRows = useMemo(() => { + return rows.filter((row) => { + if (sourceFilter !== 'all' && row.source !== sourceFilter) return false; + if (scenarioFilter !== 'all' && row.scenario !== scenarioFilter) return false; + if (statusFilter !== 'all' && row.status !== statusFilter) return false; + if (searchFilter.trim()) { + const needle = searchFilter.toLowerCase(); + if (!row.userPreview.toLowerCase().includes(needle)) return false; + } + return true; + }); + }, [rows, sourceFilter, scenarioFilter, statusFilter, searchFilter]); + + const openTrace = traces.find((trace) => trace.chat.trace_id === openTraceId) || traces[0]; + + return ( +
+
+
Scenario Set
+
+ + + + + + + + + + + + + {scenarioList.map((scenario) => ( + + + + + + + + + ))} + +
IDNameDescriptionToolsEnabledNotes
{scenario.id}{scenario.name}{scenario.description || '—'}{scenario.tools?.join(', ') || '—'}{scenario.enabled === false ? 'no' : 'yes'}{scenario.notes || '—'}
+
+
+ +
+
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + {filteredRows.map((row) => { + const isOpen = row.id === openTraceId; + const statusKind = row.status === 'success' ? 'success' : row.status === 'fail' ? 'fail' : 'neutral'; + const commentValue = comments[row.id] || ''; + return ( + setOpenTraceId(row.id)} + style={{ + cursor: 'pointer', + background: isOpen ? '#eef4ff' : '#fff', + borderBottom: '1px solid #eee', + }} + > + + + + + + + + + + + + + +
SourceScenarioStatusLatencyTokensCostCacheToolsModelModeWorkflowUser InputTimeComment
+ + {row.source === 'live' ? '🟢 Live' : '🔬 Scenario'} + + {row.scenario}{row.status}{row.latency} ms{row.tokens}{row.cost == null ? 'n/a' : `$${row.cost.toFixed(4)}`}{row.cache} ({row.cacheTokens}){row.toolCount}{row.model}{row.mode}{row.workflow}{row.userPreview}{row.timestamp} +