“BeeRad”
213c054c9f
- ship the expanded left-nav and pane header workspace updates - add dimension-driven browse flow, map view upgrades, and delete-node tooling - expand eval coverage and refresh the public UI docs for the new layout Generated with Codex
489 lines
15 KiB
TypeScript
489 lines
15 KiB
TypeScript
import fs from 'fs';
|
|
import path from 'path';
|
|
import Database from 'better-sqlite3';
|
|
import { randomUUID } from 'crypto';
|
|
import { scenarios } from './scenarios';
|
|
import { EvalCategory, Scenario } from './types';
|
|
|
|
type EvalChatRow = {
|
|
trace_id: string;
|
|
scenario_id: string;
|
|
assistant_message: string | null;
|
|
latency_ms: number | null;
|
|
input_tokens: number | null;
|
|
output_tokens: number | null;
|
|
total_tokens: number | null;
|
|
estimated_cost_usd: number | null;
|
|
};
|
|
|
|
type EvalToolCallRow = {
|
|
tool_name: string;
|
|
args_json: string | null;
|
|
};
|
|
|
|
type EvalResult = {
|
|
scenario: string;
|
|
passed: boolean;
|
|
failures: string[];
|
|
warnings: string[];
|
|
latencyMs?: number | null;
|
|
totalTokens?: number | null;
|
|
estimatedCostUsd?: number | null;
|
|
triggerStats?: {
|
|
tp: number;
|
|
fp: number;
|
|
fn: number;
|
|
};
|
|
};
|
|
|
|
const BASE_URL = process.env.RAH_EVALS_BASE_URL || 'http://localhost:3000';
|
|
const DATASET_ENV = process.env.RAH_EVALS_DATASET_ID;
|
|
const CATEGORY_ENV = (process.env.RAH_EVALS_CATEGORY || process.env.RAH_EVALS_SUITE || 'all').toLowerCase();
|
|
const WAIT_TIMEOUT_MS = Number(process.env.RAH_EVALS_TIMEOUT_MS || 60000);
|
|
const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
|
|
const RAH_DB_PATH = process.env.SQLITE_DB_PATH || path.join(
|
|
process.env.HOME || '~',
|
|
'Library/Application Support/RA-H/db/rah.sqlite'
|
|
);
|
|
|
|
function loadDatasetId() {
|
|
if (DATASET_ENV) return DATASET_ENV;
|
|
const datasetPath = path.join(process.cwd(), 'tests', 'evals', 'dataset.json');
|
|
const raw = fs.readFileSync(datasetPath, 'utf-8');
|
|
const parsed = JSON.parse(raw);
|
|
return parsed.id || 'default';
|
|
}
|
|
|
|
type EvalCategoryFilter = 'all' | EvalCategory;
|
|
|
|
type FocusedNodeContext = {
|
|
id: number;
|
|
title: string;
|
|
description: string | null;
|
|
link: string | null;
|
|
chunk_status: string | null;
|
|
chunk: string | null;
|
|
metadata: string | null;
|
|
};
|
|
|
|
function getDefaultScenarioCategories(scenario: Scenario): EvalCategory[] {
|
|
if (scenario.id.includes('search') || scenario.id.includes('quote')) {
|
|
return ['search'];
|
|
}
|
|
if (scenario.id.includes('skill')) {
|
|
return ['skills'];
|
|
}
|
|
if (scenario.id.includes('extract') || scenario.id.includes('ingest')) {
|
|
return ['ingestion', 'tools'];
|
|
}
|
|
return ['database', 'tools'];
|
|
}
|
|
|
|
function shouldRunScenario(scenario: Scenario, category: EvalCategoryFilter) {
|
|
if (category === 'all') return true;
|
|
const categories = scenario.categories && scenario.categories.length > 0
|
|
? scenario.categories
|
|
: getDefaultScenarioCategories(scenario);
|
|
return categories.includes(category);
|
|
}
|
|
|
|
function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): number | null {
|
|
if (!query) return null;
|
|
if (!fs.existsSync(RAH_DB_PATH)) return null;
|
|
const db = new Database(RAH_DB_PATH, { readonly: true, fileMustExist: true });
|
|
if (query.titleEquals) {
|
|
const row = db.prepare(`
|
|
SELECT id
|
|
FROM nodes
|
|
WHERE title = ?
|
|
ORDER BY datetime(created_at) DESC
|
|
LIMIT 1
|
|
`).get(query.titleEquals) as { id?: number } | undefined;
|
|
return row?.id ?? null;
|
|
}
|
|
if (query.titleContains) {
|
|
const row = db.prepare(`
|
|
SELECT id
|
|
FROM nodes
|
|
WHERE title LIKE ?
|
|
ORDER BY datetime(created_at) DESC
|
|
LIMIT 1
|
|
`).get(`%${query.titleContains}%`) as { id?: number } | undefined;
|
|
return row?.id ?? null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function loadFocusedNodeContext(nodeId: number | null): FocusedNodeContext[] {
|
|
if (!nodeId || !fs.existsSync(RAH_DB_PATH)) return [];
|
|
const db = new Database(RAH_DB_PATH, { readonly: true, fileMustExist: true });
|
|
const row = db.prepare(`
|
|
SELECT id, title, description, link, chunk_status, chunk, metadata
|
|
FROM nodes
|
|
WHERE id = ?
|
|
LIMIT 1
|
|
`).get(nodeId) as FocusedNodeContext | undefined;
|
|
return row ? [row] : [];
|
|
}
|
|
|
|
async function drainResponse(response: Response) {
|
|
if (!response.body) return;
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder();
|
|
while (true) {
|
|
const { value, done } = await reader.read();
|
|
if (done) break;
|
|
decoder.decode(value, { stream: true });
|
|
}
|
|
}
|
|
|
|
function openEvalDb() {
|
|
if (!fs.existsSync(LOG_DB_PATH)) {
|
|
return null;
|
|
}
|
|
return new Database(LOG_DB_PATH, { readonly: true, fileMustExist: true });
|
|
}
|
|
|
|
function getEvalChatRow(db: Database.Database, traceId: string, scenarioId: string) {
|
|
return db.prepare(`
|
|
SELECT trace_id, scenario_id, assistant_message, latency_ms, input_tokens, output_tokens, total_tokens, estimated_cost_usd
|
|
FROM llm_chats
|
|
WHERE trace_id = ? AND scenario_id = ?
|
|
ORDER BY id DESC
|
|
LIMIT 1
|
|
`).get(traceId, scenarioId) as EvalChatRow | undefined;
|
|
}
|
|
|
|
function getEvalToolCalls(db: Database.Database, traceId: string, scenarioId: string) {
|
|
return db.prepare(`
|
|
SELECT tool_name, args_json
|
|
FROM tool_calls
|
|
WHERE trace_id = ? AND scenario_id = ?
|
|
`).all(traceId, scenarioId) as EvalToolCallRow[];
|
|
}
|
|
|
|
async function waitForEvalRow(traceId: string, scenarioId: string, timeoutMs = 10000) {
|
|
const startedAt = Date.now();
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
const db = openEvalDb();
|
|
if (db) {
|
|
const row = getEvalChatRow(db, traceId, scenarioId);
|
|
if (row) return { row, db };
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
}
|
|
return { row: undefined, db: null };
|
|
}
|
|
|
|
async function ensureServerReady() {
|
|
try {
|
|
const response = await fetch(`${BASE_URL}/api/health/ping`);
|
|
if (!response.ok) {
|
|
throw new Error(`Health check returned HTTP ${response.status}`);
|
|
}
|
|
} catch (error) {
|
|
throw new Error(
|
|
`RA-H dev server is not reachable at ${BASE_URL}. Start it with "npm run dev:evals".` +
|
|
` ${error instanceof Error ? error.message : String(error)}`
|
|
);
|
|
}
|
|
}
|
|
|
|
async function ensureEvalsEnabled() {
|
|
try {
|
|
const response = await fetch(`${BASE_URL}/evals`, {
|
|
redirect: 'manual',
|
|
});
|
|
|
|
if (response.status === 404) {
|
|
throw new Error('The /evals route returned 404');
|
|
}
|
|
|
|
if (!response.ok && response.status !== 307 && response.status !== 308) {
|
|
throw new Error(`The /evals route returned HTTP ${response.status}`);
|
|
}
|
|
} catch (error) {
|
|
throw new Error(
|
|
`Eval logging does not appear to be enabled on the dev server. Start the server with "npm run dev:evals" before running "npm run evals".` +
|
|
` ${error instanceof Error ? error.message : String(error)}`
|
|
);
|
|
}
|
|
}
|
|
|
|
function normalizeContains(text: string) {
|
|
return text.toLowerCase();
|
|
}
|
|
|
|
function normalizeSkillId(value: string) {
|
|
return value
|
|
.trim()
|
|
.toLowerCase()
|
|
.replace(/[\s_]+/g, '-')
|
|
.replace(/[^a-z0-9-]/g, '')
|
|
.replace(/-+/g, '-')
|
|
.replace(/^-|-$/g, '');
|
|
}
|
|
|
|
function safeJsonParse(value: string | null): Record<string, unknown> | null {
|
|
if (!value) return null;
|
|
try {
|
|
const parsed = JSON.parse(value);
|
|
if (parsed && typeof parsed === 'object') {
|
|
return parsed as Record<string, unknown>;
|
|
}
|
|
return null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function getSkillsRead(toolCalls: EvalToolCallRow[]) {
|
|
const names = new Set<string>();
|
|
for (const call of toolCalls) {
|
|
if (call.tool_name !== 'readSkill' && call.tool_name !== 'rah_read_skill') {
|
|
continue;
|
|
}
|
|
const parsed = safeJsonParse(call.args_json);
|
|
const rawName = parsed?.name;
|
|
if (typeof rawName === 'string' && rawName.trim().length > 0) {
|
|
names.add(normalizeSkillId(rawName));
|
|
}
|
|
}
|
|
return names;
|
|
}
|
|
|
|
function checkScenario(
|
|
scenario: Scenario,
|
|
chatRow: EvalChatRow,
|
|
toolCalls: EvalToolCallRow[]
|
|
): EvalResult {
|
|
const failures: string[] = [];
|
|
const warnings: string[] = [];
|
|
const expect = scenario.expect || {};
|
|
const toolNames = toolCalls.map(call => call.tool_name);
|
|
const responseText = chatRow.assistant_message || '';
|
|
const skillsRead = getSkillsRead(toolCalls);
|
|
const requiredSkills = new Set((expect.skillsRead || []).map(normalizeSkillId));
|
|
const requiredSkillsSoft = new Set((expect.skillsReadSoft || []).map(normalizeSkillId));
|
|
const forbiddenSkills = new Set((expect.skillsNotRead || []).map(normalizeSkillId));
|
|
const forbiddenSkillsSoft = new Set((expect.skillsNotReadSoft || []).map(normalizeSkillId));
|
|
const triggerStats = { tp: 0, fp: 0, fn: 0 };
|
|
|
|
(expect.toolsCalled || []).forEach(tool => {
|
|
if (!toolNames.includes(tool)) {
|
|
failures.push(`Expected tool "${tool}" not called`);
|
|
}
|
|
});
|
|
|
|
(expect.toolsCalledSoft || []).forEach(tool => {
|
|
if (!toolNames.includes(tool)) {
|
|
warnings.push(`(soft) Expected tool "${tool}" not called`);
|
|
}
|
|
});
|
|
|
|
(expect.toolsNotCalled || []).forEach(tool => {
|
|
if (toolNames.includes(tool)) {
|
|
failures.push(`Tool "${tool}" should not be called`);
|
|
}
|
|
});
|
|
|
|
(expect.toolsNotCalledSoft || []).forEach(tool => {
|
|
if (toolNames.includes(tool)) {
|
|
warnings.push(`(soft) Tool "${tool}" should not be called`);
|
|
}
|
|
});
|
|
|
|
requiredSkills.forEach(skill => {
|
|
if (!skillsRead.has(skill)) {
|
|
failures.push(`Expected skill "${skill}" not read`);
|
|
triggerStats.fn += 1;
|
|
} else {
|
|
triggerStats.tp += 1;
|
|
}
|
|
});
|
|
|
|
requiredSkillsSoft.forEach(skill => {
|
|
if (!skillsRead.has(skill)) {
|
|
warnings.push(`(soft) Expected skill "${skill}" not read`);
|
|
}
|
|
});
|
|
|
|
forbiddenSkills.forEach(skill => {
|
|
if (skillsRead.has(skill)) {
|
|
failures.push(`Skill "${skill}" should not be read`);
|
|
triggerStats.fp += 1;
|
|
}
|
|
});
|
|
|
|
forbiddenSkillsSoft.forEach(skill => {
|
|
if (skillsRead.has(skill)) {
|
|
warnings.push(`(soft) Skill "${skill}" should not be read`);
|
|
}
|
|
});
|
|
|
|
if (requiredSkills.size > 0) {
|
|
skillsRead.forEach(skill => {
|
|
if (!requiredSkills.has(skill)) {
|
|
triggerStats.fp += 1;
|
|
}
|
|
});
|
|
}
|
|
|
|
(expect.responseContains || []).forEach(text => {
|
|
if (!normalizeContains(responseText).includes(normalizeContains(text))) {
|
|
failures.push(`Response missing "${text}"`);
|
|
}
|
|
});
|
|
|
|
(expect.responseContainsSoft || []).forEach(text => {
|
|
if (!normalizeContains(responseText).includes(normalizeContains(text))) {
|
|
warnings.push(`(soft) Response missing "${text}"`);
|
|
}
|
|
});
|
|
|
|
(expect.responseNotContains || []).forEach(text => {
|
|
if (normalizeContains(responseText).includes(normalizeContains(text))) {
|
|
failures.push(`Response should not contain "${text}"`);
|
|
}
|
|
});
|
|
|
|
if (typeof expect.maxLatencyMs === 'number' && chatRow.latency_ms !== null) {
|
|
if (chatRow.latency_ms > expect.maxLatencyMs) {
|
|
failures.push(`Latency ${chatRow.latency_ms}ms exceeded ${expect.maxLatencyMs}ms`);
|
|
}
|
|
}
|
|
|
|
if (typeof expect.maxTotalTokens === 'number' && chatRow.total_tokens !== null) {
|
|
if (chatRow.total_tokens > expect.maxTotalTokens) {
|
|
failures.push(`Total tokens ${chatRow.total_tokens} exceeded ${expect.maxTotalTokens}`);
|
|
}
|
|
}
|
|
|
|
if (typeof expect.maxEstimatedCostUsd === 'number' && chatRow.estimated_cost_usd !== null) {
|
|
if (chatRow.estimated_cost_usd > expect.maxEstimatedCostUsd) {
|
|
failures.push(`Estimated cost ${chatRow.estimated_cost_usd} exceeded ${expect.maxEstimatedCostUsd}`);
|
|
}
|
|
}
|
|
|
|
return {
|
|
scenario: scenario.name,
|
|
passed: failures.length === 0,
|
|
failures,
|
|
warnings,
|
|
latencyMs: chatRow.latency_ms,
|
|
totalTokens: chatRow.total_tokens,
|
|
estimatedCostUsd: chatRow.estimated_cost_usd,
|
|
triggerStats: triggerStats.tp || triggerStats.fp || triggerStats.fn ? triggerStats : undefined,
|
|
};
|
|
}
|
|
|
|
async function runScenario(scenario: Scenario, datasetId: string): Promise<EvalResult> {
|
|
const traceId = `eval_${Date.now()}_${randomUUID().slice(0, 8)}`;
|
|
const resolvedFocusedNodeId =
|
|
scenario.input.focusedNodeId ?? resolveFocusedNodeId(scenario.input.focusedNodeQuery);
|
|
const openTabs = loadFocusedNodeContext(resolvedFocusedNodeId ?? null);
|
|
const response = await fetch(`${BASE_URL}/api/rah/chat`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
messages: [{ role: 'user', content: scenario.input.message }],
|
|
openTabs,
|
|
activeTabId: resolvedFocusedNodeId ?? null,
|
|
currentView: 'nodes',
|
|
traceId,
|
|
evals: {
|
|
datasetId,
|
|
scenarioId: scenario.id,
|
|
},
|
|
}),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
return {
|
|
scenario: scenario.name,
|
|
passed: false,
|
|
failures: [`HTTP ${response.status} from /api/rah/chat`],
|
|
warnings: [],
|
|
};
|
|
}
|
|
|
|
await drainResponse(response);
|
|
|
|
const { row, db } = await waitForEvalRow(traceId, scenario.id, WAIT_TIMEOUT_MS);
|
|
if (!row) {
|
|
return {
|
|
scenario: scenario.name,
|
|
passed: false,
|
|
failures: ['Timed out waiting for eval logs'],
|
|
warnings: [],
|
|
};
|
|
}
|
|
|
|
const toolCalls = db ? getEvalToolCalls(db, traceId, scenario.id) : [];
|
|
return checkScenario(scenario, row, toolCalls);
|
|
}
|
|
|
|
async function runAll() {
|
|
const category: EvalCategoryFilter = ['database', 'tools', 'skills', 'search', 'ingestion'].includes(CATEGORY_ENV)
|
|
? (CATEGORY_ENV as EvalCategoryFilter)
|
|
: 'all';
|
|
const datasetId = loadDatasetId();
|
|
await ensureServerReady();
|
|
await ensureEvalsEnabled();
|
|
const runnable = scenarios.filter(s => s.enabled !== false && shouldRunScenario(s, category));
|
|
console.log(`Running ${runnable.length} scenarios (dataset: ${datasetId}, category: ${category})...\n`);
|
|
|
|
const results: EvalResult[] = [];
|
|
for (const scenario of runnable) {
|
|
const result = await runScenario(scenario, datasetId);
|
|
results.push(result);
|
|
const icon = result.passed ? '✓' : '✗';
|
|
const parts: string[] = [];
|
|
if (typeof result.latencyMs === 'number') parts.push(`${result.latencyMs}ms`);
|
|
if (typeof result.totalTokens === 'number') parts.push(`${result.totalTokens} tok`);
|
|
if (typeof result.estimatedCostUsd === 'number') parts.push(`$${result.estimatedCostUsd.toFixed(4)}`);
|
|
const details = parts.length > 0 ? ` (${parts.join(', ')})` : '';
|
|
console.log(`${icon} ${result.scenario}${details}`);
|
|
result.failures.forEach(failure => console.log(` - ${failure}`));
|
|
result.warnings.forEach(warning => console.log(` - ${warning}`));
|
|
}
|
|
|
|
const failed = results.filter(result => !result.passed);
|
|
const warnings = results.filter(result => result.warnings.length > 0);
|
|
const triggerTotals = results.reduce(
|
|
(acc, result) => {
|
|
if (result.triggerStats) {
|
|
acc.tp += result.triggerStats.tp;
|
|
acc.fp += result.triggerStats.fp;
|
|
acc.fn += result.triggerStats.fn;
|
|
}
|
|
return acc;
|
|
},
|
|
{ tp: 0, fp: 0, fn: 0 }
|
|
);
|
|
const triggerPrecision = triggerTotals.tp + triggerTotals.fp > 0
|
|
? triggerTotals.tp / (triggerTotals.tp + triggerTotals.fp)
|
|
: null;
|
|
const triggerRecall = triggerTotals.tp + triggerTotals.fn > 0
|
|
? triggerTotals.tp / (triggerTotals.tp + triggerTotals.fn)
|
|
: null;
|
|
console.log('\nSummary');
|
|
console.log(`- Passed: ${results.length - failed.length}`);
|
|
console.log(`- Failed: ${failed.length}`);
|
|
console.log(`- With warnings: ${warnings.length}`);
|
|
if (triggerPrecision !== null || triggerRecall !== null) {
|
|
console.log(`- Trigger TP/FP/FN: ${triggerTotals.tp}/${triggerTotals.fp}/${triggerTotals.fn}`);
|
|
console.log(`- Trigger precision: ${triggerPrecision === null ? 'n/a' : triggerPrecision.toFixed(3)}`);
|
|
console.log(`- Trigger recall: ${triggerRecall === null ? 'n/a' : triggerRecall.toFixed(3)}`);
|
|
}
|
|
|
|
if (failed.length > 0) {
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
runAll().catch(error => {
|
|
console.error('Eval runner failed:', error);
|
|
process.exit(1);
|
|
});
|