feat: sync pane workspace overhaul from private repo
- ship the expanded left-nav and pane header workspace updates - add dimension-driven browse flow, map view upgrades, and delete-node tooling - expand eval coverage and refresh the public UI docs for the new layout Generated with Codex
This commit is contained in:
@@ -1,11 +1,13 @@
|
||||
{
|
||||
"id": "golden-ra-h-core-v2",
|
||||
"name": "Golden RA-H Core v2",
|
||||
"description": "Slim 5-scenario eval set for core RA-H usage: focused graph writes, skill-guided writes, indexed node search, chunk-grounded insight creation, and hub traversal.",
|
||||
"description": "Core RA-H eval set covering database, tools, skills, search, and ingestion behavior, including graph writes, indexed node search, retrieval regressions, chunk-grounded insight creation, and hub traversal.",
|
||||
"inherits": "golden-v1",
|
||||
"version": 2,
|
||||
"focus": [
|
||||
"tool-call correctness",
|
||||
"category-based coverage",
|
||||
"search regressions",
|
||||
"skill usage only when appropriate",
|
||||
"core retrieval/write behavior",
|
||||
"latency/tokens/cost guards"
|
||||
|
||||
+22
-22
@@ -3,7 +3,7 @@ import path from 'path';
|
||||
import Database from 'better-sqlite3';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { scenarios } from './scenarios';
|
||||
import { Scenario } from './types';
|
||||
import { EvalCategory, Scenario } from './types';
|
||||
|
||||
type EvalChatRow = {
|
||||
trace_id: string;
|
||||
@@ -38,7 +38,7 @@ type EvalResult = {
|
||||
|
||||
const BASE_URL = process.env.RAH_EVALS_BASE_URL || 'http://localhost:3000';
|
||||
const DATASET_ENV = process.env.RAH_EVALS_DATASET_ID;
|
||||
const SUITE_ENV = (process.env.RAH_EVALS_SUITE || 'all').toLowerCase();
|
||||
const CATEGORY_ENV = (process.env.RAH_EVALS_CATEGORY || process.env.RAH_EVALS_SUITE || 'all').toLowerCase();
|
||||
const WAIT_TIMEOUT_MS = Number(process.env.RAH_EVALS_TIMEOUT_MS || 60000);
|
||||
const LOG_DB_PATH = path.join(process.cwd(), 'logs', 'evals.sqlite');
|
||||
const RAH_DB_PATH = process.env.SQLITE_DB_PATH || path.join(
|
||||
@@ -54,7 +54,7 @@ function loadDatasetId() {
|
||||
return parsed.id || 'default';
|
||||
}
|
||||
|
||||
type EvalSuite = 'all' | 'tools' | 'skills' | 'traversal' | 'internal' | 'external';
|
||||
type EvalCategoryFilter = 'all' | EvalCategory;
|
||||
|
||||
type FocusedNodeContext = {
|
||||
id: number;
|
||||
@@ -66,25 +66,25 @@ type FocusedNodeContext = {
|
||||
metadata: string | null;
|
||||
};
|
||||
|
||||
function getDefaultScenarioSuites(scenario: Scenario): string[] {
|
||||
const suites = ['internal'];
|
||||
if (scenario.id.startsWith('skill-trigger-')) {
|
||||
suites.push('skills');
|
||||
} else {
|
||||
suites.push('tools');
|
||||
function getDefaultScenarioCategories(scenario: Scenario): EvalCategory[] {
|
||||
if (scenario.id.includes('search') || scenario.id.includes('quote')) {
|
||||
return ['search'];
|
||||
}
|
||||
if (scenario.id.includes('traverse')) {
|
||||
suites.push('traversal');
|
||||
if (scenario.id.includes('skill')) {
|
||||
return ['skills'];
|
||||
}
|
||||
return suites;
|
||||
if (scenario.id.includes('extract') || scenario.id.includes('ingest')) {
|
||||
return ['ingestion', 'tools'];
|
||||
}
|
||||
return ['database', 'tools'];
|
||||
}
|
||||
|
||||
function shouldRunScenario(scenario: Scenario, suite: EvalSuite) {
|
||||
if (suite === 'all') return true;
|
||||
const suites = scenario.suites && scenario.suites.length > 0
|
||||
? scenario.suites
|
||||
: getDefaultScenarioSuites(scenario);
|
||||
return suites.includes(suite);
|
||||
function shouldRunScenario(scenario: Scenario, category: EvalCategoryFilter) {
|
||||
if (category === 'all') return true;
|
||||
const categories = scenario.categories && scenario.categories.length > 0
|
||||
? scenario.categories
|
||||
: getDefaultScenarioCategories(scenario);
|
||||
return categories.includes(category);
|
||||
}
|
||||
|
||||
function resolveFocusedNodeId(query: Scenario['input']['focusedNodeQuery']): number | null {
|
||||
@@ -424,14 +424,14 @@ async function runScenario(scenario: Scenario, datasetId: string): Promise<EvalR
|
||||
}
|
||||
|
||||
async function runAll() {
|
||||
const suite: EvalSuite = ['skills', 'tools', 'traversal', 'internal', 'external'].includes(SUITE_ENV)
|
||||
? (SUITE_ENV as EvalSuite)
|
||||
const category: EvalCategoryFilter = ['database', 'tools', 'skills', 'search', 'ingestion'].includes(CATEGORY_ENV)
|
||||
? (CATEGORY_ENV as EvalCategoryFilter)
|
||||
: 'all';
|
||||
const datasetId = loadDatasetId();
|
||||
await ensureServerReady();
|
||||
await ensureEvalsEnabled();
|
||||
const runnable = scenarios.filter(s => s.enabled !== false && shouldRunScenario(s, suite));
|
||||
console.log(`Running ${runnable.length} scenarios (dataset: ${datasetId}, suite: ${suite})...\n`);
|
||||
const runnable = scenarios.filter(s => s.enabled !== false && shouldRunScenario(s, category));
|
||||
console.log(`Running ${runnable.length} scenarios (dataset: ${datasetId}, category: ${category})...\n`);
|
||||
|
||||
const results: EvalResult[] = [];
|
||||
for (const scenario of runnable) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { scenario as focusedGraphWrite } from './focused-graph-write';
|
||||
import { scenario as skillGuidedWrite } from './skill-guided-write';
|
||||
import { scenario as nodeIndexSearch } from './node-index-search';
|
||||
import { scenario as searchDocToLora } from './search-doc-to-lora';
|
||||
import { scenario as chunkQuoteInsight } from './chunk-quote-insight';
|
||||
import { scenario as hubTraversal } from './hub-traversal';
|
||||
|
||||
@@ -8,6 +9,7 @@ export const scenarios = [
|
||||
focusedGraphWrite,
|
||||
skillGuidedWrite,
|
||||
nodeIndexSearch,
|
||||
searchDocToLora,
|
||||
chunkQuoteInsight,
|
||||
hubTraversal,
|
||||
];
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { Scenario } from '../types';
|
||||
|
||||
export const scenario: Scenario = {
|
||||
id: 'search-doc-to-lora',
|
||||
name: 'Search Doc-to-LoRA',
|
||||
description: 'Free-text graph search should find the Doc-to-LoRA node by title without speculative dimension constraints or unnecessary web search.',
|
||||
categories: ['search'],
|
||||
tools: ['queryNodes'],
|
||||
input: {
|
||||
message: 'Find the recent Doc-to-LoRA stuff in my graph. Just return the matching graph node or nodes.',
|
||||
},
|
||||
expect: {
|
||||
toolsCalledSoft: ['queryNodes'],
|
||||
toolsNotCalled: ['webSearch'],
|
||||
responseContains: ['Doc-to-LoRA: Learning to Instantly Internalize Contexts'],
|
||||
maxLatencyMs: 15000,
|
||||
maxTotalTokens: 8000,
|
||||
maxEstimatedCostUsd: 0.08,
|
||||
},
|
||||
notes: 'Regression for the failure where free-text search was over-constrained by guessed dimensions and missed the existing Doc-to-LoRA node.',
|
||||
};
|
||||
@@ -1,3 +1,5 @@
|
||||
export type EvalCategory = 'database' | 'tools' | 'skills' | 'search' | 'ingestion';
|
||||
|
||||
export type ScenarioExpectations = {
|
||||
skillsRead?: string[];
|
||||
skillsReadSoft?: string[];
|
||||
@@ -32,6 +34,7 @@ export type Scenario = {
|
||||
expect?: ScenarioExpectations;
|
||||
description?: string;
|
||||
tools?: string[];
|
||||
categories?: EvalCategory[];
|
||||
suites?: string[];
|
||||
enabled?: boolean;
|
||||
notes?: string;
|
||||
|
||||
Reference in New Issue
Block a user