Initial commit: RA-H Open Source Edition
Local-first knowledge management system with BYO API keys. Features: - 3-panel UI (Nodes | Focus | Helpers) - SQLite + sqlite-vec for vector search - Agent system (Easy/Hard mode orchestrators) - Content extraction (YouTube, PDF, web) - Integrate workflow for connection discovery - Dimension system with auto-assignment Tech stack: - Next.js 15 + TypeScript + Tailwind CSS - Anthropic (Claude) + OpenAI (GPT) via Vercel AI SDK Setup: npm install && npm rebuild better-sqlite3 scripts/dev/bootstrap-local.sh npm run dev MIT License
This commit is contained in:
“BeeRad”
Executable
+225
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Test script for Python to TypeScript migration
|
||||
* Tests extractors and embeddings using TypeScript modules
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Module = require('module');
|
||||
const ts = require('typescript');
|
||||
const { getSubtitles } = require('youtube-captions-scraper');
|
||||
|
||||
function isNetworkError(error) {
|
||||
if (!error) return false;
|
||||
const message = typeof error === 'string' ? error : error.message || '';
|
||||
const code = error.code || '';
|
||||
return [
|
||||
'fetch failed',
|
||||
'ENOTFOUND',
|
||||
'ECONNRESET',
|
||||
'EAI_AGAIN',
|
||||
'ECONNREFUSED',
|
||||
'ETIMEDOUT'
|
||||
].some(token => message.includes(token)) || ['ENOTFOUND', 'ECONNRESET', 'EAI_AGAIN', 'ECONNREFUSED', 'ETIMEDOUT'].includes(code);
|
||||
}
|
||||
|
||||
function requireTs(modulePath) {
|
||||
const resolved = path.resolve(__dirname, '..', modulePath);
|
||||
const source = fs.readFileSync(resolved, 'utf8');
|
||||
const transpiled = ts.transpileModule(source, {
|
||||
compilerOptions: {
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
target: ts.ScriptTarget.ES2020,
|
||||
esModuleInterop: true,
|
||||
moduleResolution: ts.ModuleResolutionKind.Node16,
|
||||
resolveJsonModule: true,
|
||||
skipLibCheck: true
|
||||
},
|
||||
fileName: resolved
|
||||
});
|
||||
|
||||
const tempModule = new Module(resolved, module);
|
||||
tempModule.filename = resolved;
|
||||
tempModule.paths = Module._nodeModulePaths(path.dirname(resolved));
|
||||
tempModule._compile(transpiled.outputText, resolved);
|
||||
return tempModule.exports;
|
||||
}
|
||||
|
||||
async function testTranscriptLibrary() {
|
||||
console.log('🔍 Testing youtube-captions-scraper library directly...');
|
||||
|
||||
const testUrls = [
|
||||
'dQw4w9WgXcQ', // Rick Roll
|
||||
'MnrJzXM7a6o', // TED Talk
|
||||
'jNQXAC9IVRw' // Popular video
|
||||
];
|
||||
|
||||
let sawNetworkError = false;
|
||||
let sawEmptyTranscript = false;
|
||||
|
||||
for (const videoId of testUrls) {
|
||||
console.log(`\nTesting video: ${videoId}`);
|
||||
try {
|
||||
const transcript = await getSubtitles({ videoID: videoId });
|
||||
if (!transcript || transcript.length === 0) {
|
||||
sawEmptyTranscript = true;
|
||||
console.log('⚠️ Library returned 0 segments (likely geo-blocked). Trying next video...');
|
||||
continue;
|
||||
}
|
||||
console.log(`✅ Success! Got ${transcript.length} segments`);
|
||||
console.log('First segment:', transcript[0]);
|
||||
console.log('Sample formatted:', `[${transcript[0].start.toFixed(1)}s] ${transcript[0].text}`);
|
||||
return 'passed'; // Exit on first success
|
||||
} catch (error) {
|
||||
if (isNetworkError(error)) {
|
||||
sawNetworkError = true;
|
||||
console.log('⚠️ Network unavailable, skipping this video');
|
||||
} else {
|
||||
console.log('❌ Failed:', error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (sawNetworkError) {
|
||||
console.log('\n⚠️ Network unavailable for transcript tests, marking as SKIPPED');
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
if (sawEmptyTranscript) {
|
||||
console.log('\n⚠️ Library could not fetch transcripts for any sample video (likely YouTube changes). Marking as SKIPPED.');
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
console.log('\n❌ No transcripts could be extracted from any test video');
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
console.log('🧪 Testing TypeScript Migration...\n');
|
||||
|
||||
async function testYouTubeExtractor() {
|
||||
console.log('📺 Testing YouTube Extractor...');
|
||||
try {
|
||||
const { extractYouTube } = requireTs('src/services/typescript/extractors/youtube.ts');
|
||||
const result = await extractYouTube('https://www.youtube.com/watch?v=dQw4w9WgXcQ');
|
||||
console.log('✅ YouTube extraction successful');
|
||||
console.log(` Title: ${result.metadata.title}`);
|
||||
console.log(` Channel: ${result.metadata.author_name}`);
|
||||
console.log(` Transcript length: ${result.chunk.length} chars\n`);
|
||||
return 'passed';
|
||||
} catch (error) {
|
||||
if (isNetworkError(error)) {
|
||||
console.warn('⚠️ Network unavailable, skipping YouTube extractor test\n');
|
||||
return 'skipped';
|
||||
}
|
||||
console.error('❌ YouTube extraction failed:', error.message, '\n');
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
async function testWebsiteExtractor() {
|
||||
console.log('🌐 Testing Website Extractor...');
|
||||
try {
|
||||
const { extractWebsite } = requireTs('src/services/typescript/extractors/website.ts');
|
||||
const result = await extractWebsite('https://example.com');
|
||||
console.log('✅ Website extraction successful');
|
||||
console.log(` Title: ${result.metadata.title}`);
|
||||
console.log(` Content length: ${result.chunk.length} chars\n`);
|
||||
return 'passed';
|
||||
} catch (error) {
|
||||
if (isNetworkError(error)) {
|
||||
console.warn('⚠️ Network unavailable, skipping website extractor test\n');
|
||||
return 'skipped';
|
||||
}
|
||||
console.error('❌ Website extraction failed:', error.message, '\n');
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
async function testPDFExtractor() {
|
||||
console.log('📄 Testing PDF Extractor...');
|
||||
try {
|
||||
const { extractPaper } = requireTs('src/services/typescript/extractors/paper.ts');
|
||||
// Using a sample PDF URL
|
||||
const samplePdf = 'https://raw.githubusercontent.com/mozilla/pdf.js/master/web/compressed.tracemonkey-pldi-09.pdf';
|
||||
const result = await extractPaper(samplePdf);
|
||||
console.log('✅ PDF extraction successful');
|
||||
console.log(` Pages: ${result.metadata.pages}`);
|
||||
console.log(` Text length: ${result.metadata.text_length} chars\n`);
|
||||
return 'passed';
|
||||
} catch (error) {
|
||||
if (isNetworkError(error)) {
|
||||
console.warn('⚠️ Network unavailable, skipping PDF extractor test\n');
|
||||
return 'skipped';
|
||||
}
|
||||
console.error('❌ PDF extraction failed:', error.message, '\n');
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
async function testNodeEmbedding() {
|
||||
console.log('🔮 Testing Node Embedding...');
|
||||
try {
|
||||
// This would require a real node ID and database connection
|
||||
console.log('⚠️ Skipping node embedding test (requires database and API key)\n');
|
||||
return 'skipped';
|
||||
} catch (error) {
|
||||
console.error('❌ Node embedding failed:', error.message, '\n');
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('Environment:');
|
||||
console.log(` Node version: ${process.version}\n`);
|
||||
|
||||
// First test the library directly
|
||||
const libraryStatus = await testTranscriptLibrary();
|
||||
|
||||
const results = {
|
||||
library: libraryStatus,
|
||||
youtube: await testYouTubeExtractor(),
|
||||
website: await testWebsiteExtractor(),
|
||||
pdf: await testPDFExtractor(),
|
||||
embedding: await testNodeEmbedding()
|
||||
};
|
||||
|
||||
console.log('📊 Test Results:');
|
||||
console.log('─────────────────');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const [test, result] of Object.entries(results)) {
|
||||
if (result === 'passed') {
|
||||
console.log(` ${test}: ✅ PASSED`);
|
||||
passed++;
|
||||
} else if (result === 'skipped') {
|
||||
console.log(` ${test}: ⚠️ SKIPPED`);
|
||||
skipped++;
|
||||
} else {
|
||||
console.log(` ${test}: ❌ FAILED`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n📈 Summary:');
|
||||
console.log(` Passed: ${passed}`);
|
||||
console.log(` Failed: ${failed}`);
|
||||
console.log(` Skipped: ${skipped}`);
|
||||
|
||||
if (failed === 0) {
|
||||
console.log('\n🎉 No hard failures. Re-run skipped tests when network/database access is available.');
|
||||
} else {
|
||||
console.log('\n⚠️ Some tests failed. Check the errors above.');
|
||||
}
|
||||
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
// Run tests
|
||||
runTests().catch(error => {
|
||||
console.error('Fatal error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user