#!/usr/bin/env node /** * Test script for Python to TypeScript migration * Tests extractors and embeddings using TypeScript modules */ const fs = require('fs'); const path = require('path'); const Module = require('module'); const ts = require('typescript'); const { getSubtitles } = require('youtube-captions-scraper'); function isNetworkError(error) { if (!error) return false; const message = typeof error === 'string' ? error : error.message || ''; const code = error.code || ''; return [ 'fetch failed', 'ENOTFOUND', 'ECONNRESET', 'EAI_AGAIN', 'ECONNREFUSED', 'ETIMEDOUT' ].some(token => message.includes(token)) || ['ENOTFOUND', 'ECONNRESET', 'EAI_AGAIN', 'ECONNREFUSED', 'ETIMEDOUT'].includes(code); } function requireTs(modulePath) { const resolved = path.resolve(__dirname, '..', modulePath); const source = fs.readFileSync(resolved, 'utf8'); const transpiled = ts.transpileModule(source, { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020, esModuleInterop: true, moduleResolution: ts.ModuleResolutionKind.Node16, resolveJsonModule: true, skipLibCheck: true }, fileName: resolved }); const tempModule = new Module(resolved, module); tempModule.filename = resolved; tempModule.paths = Module._nodeModulePaths(path.dirname(resolved)); tempModule._compile(transpiled.outputText, resolved); return tempModule.exports; } async function testTranscriptLibrary() { console.log('๐Ÿ” Testing youtube-captions-scraper library directly...'); const testUrls = [ 'dQw4w9WgXcQ', // Rick Roll 'MnrJzXM7a6o', // TED Talk 'jNQXAC9IVRw' // Popular video ]; let sawNetworkError = false; let sawEmptyTranscript = false; for (const videoId of testUrls) { console.log(`\nTesting video: ${videoId}`); try { const transcript = await getSubtitles({ videoID: videoId }); if (!transcript || transcript.length === 0) { sawEmptyTranscript = true; console.log('โš ๏ธ Library returned 0 segments (likely geo-blocked). Trying next video...'); continue; } console.log(`โœ… Success! Got ${transcript.length} segments`); console.log('First segment:', transcript[0]); console.log('Sample formatted:', `[${transcript[0].start.toFixed(1)}s] ${transcript[0].text}`); return 'passed'; // Exit on first success } catch (error) { if (isNetworkError(error)) { sawNetworkError = true; console.log('โš ๏ธ Network unavailable, skipping this video'); } else { console.log('โŒ Failed:', error.message); } } } if (sawNetworkError) { console.log('\nโš ๏ธ Network unavailable for transcript tests, marking as SKIPPED'); return 'skipped'; } if (sawEmptyTranscript) { console.log('\nโš ๏ธ Library could not fetch transcripts for any sample video (likely YouTube changes). Marking as SKIPPED.'); return 'skipped'; } console.log('\nโŒ No transcripts could be extracted from any test video'); return 'failed'; } console.log('๐Ÿงช Testing TypeScript Migration...\n'); async function testYouTubeExtractor() { console.log('๐Ÿ“บ Testing YouTube Extractor...'); try { const { extractYouTube } = requireTs('src/services/typescript/extractors/youtube.ts'); const result = await extractYouTube('https://www.youtube.com/watch?v=dQw4w9WgXcQ'); console.log('โœ… YouTube extraction successful'); console.log(` Title: ${result.metadata.title}`); console.log(` Channel: ${result.metadata.author_name}`); console.log(` Transcript length: ${result.chunk.length} chars\n`); return 'passed'; } catch (error) { if (isNetworkError(error)) { console.warn('โš ๏ธ Network unavailable, skipping YouTube extractor test\n'); return 'skipped'; } console.error('โŒ YouTube extraction failed:', error.message, '\n'); return 'failed'; } } async function testWebsiteExtractor() { console.log('๐ŸŒ Testing Website Extractor...'); try { const { extractWebsite } = requireTs('src/services/typescript/extractors/website.ts'); const result = await extractWebsite('https://example.com'); console.log('โœ… Website extraction successful'); console.log(` Title: ${result.metadata.title}`); console.log(` Content length: ${result.chunk.length} chars\n`); return 'passed'; } catch (error) { if (isNetworkError(error)) { console.warn('โš ๏ธ Network unavailable, skipping website extractor test\n'); return 'skipped'; } console.error('โŒ Website extraction failed:', error.message, '\n'); return 'failed'; } } async function testPDFExtractor() { console.log('๐Ÿ“„ Testing PDF Extractor...'); try { const { extractPaper } = requireTs('src/services/typescript/extractors/paper.ts'); // Using a sample PDF URL const samplePdf = 'https://raw.githubusercontent.com/mozilla/pdf.js/master/web/compressed.tracemonkey-pldi-09.pdf'; const result = await extractPaper(samplePdf); console.log('โœ… PDF extraction successful'); console.log(` Pages: ${result.metadata.pages}`); console.log(` Text length: ${result.metadata.text_length} chars\n`); return 'passed'; } catch (error) { if (isNetworkError(error)) { console.warn('โš ๏ธ Network unavailable, skipping PDF extractor test\n'); return 'skipped'; } console.error('โŒ PDF extraction failed:', error.message, '\n'); return 'failed'; } } async function testNodeEmbedding() { console.log('๐Ÿ”ฎ Testing Node Embedding...'); try { // This would require a real node ID and database connection console.log('โš ๏ธ Skipping node embedding test (requires database and API key)\n'); return 'skipped'; } catch (error) { console.error('โŒ Node embedding failed:', error.message, '\n'); return 'failed'; } } async function runTests() { console.log('Environment:'); console.log(` Node version: ${process.version}\n`); // First test the library directly const libraryStatus = await testTranscriptLibrary(); const results = { library: libraryStatus, youtube: await testYouTubeExtractor(), website: await testWebsiteExtractor(), pdf: await testPDFExtractor(), embedding: await testNodeEmbedding() }; console.log('๐Ÿ“Š Test Results:'); console.log('โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€'); let passed = 0; let failed = 0; let skipped = 0; for (const [test, result] of Object.entries(results)) { if (result === 'passed') { console.log(` ${test}: โœ… PASSED`); passed++; } else if (result === 'skipped') { console.log(` ${test}: โš ๏ธ SKIPPED`); skipped++; } else { console.log(` ${test}: โŒ FAILED`); failed++; } } console.log('\n๐Ÿ“ˆ Summary:'); console.log(` Passed: ${passed}`); console.log(` Failed: ${failed}`); console.log(` Skipped: ${skipped}`); if (failed === 0) { console.log('\n๐ŸŽ‰ No hard failures. Re-run skipped tests when network/database access is available.'); } else { console.log('\nโš ๏ธ Some tests failed. Check the errors above.'); } process.exit(failed > 0 ? 1 : 0); } // Run tests runTests().catch(error => { console.error('Fatal error:', error); process.exit(1); });