feat: finalize local OpenAI key flow and MCP docs
- add server-side .env.local OpenAI key management for the open-source app - route AI features through the preferred local key path and lazy-load embed recovery - rewrite README and docs for current MCP setup, schema, and fully-local guidance Generated with Claude Code
This commit is contained in:
@@ -28,13 +28,13 @@ export default function FirstRunModal() {
|
||||
<div style={sectionStyle}>
|
||||
<p style={sectionDescStyle}>
|
||||
To use AI features (embeddings, auto-descriptions, smart tagging),
|
||||
add your OpenAI API key to <code style={codeStyle}>.env.local</code>:
|
||||
add your OpenAI API key in <strong>Settings → API Keys</strong> or directly in <code style={codeStyle}>.env.local</code>:
|
||||
</p>
|
||||
<div style={codeBlockStyle}>
|
||||
<code>OPENAI_API_KEY=sk-your-key-here</code>
|
||||
</div>
|
||||
<p style={costNoteStyle}>
|
||||
Then restart the app. Average cost for heavy use is less than $0.10/day.
|
||||
Settings writes to <code style={codeStyle}>.env.local</code> for you. Average cost for heavy use is less than $0.10/day.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -5,17 +5,80 @@ import { openExternalUrl } from '@/utils/openExternalUrl';
|
||||
|
||||
export default function ApiKeysViewer() {
|
||||
const [status, setStatus] = useState<'checking' | 'configured' | 'not-set'>('checking');
|
||||
const [keyInput, setKeyInput] = useState('');
|
||||
const [maskedKey, setMaskedKey] = useState<string | null>(null);
|
||||
const [envPath, setEnvPath] = useState<string>('.env.local');
|
||||
const [message, setMessage] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
// Check via health endpoint (server-side check of process.env)
|
||||
fetch('/api/health')
|
||||
fetch('/api/settings/openai-key')
|
||||
.then(res => res.json())
|
||||
.then(data => {
|
||||
setStatus(data.aiFeatures?.startsWith('enabled') ? 'configured' : 'not-set');
|
||||
setStatus(data.configured ? 'configured' : 'not-set');
|
||||
setMaskedKey(data.maskedKey ?? null);
|
||||
setEnvPath(data.envPath ?? '.env.local');
|
||||
})
|
||||
.catch(() => setStatus('not-set'));
|
||||
}, []);
|
||||
|
||||
const saveKey = async () => {
|
||||
setSaving(true);
|
||||
setMessage(null);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/settings/openai-key', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ key: keyInput }),
|
||||
});
|
||||
const payload = await response.json();
|
||||
if (!response.ok) {
|
||||
throw new Error(payload.error || 'Failed to save API key');
|
||||
}
|
||||
|
||||
setStatus('configured');
|
||||
setMaskedKey(payload.maskedKey ?? null);
|
||||
setEnvPath(payload.envPath ?? envPath);
|
||||
setKeyInput('');
|
||||
setMessage('Saved to .env.local and updated the running app.');
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to save API key');
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const removeKey = async () => {
|
||||
setSaving(true);
|
||||
setMessage(null);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/settings/openai-key', {
|
||||
method: 'DELETE',
|
||||
});
|
||||
const payload = await response.json();
|
||||
if (!response.ok) {
|
||||
throw new Error(payload.error || 'Failed to remove API key');
|
||||
}
|
||||
|
||||
setStatus('not-set');
|
||||
setMaskedKey(null);
|
||||
setKeyInput('');
|
||||
setEnvPath(payload.envPath ?? envPath);
|
||||
setMessage('Removed from .env.local and cleared the running app key.');
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to remove API key');
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle}>
|
||||
{/* Features explanation */}
|
||||
@@ -50,13 +113,68 @@ export default function ApiKeysViewer() {
|
||||
|
||||
<div style={instructionsStyle}>
|
||||
<p style={{ margin: 0, marginBottom: 8 }}>
|
||||
Add your key to <code style={codeInlineStyle}>.env.local</code> in the project root:
|
||||
Save your key here to write it into <code style={codeInlineStyle}>{envPath}</code>.
|
||||
You can still edit the file directly if you prefer.
|
||||
</p>
|
||||
<div style={codeBlockStyle}>
|
||||
<code>OPENAI_API_KEY=sk-your-key-here</code>
|
||||
</div>
|
||||
<p style={{ margin: 0, fontSize: 12, color: 'var(--settings-muted)' }}>
|
||||
Restart the app after changing the key.
|
||||
<label style={fieldLabelStyle}>
|
||||
{status === 'configured' ? 'Replace OpenAI API key' : 'OpenAI API key'}
|
||||
</label>
|
||||
<input
|
||||
type="password"
|
||||
value={keyInput}
|
||||
onChange={(e) => setKeyInput(e.target.value)}
|
||||
placeholder={status === 'configured' ? 'Paste a new key to replace the current one' : 'sk-...'}
|
||||
style={inputStyle}
|
||||
autoCapitalize="off"
|
||||
autoCorrect="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
<div style={actionsStyle}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={saveKey}
|
||||
disabled={saving || keyInput.trim().length === 0}
|
||||
style={{
|
||||
...primaryButtonStyle,
|
||||
opacity: saving || keyInput.trim().length === 0 ? 0.5 : 1,
|
||||
cursor: saving || keyInput.trim().length === 0 ? 'not-allowed' : 'pointer',
|
||||
}}
|
||||
>
|
||||
{saving ? 'Saving…' : status === 'configured' ? 'Replace key' : 'Save key'}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={removeKey}
|
||||
disabled={saving || status !== 'configured'}
|
||||
style={{
|
||||
...secondaryButtonStyle,
|
||||
opacity: saving || status !== 'configured' ? 0.5 : 1,
|
||||
cursor: saving || status !== 'configured' ? 'not-allowed' : 'pointer',
|
||||
}}
|
||||
>
|
||||
Remove key
|
||||
</button>
|
||||
</div>
|
||||
{maskedKey && (
|
||||
<p style={{ margin: '10px 0 0', fontSize: 12, color: 'var(--settings-muted)' }}>
|
||||
Current key: <code style={codeInlineStyle}>{maskedKey}</code>
|
||||
</p>
|
||||
)}
|
||||
{message && (
|
||||
<p style={{ margin: '10px 0 0', fontSize: 12, color: 'var(--settings-text)' }}>
|
||||
{message}
|
||||
</p>
|
||||
)}
|
||||
{error && (
|
||||
<p style={{ margin: '10px 0 0', fontSize: 12, color: 'var(--settings-danger)' }}>
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
<p style={{ margin: '10px 0 0', fontSize: 12, color: 'var(--settings-muted)' }}>
|
||||
This open-source build uses your own local key only. No Railway key path is used here.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
@@ -143,6 +261,14 @@ const instructionsStyle: CSSProperties = {
|
||||
lineHeight: 1.5,
|
||||
};
|
||||
|
||||
const fieldLabelStyle: CSSProperties = {
|
||||
display: 'block',
|
||||
fontSize: 12,
|
||||
fontWeight: 500,
|
||||
color: 'var(--settings-text)',
|
||||
marginBottom: 6,
|
||||
};
|
||||
|
||||
const codeInlineStyle: CSSProperties = {
|
||||
background: 'var(--settings-code-bg)',
|
||||
padding: '2px 6px',
|
||||
@@ -163,6 +289,44 @@ const codeBlockStyle: CSSProperties = {
|
||||
marginBottom: 8,
|
||||
};
|
||||
|
||||
const inputStyle: CSSProperties = {
|
||||
width: '100%',
|
||||
background: 'var(--settings-code-bg)',
|
||||
border: '1px solid var(--settings-border)',
|
||||
borderRadius: 6,
|
||||
padding: '10px 12px',
|
||||
color: 'var(--settings-text)',
|
||||
fontSize: 13,
|
||||
fontFamily: 'monospace',
|
||||
marginBottom: 12,
|
||||
};
|
||||
|
||||
const actionsStyle: CSSProperties = {
|
||||
display: 'flex',
|
||||
gap: 10,
|
||||
flexWrap: 'wrap',
|
||||
};
|
||||
|
||||
const primaryButtonStyle: CSSProperties = {
|
||||
background: 'var(--settings-text)',
|
||||
color: 'var(--settings-bg)',
|
||||
border: '1px solid var(--settings-text)',
|
||||
borderRadius: 6,
|
||||
padding: '9px 14px',
|
||||
fontSize: 12,
|
||||
fontWeight: 600,
|
||||
};
|
||||
|
||||
const secondaryButtonStyle: CSSProperties = {
|
||||
background: 'transparent',
|
||||
color: 'var(--settings-text)',
|
||||
border: '1px solid var(--settings-border-strong)',
|
||||
borderRadius: 6,
|
||||
padding: '9px 14px',
|
||||
fontSize: 12,
|
||||
fontWeight: 500,
|
||||
};
|
||||
|
||||
const helpStyle: CSSProperties = {
|
||||
fontSize: 12,
|
||||
color: 'var(--settings-muted)',
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { generateText } from 'ai';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { createLocalOpenAIProvider } from '@/services/openai/localProvider';
|
||||
|
||||
export interface TranscriptSummaryResult {
|
||||
subject?: string;
|
||||
@@ -48,8 +48,9 @@ export async function summarizeTranscript(transcript: string): Promise<Transcrip
|
||||
: transcript;
|
||||
|
||||
try {
|
||||
const provider = createLocalOpenAIProvider();
|
||||
const response = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt: buildPrompt(limited),
|
||||
maxOutputTokens: 600,
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { generateText } from 'ai';
|
||||
import { openai as openaiProvider } from '@ai-sdk/openai';
|
||||
import { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { hasPreferredOpenAiKey, getPreferredOpenAiKey } from '../storage/openaiKeyServer';
|
||||
import type { CanonicalNodeMetadata } from '@/types/database';
|
||||
|
||||
export interface DescriptionInput {
|
||||
@@ -23,7 +23,7 @@ export interface DescriptionInput {
|
||||
* The result must cover what the artifact is, why it is in the graph, and workflow status.
|
||||
*/
|
||||
export async function generateDescription(input: DescriptionInput): Promise<string> {
|
||||
if (!hasValidOpenAiKey()) {
|
||||
if (!hasPreferredOpenAiKey()) {
|
||||
console.log(`[DescriptionService] No valid OpenAI key, using fallback for: "${input.title}"`);
|
||||
return `${input.title}. Added via Quick Add with no further context yet, so the reason it belongs in the graph is not fully inferred. It has not been reviewed yet.`.slice(0, 500);
|
||||
}
|
||||
@@ -33,8 +33,9 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
|
||||
|
||||
console.log(`[DescriptionService] Generating description for: "${input.title}"`);
|
||||
|
||||
const provider = createOpenAI({ apiKey: getPreferredOpenAiKey() });
|
||||
const response = await generateText({
|
||||
model: openaiProvider('gpt-4o-mini'),
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 100,
|
||||
temperature: 0.3,
|
||||
@@ -52,8 +53,6 @@ export async function generateDescription(input: DescriptionInput): Promise<stri
|
||||
}
|
||||
}
|
||||
|
||||
export { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
|
||||
function buildDescriptionPrompt(input: DescriptionInput): string {
|
||||
const sourceMetadata = input.metadata?.source_metadata as Record<string, unknown> | undefined;
|
||||
const sourceType = typeof input.metadata?.type === 'string'
|
||||
|
||||
@@ -3,10 +3,10 @@ import { Edge, EdgeContext, EdgeData, EdgeCreatedVia, NodeConnection, Node } fro
|
||||
import { eventBroadcaster } from '../events';
|
||||
import { nodeService } from './nodes';
|
||||
import { generateText } from 'ai';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { z } from 'zod';
|
||||
import { validateEdgeExplanation } from './quality';
|
||||
import { hasValidOpenAiKey } from '../storage/apiKeys';
|
||||
import { getPreferredOpenAiKey, hasPreferredOpenAiKey } from '../storage/openaiKeyServer';
|
||||
|
||||
const inferredEdgeContextSchema = z.object({
|
||||
type: z.enum(['created_by', 'part_of', 'source_of', 'related_to']),
|
||||
@@ -75,12 +75,13 @@ async function inferEdgeContext(params: {
|
||||
].join('\n');
|
||||
|
||||
try {
|
||||
if (!hasValidOpenAiKey()) {
|
||||
if (!hasPreferredOpenAiKey()) {
|
||||
return { type: 'related_to', confidence: 0.2, swap_direction: false };
|
||||
}
|
||||
|
||||
const provider = createOpenAI({ apiKey: getPreferredOpenAiKey() });
|
||||
const { text } = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt,
|
||||
temperature: 0.0,
|
||||
maxOutputTokens: 120,
|
||||
@@ -141,7 +142,7 @@ async function autoInferEdge(params: {
|
||||
].join('\n');
|
||||
|
||||
try {
|
||||
if (!hasValidOpenAiKey()) {
|
||||
if (!hasPreferredOpenAiKey()) {
|
||||
return {
|
||||
explanation: `Connection to ${toNode.title}; exact relationship uncertain.`,
|
||||
type: 'related_to',
|
||||
@@ -150,8 +151,9 @@ async function autoInferEdge(params: {
|
||||
};
|
||||
}
|
||||
|
||||
const provider = createOpenAI({ apiKey: getPreferredOpenAiKey() });
|
||||
const { text } = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt,
|
||||
temperature: 0.0,
|
||||
maxOutputTokens: 150,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import OpenAI from 'openai';
|
||||
import { getPreferredOpenAiKey } from './storage/openaiKeyServer';
|
||||
|
||||
function getOpenAiClient(): OpenAI {
|
||||
const apiKey = process.env.OPENAI_API_KEY;
|
||||
const apiKey = getPreferredOpenAiKey();
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API key not configured. Add OPENAI_API_KEY to your .env.local file.');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { getPreferredOpenAiKey } from '@/services/storage/openaiKeyServer';
|
||||
|
||||
export function createLocalOpenAIProvider() {
|
||||
const apiKey = getPreferredOpenAiKey();
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API key not configured. Add your key in Settings or .env.local.');
|
||||
}
|
||||
|
||||
return createOpenAI({ apiKey });
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
const OPENAI_KEY_PREFIXES = ['sk-', 'sk-proj-'];
|
||||
const OPENAI_KEY_PLACEHOLDER = 'your-openai-api-key-here';
|
||||
|
||||
export function getEnvLocalPath(): string {
|
||||
return path.join(process.cwd(), '.env.local');
|
||||
}
|
||||
|
||||
export function isValidOpenAiKey(key: string | null | undefined): boolean {
|
||||
if (!key) return false;
|
||||
const trimmed = key.trim();
|
||||
if (!trimmed || trimmed === OPENAI_KEY_PLACEHOLDER) return false;
|
||||
return OPENAI_KEY_PREFIXES.some((prefix) => trimmed.startsWith(prefix)) && trimmed.length > 20;
|
||||
}
|
||||
|
||||
export function maskOpenAiKey(key: string | null | undefined): string | null {
|
||||
if (!key) return null;
|
||||
const trimmed = key.trim();
|
||||
if (trimmed.length <= 8) return '••••';
|
||||
return `${'•'.repeat(Math.max(8, trimmed.length - 4))}${trimmed.slice(-4)}`;
|
||||
}
|
||||
|
||||
async function readEnvLocalFile(): Promise<string> {
|
||||
const envPath = getEnvLocalPath();
|
||||
try {
|
||||
return await fs.readFile(envPath, 'utf8');
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') {
|
||||
return '';
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readStoredOpenAiKey(): Promise<string | null> {
|
||||
const contents = await readEnvLocalFile();
|
||||
const lines = contents.split(/\r?\n/);
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
if (!trimmed.startsWith('OPENAI_API_KEY=')) continue;
|
||||
const rawValue = trimmed.slice('OPENAI_API_KEY='.length).trim();
|
||||
const unquoted =
|
||||
(rawValue.startsWith('"') && rawValue.endsWith('"')) ||
|
||||
(rawValue.startsWith("'") && rawValue.endsWith("'"))
|
||||
? rawValue.slice(1, -1)
|
||||
: rawValue;
|
||||
return unquoted || null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function writeOpenAiKeyToEnvLocal(key: string | null): Promise<void> {
|
||||
const envPath = getEnvLocalPath();
|
||||
const existing = await readEnvLocalFile();
|
||||
const lines = existing ? existing.split(/\r?\n/) : [];
|
||||
|
||||
const nextLines: string[] = [];
|
||||
let handled = false;
|
||||
|
||||
for (const line of lines) {
|
||||
if (/^\s*OPENAI_API_KEY\s*=/.test(line)) {
|
||||
if (!handled && key) {
|
||||
nextLines.push(`OPENAI_API_KEY=${key}`);
|
||||
}
|
||||
handled = true;
|
||||
continue;
|
||||
}
|
||||
nextLines.push(line);
|
||||
}
|
||||
|
||||
if (!handled && key) {
|
||||
if (nextLines.length > 0 && nextLines[nextLines.length - 1]?.trim() !== '') {
|
||||
nextLines.push('');
|
||||
}
|
||||
nextLines.push(`OPENAI_API_KEY=${key}`);
|
||||
}
|
||||
|
||||
const normalized = nextLines.join('\n').replace(/\n{3,}/g, '\n\n');
|
||||
const finalContents = normalized.length > 0 ? `${normalized}\n` : '';
|
||||
await fs.writeFile(envPath, finalContents, 'utf8');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
const PLACEHOLDER = 'your-openai-api-key-here';
|
||||
|
||||
function parseOpenAiKeyFromEnvFile(contents: string): string | undefined {
|
||||
for (const line of contents.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
if (!trimmed.startsWith('OPENAI_API_KEY=')) continue;
|
||||
const raw = trimmed.slice('OPENAI_API_KEY='.length).trim();
|
||||
const value =
|
||||
(raw.startsWith('"') && raw.endsWith('"')) ||
|
||||
(raw.startsWith("'") && raw.endsWith("'"))
|
||||
? raw.slice(1, -1)
|
||||
: raw;
|
||||
if (!value || value === PLACEHOLDER) return undefined;
|
||||
return value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function getPreferredOpenAiKey(): string | undefined {
|
||||
const envPath = path.join(process.cwd(), '.env.local');
|
||||
try {
|
||||
const fileKey = parseOpenAiKeyFromEnvFile(fs.readFileSync(envPath, 'utf8'));
|
||||
if (fileKey) return fileKey;
|
||||
} catch {
|
||||
// Ignore missing/unreadable .env.local and fall back to process env.
|
||||
}
|
||||
|
||||
const envKey = process.env.OPENAI_API_KEY;
|
||||
if (!envKey || envKey === PLACEHOLDER) return undefined;
|
||||
return envKey;
|
||||
}
|
||||
|
||||
export function hasPreferredOpenAiKey(): boolean {
|
||||
const key = getPreferredOpenAiKey();
|
||||
return Boolean(key && key.startsWith('sk-') && key.length > 20);
|
||||
}
|
||||
@@ -6,6 +6,7 @@
|
||||
import OpenAI from 'openai';
|
||||
import { generateText } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { getPreferredOpenAiKey } from '@/services/storage/openaiKeyServer';
|
||||
import {
|
||||
createDatabaseConnection,
|
||||
serializeFloat32Vector,
|
||||
@@ -38,7 +39,7 @@ export class NodeEmbedder {
|
||||
private failedCount: number = 0;
|
||||
|
||||
constructor() {
|
||||
const apiKey = process.env.OPENAI_API_KEY;
|
||||
const apiKey = getPreferredOpenAiKey();
|
||||
if (!apiKey) {
|
||||
throw new Error('OPENAI_API_KEY environment variable is not set');
|
||||
}
|
||||
|
||||
@@ -5,13 +5,11 @@
|
||||
|
||||
import OpenAI from 'openai';
|
||||
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
|
||||
import { getPreferredOpenAiKey } from '@/services/storage/openaiKeyServer';
|
||||
import {
|
||||
createDatabaseConnection,
|
||||
getDbVectorCapability,
|
||||
serializeFloat32Vector,
|
||||
batchProcess
|
||||
} from './sqlite-vec';
|
||||
import type { VectorCapability } from '@/services/database/sqlite-runtime';
|
||||
|
||||
interface Node {
|
||||
id: number;
|
||||
@@ -39,18 +37,16 @@ export class UniversalEmbedder {
|
||||
private openaiClient: OpenAI;
|
||||
private db: ReturnType<typeof createDatabaseConnection>;
|
||||
private textSplitter: RecursiveCharacterTextSplitter;
|
||||
private readonly vectorCapability: VectorCapability;
|
||||
private vecChunksInsertSQL: string | null = null;
|
||||
|
||||
constructor() {
|
||||
const apiKey = process.env.OPENAI_API_KEY;
|
||||
const apiKey = getPreferredOpenAiKey();
|
||||
if (!apiKey) {
|
||||
throw new Error('OPENAI_API_KEY environment variable is not set');
|
||||
}
|
||||
|
||||
this.openaiClient = new OpenAI({ apiKey });
|
||||
this.db = createDatabaseConnection();
|
||||
this.vectorCapability = getDbVectorCapability(this.db);
|
||||
|
||||
// Configure text splitter (same as old KMS system)
|
||||
this.textSplitter = new RecursiveCharacterTextSplitter({
|
||||
@@ -91,14 +87,12 @@ export class UniversalEmbedder {
|
||||
const chunkIds = this.db.prepare('SELECT id FROM chunks WHERE node_id = ?').all(nodeId) as Array<{ id: number }>;
|
||||
|
||||
// Delete from vec_chunks first, one by one to ensure they're removed
|
||||
if (this.vectorCapability.available) {
|
||||
for (const chunk of chunkIds) {
|
||||
try {
|
||||
const deleteVecStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
|
||||
deleteVecStmt.run(BigInt(chunk.id));
|
||||
} catch (error) {
|
||||
console.warn(`Could not delete vec_chunk ${chunk.id}:`, error);
|
||||
}
|
||||
for (const chunk of chunkIds) {
|
||||
try {
|
||||
const deleteVecStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
|
||||
deleteVecStmt.run(BigInt(chunk.id));
|
||||
} catch (error) {
|
||||
console.warn(`Could not delete vec_chunk ${chunk.id}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,20 +131,18 @@ export class UniversalEmbedder {
|
||||
|
||||
const chunkId = Number(result.lastInsertRowid);
|
||||
|
||||
if (this.vectorCapability.available) {
|
||||
try {
|
||||
const vectorString = `[${embedding.join(',')}]`;
|
||||
try {
|
||||
const vectorString = `[${embedding.join(',')}]`;
|
||||
try {
|
||||
const deleteStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
|
||||
deleteStmt.run(BigInt(chunkId));
|
||||
} catch {}
|
||||
const deleteStmt = this.db.prepare('DELETE FROM vec_chunks WHERE chunk_id = ?');
|
||||
deleteStmt.run(BigInt(chunkId));
|
||||
} catch {}
|
||||
|
||||
const sql = 'INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)';
|
||||
const vecInsertStmt = this.db.prepare(sql);
|
||||
vecInsertStmt.run(BigInt(chunkId), vectorString);
|
||||
} catch (error) {
|
||||
console.warn(`Could not insert into vec_chunks for chunk ${chunkId}:`, error);
|
||||
}
|
||||
const sql = this.resolveVecChunksInsertSQL();
|
||||
const vecInsertStmt = this.db.prepare(sql);
|
||||
vecInsertStmt.run(BigInt(chunkId), vectorString);
|
||||
} catch (error) {
|
||||
console.warn(`Could not insert into vec_chunks for chunk ${chunkId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
import { extractPaper } from '@/services/typescript/extractors/paper';
|
||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
||||
import { validateExplicitDescription } from '@/services/database/quality';
|
||||
import { createLocalOpenAIProvider } from '@/services/openai/localProvider';
|
||||
|
||||
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
|
||||
const normalizedCandidate = typeof candidate === 'string'
|
||||
@@ -25,6 +25,7 @@ function ensureNodeDescription(candidate: string | undefined, fallbackLead: stri
|
||||
// AI-powered content analysis
|
||||
async function analyzeContentWithAI(title: string, description: string, contentType: string) {
|
||||
try {
|
||||
const provider = createLocalOpenAIProvider();
|
||||
const prompt = `Analyze this ${contentType} content and provide classification.
|
||||
|
||||
Title: "${title}"
|
||||
@@ -57,7 +58,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
|
||||
}`;
|
||||
|
||||
const response = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 800
|
||||
});
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
import { extractWebsite } from '@/services/typescript/extractors/website';
|
||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
||||
import { validateExplicitDescription } from '@/services/database/quality';
|
||||
import { createLocalOpenAIProvider } from '@/services/openai/localProvider';
|
||||
|
||||
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
|
||||
const normalizedCandidate = typeof candidate === 'string'
|
||||
@@ -40,6 +40,7 @@ async function analyzeContentWithAI(
|
||||
contentType: string
|
||||
) {
|
||||
try {
|
||||
const provider = createLocalOpenAIProvider();
|
||||
const prompt = `Analyze this ${contentType} content and provide classification.
|
||||
|
||||
Title: "${title}"
|
||||
@@ -71,7 +72,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
|
||||
}`;
|
||||
|
||||
const response = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 800
|
||||
});
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { openai } from '@ai-sdk/openai';
|
||||
import { generateText } from 'ai';
|
||||
import { extractYouTube } from '@/services/typescript/extractors/youtube';
|
||||
import { getInternalApiBaseUrl } from '@/services/runtime/apiBase';
|
||||
import { formatNodeForChat } from '../infrastructure/nodeFormatter';
|
||||
import { validateExplicitDescription } from '@/services/database/quality';
|
||||
import { createLocalOpenAIProvider } from '@/services/openai/localProvider';
|
||||
|
||||
function ensureNodeDescription(candidate: string | undefined, fallbackLead: string): string {
|
||||
const normalizedCandidate = typeof candidate === 'string'
|
||||
@@ -29,6 +29,7 @@ async function analyzeContentWithAI(
|
||||
contentType: string
|
||||
) {
|
||||
try {
|
||||
const provider = createLocalOpenAIProvider();
|
||||
const prompt = `Analyze this ${contentType} content and provide classification.
|
||||
|
||||
Title: "${title}"
|
||||
@@ -60,7 +61,7 @@ Respond with ONLY valid JSON (no markdown, no code blocks):
|
||||
}`;
|
||||
|
||||
const response = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 800
|
||||
});
|
||||
@@ -113,8 +114,9 @@ ${excerpt}
|
||||
`;
|
||||
|
||||
try {
|
||||
const provider = createLocalOpenAIProvider();
|
||||
const response = await generateText({
|
||||
model: openai('gpt-4o-mini'),
|
||||
model: provider('gpt-4o-mini'),
|
||||
prompt,
|
||||
maxOutputTokens: 400
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user