Initial commit: RA-H Open Source Edition
Local-first knowledge management system with BYO API keys. Features: - 3-panel UI (Nodes | Focus | Helpers) - SQLite + sqlite-vec for vector search - Agent system (Easy/Hard mode orchestrators) - Content extraction (YouTube, PDF, web) - Integrate workflow for connection discovery - Dimension system with auto-assignment Tech stack: - Next.js 15 + TypeScript + Tailwind CSS - Anthropic (Claude) + OpenAI (GPT) via Vercel AI SDK Setup: npm install && npm rebuild better-sqlite3 scripts/dev/bootstrap-local.sh npm run dev MIT License
This commit is contained in:
“BeeRad”
@@ -0,0 +1,300 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
export type TTSStatus = 'idle' | 'loading' | 'speaking';
|
||||
|
||||
interface UseAssistantTTSOptions {
|
||||
voice?: string;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechComplete?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
}
|
||||
|
||||
interface SpeakRequestMetadata {
|
||||
sessionId?: string | null;
|
||||
helper?: string | null;
|
||||
requestId?: string;
|
||||
messageId?: string | null;
|
||||
}
|
||||
|
||||
interface SpeakOptions {
|
||||
flush?: boolean;
|
||||
metadata?: SpeakRequestMetadata;
|
||||
}
|
||||
|
||||
type SpeakQueueItem = {
|
||||
text: string;
|
||||
metadata?: SpeakRequestMetadata;
|
||||
};
|
||||
|
||||
export function useAssistantTTS(options: UseAssistantTTSOptions = {}) {
|
||||
const [status, setStatus] = useState<TTSStatus>('idle');
|
||||
const queueRef = useRef<SpeakQueueItem[]>([]);
|
||||
const isProcessingRef = useRef(false);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const objectUrlRef = useRef<string | null>(null);
|
||||
const mediaSourceRef = useRef<MediaSource | null>(null);
|
||||
const sourceBufferRef = useRef<SourceBuffer | null>(null);
|
||||
const chunkQueueRef = useRef<ArrayBuffer[]>([]);
|
||||
const updateEndHandlerRef = useRef<(() => void) | null>(null);
|
||||
const readerDoneRef = useRef(false);
|
||||
const processQueueRef = useRef<(() => void) | null>(null);
|
||||
|
||||
const onSpeechStartRef = useRef(options.onSpeechStart);
|
||||
const onSpeechCompleteRef = useRef(options.onSpeechComplete);
|
||||
const onErrorRef = useRef(options.onError);
|
||||
const voiceRef = useRef(options.voice);
|
||||
|
||||
useEffect(() => {
|
||||
onSpeechStartRef.current = options.onSpeechStart;
|
||||
}, [options.onSpeechStart]);
|
||||
|
||||
useEffect(() => {
|
||||
onSpeechCompleteRef.current = options.onSpeechComplete;
|
||||
}, [options.onSpeechComplete]);
|
||||
|
||||
useEffect(() => {
|
||||
onErrorRef.current = options.onError;
|
||||
}, [options.onError]);
|
||||
|
||||
useEffect(() => {
|
||||
voiceRef.current = options.voice;
|
||||
}, [options.voice]);
|
||||
|
||||
const setStatusSafe = useCallback((next: TTSStatus) => {
|
||||
setStatus(next);
|
||||
}, []);
|
||||
|
||||
const cleanupMedia = useCallback(() => {
|
||||
if (sourceBufferRef.current && updateEndHandlerRef.current) {
|
||||
try {
|
||||
sourceBufferRef.current.removeEventListener('updateend', updateEndHandlerRef.current);
|
||||
} catch (err) {
|
||||
console.warn('[TTS] Failed to detach source buffer listener', err);
|
||||
}
|
||||
}
|
||||
updateEndHandlerRef.current = null;
|
||||
sourceBufferRef.current = null;
|
||||
mediaSourceRef.current = null;
|
||||
if (objectUrlRef.current) {
|
||||
URL.revokeObjectURL(objectUrlRef.current);
|
||||
objectUrlRef.current = null;
|
||||
}
|
||||
chunkQueueRef.current = [];
|
||||
readerDoneRef.current = false;
|
||||
}, []);
|
||||
|
||||
const handleError = useCallback((error: Error) => {
|
||||
console.error('[TTS] Playback error', error);
|
||||
onErrorRef.current?.(error);
|
||||
}, []);
|
||||
|
||||
const stopCurrentPlayback = useCallback(() => {
|
||||
abortRef.current?.abort();
|
||||
abortRef.current = null;
|
||||
const audio = audioRef.current;
|
||||
if (audio) {
|
||||
audio.pause();
|
||||
audio.removeAttribute('src');
|
||||
try {
|
||||
audio.load();
|
||||
} catch (err) {
|
||||
console.warn('[TTS] Failed to reset audio element', err);
|
||||
}
|
||||
}
|
||||
cleanupMedia();
|
||||
isProcessingRef.current = false;
|
||||
}, [cleanupMedia]);
|
||||
|
||||
const ensureAudioElement = useCallback(() => {
|
||||
if (audioRef.current) {
|
||||
return audioRef.current;
|
||||
}
|
||||
const audio = new Audio();
|
||||
audio.autoplay = true;
|
||||
audio.preload = 'auto';
|
||||
audio.addEventListener('playing', () => {
|
||||
setStatusSafe('speaking');
|
||||
onSpeechStartRef.current?.();
|
||||
});
|
||||
audio.addEventListener('ended', () => {
|
||||
cleanupMedia();
|
||||
isProcessingRef.current = false;
|
||||
setStatusSafe('idle');
|
||||
onSpeechCompleteRef.current?.();
|
||||
processQueueRef.current?.();
|
||||
});
|
||||
audio.addEventListener('error', () => {
|
||||
handleError(new Error('Audio playback failed'));
|
||||
stopCurrentPlayback();
|
||||
processQueueRef.current?.();
|
||||
});
|
||||
audioRef.current = audio;
|
||||
return audio;
|
||||
}, [cleanupMedia, handleError, setStatusSafe, stopCurrentPlayback]);
|
||||
|
||||
const flushChunks = useCallback(() => {
|
||||
const sourceBuffer = sourceBufferRef.current;
|
||||
const mediaSource = mediaSourceRef.current;
|
||||
if (!sourceBuffer || !mediaSource || sourceBuffer.updating) {
|
||||
return;
|
||||
}
|
||||
const nextChunk = chunkQueueRef.current.shift();
|
||||
if (nextChunk) {
|
||||
try {
|
||||
sourceBuffer.appendBuffer(nextChunk);
|
||||
} catch (error) {
|
||||
handleError(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (readerDoneRef.current && !sourceBuffer.updating) {
|
||||
try {
|
||||
mediaSource.endOfStream();
|
||||
} catch (error) {
|
||||
console.warn('[TTS] Failed to end media source stream', error);
|
||||
}
|
||||
}
|
||||
}, [handleError]);
|
||||
|
||||
const processQueue = useCallback(async () => {
|
||||
if (isProcessingRef.current) return;
|
||||
const next = queueRef.current.shift();
|
||||
if (!next) {
|
||||
setStatusSafe('idle');
|
||||
return;
|
||||
}
|
||||
isProcessingRef.current = true;
|
||||
setStatusSafe('loading');
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
const payload: Record<string, any> = {
|
||||
text: next.text,
|
||||
voice: voiceRef.current,
|
||||
};
|
||||
|
||||
if (next.metadata?.sessionId) {
|
||||
payload.sessionId = next.metadata.sessionId;
|
||||
}
|
||||
if (next.metadata?.helper) {
|
||||
payload.helper = next.metadata.helper;
|
||||
}
|
||||
if (next.metadata?.requestId) {
|
||||
payload.requestId = next.metadata.requestId;
|
||||
}
|
||||
if (next.metadata?.messageId) {
|
||||
payload.messageId = next.metadata.messageId;
|
||||
}
|
||||
|
||||
const response = await fetch('/api/voice/tts', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error((await response.text().catch(() => '')) || 'Failed to synthesize audio');
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const audio = ensureAudioElement();
|
||||
cleanupMedia();
|
||||
const mimeType = response.headers.get('Content-Type') || 'audio/mpeg';
|
||||
const mediaSource = new MediaSource();
|
||||
mediaSourceRef.current = mediaSource;
|
||||
const objectUrl = URL.createObjectURL(mediaSource);
|
||||
objectUrlRef.current = objectUrl;
|
||||
audio.src = objectUrl;
|
||||
audio.load();
|
||||
|
||||
const onSourceOpen = () => {
|
||||
mediaSource.removeEventListener('sourceopen', onSourceOpen);
|
||||
let sourceBuffer: SourceBuffer;
|
||||
try {
|
||||
sourceBuffer = mediaSource.addSourceBuffer(mimeType);
|
||||
} catch {
|
||||
throw new Error(`Unsupported audio format: ${mimeType}`);
|
||||
}
|
||||
sourceBufferRef.current = sourceBuffer;
|
||||
const handleUpdateEnd = () => flushChunks();
|
||||
updateEndHandlerRef.current = handleUpdateEnd;
|
||||
sourceBuffer.addEventListener('updateend', handleUpdateEnd);
|
||||
flushChunks();
|
||||
};
|
||||
|
||||
mediaSource.addEventListener('sourceopen', onSourceOpen);
|
||||
|
||||
const pump = async () => {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
readerDoneRef.current = true;
|
||||
flushChunks();
|
||||
break;
|
||||
}
|
||||
if (value) {
|
||||
const buffer = value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
|
||||
chunkQueueRef.current.push(buffer);
|
||||
flushChunks();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
pump().catch((error) => {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
if ((err as DOMException).name !== 'AbortError') {
|
||||
handleError(err);
|
||||
}
|
||||
stopCurrentPlayback();
|
||||
processQueueRef.current?.();
|
||||
});
|
||||
|
||||
audio.play().catch((error) => {
|
||||
handleError(error instanceof Error ? error : new Error(String(error)));
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
isProcessingRef.current = false;
|
||||
setStatusSafe('idle');
|
||||
handleError(error instanceof Error ? error : new Error(String(error)));
|
||||
processQueueRef.current?.();
|
||||
}
|
||||
}, [cleanupMedia, ensureAudioElement, flushChunks, handleError, setStatusSafe, stopCurrentPlayback]);
|
||||
|
||||
processQueueRef.current = processQueue;
|
||||
|
||||
const speak = useCallback(
|
||||
(text: string, options?: SpeakOptions) => {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return;
|
||||
if (options?.flush) {
|
||||
queueRef.current = [{ text: trimmed, metadata: options.metadata }];
|
||||
stopCurrentPlayback();
|
||||
} else {
|
||||
queueRef.current.push({ text: trimmed, metadata: options?.metadata });
|
||||
}
|
||||
processQueue();
|
||||
},
|
||||
[processQueue, stopCurrentPlayback]
|
||||
);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
queueRef.current = [];
|
||||
stopCurrentPlayback();
|
||||
setStatusSafe('idle');
|
||||
}, [setStatusSafe, stopCurrentPlayback]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stop();
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return {
|
||||
status,
|
||||
speak,
|
||||
stop,
|
||||
} as const;
|
||||
}
|
||||
@@ -0,0 +1,570 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useRef } from 'react';
|
||||
|
||||
export type RealtimeConnectionState = 'idle' | 'connecting' | 'ready' | 'capturing';
|
||||
|
||||
interface VoiceRealtimeCallbacks {
|
||||
onStatusChange?: (status: 'idle' | 'listening' | 'thinking' | 'speaking') => void;
|
||||
onInterimTranscript?: (text: string) => void;
|
||||
onFinalTranscript?: (text: string) => void;
|
||||
onAmplitude?: (value: number) => void;
|
||||
onError?: (error: Error) => void;
|
||||
}
|
||||
|
||||
interface UseRealtimeVoiceClientOptions {
|
||||
getAuthToken?: () => string | null | undefined;
|
||||
fetchEphemeralToken?: (
|
||||
authToken: string | null
|
||||
) => Promise<{ client_secret: { value: string }; model: string; voice: string }>;
|
||||
silenceThresholdMs?: number;
|
||||
silenceAmplitudeCutoff?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_SILENCE_THRESHOLD_MS = 800;
|
||||
const DEFAULT_SILENCE_AMPLITUDE = 0.0015;
|
||||
|
||||
type Nullable<T> = T | null;
|
||||
|
||||
function calculateRms(buffer: Float32Array) {
|
||||
if (!buffer.length) return 0;
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < buffer.length; i += 1) {
|
||||
const value = buffer[i];
|
||||
sumSquares += value * value;
|
||||
}
|
||||
return Math.sqrt(sumSquares / buffer.length);
|
||||
}
|
||||
|
||||
|
||||
export function useRealtimeVoiceClient(
|
||||
callbacks: VoiceRealtimeCallbacks,
|
||||
options: UseRealtimeVoiceClientOptions = {}
|
||||
) {
|
||||
const { onStatusChange, onInterimTranscript, onFinalTranscript, onAmplitude, onError } = callbacks;
|
||||
|
||||
const { getAuthToken, fetchEphemeralToken, silenceThresholdMs, silenceAmplitudeCutoff } = options;
|
||||
|
||||
const connectionStateRef = useRef<RealtimeConnectionState>('idle');
|
||||
const peerConnectionRef = useRef<Nullable<RTCPeerConnection>>(null);
|
||||
const dataChannelRef = useRef<Nullable<RTCDataChannel>>(null);
|
||||
const audioContextRef = useRef<Nullable<AudioContext>>(null);
|
||||
const processorNodeRef = useRef<Nullable<ScriptProcessorNode>>(null);
|
||||
const mediaStreamRef = useRef<Nullable<MediaStream>>(null);
|
||||
const mediaSourceRef = useRef<Nullable<MediaStreamAudioSourceNode>>(null);
|
||||
const awaitingTranscriptRef = useRef(false);
|
||||
const hasUncommittedAudioRef = useRef(false);
|
||||
const lastSpeechAtRef = useRef<number | null>(null);
|
||||
const destroyedRef = useRef(false);
|
||||
const channelReadyRef = useRef(false);
|
||||
|
||||
const silenceWindowMs = silenceThresholdMs ?? DEFAULT_SILENCE_THRESHOLD_MS;
|
||||
const amplitudeGate = silenceAmplitudeCutoff ?? DEFAULT_SILENCE_AMPLITUDE;
|
||||
|
||||
const onStatusChangeRef = useRef(onStatusChange);
|
||||
const onInterimTranscriptRef = useRef(onInterimTranscript);
|
||||
const onFinalTranscriptRef = useRef(onFinalTranscript);
|
||||
const onAmplitudeRef = useRef(onAmplitude);
|
||||
const onErrorRef = useRef(onError);
|
||||
|
||||
useEffect(() => {
|
||||
onStatusChangeRef.current = onStatusChange;
|
||||
}, [onStatusChange]);
|
||||
|
||||
useEffect(() => {
|
||||
onInterimTranscriptRef.current = onInterimTranscript;
|
||||
}, [onInterimTranscript]);
|
||||
|
||||
useEffect(() => {
|
||||
onFinalTranscriptRef.current = onFinalTranscript;
|
||||
}, [onFinalTranscript]);
|
||||
|
||||
useEffect(() => {
|
||||
onAmplitudeRef.current = onAmplitude;
|
||||
}, [onAmplitude]);
|
||||
|
||||
useEffect(() => {
|
||||
onErrorRef.current = onError;
|
||||
}, [onError]);
|
||||
|
||||
const setConnectionState = useCallback((next: RealtimeConnectionState) => {
|
||||
connectionStateRef.current = next;
|
||||
}, []);
|
||||
|
||||
const teardownInputNodes = useCallback(() => {
|
||||
processorNodeRef.current?.disconnect();
|
||||
mediaSourceRef.current?.disconnect();
|
||||
processorNodeRef.current?.removeEventListener('audioprocess', () => undefined);
|
||||
processorNodeRef.current = null;
|
||||
mediaSourceRef.current = null;
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getTracks().forEach((track) => track.stop());
|
||||
mediaStreamRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const closePeerConnection = useCallback(() => {
|
||||
if (dataChannelRef.current) {
|
||||
try {
|
||||
dataChannelRef.current.close();
|
||||
} catch (err) {
|
||||
console.warn('[VoiceRealtime] Failed to close data channel:', err);
|
||||
}
|
||||
}
|
||||
dataChannelRef.current = null;
|
||||
|
||||
if (peerConnectionRef.current) {
|
||||
try {
|
||||
peerConnectionRef.current.ontrack = null;
|
||||
peerConnectionRef.current.onconnectionstatechange = null;
|
||||
peerConnectionRef.current.close();
|
||||
} catch (err) {
|
||||
console.warn('[VoiceRealtime] Failed to close peer connection:', err);
|
||||
}
|
||||
}
|
||||
peerConnectionRef.current = null;
|
||||
|
||||
channelReadyPromiseRef.current = null;
|
||||
channelReadyResolveRef.current = null;
|
||||
}, []);
|
||||
|
||||
const resetState = useCallback(() => {
|
||||
awaitingTranscriptRef.current = false;
|
||||
hasUncommittedAudioRef.current = false;
|
||||
lastSpeechAtRef.current = null;
|
||||
channelReadyPromiseRef.current = null;
|
||||
channelReadyResolveRef.current = null;
|
||||
}, []);
|
||||
|
||||
const notifyError = useCallback((message: string | Error) => {
|
||||
const error = message instanceof Error ? message : new Error(message);
|
||||
onErrorRef.current?.(error);
|
||||
}, []);
|
||||
|
||||
const channelReadyPromiseRef = useRef<Promise<void> | null>(null);
|
||||
const channelReadyResolveRef = useRef<(() => void) | null>(null);
|
||||
|
||||
const ensureChannelReady = useCallback(async () => {
|
||||
const channel = dataChannelRef.current;
|
||||
if (channel?.readyState === 'open') return;
|
||||
if (!channelReadyPromiseRef.current) {
|
||||
channelReadyPromiseRef.current = new Promise<void>((resolve) => {
|
||||
channelReadyResolveRef.current = resolve;
|
||||
});
|
||||
}
|
||||
await channelReadyPromiseRef.current;
|
||||
}, []);
|
||||
|
||||
const sendEvent = useCallback(
|
||||
async (event: Record<string, unknown>) => {
|
||||
await ensureChannelReady();
|
||||
const channel = dataChannelRef.current;
|
||||
if (!channel || channel.readyState !== 'open') {
|
||||
throw new Error('Realtime data channel is not open');
|
||||
}
|
||||
channel.send(JSON.stringify(event));
|
||||
},
|
||||
[ensureChannelReady]
|
||||
);
|
||||
|
||||
const initialiseMicrophone = useCallback(async () => {
|
||||
if (typeof window === 'undefined') {
|
||||
throw new Error('Voice not supported in this environment');
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
echoCancellation: false,
|
||||
autoGainControl: false,
|
||||
noiseSuppression: false,
|
||||
},
|
||||
});
|
||||
|
||||
if (destroyedRef.current) {
|
||||
stream.getTracks().forEach((track) => {
|
||||
try {
|
||||
track.stop();
|
||||
} catch (err) {
|
||||
console.warn('[VoiceRealtime] Failed to stop track after destroy', err);
|
||||
}
|
||||
});
|
||||
return stream;
|
||||
}
|
||||
|
||||
const audioContext = new AudioContext();
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
const processor = audioContext.createScriptProcessor(2048, 1, 1);
|
||||
|
||||
processor.onaudioprocess = (event) => {
|
||||
if (destroyedRef.current) return;
|
||||
const inputBuffer = event.inputBuffer.getChannelData(0);
|
||||
const amplitude = calculateRms(inputBuffer);
|
||||
onAmplitudeRef.current?.(Math.min(1, amplitude * 8));
|
||||
|
||||
const now = Date.now();
|
||||
const channelReady = channelReadyRef.current;
|
||||
if (amplitude > amplitudeGate && channelReady) {
|
||||
hasUncommittedAudioRef.current = true;
|
||||
lastSpeechAtRef.current = now;
|
||||
if (!awaitingTranscriptRef.current) {
|
||||
onStatusChangeRef.current?.('listening');
|
||||
}
|
||||
} else if (
|
||||
channelReady &&
|
||||
hasUncommittedAudioRef.current &&
|
||||
!awaitingTranscriptRef.current &&
|
||||
lastSpeechAtRef.current &&
|
||||
now - lastSpeechAtRef.current > silenceWindowMs
|
||||
) {
|
||||
awaitingTranscriptRef.current = true;
|
||||
hasUncommittedAudioRef.current = false;
|
||||
lastSpeechAtRef.current = null;
|
||||
onStatusChangeRef.current?.('thinking');
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(audioContext.destination);
|
||||
|
||||
audioContextRef.current = audioContext;
|
||||
processorNodeRef.current = processor;
|
||||
mediaSourceRef.current = source;
|
||||
mediaStreamRef.current = stream;
|
||||
return stream;
|
||||
}, [amplitudeGate, silenceWindowMs]);
|
||||
|
||||
const disconnect = useCallback(() => {
|
||||
destroyedRef.current = true;
|
||||
teardownInputNodes();
|
||||
closePeerConnection();
|
||||
audioContextRef.current?.close().catch(() => undefined);
|
||||
audioContextRef.current = null;
|
||||
resetState();
|
||||
channelReadyRef.current = false;
|
||||
setConnectionState('idle');
|
||||
onStatusChangeRef.current?.('idle');
|
||||
onAmplitudeRef.current?.(0);
|
||||
}, [closePeerConnection, resetState, setConnectionState, teardownInputNodes]);
|
||||
|
||||
const extractTextDelta = useCallback((payload: unknown): string => {
|
||||
if (!payload) return '';
|
||||
if (typeof payload === 'string') return payload;
|
||||
if (typeof payload !== 'object') return '';
|
||||
|
||||
const record = payload as Record<string, unknown>;
|
||||
if (typeof record.delta === 'string') return record.delta;
|
||||
if (typeof record.text === 'string') return record.text;
|
||||
|
||||
const outputText = record.output_text as Record<string, unknown> | undefined;
|
||||
if (typeof outputText?.text === 'string') return outputText.text;
|
||||
|
||||
if (Array.isArray(record.output)) {
|
||||
return record.output
|
||||
.flatMap((item) =>
|
||||
Array.isArray((item as Record<string, unknown>)?.content)
|
||||
? ((item as Record<string, unknown>).content as unknown[])
|
||||
: []
|
||||
)
|
||||
.map((content) => {
|
||||
if (!content || typeof content !== 'object') return '';
|
||||
const entry = content as Record<string, unknown>;
|
||||
const nestedText = entry.text as Record<string, unknown> | string | undefined;
|
||||
if (typeof nestedText === 'string') return nestedText;
|
||||
if (typeof nestedText === 'object' && nestedText !== null && typeof nestedText.value === 'string') {
|
||||
return nestedText.value;
|
||||
}
|
||||
return '';
|
||||
})
|
||||
.filter(Boolean)
|
||||
.join('');
|
||||
}
|
||||
|
||||
if (Array.isArray(record.content)) {
|
||||
return record.content
|
||||
.map((content) => {
|
||||
if (!content || typeof content !== 'object') return '';
|
||||
const entry = content as Record<string, unknown>;
|
||||
const nestedText = entry.text as Record<string, unknown> | string | undefined;
|
||||
if (typeof nestedText === 'string') return nestedText;
|
||||
if (typeof nestedText === 'object' && nestedText !== null && typeof nestedText.value === 'string') {
|
||||
return nestedText.value;
|
||||
}
|
||||
return '';
|
||||
})
|
||||
.filter(Boolean)
|
||||
.join('');
|
||||
}
|
||||
|
||||
return '';
|
||||
}, []);
|
||||
|
||||
const handleDataMessage = useCallback(
|
||||
(raw: string) => {
|
||||
try {
|
||||
const data = JSON.parse(raw);
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
console.debug('[VoiceRealtime] Event', data.type, data);
|
||||
}
|
||||
|
||||
if (data.type === 'conversation.item.input_audio_transcription.completed') {
|
||||
const transcriptSource =
|
||||
typeof data.transcript === 'string'
|
||||
? data.transcript
|
||||
: extractTextDelta(data.item ?? data.content ?? data);
|
||||
const transcript = transcriptSource?.trim();
|
||||
awaitingTranscriptRef.current = false;
|
||||
hasUncommittedAudioRef.current = false;
|
||||
lastSpeechAtRef.current = null;
|
||||
if (transcript) {
|
||||
console.info('[VoiceRealtime] Transcript completed', transcript);
|
||||
onInterimTranscriptRef.current?.('');
|
||||
onFinalTranscriptRef.current?.(transcript);
|
||||
} else {
|
||||
console.warn('[VoiceRealtime] Received empty transcription event');
|
||||
onInterimTranscriptRef.current?.('');
|
||||
}
|
||||
onStatusChangeRef.current?.('listening');
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.type === 'error' && data.error) {
|
||||
console.error('[VoiceRealtime] Server error', data.error);
|
||||
}
|
||||
|
||||
if (data.type === 'response.error' && data.error) {
|
||||
console.error('[VoiceRealtime] Response error', data.error);
|
||||
}
|
||||
} catch (err) {
|
||||
notifyError(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
},
|
||||
[extractTextDelta, notifyError]
|
||||
);
|
||||
|
||||
const waitForIceGatheringComplete = useCallback((pc: RTCPeerConnection, timeoutMs = 2000) => {
|
||||
if (pc.iceGatheringState === 'complete') {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve) => {
|
||||
let resolved = false;
|
||||
let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const finish = () => {
|
||||
if (resolved) return;
|
||||
resolved = true;
|
||||
pc.removeEventListener('icegatheringstatechange', handleStateChange);
|
||||
if (timeoutHandle) {
|
||||
clearTimeout(timeoutHandle);
|
||||
timeoutHandle = null;
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
|
||||
const handleStateChange = () => {
|
||||
if (pc.iceGatheringState === 'complete') {
|
||||
finish();
|
||||
}
|
||||
};
|
||||
|
||||
if (timeoutMs) {
|
||||
timeoutHandle = setTimeout(() => {
|
||||
console.warn('[VoiceRealtime] ICE gathering timeout reached, proceeding with partial candidates');
|
||||
finish();
|
||||
}, timeoutMs);
|
||||
}
|
||||
|
||||
pc.addEventListener('icegatheringstatechange', handleStateChange);
|
||||
});
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
try {
|
||||
destroyedRef.current = false;
|
||||
resetState();
|
||||
setConnectionState('connecting');
|
||||
|
||||
const authToken = getAuthToken?.() ?? null;
|
||||
const fetchToken = fetchEphemeralToken
|
||||
? fetchEphemeralToken
|
||||
: async (token: string | null) => {
|
||||
const response = await fetch('/api/realtime/ephemeral-token', {
|
||||
method: 'POST',
|
||||
headers: token ? { Authorization: `Bearer ${token}` } : undefined,
|
||||
});
|
||||
if (!response.ok) {
|
||||
const payload = await response.json().catch(() => ({}));
|
||||
throw new Error(payload.error || `Failed to mint ephemeral token (${response.status})`);
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
|
||||
const sessionPromise = fetchToken(authToken);
|
||||
const microphonePromise = initialiseMicrophone();
|
||||
|
||||
const session = await sessionPromise;
|
||||
const clientSecret = session?.client_secret?.value || session?.client_secret;
|
||||
if (!clientSecret || typeof clientSecret !== 'string') {
|
||||
throw new Error('Realtime session did not include a client_secret');
|
||||
}
|
||||
|
||||
const pc = new RTCPeerConnection({
|
||||
iceServers: [{ urls: ['stun:stun.l.google.com:19302'] }],
|
||||
});
|
||||
peerConnectionRef.current = pc;
|
||||
|
||||
const channel = pc.createDataChannel('oai-events');
|
||||
dataChannelRef.current = channel;
|
||||
channelReadyPromiseRef.current = new Promise<void>((resolve) => {
|
||||
channelReadyResolveRef.current = resolve;
|
||||
});
|
||||
channelReadyRef.current = false;
|
||||
|
||||
channel.onmessage = (event) => {
|
||||
handleDataMessage(event.data);
|
||||
};
|
||||
|
||||
channel.onopen = () => {
|
||||
console.info('[VoiceRealtime] Data channel open');
|
||||
setConnectionState('ready');
|
||||
channelReadyResolveRef.current?.();
|
||||
channelReadyResolveRef.current = null;
|
||||
channelReadyRef.current = true;
|
||||
|
||||
void sendEvent({
|
||||
type: 'session.update',
|
||||
session: {
|
||||
modalities: ['text'],
|
||||
input_audio_transcription: {
|
||||
model: 'whisper-1',
|
||||
},
|
||||
turn_detection: {
|
||||
type: 'server_vad',
|
||||
threshold: 0.5,
|
||||
silence_duration_ms: 1800,
|
||||
prefix_padding_ms: 300,
|
||||
create_response: false,
|
||||
},
|
||||
instructions: 'You are the RA-H voice transport. Only transcribe user speech accurately, never speak responses.',
|
||||
},
|
||||
}).catch((err) => {
|
||||
console.error('[VoiceRealtime] Failed to send session update:', err);
|
||||
});
|
||||
};
|
||||
|
||||
channel.onerror = (event) => {
|
||||
console.error('[VoiceRealtime] Data channel error:', event);
|
||||
notifyError(new Error('Realtime connection error'));
|
||||
channelReadyRef.current = false;
|
||||
disconnect();
|
||||
};
|
||||
|
||||
channel.onclose = () => {
|
||||
console.warn('[VoiceRealtime] Data channel closed');
|
||||
channelReadyPromiseRef.current = null;
|
||||
channelReadyResolveRef.current = null;
|
||||
channelReadyRef.current = false;
|
||||
if (!destroyedRef.current) {
|
||||
notifyError(new Error('Realtime data channel closed unexpectedly'));
|
||||
disconnect();
|
||||
}
|
||||
};
|
||||
|
||||
pc.onconnectionstatechange = () => {
|
||||
const state = pc.connectionState;
|
||||
console.info('[VoiceRealtime] Peer connection state:', state);
|
||||
if (state === 'connected') {
|
||||
onStatusChangeRef.current?.('listening');
|
||||
setConnectionState('capturing');
|
||||
}
|
||||
if (state === 'failed' || state === 'closed' || state === 'disconnected') {
|
||||
if (!destroyedRef.current) {
|
||||
notifyError(new Error(`Realtime connection ${state}`));
|
||||
}
|
||||
disconnect();
|
||||
}
|
||||
};
|
||||
|
||||
pc.oniceconnectionstatechange = () => {
|
||||
console.info('[VoiceRealtime] ICE connection state:', pc.iceConnectionState);
|
||||
};
|
||||
|
||||
pc.ontrack = () => undefined;
|
||||
|
||||
const mediaStream = await microphonePromise;
|
||||
mediaStream?.getAudioTracks().forEach((track) => {
|
||||
if (peerConnectionRef.current?.signalingState !== 'closed') {
|
||||
peerConnectionRef.current?.addTrack(track, mediaStream);
|
||||
}
|
||||
});
|
||||
|
||||
const offer = await pc.createOffer();
|
||||
await pc.setLocalDescription(offer);
|
||||
await waitForIceGatheringComplete(pc, 2000);
|
||||
|
||||
const localDescription = pc.localDescription;
|
||||
if (!localDescription) {
|
||||
throw new Error('Failed to create local description for realtime call');
|
||||
}
|
||||
|
||||
const response = await fetch('https://api.openai.com/v1/realtime/calls', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${clientSecret}`,
|
||||
'Content-Type': 'application/sdp',
|
||||
'OpenAI-Beta': 'realtime=v1',
|
||||
},
|
||||
body: localDescription.sdp,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
let errorDetail = '';
|
||||
try {
|
||||
const text = await response.text();
|
||||
errorDetail = text;
|
||||
const maybeJson = text ? JSON.parse(text) : null;
|
||||
if (maybeJson?.error?.message) {
|
||||
errorDetail = maybeJson.error.message;
|
||||
}
|
||||
} catch {
|
||||
// ignore JSON parse errors and fall back to raw text
|
||||
}
|
||||
const friendly = errorDetail || response.statusText || 'Unknown realtime error';
|
||||
console.error('[VoiceRealtime] Failed to start realtime call:', friendly, { status: response.status });
|
||||
throw new Error(`Failed to establish realtime call (${response.status}): ${friendly}`);
|
||||
}
|
||||
|
||||
const answerSdp = await response.text();
|
||||
await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
|
||||
|
||||
onStatusChangeRef.current?.('listening');
|
||||
setConnectionState('capturing');
|
||||
} catch (err) {
|
||||
disconnect();
|
||||
notifyError(err instanceof Error ? err : new Error(String(err)));
|
||||
throw err;
|
||||
}
|
||||
}, [disconnect, fetchEphemeralToken, getAuthToken, handleDataMessage, initialiseMicrophone, notifyError, resetState, sendEvent, setConnectionState, waitForIceGatheringComplete]);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
disconnect();
|
||||
}, [disconnect]);
|
||||
|
||||
const latestStopRef = useRef<() => void>(() => {});
|
||||
|
||||
useEffect(() => {
|
||||
latestStopRef.current = stop;
|
||||
}, [stop]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
latestStopRef.current();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
} as const;
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
"use client";
|
||||
|
||||
import { useRef, useState } from 'react';
|
||||
|
||||
export enum MessageRole {
|
||||
USER = 'user',
|
||||
ASSISTANT = 'assistant',
|
||||
TOOL = 'tool',
|
||||
SYSTEM = 'system',
|
||||
THINKING = 'thinking',
|
||||
}
|
||||
|
||||
export interface ChatMessage {
|
||||
id: string;
|
||||
role: MessageRole.USER | MessageRole.ASSISTANT | MessageRole.TOOL | MessageRole.SYSTEM | MessageRole.THINKING;
|
||||
content: string;
|
||||
timestamp: Date;
|
||||
toolName?: string;
|
||||
status?: 'processing' | 'delivered' | 'error' | 'starting' | 'running' | 'complete';
|
||||
toolArgs?: any;
|
||||
toolResult?: any;
|
||||
}
|
||||
|
||||
interface SendParams {
|
||||
text: string;
|
||||
history: ChatMessage[];
|
||||
openTabs: any[];
|
||||
activeTabId: number | null;
|
||||
currentView?: 'nodes' | 'memory';
|
||||
sessionId: string;
|
||||
mode: 'easy' | 'hard';
|
||||
}
|
||||
|
||||
interface UseSSEChatOptions {
|
||||
getAuthToken?: () => string | null | undefined;
|
||||
beforeRequest?: () => boolean;
|
||||
onRequestError?: (error: unknown, response?: Response) => boolean | void;
|
||||
onStreamComplete?: () => void | Promise<void>;
|
||||
getApiKeys?: () => { openai?: string; anthropic?: string } | undefined;
|
||||
}
|
||||
|
||||
export function useSSEChat(
|
||||
endpoint: string,
|
||||
setMessages: (updater: (prev: ChatMessage[]) => ChatMessage[]) => void,
|
||||
options: UseSSEChatOptions = {}
|
||||
) {
|
||||
const { getAuthToken, beforeRequest, onRequestError, onStreamComplete, getApiKeys } = options;
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const abortControllerRef = useRef<AbortController | null>(null);
|
||||
|
||||
const abort = () => {
|
||||
abortControllerRef.current?.abort();
|
||||
};
|
||||
|
||||
const send = async ({ text, history, openTabs, activeTabId, currentView, sessionId, mode }: SendParams) => {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed || isLoading) return;
|
||||
if (beforeRequest && !beforeRequest()) return;
|
||||
|
||||
setIsLoading(true);
|
||||
setError(null);
|
||||
|
||||
const userMessage: ChatMessage = {
|
||||
id: crypto.randomUUID(),
|
||||
role: MessageRole.USER,
|
||||
content: trimmed,
|
||||
timestamp: new Date()
|
||||
};
|
||||
const assistantMessage: ChatMessage = {
|
||||
id: crypto.randomUUID(),
|
||||
role: MessageRole.ASSISTANT,
|
||||
content: '',
|
||||
timestamp: new Date()
|
||||
};
|
||||
|
||||
setMessages((prev) => [...prev, userMessage, assistantMessage]);
|
||||
|
||||
let handledError = false;
|
||||
let currentAssistantMessage = assistantMessage;
|
||||
|
||||
try {
|
||||
abortControllerRef.current = new AbortController();
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
const token = getAuthToken?.();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
messages: history
|
||||
.concat(userMessage)
|
||||
.filter((m) => [MessageRole.USER, MessageRole.ASSISTANT, MessageRole.SYSTEM].includes(m.role))
|
||||
.map((m) => ({ role: m.role, parts: [{ type: 'text', text: m.content }] })),
|
||||
openTabs,
|
||||
activeTabId,
|
||||
currentView,
|
||||
sessionId,
|
||||
mode,
|
||||
apiKeys: getApiKeys?.(),
|
||||
}),
|
||||
signal: abortControllerRef.current.signal
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = new Error(`HTTP error! status: ${response.status}`);
|
||||
handledError = Boolean(onRequestError?.(error, response));
|
||||
setMessages((prev) =>
|
||||
prev.filter((m) => !(m.id === currentAssistantMessage.id && m.content === ''))
|
||||
);
|
||||
if (handledError) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
if (!reader) throw new Error('No response body');
|
||||
|
||||
let fullContent = '';
|
||||
let toolCallsActive: Record<string, string> = {};
|
||||
let hasToolCalls = false;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const chunk = decoder.decode(value, { stream: true });
|
||||
const lines = chunk.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.startsWith('data: ')) continue;
|
||||
try {
|
||||
const data = JSON.parse(line.substring(6));
|
||||
|
||||
if (data.type === 'text-delta' && data.delta) {
|
||||
fullContent += data.delta;
|
||||
setMessages((prev) => prev.map((m) => (m.id === currentAssistantMessage.id ? { ...m, content: fullContent } : m)));
|
||||
}
|
||||
|
||||
if (data.type === 'tool-input-start') {
|
||||
hasToolCalls = true;
|
||||
toolCallsActive[data.toolCallId] = data.toolName;
|
||||
const toolMessage: ChatMessage = {
|
||||
id: `tool-${data.toolCallId}`,
|
||||
role: MessageRole.TOOL,
|
||||
content: data.toolName,
|
||||
timestamp: new Date(),
|
||||
toolName: data.toolName,
|
||||
status: 'running',
|
||||
toolArgs: (data.args ?? data.input ?? data.parameters ?? null)
|
||||
};
|
||||
setMessages((prev) => {
|
||||
const filtered = prev.filter((m) => m.id !== toolMessage.id);
|
||||
const index = filtered.findIndex((m) => m.id === currentAssistantMessage.id);
|
||||
return [...filtered.slice(0, index), toolMessage, ...filtered.slice(index)];
|
||||
});
|
||||
}
|
||||
|
||||
if (data.type === 'tool-output-available') {
|
||||
delete toolCallsActive[data.toolCallId];
|
||||
setMessages((prev) =>
|
||||
prev.map((m) =>
|
||||
m.id === `tool-${data.toolCallId}`
|
||||
? { ...m, content: `${m.toolName} ✓`, status: 'complete', toolResult: (data.result ?? data.output ?? null) }
|
||||
: m
|
||||
)
|
||||
);
|
||||
|
||||
if (Object.keys(toolCallsActive).length === 0 && hasToolCalls) {
|
||||
const newAssistantMessage: ChatMessage = {
|
||||
id: crypto.randomUUID(),
|
||||
role: MessageRole.ASSISTANT,
|
||||
content: '',
|
||||
timestamp: new Date()
|
||||
};
|
||||
currentAssistantMessage = newAssistantMessage;
|
||||
fullContent = '';
|
||||
setMessages((prev) => [...prev, newAssistantMessage]);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore malformed lines
|
||||
}
|
||||
}
|
||||
}
|
||||
if (onStreamComplete) {
|
||||
await onStreamComplete();
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err?.name !== 'AbortError') {
|
||||
if (!handledError) {
|
||||
handledError = Boolean(onRequestError?.(err));
|
||||
}
|
||||
if (!handledError) {
|
||||
setError(err?.message || 'Failed to send message');
|
||||
}
|
||||
setMessages((prev) =>
|
||||
prev.filter((m) => !(m.id === currentAssistantMessage.id && m.content === ''))
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
};
|
||||
|
||||
return { isLoading, error, send, abort };
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import type { TTSStatus } from './useAssistantTTS';
|
||||
|
||||
interface UseVoiceInterruptionOptions {
|
||||
amplitude: number;
|
||||
isVoiceActive: boolean;
|
||||
ttsStatus: TTSStatus;
|
||||
threshold?: number;
|
||||
holdDurationMs?: number;
|
||||
cooldownMs?: number;
|
||||
onInterruption: () => void;
|
||||
}
|
||||
|
||||
const DEFAULT_THRESHOLD = 0.18;
|
||||
const DEFAULT_HOLD_MS = 120;
|
||||
const DEFAULT_COOLDOWN_MS = 800;
|
||||
|
||||
export function useVoiceInterruption(options: UseVoiceInterruptionOptions) {
|
||||
const {
|
||||
amplitude,
|
||||
isVoiceActive,
|
||||
ttsStatus,
|
||||
onInterruption,
|
||||
threshold = DEFAULT_THRESHOLD,
|
||||
holdDurationMs = DEFAULT_HOLD_MS,
|
||||
cooldownMs = DEFAULT_COOLDOWN_MS,
|
||||
} = options;
|
||||
|
||||
const [isInterrupting, setIsInterrupting] = useState(false);
|
||||
const detectionTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const lastInterruptionAtRef = useRef(0);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (detectionTimeoutRef.current) {
|
||||
clearTimeout(detectionTimeoutRef.current);
|
||||
detectionTimeoutRef.current = null;
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const clearDetection = () => {
|
||||
if (detectionTimeoutRef.current) {
|
||||
clearTimeout(detectionTimeoutRef.current);
|
||||
detectionTimeoutRef.current = null;
|
||||
}
|
||||
};
|
||||
|
||||
if (!isVoiceActive || ttsStatus === 'idle') {
|
||||
clearDetection();
|
||||
if (isInterrupting) {
|
||||
setIsInterrupting(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (amplitude < threshold) {
|
||||
clearDetection();
|
||||
if (isInterrupting) {
|
||||
setIsInterrupting(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now - lastInterruptionAtRef.current < cooldownMs) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (detectionTimeoutRef.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
detectionTimeoutRef.current = setTimeout(() => {
|
||||
lastInterruptionAtRef.current = Date.now();
|
||||
setIsInterrupting(true);
|
||||
onInterruption();
|
||||
}, holdDurationMs);
|
||||
}, [amplitude, cooldownMs, holdDurationMs, isInterrupting, isVoiceActive, onInterruption, threshold, ttsStatus]);
|
||||
|
||||
return { isInterrupting } as const;
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { useCallback, useReducer } from 'react';
|
||||
|
||||
export type VoiceSessionStatus = 'idle' | 'listening' | 'thinking' | 'speaking';
|
||||
|
||||
export interface VoiceTranscriptSegment {
|
||||
id: string;
|
||||
text: string;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
interface VoiceSessionState {
|
||||
isActive: boolean;
|
||||
status: VoiceSessionStatus;
|
||||
interimTranscript: string;
|
||||
segments: VoiceTranscriptSegment[];
|
||||
amplitude: number;
|
||||
startedAt: number | null;
|
||||
}
|
||||
|
||||
type VoiceSessionAction =
|
||||
| { type: 'start' }
|
||||
| { type: 'stop' }
|
||||
| { type: 'set-status'; status: VoiceSessionStatus }
|
||||
| { type: 'set-amplitude'; amplitude: number }
|
||||
| { type: 'set-interim'; transcript: string }
|
||||
| { type: 'append-segment'; text: string }
|
||||
| { type: 'replace-segments'; segments: VoiceTranscriptSegment[] }
|
||||
| { type: 'reset-transcript' };
|
||||
|
||||
const initialState: VoiceSessionState = {
|
||||
isActive: false,
|
||||
status: 'idle',
|
||||
interimTranscript: '',
|
||||
segments: [],
|
||||
amplitude: 0,
|
||||
startedAt: null,
|
||||
};
|
||||
|
||||
function reducer(state: VoiceSessionState, action: VoiceSessionAction): VoiceSessionState {
|
||||
switch (action.type) {
|
||||
case 'start':
|
||||
return {
|
||||
...state,
|
||||
isActive: true,
|
||||
status: 'listening',
|
||||
interimTranscript: '',
|
||||
segments: [],
|
||||
amplitude: 0,
|
||||
startedAt: Date.now(),
|
||||
};
|
||||
case 'stop':
|
||||
return {
|
||||
...state,
|
||||
isActive: false,
|
||||
status: 'idle',
|
||||
amplitude: 0,
|
||||
interimTranscript: '',
|
||||
startedAt: null,
|
||||
};
|
||||
case 'set-status':
|
||||
return {
|
||||
...state,
|
||||
status: action.status,
|
||||
};
|
||||
case 'set-amplitude':
|
||||
return {
|
||||
...state,
|
||||
amplitude: Math.max(0, Math.min(1, action.amplitude)),
|
||||
};
|
||||
case 'set-interim':
|
||||
return {
|
||||
...state,
|
||||
interimTranscript: action.transcript,
|
||||
};
|
||||
case 'append-segment': {
|
||||
const trimmed = action.text.trim();
|
||||
if (!trimmed) return state;
|
||||
const segment: VoiceTranscriptSegment = {
|
||||
id: `voice-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
text: trimmed,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
return {
|
||||
...state,
|
||||
segments: [...state.segments, segment],
|
||||
};
|
||||
}
|
||||
case 'replace-segments':
|
||||
return {
|
||||
...state,
|
||||
segments: action.segments,
|
||||
};
|
||||
case 'reset-transcript':
|
||||
return {
|
||||
...state,
|
||||
interimTranscript: '',
|
||||
segments: [],
|
||||
};
|
||||
default:
|
||||
return state;
|
||||
}
|
||||
}
|
||||
|
||||
export function useVoiceSession() {
|
||||
const [state, dispatch] = useReducer(reducer, initialState);
|
||||
|
||||
const startSession = useCallback(() => {
|
||||
dispatch({ type: 'start' });
|
||||
}, []);
|
||||
|
||||
const stopSession = useCallback(() => {
|
||||
dispatch({ type: 'stop' });
|
||||
}, []);
|
||||
|
||||
const setStatus = useCallback((status: VoiceSessionStatus) => {
|
||||
dispatch({ type: 'set-status', status });
|
||||
}, []);
|
||||
|
||||
const setAmplitude = useCallback((amplitude: number) => {
|
||||
dispatch({ type: 'set-amplitude', amplitude });
|
||||
}, []);
|
||||
|
||||
const setInterimTranscript = useCallback((transcript: string) => {
|
||||
dispatch({ type: 'set-interim', transcript });
|
||||
}, []);
|
||||
|
||||
const appendFinalTranscript = useCallback((text: string) => {
|
||||
dispatch({ type: 'append-segment', text });
|
||||
}, []);
|
||||
|
||||
const resetTranscript = useCallback(() => {
|
||||
dispatch({ type: 'reset-transcript' });
|
||||
}, []);
|
||||
|
||||
const replaceSegments = useCallback((segments: VoiceTranscriptSegment[]) => {
|
||||
dispatch({ type: 'replace-segments', segments });
|
||||
}, []);
|
||||
|
||||
return {
|
||||
...state,
|
||||
startSession,
|
||||
stopSession,
|
||||
setStatus,
|
||||
setAmplitude,
|
||||
setInterimTranscript,
|
||||
appendFinalTranscript,
|
||||
resetTranscript,
|
||||
replaceSegments,
|
||||
} as const;
|
||||
}
|
||||
Reference in New Issue
Block a user