chimerical/odidere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor STT architecture

Browse Source
This commit is contained in:
dwrz
2026-06-15 10:51:17 +00:00
parent 70e1a995f3
commit c71300b1bf
12 changed files with 321 additions and 634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

306
internal/service/static/main.js
View File

@@ -6,6 +6,21 @@ const STORAGE_KEY = 'odidere_history';
const SYSTEM_MESSAGE_KEY = 'odidere_system_message';
const VOICE_KEY = 'odidere_voice';
/**
* VOICE_MAP maps whisper language names to default Kokoro voices.
* Used for auto-selecting a voice when the selector is set to "Auto".
*/
const VOICE_MAP = {
chinese: 'zf_xiaobei',
english: 'af_heart',
french: 'ff_siwis',
hindi: 'hf_alpha',
italian: 'if_sara',
japanese: 'jf_alpha',
korean: 'kf_sarah',
portuguese: 'pf_dora',
spanish: 'ef_dora',
};
/**
* Odidere is the main application class for the voice assistant UI.
* It manages audio recording, chat history, and communication with the API.
@@ -25,6 +40,7 @@ class Odidere {
this.currentController = null;
this.isProcessing = false;
this.isRecording = false;
this.isTranscribing = false;
this.isMuted = false;
this.isResetPending = false;
this.isScrollPending = false;
@@ -45,6 +61,9 @@ class Odidere {
this.isPlaying = false;
this.currentMessageId = null;
// STT state
this.sttUrl = document.querySelector('meta[name="stt-url"]')?.content || '';
// Auto-scroll: enabled by default, disabled when user scrolls up,
// re-enabled when they scroll back to bottom or send a new message.
this.isAutoScrollEnabled = true;
@@ -64,6 +83,7 @@ class Odidere {
this.$textInput = document.getElementById('text-input');
this.$voice = document.getElementById('voice');
this.$mute = document.getElementById('mute');
this.$transcribe = document.getElementById('transcribe');
// Settings modal
this.$settings = document.getElementById('settings');
@@ -192,6 +212,35 @@ class Odidere {
this.#handleAttachments(e.target.files),
);
// Transcribe button: hold to record, release to transcribe and populate textarea.
this.$transcribe.addEventListener(
'touchstart',
this.#handleTranscribeTouchStart,
{
passive: false,
},
);
this.$transcribe.addEventListener(
'touchend',
this.#handleTranscribeTouchEnd,
);
this.$transcribe.addEventListener(
'touchcancel',
this.#handleTranscribeTouchEnd,
);
// Prevent context menu on long press.
this.$transcribe.addEventListener('contextmenu', (e) => e.preventDefault());
this.$transcribe.addEventListener(
'mousedown',
this.#handleTranscribeMouseDown,
);
this.$transcribe.addEventListener('mouseup', this.#handleTranscribeMouseUp);
this.$transcribe.addEventListener(
'mouseleave',
this.#handleTranscribeMouseUp,
);
// Save selections on change.
this.$model.addEventListener('change', () => {
localStorage.setItem(MODEL_KEY, this.$model.value);
@@ -830,6 +879,122 @@ class Odidere {
this.#setLoadingState(false);
}
// ====================
// STT: FRONTEND TRANSCRIPTION
// ====================
/**
* #transcribeAudio sends an audio blob to whisper-server directly from
* the browser and returns the transcription result.
* @param {Blob} audioBlob
* @returns {Promise<{text: string, detectedLanguage: string}>}
*/
async #transcribeAudio(audioBlob) {
if (!this.sttUrl) {
throw new Error('STT URL not configured');
}
const formData = new FormData();
formData.append('file', audioBlob, 'audio.webm');
formData.append('response_format', 'verbose_json');
const res = await fetch(`${this.sttUrl}/inference`, {
method: 'POST',
body: formData,
});
if (!res.ok) {
const errText = await res.text().catch(() => '');
throw new Error(`STT error ${res.status}: ${errText}`);
}
const data = await res.json();
const text = (data.text || '').trim();
const detectedLanguage = data.detected_language || data.language || '';
return { text, detectedLanguage };
}
/**
* #handleTranscribeTouchStart handles touch start on the transcribe button.
* Begins recording for transcription (hold-to-record, like PTT).
* @param {TouchEvent} event
*/
#handleTranscribeTouchStart = (event) => {
event.preventDefault();
event.stopPropagation();
this.#startTranscribeRecording();
};
/**
* #handleTranscribeTouchEnd handles touch end on the transcribe button.
* Stops recording; the shared stop handler will populate the textarea.
* @param {TouchEvent} event
*/
#handleTranscribeTouchEnd = (event) => {
event.stopPropagation();
this.#stopTranscribeRecording();
};
/**
* #handleTranscribeMouseDown handles mouse down on the transcribe button.
* Ignores events that originate from touch to avoid double-firing.
* @param {MouseEvent} event
*/
#handleTranscribeMouseDown = (event) => {
if (event.sourceCapabilities?.firesTouchEvents) return;
event.preventDefault();
event.stopPropagation();
this.#startTranscribeRecording();
};
/**
* #handleTranscribeMouseUp handles mouse up on the transcribe button.
* @param {MouseEvent} event
*/
#handleTranscribeMouseUp = (event) => {
if (event.sourceCapabilities?.firesTouchEvents) return;
event.stopPropagation();
this.#stopTranscribeRecording();
};
/**
* #startTranscribeRecording begins recording for transcription mode.
* Unlike PTT, this populates the textarea on stop rather than sending.
*/
async #startTranscribeRecording() {
if (this.isTranscribing) return;
if (this.isRecording) return;
if (this.isProcessing) return;
this.#stopCurrentAudio();
this.#stopTTS();
// Initialize MediaRecorder on first use (shared with PTT).
if (!this.mediaRecorder) {
const success = await this.#initMediaRecorder();
if (!success) return;
}
this.audioChunks = [];
this.mediaRecorder.start();
this.isTranscribing = true;
this.$transcribe.classList.add('compose__action-btn--recording');
}
/**
* #stopTranscribeRecording stops the media recorder for transcription mode.
* The shared 'stop' event handler (set up in #initMediaRecorder) will
* fire and clear UI state since this.isTranscribing is still true.
*/
#stopTranscribeRecording() {
if (!this.isTranscribing) return;
if (!this.mediaRecorder) return;
// Keep isTranscribing=true so the stop handler knows to populate textarea.
// The stop handler is responsible for clearing the recording class,
// so we do not remove it here.
this.mediaRecorder.stop();
}
// ====================
// AUDIO RECORDING
// ====================
@@ -840,6 +1005,7 @@ class Odidere {
*/
async #startRecording() {
if (this.isRecording) return;
if (this.isTranscribing) return;
if (this.isProcessing) return;
this.#stopCurrentAudio();
@@ -854,7 +1020,6 @@ class Odidere {
this.audioChunks = [];
this.mediaRecorder.start();
this.#setRecordingState(true);
this.#acquireWakeLock();
}
/**
@@ -864,13 +1029,17 @@ class Odidere {
if (!this.isRecording) return;
if (!this.mediaRecorder) return;
// Do not clear recording state here. The async 'stop' event handler
// is responsible for clearing it (#setRecordingState(false)) after
// the audio blob has been assembled.
this.mediaRecorder.stop();
this.#setRecordingState(false);
}
/**
* #initMediaRecorder initializes the MediaRecorder with microphone access.
* Prefers opus codec for better compression if available.
* The stop handler branches on this.isTranscribing: when true it populates
* the textarea; when false (PTT flow) it sends the request.
* @returns {Promise<boolean>} true if successful
*/
async #initMediaRecorder() {
@@ -891,22 +1060,87 @@ class Odidere {
this.mediaRecorder.addEventListener('stop', async () => {
if (this.audioChunks.length === 0) return;
// Capture the mode synchronously before any async work.
// The 'stop' event fires asynchronously, so this.isTranscribing
// may have changed by the time we await below.
const wasTranscribing = this.isTranscribing;
const audioBlob = new Blob(this.audioChunks, {
type: this.mediaRecorder.mimeType,
});
this.audioChunks = [];
// Capture and clear text input to send with audio.
const text = this.$textInput.value.trim();
this.$textInput.value = '';
this.$textInput.style.height = 'auto';
// Clear the flag now that the stop event has fired.
if (wasTranscribing) {
this.isTranscribing = false;
this.$transcribe.classList.remove('compose__action-btn--recording');
} else {
this.#setRecordingState(false);
}
await this.#sendRequest({ audio: audioBlob, text });
// Transcribe the audio.
try {
const { text: transcription, detectedLanguage } =
await this.#transcribeAudio(audioBlob);
// Resolve the effective voice for this message.
// If the selector is "Auto", pick a default for the detected language.
// This does NOT change the dropdown - the voice is per-message only.
let voice = this.$voice.value;
if (!voice && detectedLanguage) {
voice = VOICE_MAP[detectedLanguage] || '';
}
// Branch: transcribe-only (populate textarea) vs PTT (send).
if (wasTranscribing) {
// Transcribe mode: populate textarea, do not send.
this.$transcribe.classList.add('compose__action-btn--transcribing');
try {
if (transcription) {
const existing = this.$textInput.value;
if (existing.trim()) {
this.$textInput.value = `${existing}\n${transcription}`;
} else {
this.$textInput.value = transcription;
}
this.#handleTextareaInput();
}
} finally {
this.$transcribe.classList.remove(
'compose__action-btn--transcribing',
);
this.$textInput.focus();
}
} else {
// PTT mode: capture text, clear input, send request.
const text = this.$textInput.value.trim();
this.$textInput.value = '';
this.$textInput.style.height = 'auto';
const combined = text ? `${text}\n${transcription}` : transcription;
await this.#sendRequest({
text: combined,
detectedLanguage,
voice,
});
}
} catch (e) {
console.error('transcription failed:', e);
this.#renderError(`Transcription failed: ${e.message}`);
if (!wasTranscribing) {
this.#setLoadingState(false);
}
}
});
this.mediaRecorder.addEventListener('error', (event) => {
console.error('MediaRecorder error:', event.error);
this.#setRecordingState(false);
if (this.isTranscribing) {
this.isTranscribing = false;
this.$transcribe.classList.remove('compose__action-btn--recording');
} else {
this.#setRecordingState(false);
}
this.audioChunks = [];
this.#renderError(
`Recording error: ${event.error?.message || 'Unknown error'}`,
@@ -1181,25 +1415,24 @@ class Odidere {
*
* For text-only input, renders the user message immediately for
* responsiveness.
* For audio input, waits for the server's transcription event before
* rendering the user message.
*
* Messages are rendered incrementally as SSE events arrive:
* - user (transcription), assistant (tool calls), tool (results),
* - user, assistant (tool calls), tool (results),
* and a final assistant message with content.
*
* @param {Object} options
* @param {Blob} [options.audio] - Recorded audio blob
* @param {string} [options.text] - Text input
* @param {string} [options.detectedLanguage] - Detected language from STT
* @param {string} [options.voice] - Resolved voice for this message (from auto-selection)
*/
async #sendRequest({ audio = null, text = '' }) {
async #sendRequest({ text = '', detectedLanguage = '', voice }) {
this.#setLoadingState(true);
// Re-enable auto-scroll when the user sends a new message.
this.isAutoScrollEnabled = true;
this.currentController = new AbortController();
try {
const hasNewInput = text || this.attachments.length > 0 || audio;
const hasNewInput = text || this.attachments.length > 0;
// Build messages array from active path + current user message.
const activePath = this.getActivePath();
@@ -1221,25 +1454,24 @@ class Odidere {
const payload = {
messages,
voice: this.$voice.value,
voice: voice ?? this.$voice.value,
model: this.$model.value,
};
const systemMessage = localStorage.getItem(SYSTEM_MESSAGE_KEY);
if (systemMessage) {
payload.system_message = systemMessage;
}
if (audio) {
payload.audio = await this.#toBase64(audio);
}
// Clear attachments after building payload (before async operations).
this.#clearAttachments();
// For text-only requests with new input, add to history and render
// immediately. Skip when continuing conversation (no new user message).
if (!audio && hasNewInput) {
const appended = this.#appendHistory([userMessage]);
this.#renderMessages(appended);
if (hasNewInput) {
const meta = {};
if (detectedLanguage) meta.language = detectedLanguage;
const appended = this.#appendHistory([userMessage], meta);
this.#renderMessages(appended, meta);
}
const res = await fetch(STREAM_ENDPOINT, {
@@ -1296,37 +1528,6 @@ class Odidere {
}
const message = event.message;
// Handle user message from server (audio transcription).
if (message.role === 'user' && event.transcription) {
const displayParts = [];
if (text) displayParts.push(text);
if (event.transcription) displayParts.push(event.transcription);
const displayContent = displayParts.join('\n\n');
let historyContent;
if (typeof userMessage.content === 'string') {
historyContent = displayContent;
} else {
historyContent = [...userMessage.content];
if (event.transcription) {
historyContent.push({
type: 'text',
text: event.transcription,
});
}
}
const audioUserMessage = {
id: userMessage.id,
role: 'user',
content: historyContent,
meta: { language: event.detected_language },
};
this.#appendHistory([audioUserMessage]);
this.#renderMessages([audioUserMessage]);
continue;
}
// Stash assistant messages with tool calls; render once all
// tool results have arrived so the output is visible in the UI.
@@ -1488,6 +1689,7 @@ class Odidere {
this.$chatLoading.hidden = !loading;
// Keep the send button enabled during loading so it can act as a stop button.
this.$ptt.disabled = loading;
this.$transcribe.disabled = loading;
// Clear any pending stop state when loading ends.
if (!loading && this.isStopPending) {
this.isStopPending = false;