From c17c48f269e4edc65c5882ba89f71b7bda1b10fe Mon Sep 17 00:00:00 2001 From: dwrz Date: Sat, 13 Jun 2026 18:55:16 +0000 Subject: [PATCH] Refactor TTS architecture --- internal/config/config.go | 34 ++- internal/service/service.go | 113 +++------ internal/service/static/main.css | 46 ++-- internal/service/static/main.js | 190 ++++++++++++-- internal/service/templates/static/head.gohtml | 2 + .../service/templates/static/templates.gohtml | 8 + internal/tts/tts.go | 208 --------------- internal/tts/tts_test.go | 236 ------------------ 8 files changed, 274 insertions(+), 563 deletions(-) delete mode 100644 internal/tts/tts.go delete mode 100644 internal/tts/tts_test.go diff --git a/internal/config/config.go b/internal/config/config.go index 6d2c521..63d3231 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,11 +11,41 @@ import ( "code.chimeric.al/chimerical/odidere/internal/llm" "code.chimeric.al/chimerical/odidere/internal/stt" "code.chimeric.al/chimerical/odidere/internal/tool" - "code.chimeric.al/chimerical/odidere/internal/tts" "gopkg.in/yaml.v3" ) +// TTS holds the configuration for TTS. +type TTS struct { + // URL is the base URL of kokoro-fastapi. + // For example, "http://localhost:8880". + URL string `yaml:"url"` + // Voice is the default voice ID to use (e.g., "af_heart"). + Voice string `yaml:"voice"` + // Timeout is the maximum duration for a synthesis request. + // Defaults to 60s if empty. + Timeout string `yaml:"timeout"` + // VoiceMap maps whisper language names to voice IDs. + // Used by SelectVoice to auto-select voices based on detected + // language. + VoiceMap map[string]string `yaml:"voice_map"` +} + +func (cfg TTS) Validate() error { + if cfg.URL == "" { + return fmt.Errorf("missing URL") + } + if cfg.Voice == "" { + return fmt.Errorf("missing voice") + } + if cfg.Timeout != "" { + if _, err := time.ParseDuration(cfg.Timeout); err != nil { + return fmt.Errorf("invalid timeout: %w", err) + } + } + return nil +} + // Config holds all application configuration sections. type Config struct { // Address is the host:port to listen on. @@ -36,7 +66,7 @@ type Config struct { // Tools defines external commands available to the LLM. Tools []tool.Tool `yaml:"tools"` // TTS configures the text-to-speech client. - TTS tts.Config `yaml:"tts"` + TTS TTS `yaml:"tts"` } // ApplyDefaults sets default values for optional configuration fields. diff --git a/internal/service/service.go b/internal/service/service.go index b53ebe9..6db06ec 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -1,5 +1,5 @@ // Package service orchestrates the odidere voice assistant server. -// It coordinates the HTTP server, STT/LLM/TTS clients, and handles +// It coordinates the HTTP server, STT/LLM clients, and handles // graceful shutdown. package service @@ -26,7 +26,6 @@ import ( "code.chimeric.al/chimerical/odidere/internal/service/templates" "code.chimeric.al/chimerical/odidere/internal/stt" "code.chimeric.al/chimerical/odidere/internal/tool" - "code.chimeric.al/chimerical/odidere/internal/tts" "github.com/google/uuid" "github.com/robfig/cron/v3" @@ -42,6 +41,19 @@ const ( ipKey key = "ip" ) +// defaultVoices maps whisper language names to default Kokoro voices. +var defaultVoices = map[string]string{ + "chinese": "zf_xiaobei", + "english": "af_heart", + "french": "ff_siwis", + "hindi": "hf_alpha", + "italian": "if_sara", + "japanese": "jf_alpha", + "korean": "kf_sarah", + "portuguese": "pf_dora", + "spanish": "ef_dora", +} + //go:embed all:static/* var static embed.FS @@ -58,7 +70,6 @@ type Service struct { stt *stt.Client tmpl *template.Template tools *tool.Registry - tts *tts.Client } // New creates a Service from the provided configuration. @@ -91,13 +102,6 @@ func New(cfg *config.Config, log *slog.Logger) (*Service, error) { } svc.llm = llmClient - // Create TTS client. - ttsClient, err := tts.NewClient(cfg.TTS, log) - if err != nil { - return nil, fmt.Errorf("create TTS client: %v", err) - } - svc.tts = ttsClient - // Convert job configs to jobs. jobs := make([]*job.Job, 0, len(cfg.Jobs)) for _, jc := range cfg.Jobs { @@ -133,7 +137,6 @@ func New(cfg *config.Config, log *slog.Logger) (*Service, error) { ) svc.mux.HandleFunc("POST /v1/chat/voice", svc.voice) svc.mux.HandleFunc("POST /v1/chat/voice/stream", svc.voiceStream) - svc.mux.HandleFunc("GET /v1/voices", svc.voices) svc.mux.HandleFunc("GET /v1/models", svc.models) svc.server = &http.Server{ @@ -236,7 +239,7 @@ func (svc *Service) Run(ctx context.Context) error { ), ), slog.Group( - "tts", + "tts_url", slog.String("url", svc.cfg.TTS.URL), slog.String("default_voice", svc.cfg.TTS.Voice), ), @@ -352,6 +355,19 @@ func (svc *Service) JobCount() int { return len(svc.jobs) } +// selectVoice returns the voice ID for the given language. +// It uses the configured voice map, falling back to defaults. +func (svc *Service) selectVoice(lang string) string { + voiceMap := svc.cfg.TTS.VoiceMap + if len(voiceMap) == 0 { + voiceMap = defaultVoices + } + if voice, ok := voiceMap[lang]; ok { + return voice + } + return "" +} + func (svc *Service) home(w http.ResponseWriter, r *http.Request) { var ( ctx = r.Context() @@ -364,7 +380,13 @@ func (svc *Service) home(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Type", "text/html; charset=utf-8") if err := svc.tmpl.ExecuteTemplate( - w, "index.gohtml", nil, + w, "index.gohtml", struct { + TTSURL string + TTSDefaultVoice string + }{ + TTSURL: svc.cfg.TTS.URL, + TTSDefaultVoice: svc.cfg.TTS.Voice, + }, ); err != nil { log.ErrorContext( ctx, "template error", slog.Any("error", err), @@ -398,8 +420,6 @@ type Request struct { // Response is the response format for chat and voice endpoints. type Response struct { - // Audio is the base64-encoded WAV audio response. - Audio string `json:"audio,omitempty"` // DetectedLanguage is the language detected in the input speech. DetectedLanguage string `json:"detected_language,omitempty"` // Messages is the full list of messages generated during the query, @@ -565,7 +585,7 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) { // Determine voice to use. var voice = req.Voice if req.Voice == "" && detectedLang != "" { - if autoVoice := svc.tts.SelectVoice( + if autoVoice := svc.selectVoice( detectedLang, ); autoVoice != "" { voice = autoVoice @@ -581,17 +601,8 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) { ) } - // Generate audio response with selected voice. - audio, err := svc.tts.Synthesize(ctx, final.Content, voice) - if err != nil { - log.ErrorContext(ctx, "TTS failed", slog.Any("error", err)) - http.Error(w, "TTS error", http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(Response{ - Audio: base64.StdEncoding.EncodeToString(audio), DetectedLanguage: detectedLang, Messages: msgs, Model: model, @@ -608,8 +619,6 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) { // StreamMessage is the SSE event payload for the streaming voice endpoint. type StreamMessage struct { - // Audio is the base64-encoded WAV audio response. - Audio string `json:"audio,omitempty"` // DetectedLanguage is the language detected in the input speech. DetectedLanguage string `json:"detected_language,omitempty"` // Error is an error message, if any. @@ -784,7 +793,7 @@ func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) { // Determine voice to use. var voice = req.Voice if req.Voice == "" && detectedLang != "" { - if autoVoice := svc.tts.SelectVoice( + if autoVoice := svc.selectVoice( detectedLang, ); autoVoice != "" { voice = autoVoice @@ -834,20 +843,6 @@ func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) { return } - // Synthesize TTS for the final assistant message. - if last.Message.Content != "" { - audio, err := svc.tts.Synthesize( - ctx, last.Message.Content, voice, - ) - if err != nil { - log.ErrorContext( - ctx, "TTS failed", slog.Any("error", err), - ) - last.Error = fmt.Sprintf("TTS error: %v", err) - } else { - last.Audio = base64.StdEncoding.EncodeToString(audio) - } - } last.Model = model last.Voice = voice send(last) @@ -890,37 +885,3 @@ func (svc *Service) models(w http.ResponseWriter, r *http.Request) { ) } } - -// voices returns available TTS voices. -func (svc *Service) voices(w http.ResponseWriter, r *http.Request) { - var ( - ctx = r.Context() - log = ctx.Value(logKey).(*slog.Logger) - ) - - voices, err := svc.tts.ListVoices(ctx) - if err != nil { - log.ErrorContext( - ctx, - "failed to list voices", - slog.Any("error", err), - ) - http.Error( - w, - "failed to list voices", - http.StatusInternalServerError, - ) - return - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(map[string][]string{ - "voices": voices, - }); err != nil { - log.ErrorContext( - ctx, - "failed to encode voices response", - slog.Any("error", err), - ) - } -} diff --git a/internal/service/static/main.css b/internal/service/static/main.css index 37efbee..750700a 100644 --- a/internal/service/static/main.css +++ b/internal/service/static/main.css @@ -144,6 +144,13 @@ body { padding: var(--s-2) var(--s-1); } +.collapsible__summary::before { + content: "▶"; + font-size: 0.625em; + color: var(--color-text-muted); + transition: transform 0.15s ease; +} + .collapsible[open] .collapsible__summary::before { transform: rotate(90deg); } @@ -204,13 +211,6 @@ body { display: none; } -.collapsible__summary::before { - content: "▶"; - font-size: 0.625em; - color: var(--color-text-muted); - transition: transform 0.15s ease; -} - .collapsible__summary .icon { width: var(--icon-size); height: var(--icon-size); @@ -530,10 +530,24 @@ body { flex-direction: column; } +.message__icon { + width: var(--action-icon-size); + height: var(--action-icon-size); + flex-shrink: 0; + color: var(--color-text-muted); +} + .message--assistant .message__icon { color: var(--color-primary); } +.message__debug { + display: none; + padding: var(--s-2) var(--s-1); + border-top: 1px dotted var(--color-border-light); + background: var(--color-surface); +} + .message--debug-open .message__debug { display: block; } @@ -579,6 +593,10 @@ body { color: var(--color-primary); } +.message__action-btn--play.playing { + color: var(--color-red); +} + @media (hover: hover) { .message__action-btn--delete:hover { color: var(--color-red); @@ -637,13 +655,6 @@ body { border-radius: var(--radius); } -.message__debug { - display: none; - padding: var(--s-2) var(--s-1); - border-top: 1px dotted var(--color-border-light); - background: var(--color-surface); -} - .message__debug-list { display: grid; grid-template-columns: auto 1fr; @@ -663,13 +674,6 @@ body { white-space: nowrap; } -.message__icon { - width: var(--action-icon-size); - height: var(--action-icon-size); - flex-shrink: 0; - color: var(--color-text-muted); -} - /* ==================== */ /* Settings Modal */ /* ==================== */ diff --git a/internal/service/static/main.js b/internal/service/static/main.js index 8e88a94..5759d36 100644 --- a/internal/service/static/main.js +++ b/internal/service/static/main.js @@ -4,7 +4,6 @@ const MODELS_ENDPOINT = '/v1/models'; const MODEL_KEY = 'odidere_model'; const STORAGE_KEY = 'odidere_history'; const SYSTEM_MESSAGE_KEY = 'odidere_system_message'; -const VOICES_ENDPOINT = '/v1/voices'; const VOICE_KEY = 'odidere_voice'; /** @@ -38,6 +37,14 @@ class Odidere { this.leafId = null; this.rootId = null; + // TTS state + this.ttsUrl = document.querySelector('meta[name="tts-url"]')?.content || ''; + this.ttsDefaultVoice = + document.querySelector('meta[name="tts-default-voice"]')?.content || ''; + this.playQueue = []; + this.isPlaying = false; + this.currentMessageId = null; + // Auto-scroll: enabled by default, disabled when user scrolls up, // re-enabled when they scroll back to bottom or send a new message. this.isAutoScrollEnabled = true; @@ -92,6 +99,7 @@ class Odidere { track.stop(); } this.#stopCurrentAudio(); + this.#stopTTS(); this.currentController?.abort(); this.#releaseWakeLock(); } @@ -102,6 +110,7 @@ class Odidere { */ reset() { this.#stopCurrentAudio(); + this.#stopTTS(); if (this.currentController) { this.currentController.abort(); @@ -234,7 +243,7 @@ class Odidere { // ==================== // CONVERSATION TREE - // ===================== + // ==================== /** * #loadHistory loads chat history from localStorage and renders it. @@ -466,7 +475,7 @@ class Odidere { let current = startId; while (current) { const msg = this.messagesMap.get(current); - if (!msg || !msg.children || msg.children.length === 0) break; + if (!msg?.children || msg.children.length === 0) break; // Follow the first child. With branching, this picks the first branch. current = msg.children[0]; } @@ -811,6 +820,7 @@ class Odidere { */ #stopConversation() { this.#stopCurrentAudio(); + this.#stopTTS(); if (this.currentController) { this.currentController.abort(); @@ -833,6 +843,7 @@ class Odidere { if (this.isProcessing) return; this.#stopCurrentAudio(); + this.#stopTTS(); // Initialize MediaRecorder on first use if (!this.mediaRecorder) { @@ -911,20 +922,92 @@ class Odidere { } // ==================== - // AUDIO PLAYBACK + // TTS PLAYBACK // ==================== + /** - * #playAudio decodes and plays base64-encoded WAV audio. - * @param {string} base64Audio + * #enqueueTTS adds a message to the TTS playback queue and starts + * playback if not already playing. + * @param {string} text + * @param {string} voice + * @param {string} messageId */ - async #playAudio(base64Audio) { + #enqueueTTS(text, voice, messageId) { + if (!text || !this.ttsUrl) return; + this.playQueue.push({ text, voice, messageId }); + if (!this.isPlaying) { + this.#processQueue(); + } + } + + /** + * #processQueue pops the next item from the TTS queue and synthesizes it. + */ + async #processQueue() { + if (this.playQueue.length === 0) { + this.isPlaying = false; + this.currentMessageId = null; + return; + } + + this.isPlaying = true; + const item = this.playQueue.shift(); + this.currentMessageId = item.messageId; + + // Update the play button on the message to show stop state. + this.#setPlayButtonState(item.messageId, true); + + await this.#synthesizeAndPlay(item.text, item.voice, item.messageId); + + // Reset this message's button back to play state before moving on. + this.#setPlayButtonState(item.messageId, false); + + // Move to next item in queue. + this.#processQueue(); + } + + /** + * #stopTTS stops current audio playback, clears the queue, and resets UI. + */ + #stopTTS() { + this.#stopCurrentAudio(); + this.playQueue = []; + this.isPlaying = false; + + // Reset all play buttons. + if (this.currentMessageId) { + this.#setPlayButtonState(this.currentMessageId, false); + } + this.currentMessageId = null; + } + + /** + * #synthesizeAndPlay calls kokoro-fastapi to synthesize text and plays + * the resulting audio. On error, logs and returns without blocking the queue. + * @param {string} text + * @param {string} voice + * @param {string} _messageId + */ + async #synthesizeAndPlay(text, voice, _messageId) { try { this.#stopCurrentAudio(); - const audioData = Uint8Array.from(atob(base64Audio), (c) => - c.charCodeAt(0), - ); - const audioBlob = new Blob([audioData], { type: 'audio/wav' }); + const res = await fetch(`${this.ttsUrl}/v1/audio/speech`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: 'tts-1', + input: text, + voice: voice || this.$voice.value || this.ttsDefaultVoice, + response_format: 'wav', + }), + }); + + if (!res.ok) { + throw new Error(`TTS error ${res.status}`); + } + + const audioBlob = await res.blob(); const audioUrl = URL.createObjectURL(audioBlob); this.currentAudioUrl = audioUrl; @@ -951,11 +1034,59 @@ class Odidere { this.currentAudio.play().catch(reject); }); } catch (e) { - console.error('failed to play audio:', e); + console.error('TTS synthesis failed:', e); this.#cleanupAudio(); } } + /** + * #setPlayButtonState updates the play/stop button icon and state for a + * given message. + * @param {string} messageId + * @param {boolean} playing + */ + #setPlayButtonState(messageId, playing) { + const $msg = this.$chat.querySelector(`[data-id="${messageId}"]`); + if (!$msg) return; + const $btn = $msg.querySelector('[data-action="play"]'); + if (!$btn) return; + + if (playing) { + $btn.classList.add('playing'); + $btn.innerHTML = ``; + $btn.setAttribute('aria-label', 'Stop'); + } else { + $btn.classList.remove('playing'); + $btn.innerHTML = ``; + $btn.setAttribute('aria-label', 'Play'); + } + } + + /** + * #handlePlayClick handles the play/stop button on an assistant message. + * If not playing: stop any current audio, enqueue this message, start playing. + * If currently playing this message: stop playback, clear remaining queue, reset. + * @param {MouseEvent} event + * @param {string} messageId + * @param {string} text + * @param {string} voice + */ + #handlePlayClick(event, messageId, text, voice) { + event.stopPropagation(); + + if (this.currentMessageId === messageId) { + // Currently playing this message: stop everything. + this.#stopTTS(); + } else { + // Not playing this message: stop current playback, enqueue this one. + this.#stopTTS(); + this.#enqueueTTS(text, voice, messageId); + } + } + + // ==================== + // AUDIO PLAYBACK + // ==================== /** * #stopCurrentAudio stops and cleans up any playing audio. */ @@ -1002,13 +1133,17 @@ class Odidere { */ async #fetchVoices() { try { - const res = await fetch(VOICES_ENDPOINT); + const res = await fetch(`${this.ttsUrl}/v1/audio/voices`); if (!res.ok) throw new Error(`${res.status}`); const data = await res.json(); // Filter out legacy v0 voices. - const filtered = data.voices.filter((v) => !v.includes('_v0')); - this.#populateVoiceSelect(filtered); + const filtered = data.voices.filter((v) => { + const id = typeof v === 'string' ? v : v.id; + return !id.includes('_v0'); + }); + const voiceIds = filtered.map((v) => (typeof v === 'string' ? v : v.id)); + this.#populateVoiceSelect(voiceIds); } catch (e) { console.error('failed to fetch voices:', e); this.$voice.innerHTML = ''; @@ -1034,6 +1169,7 @@ class Odidere { if (this.isProcessing) return; this.#stopCurrentAudio(); + this.#stopTTS(); this.$textInput.value = ''; this.$textInput.style.height = 'auto'; @@ -1050,7 +1186,7 @@ class Odidere { * * Messages are rendered incrementally as SSE events arrive: * - user (transcription), assistant (tool calls), tool (results), - * and a final assistant message with synthesized audio. + * and a final assistant message with content. * * @param {Object} options * @param {Blob} [options.audio] - Recorded audio blob @@ -1234,10 +1370,13 @@ class Odidere { const appended = this.#appendHistory([message], meta); this.#renderMessages(appended, meta); - // For the final assistant message with audio, play it. - if (event.audio) { - this.#setLoadingState(false); - await this.#playAudio(event.audio); + // Enqueue TTS for the final assistant message with content. + if (message.content) { + this.#enqueueTTS( + message.content, + event.voice || this.$voice.value || '', + message.id, + ); } } } @@ -1766,6 +1905,17 @@ class Odidere { $inspect.setAttribute('aria-expanded', String(isOpen)); }); + // Play button: only for messages with non-empty content. + const $play = $msg.querySelector('[data-action="play"]'); + if (message.content) { + const voice = meta?.voice || this.$voice.value || ''; + $play.addEventListener('click', (e) => + this.#handlePlayClick(e, message.id, message.content, voice), + ); + } else { + $play.remove(); + } + // Assemble text from all available content. const copyParts = []; if (message.reasoning_content) copyParts.push(message.reasoning_content); diff --git a/internal/service/templates/static/head.gohtml b/internal/service/templates/static/head.gohtml index 4b11164..3027ec7 100644 --- a/internal/service/templates/static/head.gohtml +++ b/internal/service/templates/static/head.gohtml @@ -4,6 +4,8 @@ + + Odidere diff --git a/internal/service/templates/static/templates.gohtml b/internal/service/templates/static/templates.gohtml index c95ea06..e5c80f5 100644 --- a/internal/service/templates/static/templates.gohtml +++ b/internal/service/templates/static/templates.gohtml @@ -47,6 +47,14 @@
+