From c71300b1bff5f6518929ae7922e8eaffb33a73ae Mon Sep 17 00:00:00 2001 From: dwrz Date: Mon, 15 Jun 2026 10:51:17 +0000 Subject: [PATCH] Refactor STT architecture --- cmd/odidere/main.go | 1 - config.example.yaml | 11 - internal/config/config.go | 25 +- internal/config/config_test.go | 2 +- internal/service/service.go | 306 ++---------------- internal/service/static/icons.svg | 4 + internal/service/static/main.css | 13 + internal/service/static/main.js | 306 +++++++++++++++--- .../templates/static/footer/compose.gohtml | 9 + internal/service/templates/static/head.gohtml | 1 + internal/stt/stt.go | 171 ---------- internal/stt/stt_test.go | 106 ------ 12 files changed, 321 insertions(+), 634 deletions(-) delete mode 100644 internal/stt/stt.go delete mode 100644 internal/stt/stt_test.go diff --git a/cmd/odidere/main.go b/cmd/odidere/main.go index 54b70d8..5d46f02 100644 --- a/cmd/odidere/main.go +++ b/cmd/odidere/main.go @@ -1,5 +1,4 @@ // Command odidere serves the odidere voice assistant API. -// It provides STT, LLM, and TTS orchestration via HTTP endpoints. // See package service for the main orchestration logic and package // config for configuration options. package main diff --git a/config.example.yaml b/config.example.yaml index e036c44..9ac1a19 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -5,7 +5,6 @@ server: stt: url: "http://localhost:8178" - timeout: "30s" llm: url: "http://localhost:8081/v1" @@ -17,16 +16,6 @@ llm: tts: url: "http://localhost:8880" voice: "af_heart" - voice_map: - english: "af_heart" # American English - chinese: "zf_xiaobei" # Mandarin Chinese - japanese: "jf_alpha" # Japanese - spanish: "ef_dora" # Spanish - french: "ff_siwis" # French - hindi: "hf_alpha" # Hindi - italian: "if_sara" # Italian - portuguese: "pf_dora" # Brazilian Portuguese - korean: "kf_sarah" # Korean timeout: "60s" tools: diff --git a/internal/config/config.go b/internal/config/config.go index 63d3231..933f359 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,7 +9,6 @@ import ( "code.chimeric.al/chimerical/odidere/internal/job" "code.chimeric.al/chimerical/odidere/internal/llm" - "code.chimeric.al/chimerical/odidere/internal/stt" "code.chimeric.al/chimerical/odidere/internal/tool" "gopkg.in/yaml.v3" @@ -25,10 +24,6 @@ type TTS struct { // Timeout is the maximum duration for a synthesis request. // Defaults to 60s if empty. Timeout string `yaml:"timeout"` - // VoiceMap maps whisper language names to voice IDs. - // Used by SelectVoice to auto-select voices based on detected - // language. - VoiceMap map[string]string `yaml:"voice_map"` } func (cfg TTS) Validate() error { @@ -46,6 +41,22 @@ func (cfg TTS) Validate() error { return nil } +// STT holds the configuration for the frontend-facing speech-to-text server. +// The URL is passed to the browser via a meta tag; the browser calls +// whisper-server directly. +type STT struct { + // URL is the base URL of the whisper-server. + // For example, "http://localhost:8178". + URL string `yaml:"url"` +} + +func (cfg STT) Validate() error { + if cfg.URL == "" { + return fmt.Errorf("missing URL") + } + return nil +} + // Config holds all application configuration sections. type Config struct { // Address is the host:port to listen on. @@ -59,8 +70,8 @@ type Config struct { // LLM configures the language model client. LLM llm.Config `yaml:"llm"` - // STT configures the speech-to-text client. - STT stt.Config `yaml:"stt"` + // STT configures the speech-to-text server URL passed to the frontend. + STT STT `yaml:"stt"` // Jobs defines scheduled autonomous tasks. Jobs []job.Config `yaml:"jobs"` // Tools defines external commands available to the LLM. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 302f93e..93ada20 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -215,7 +215,7 @@ llm: url: http://localhost:8080 stt: - timeout: "30s" + url: "" tts: url: http://localhost:8880 diff --git a/internal/service/service.go b/internal/service/service.go index 6db06ec..95bcc63 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -1,12 +1,11 @@ // Package service orchestrates the odidere voice assistant server. -// It coordinates the HTTP server, STT/LLM clients, and handles +// It coordinates the HTTP server, LLM client, and handles // graceful shutdown. package service import ( "context" "embed" - "encoding/base64" "encoding/json" "fmt" "html/template" @@ -24,7 +23,6 @@ import ( "code.chimeric.al/chimerical/odidere/internal/job" "code.chimeric.al/chimerical/odidere/internal/llm" "code.chimeric.al/chimerical/odidere/internal/service/templates" - "code.chimeric.al/chimerical/odidere/internal/stt" "code.chimeric.al/chimerical/odidere/internal/tool" "github.com/google/uuid" @@ -41,19 +39,6 @@ const ( ipKey key = "ip" ) -// defaultVoices maps whisper language names to default Kokoro voices. -var defaultVoices = map[string]string{ - "chinese": "zf_xiaobei", - "english": "af_heart", - "french": "ff_siwis", - "hindi": "hf_alpha", - "italian": "if_sara", - "japanese": "jf_alpha", - "korean": "kf_sarah", - "portuguese": "pf_dora", - "spanish": "ef_dora", -} - //go:embed all:static/* var static embed.FS @@ -67,7 +52,6 @@ type Service struct { log *slog.Logger mux *http.ServeMux server *http.Server - stt *stt.Client tmpl *template.Template tools *tool.Registry } @@ -88,13 +72,6 @@ func New(cfg *config.Config, log *slog.Logger) (*Service, error) { } svc.tools = registry - // Create STT client. - sttClient, err := stt.NewClient(cfg.STT, log) - if err != nil { - return nil, fmt.Errorf("create STT client: %v", err) - } - svc.stt = sttClient - // Create LLM client. llmClient, err := llm.NewClient(cfg.LLM, registry, log) if err != nil { @@ -135,8 +112,8 @@ func New(cfg *config.Config, log *slog.Logger) (*Service, error) { "/static/", http.FileServer(http.FS(staticFS)), ), ) - svc.mux.HandleFunc("POST /v1/chat/voice", svc.voice) - svc.mux.HandleFunc("POST /v1/chat/voice/stream", svc.voiceStream) + svc.mux.HandleFunc("POST /v1/chat/voice", svc.chat) + svc.mux.HandleFunc("POST /v1/chat/voice/stream", svc.chatStream) svc.mux.HandleFunc("GET /v1/models", svc.models) svc.server = &http.Server{ @@ -226,10 +203,6 @@ func (svc *Service) Run(ctx context.Context) error { "server", slog.String("address", svc.cfg.Address), ), - slog.Group( - "stt", - slog.String("url", svc.cfg.STT.URL), - ), slog.Group( "tools", slog.Int("count", len(svc.tools.List())), @@ -243,6 +216,10 @@ func (svc *Service) Run(ctx context.Context) error { slog.String("url", svc.cfg.TTS.URL), slog.String("default_voice", svc.cfg.TTS.Voice), ), + slog.Group( + "stt_url", + slog.String("url", svc.cfg.STT.URL), + ), slog.Group( "jobs", slog.Int("count", svc.JobCount()), @@ -355,19 +332,6 @@ func (svc *Service) JobCount() int { return len(svc.jobs) } -// selectVoice returns the voice ID for the given language. -// It uses the configured voice map, falling back to defaults. -func (svc *Service) selectVoice(lang string) string { - voiceMap := svc.cfg.TTS.VoiceMap - if len(voiceMap) == 0 { - voiceMap = defaultVoices - } - if voice, ok := voiceMap[lang]; ok { - return voice - } - return "" -} - func (svc *Service) home(w http.ResponseWriter, r *http.Request) { var ( ctx = r.Context() @@ -383,9 +347,11 @@ func (svc *Service) home(w http.ResponseWriter, r *http.Request) { w, "index.gohtml", struct { TTSURL string TTSDefaultVoice string + STTURL string }{ TTSURL: svc.cfg.TTS.URL, TTSDefaultVoice: svc.cfg.TTS.Voice, + STTURL: svc.cfg.STT.URL, }, ); err != nil { log.ErrorContext( @@ -404,10 +370,8 @@ func (svc *Service) status(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -// Request is the incoming request format for chat and voice endpoints. +// Request is the incoming request format for the chat endpoints. type Request struct { - // Audio is base64-encoded audio data (webm) for transcription. - Audio string `json:"audio,omitempty"` // Messages is the conversation history. Messages []openai.ChatCompletionMessage `json:"messages"` // Model is the LLM model ID. If empty, the default model is used. @@ -420,21 +384,17 @@ type Request struct { // Response is the response format for chat and voice endpoints. type Response struct { - // DetectedLanguage is the language detected in the input speech. - DetectedLanguage string `json:"detected_language,omitempty"` // Messages is the full list of messages generated during the query, // including tool calls and tool results. Messages []openai.ChatCompletionMessage `json:"messages,omitempty"` // Model is the LLM model used for the response. Model string `json:"used_model,omitempty"` - // Transcription is the transcribed user speech from the input audio. - Transcription string `json:"transcription,omitempty"` // Voice is the voice used for TTS synthesis. Voice string `json:"used_voice,omitempty"` } -// voice processes voice requests with audio input/output. -func (svc *Service) voice(w http.ResponseWriter, r *http.Request) { +// chat processes text chat requests. +func (svc *Service) chat(w http.ResponseWriter, r *http.Request) { var ( ctx = r.Context() log = ctx.Value(logKey).(*slog.Logger) @@ -462,101 +422,12 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) { slog.Any("data", req.Messages), ) - var ( - messages = req.Messages - transcription string - detectedLang string - ) - - // If audio provided, transcribe and append to last message. - if req.Audio != "" { - last := &messages[len(messages)-1] - if last.Role != openai.ChatMessageRoleUser { - http.Error( - w, - "last message must be role=user when audio is provided", - http.StatusBadRequest, - ) - return - } - - data, err := base64.StdEncoding.DecodeString(req.Audio) - if err != nil { - log.ErrorContext( - ctx, - "failed to decode audio", - slog.Any("error", err), - ) - http.Error(w, "invalid audio", http.StatusBadRequest) - return - } - - output, err := svc.stt.Transcribe(ctx, data) - if err != nil { - log.ErrorContext( - ctx, - "STT failed", - slog.Any("error", err), - ) - http.Error( - w, - "STT error", - http.StatusInternalServerError, - ) - return - } - - transcription = strings.TrimSpace(output.Text) - detectedLang = output.DetectedLanguage - if detectedLang == "" { - detectedLang = output.Language - } - log.InfoContext( - ctx, - "transcribed audio", - slog.String("text", transcription), - slog.String("language", detectedLang), - ) - - // Append transcription to last message's content. - switch { - // Already using MultiContent, append text part. - case len(last.MultiContent) > 0: - last.MultiContent = append(last.MultiContent, - openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeText, - Text: transcription, - }, - ) - last.Content = "" - - // Has string content, convert to MultiContent. - case last.Content != "": - last.MultiContent = []openai.ChatMessagePart{ - { - Type: openai.ChatMessagePartTypeText, - Text: last.Content, - }, - { - Type: openai.ChatMessagePartTypeText, - Text: transcription, - }, - } - last.Content = "" - // Empty message, just set content. - // Clear MultiContent, as required by the API spec. - default: - last.Content = transcription - last.MultiContent = nil - } - } - // Get LLM response. var model = req.Model if model == "" { model = svc.llm.DefaultModel() } - msgs, err := svc.llm.Query(ctx, messages, model, req.SystemMessage, 0) + msgs, err := svc.llm.Query(ctx, req.Messages, model, req.SystemMessage, 0) if err != nil { log.ErrorContext( ctx, @@ -582,32 +453,11 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) { slog.String("model", model), ) - // Determine voice to use. - var voice = req.Voice - if req.Voice == "" && detectedLang != "" { - if autoVoice := svc.selectVoice( - detectedLang, - ); autoVoice != "" { - voice = autoVoice - log.InfoContext(ctx, "auto-selected voice", - slog.String("language", detectedLang), - slog.String("voice", voice), - ) - } - } else if req.Voice == "" { - log.WarnContext( - ctx, - "auto-voice enabled but no language detected", - ) - } - w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(Response{ - DetectedLanguage: detectedLang, - Messages: msgs, - Model: model, - Transcription: transcription, - Voice: voice, + Messages: msgs, + Model: model, + Voice: req.Voice, }); err != nil { log.ErrorContext( ctx, @@ -617,24 +467,20 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) { } } -// StreamMessage is the SSE event payload for the streaming voice endpoint. +// StreamMessage is the SSE event payload for the streaming chat endpoint. type StreamMessage struct { - // DetectedLanguage is the language detected in the input speech. - DetectedLanguage string `json:"detected_language,omitempty"` // Error is an error message, if any. Error string `json:"error,omitempty"` // Message is the chat completion message. Message openai.ChatCompletionMessage `json:"message"` // Model is the LLM model used for the response. Model string `json:"model,omitempty"` - // Transcription is the transcribed user speech from the input audio. - Transcription string `json:"transcription,omitempty"` // Voice is the voice used for TTS synthesis. Voice string `json:"voice,omitempty"` } -// voiceStream processes voice requests with streaming SSE output. -func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) { +// chatStream processes chat requests with streaming SSE output. +func (svc *Service) chatStream(w http.ResponseWriter, r *http.Request) { var ( ctx = r.Context() log = ctx.Value(logKey).(*slog.Logger) @@ -670,90 +516,6 @@ func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) { return } - var ( - messages = req.Messages - transcription string - detectedLang string - ) - - // If audio provided, transcribe and append to last message. - if req.Audio != "" { - last := &messages[len(messages)-1] - if last.Role != openai.ChatMessageRoleUser { - http.Error( - w, - "last message must be role=user when audio is provided", - http.StatusBadRequest, - ) - return - } - - data, err := base64.StdEncoding.DecodeString(req.Audio) - if err != nil { - log.ErrorContext( - ctx, - "failed to decode audio", - slog.Any("error", err), - ) - http.Error(w, "invalid audio", http.StatusBadRequest) - return - } - - output, err := svc.stt.Transcribe(ctx, data) - if err != nil { - log.ErrorContext( - ctx, - "STT failed", - slog.Any("error", err), - ) - http.Error( - w, - "STT error", - http.StatusInternalServerError, - ) - return - } - - transcription = strings.TrimSpace(output.Text) - detectedLang = output.DetectedLanguage - if detectedLang == "" { - detectedLang = output.Language - } - log.InfoContext( - ctx, - "transcribed audio", - slog.String("text", transcription), - slog.String("language", detectedLang), - ) - - // Append transcription to last message's content. - switch { - case len(last.MultiContent) > 0: - last.MultiContent = append(last.MultiContent, - openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeText, - Text: transcription, - }, - ) - last.Content = "" - case last.Content != "": - last.MultiContent = []openai.ChatMessagePart{ - { - Type: openai.ChatMessagePartTypeText, - Text: last.Content, - }, - { - Type: openai.ChatMessagePartTypeText, - Text: transcription, - }, - } - last.Content = "" - default: - last.Content = transcription - last.MultiContent = nil - } - } - // Set SSE headers. w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") @@ -772,45 +534,19 @@ func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) { flusher.Flush() } - // If audio was transcribed, send user message with transcription. - if transcription != "" { - send(StreamMessage{ - Message: openai.ChatCompletionMessage{ - Role: openai.ChatMessageRoleUser, - Content: transcription, - }, - Transcription: transcription, - DetectedLanguage: detectedLang, - }) - } - // Get model. var model = req.Model if model == "" { model = svc.llm.DefaultModel() } - // Determine voice to use. - var voice = req.Voice - if req.Voice == "" && detectedLang != "" { - if autoVoice := svc.selectVoice( - detectedLang, - ); autoVoice != "" { - voice = autoVoice - log.InfoContext(ctx, "auto-selected voice", - slog.String("language", detectedLang), - slog.String("voice", voice), - ) - } - } - // Start streaming LLM query. var ( events = make(chan llm.StreamEvent) llmErr error ) go func() { - llmErr = svc.llm.QueryStream(ctx, messages, model, req.SystemMessage, 0, events) + llmErr = svc.llm.QueryStream(ctx, req.Messages, model, req.SystemMessage, 0, events) }() // Consume events and send as SSE. @@ -844,7 +580,7 @@ func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) { } last.Model = model - last.Voice = voice + last.Voice = req.Voice send(last) } diff --git a/internal/service/static/icons.svg b/internal/service/static/icons.svg index fe4bc3a..62ded67 100644 --- a/internal/service/static/icons.svg +++ b/internal/service/static/icons.svg @@ -57,6 +57,10 @@ + + + + diff --git a/internal/service/static/main.css b/internal/service/static/main.css index 750700a..1c9c694 100644 --- a/internal/service/static/main.css +++ b/internal/service/static/main.css @@ -279,6 +279,19 @@ body { animation: pulse 1s ease-in-out infinite; } +/* Transcribe button: recording state (same as PTT but for transcribe) */ +.compose__action-btn--recording { + color: white; + background: var(--color-recording); + animation: pulse 1s ease-in-out infinite; +} + +/* Transcribe button: transcribing state (spinning) */ +.compose__action-btn--transcribing { + color: var(--color-primary); + animation: spin 1s linear infinite; +} + .compose__action-btn--send .compose__icon--stop { display: none; } diff --git a/internal/service/static/main.js b/internal/service/static/main.js index d67f54a..7e80c1e 100644 --- a/internal/service/static/main.js +++ b/internal/service/static/main.js @@ -6,6 +6,21 @@ const STORAGE_KEY = 'odidere_history'; const SYSTEM_MESSAGE_KEY = 'odidere_system_message'; const VOICE_KEY = 'odidere_voice'; +/** + * VOICE_MAP maps whisper language names to default Kokoro voices. + * Used for auto-selecting a voice when the selector is set to "Auto". + */ +const VOICE_MAP = { + chinese: 'zf_xiaobei', + english: 'af_heart', + french: 'ff_siwis', + hindi: 'hf_alpha', + italian: 'if_sara', + japanese: 'jf_alpha', + korean: 'kf_sarah', + portuguese: 'pf_dora', + spanish: 'ef_dora', +}; /** * Odidere is the main application class for the voice assistant UI. * It manages audio recording, chat history, and communication with the API. @@ -25,6 +40,7 @@ class Odidere { this.currentController = null; this.isProcessing = false; this.isRecording = false; + this.isTranscribing = false; this.isMuted = false; this.isResetPending = false; this.isScrollPending = false; @@ -45,6 +61,9 @@ class Odidere { this.isPlaying = false; this.currentMessageId = null; + // STT state + this.sttUrl = document.querySelector('meta[name="stt-url"]')?.content || ''; + // Auto-scroll: enabled by default, disabled when user scrolls up, // re-enabled when they scroll back to bottom or send a new message. this.isAutoScrollEnabled = true; @@ -64,6 +83,7 @@ class Odidere { this.$textInput = document.getElementById('text-input'); this.$voice = document.getElementById('voice'); this.$mute = document.getElementById('mute'); + this.$transcribe = document.getElementById('transcribe'); // Settings modal this.$settings = document.getElementById('settings'); @@ -192,6 +212,35 @@ class Odidere { this.#handleAttachments(e.target.files), ); + // Transcribe button: hold to record, release to transcribe and populate textarea. + this.$transcribe.addEventListener( + 'touchstart', + this.#handleTranscribeTouchStart, + { + passive: false, + }, + ); + this.$transcribe.addEventListener( + 'touchend', + this.#handleTranscribeTouchEnd, + ); + this.$transcribe.addEventListener( + 'touchcancel', + this.#handleTranscribeTouchEnd, + ); + // Prevent context menu on long press. + this.$transcribe.addEventListener('contextmenu', (e) => e.preventDefault()); + + this.$transcribe.addEventListener( + 'mousedown', + this.#handleTranscribeMouseDown, + ); + this.$transcribe.addEventListener('mouseup', this.#handleTranscribeMouseUp); + this.$transcribe.addEventListener( + 'mouseleave', + this.#handleTranscribeMouseUp, + ); + // Save selections on change. this.$model.addEventListener('change', () => { localStorage.setItem(MODEL_KEY, this.$model.value); @@ -830,6 +879,122 @@ class Odidere { this.#setLoadingState(false); } + // ==================== + // STT: FRONTEND TRANSCRIPTION + // ==================== + /** + * #transcribeAudio sends an audio blob to whisper-server directly from + * the browser and returns the transcription result. + * @param {Blob} audioBlob + * @returns {Promise<{text: string, detectedLanguage: string}>} + */ + async #transcribeAudio(audioBlob) { + if (!this.sttUrl) { + throw new Error('STT URL not configured'); + } + + const formData = new FormData(); + formData.append('file', audioBlob, 'audio.webm'); + formData.append('response_format', 'verbose_json'); + + const res = await fetch(`${this.sttUrl}/inference`, { + method: 'POST', + body: formData, + }); + + if (!res.ok) { + const errText = await res.text().catch(() => ''); + throw new Error(`STT error ${res.status}: ${errText}`); + } + + const data = await res.json(); + const text = (data.text || '').trim(); + const detectedLanguage = data.detected_language || data.language || ''; + return { text, detectedLanguage }; + } + + /** + * #handleTranscribeTouchStart handles touch start on the transcribe button. + * Begins recording for transcription (hold-to-record, like PTT). + * @param {TouchEvent} event + */ + #handleTranscribeTouchStart = (event) => { + event.preventDefault(); + event.stopPropagation(); + this.#startTranscribeRecording(); + }; + + /** + * #handleTranscribeTouchEnd handles touch end on the transcribe button. + * Stops recording; the shared stop handler will populate the textarea. + * @param {TouchEvent} event + */ + #handleTranscribeTouchEnd = (event) => { + event.stopPropagation(); + this.#stopTranscribeRecording(); + }; + + /** + * #handleTranscribeMouseDown handles mouse down on the transcribe button. + * Ignores events that originate from touch to avoid double-firing. + * @param {MouseEvent} event + */ + #handleTranscribeMouseDown = (event) => { + if (event.sourceCapabilities?.firesTouchEvents) return; + event.preventDefault(); + event.stopPropagation(); + this.#startTranscribeRecording(); + }; + + /** + * #handleTranscribeMouseUp handles mouse up on the transcribe button. + * @param {MouseEvent} event + */ + #handleTranscribeMouseUp = (event) => { + if (event.sourceCapabilities?.firesTouchEvents) return; + event.stopPropagation(); + this.#stopTranscribeRecording(); + }; + + /** + * #startTranscribeRecording begins recording for transcription mode. + * Unlike PTT, this populates the textarea on stop rather than sending. + */ + async #startTranscribeRecording() { + if (this.isTranscribing) return; + if (this.isRecording) return; + if (this.isProcessing) return; + + this.#stopCurrentAudio(); + this.#stopTTS(); + + // Initialize MediaRecorder on first use (shared with PTT). + if (!this.mediaRecorder) { + const success = await this.#initMediaRecorder(); + if (!success) return; + } + + this.audioChunks = []; + this.mediaRecorder.start(); + this.isTranscribing = true; + this.$transcribe.classList.add('compose__action-btn--recording'); + } + + /** + * #stopTranscribeRecording stops the media recorder for transcription mode. + * The shared 'stop' event handler (set up in #initMediaRecorder) will + * fire and clear UI state since this.isTranscribing is still true. + */ + #stopTranscribeRecording() { + if (!this.isTranscribing) return; + if (!this.mediaRecorder) return; + + // Keep isTranscribing=true so the stop handler knows to populate textarea. + // The stop handler is responsible for clearing the recording class, + // so we do not remove it here. + this.mediaRecorder.stop(); + } + // ==================== // AUDIO RECORDING // ==================== @@ -840,6 +1005,7 @@ class Odidere { */ async #startRecording() { if (this.isRecording) return; + if (this.isTranscribing) return; if (this.isProcessing) return; this.#stopCurrentAudio(); @@ -854,7 +1020,6 @@ class Odidere { this.audioChunks = []; this.mediaRecorder.start(); this.#setRecordingState(true); - this.#acquireWakeLock(); } /** @@ -864,13 +1029,17 @@ class Odidere { if (!this.isRecording) return; if (!this.mediaRecorder) return; + // Do not clear recording state here. The async 'stop' event handler + // is responsible for clearing it (#setRecordingState(false)) after + // the audio blob has been assembled. this.mediaRecorder.stop(); - this.#setRecordingState(false); } /** * #initMediaRecorder initializes the MediaRecorder with microphone access. * Prefers opus codec for better compression if available. + * The stop handler branches on this.isTranscribing: when true it populates + * the textarea; when false (PTT flow) it sends the request. * @returns {Promise} true if successful */ async #initMediaRecorder() { @@ -891,22 +1060,87 @@ class Odidere { this.mediaRecorder.addEventListener('stop', async () => { if (this.audioChunks.length === 0) return; + // Capture the mode synchronously before any async work. + // The 'stop' event fires asynchronously, so this.isTranscribing + // may have changed by the time we await below. + const wasTranscribing = this.isTranscribing; + const audioBlob = new Blob(this.audioChunks, { type: this.mediaRecorder.mimeType, }); this.audioChunks = []; - // Capture and clear text input to send with audio. - const text = this.$textInput.value.trim(); - this.$textInput.value = ''; - this.$textInput.style.height = 'auto'; + // Clear the flag now that the stop event has fired. + if (wasTranscribing) { + this.isTranscribing = false; + this.$transcribe.classList.remove('compose__action-btn--recording'); + } else { + this.#setRecordingState(false); + } - await this.#sendRequest({ audio: audioBlob, text }); + // Transcribe the audio. + try { + const { text: transcription, detectedLanguage } = + await this.#transcribeAudio(audioBlob); + + // Resolve the effective voice for this message. + // If the selector is "Auto", pick a default for the detected language. + // This does NOT change the dropdown - the voice is per-message only. + let voice = this.$voice.value; + if (!voice && detectedLanguage) { + voice = VOICE_MAP[detectedLanguage] || ''; + } + + // Branch: transcribe-only (populate textarea) vs PTT (send). + if (wasTranscribing) { + // Transcribe mode: populate textarea, do not send. + this.$transcribe.classList.add('compose__action-btn--transcribing'); + try { + if (transcription) { + const existing = this.$textInput.value; + if (existing.trim()) { + this.$textInput.value = `${existing}\n${transcription}`; + } else { + this.$textInput.value = transcription; + } + this.#handleTextareaInput(); + } + } finally { + this.$transcribe.classList.remove( + 'compose__action-btn--transcribing', + ); + this.$textInput.focus(); + } + } else { + // PTT mode: capture text, clear input, send request. + const text = this.$textInput.value.trim(); + this.$textInput.value = ''; + this.$textInput.style.height = 'auto'; + + const combined = text ? `${text}\n${transcription}` : transcription; + await this.#sendRequest({ + text: combined, + detectedLanguage, + voice, + }); + } + } catch (e) { + console.error('transcription failed:', e); + this.#renderError(`Transcription failed: ${e.message}`); + if (!wasTranscribing) { + this.#setLoadingState(false); + } + } }); this.mediaRecorder.addEventListener('error', (event) => { console.error('MediaRecorder error:', event.error); - this.#setRecordingState(false); + if (this.isTranscribing) { + this.isTranscribing = false; + this.$transcribe.classList.remove('compose__action-btn--recording'); + } else { + this.#setRecordingState(false); + } this.audioChunks = []; this.#renderError( `Recording error: ${event.error?.message || 'Unknown error'}`, @@ -1181,25 +1415,24 @@ class Odidere { * * For text-only input, renders the user message immediately for * responsiveness. - * For audio input, waits for the server's transcription event before - * rendering the user message. * * Messages are rendered incrementally as SSE events arrive: - * - user (transcription), assistant (tool calls), tool (results), + * - user, assistant (tool calls), tool (results), * and a final assistant message with content. * * @param {Object} options - * @param {Blob} [options.audio] - Recorded audio blob * @param {string} [options.text] - Text input + * @param {string} [options.detectedLanguage] - Detected language from STT + * @param {string} [options.voice] - Resolved voice for this message (from auto-selection) */ - async #sendRequest({ audio = null, text = '' }) { + async #sendRequest({ text = '', detectedLanguage = '', voice }) { this.#setLoadingState(true); // Re-enable auto-scroll when the user sends a new message. this.isAutoScrollEnabled = true; this.currentController = new AbortController(); try { - const hasNewInput = text || this.attachments.length > 0 || audio; + const hasNewInput = text || this.attachments.length > 0; // Build messages array from active path + current user message. const activePath = this.getActivePath(); @@ -1221,25 +1454,24 @@ class Odidere { const payload = { messages, - voice: this.$voice.value, + voice: voice ?? this.$voice.value, model: this.$model.value, }; const systemMessage = localStorage.getItem(SYSTEM_MESSAGE_KEY); if (systemMessage) { payload.system_message = systemMessage; } - if (audio) { - payload.audio = await this.#toBase64(audio); - } // Clear attachments after building payload (before async operations). this.#clearAttachments(); // For text-only requests with new input, add to history and render // immediately. Skip when continuing conversation (no new user message). - if (!audio && hasNewInput) { - const appended = this.#appendHistory([userMessage]); - this.#renderMessages(appended); + if (hasNewInput) { + const meta = {}; + if (detectedLanguage) meta.language = detectedLanguage; + const appended = this.#appendHistory([userMessage], meta); + this.#renderMessages(appended, meta); } const res = await fetch(STREAM_ENDPOINT, { @@ -1296,37 +1528,6 @@ class Odidere { } const message = event.message; - // Handle user message from server (audio transcription). - if (message.role === 'user' && event.transcription) { - const displayParts = []; - if (text) displayParts.push(text); - if (event.transcription) displayParts.push(event.transcription); - const displayContent = displayParts.join('\n\n'); - - let historyContent; - if (typeof userMessage.content === 'string') { - historyContent = displayContent; - } else { - historyContent = [...userMessage.content]; - if (event.transcription) { - historyContent.push({ - type: 'text', - text: event.transcription, - }); - } - } - - const audioUserMessage = { - id: userMessage.id, - role: 'user', - content: historyContent, - meta: { language: event.detected_language }, - }; - - this.#appendHistory([audioUserMessage]); - this.#renderMessages([audioUserMessage]); - continue; - } // Stash assistant messages with tool calls; render once all // tool results have arrived so the output is visible in the UI. @@ -1488,6 +1689,7 @@ class Odidere { this.$chatLoading.hidden = !loading; // Keep the send button enabled during loading so it can act as a stop button. this.$ptt.disabled = loading; + this.$transcribe.disabled = loading; // Clear any pending stop state when loading ends. if (!loading && this.isStopPending) { this.isStopPending = false; diff --git a/internal/service/templates/static/footer/compose.gohtml b/internal/service/templates/static/footer/compose.gohtml index 8ff1e93..2e6a19e 100644 --- a/internal/service/templates/static/footer/compose.gohtml +++ b/internal/service/templates/static/footer/compose.gohtml @@ -19,6 +19,15 @@
+