chimerical/odidere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor STT architecture

Browse Source
This commit is contained in:
dwrz
2026-06-15 10:51:17 +00:00
parent 70e1a995f3
commit c71300b1bf
12 changed files with 321 additions and 634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/odidere/main.go
View File

@@ -1,5 +1,4 @@
// Command odidere serves the odidere voice assistant API.
// It provides STT, LLM, and TTS orchestration via HTTP endpoints.
// See package service for the main orchestration logic and package
// config for configuration options.
package main

11
config.example.yaml
View File

@@ -5,7 +5,6 @@ server:
stt:
url: "http://localhost:8178"
timeout: "30s"
llm:
url: "http://localhost:8081/v1"
@@ -17,16 +16,6 @@ llm:
tts:
url: "http://localhost:8880"
voice: "af_heart"
voice_map:
english: "af_heart" # American English
chinese: "zf_xiaobei" # Mandarin Chinese
japanese: "jf_alpha" # Japanese
spanish: "ef_dora" # Spanish
french: "ff_siwis" # French
hindi: "hf_alpha" # Hindi
italian: "if_sara" # Italian
portuguese: "pf_dora" # Brazilian Portuguese
korean: "kf_sarah" # Korean
timeout: "60s"
tools:

25
internal/config/config.go
View File

@@ -9,7 +9,6 @@ import (
"code.chimeric.al/chimerical/odidere/internal/job"
"code.chimeric.al/chimerical/odidere/internal/llm"
"code.chimeric.al/chimerical/odidere/internal/stt"
"code.chimeric.al/chimerical/odidere/internal/tool"
"gopkg.in/yaml.v3"
@@ -25,10 +24,6 @@ type TTS struct {
// Timeout is the maximum duration for a synthesis request.
// Defaults to 60s if empty.
Timeout string `yaml:"timeout"`
// VoiceMap maps whisper language names to voice IDs.
// Used by SelectVoice to auto-select voices based on detected
// language.
VoiceMap map[string]string `yaml:"voice_map"`
}
func (cfg TTS) Validate() error {
@@ -46,6 +41,22 @@ func (cfg TTS) Validate() error {
return nil
}
// STT holds the configuration for the frontend-facing speech-to-text server.
// The URL is passed to the browser via a meta tag; the browser calls
// whisper-server directly.
type STT struct {
// URL is the base URL of the whisper-server.
// For example, "http://localhost:8178".
URL string `yaml:"url"`
}
func (cfg STT) Validate() error {
if cfg.URL == "" {
return fmt.Errorf("missing URL")
}
return nil
}
// Config holds all application configuration sections.
type Config struct {
// Address is the host:port to listen on.
@@ -59,8 +70,8 @@ type Config struct {
// LLM configures the language model client.
LLM llm.Config `yaml:"llm"`
// STT configures the speech-to-text client.
STT stt.Config `yaml:"stt"`
// STT configures the speech-to-text server URL passed to the frontend.
STT STT `yaml:"stt"`
// Jobs defines scheduled autonomous tasks.
Jobs []job.Config `yaml:"jobs"`
// Tools defines external commands available to the LLM.

2
internal/config/config_test.go
View File

@@ -215,7 +215,7 @@ llm:
url: http://localhost:8080
stt:
timeout: "30s"
url: ""
tts:
url: http://localhost:8880

306
internal/service/service.go
View File

@@ -1,12 +1,11 @@
// Package service orchestrates the odidere voice assistant server.
// It coordinates the HTTP server, STT/LLM clients, and handles
// It coordinates the HTTP server, LLM client, and handles
// graceful shutdown.
package service
import (
"context"
"embed"
"encoding/base64"
"encoding/json"
"fmt"
"html/template"
@@ -24,7 +23,6 @@ import (
"code.chimeric.al/chimerical/odidere/internal/job"
"code.chimeric.al/chimerical/odidere/internal/llm"
"code.chimeric.al/chimerical/odidere/internal/service/templates"
"code.chimeric.al/chimerical/odidere/internal/stt"
"code.chimeric.al/chimerical/odidere/internal/tool"
"github.com/google/uuid"
@@ -41,19 +39,6 @@ const (
ipKey key = "ip"
)
// defaultVoices maps whisper language names to default Kokoro voices.
var defaultVoices = map[string]string{
"chinese": "zf_xiaobei",
"english": "af_heart",
"french": "ff_siwis",
"hindi": "hf_alpha",
"italian": "if_sara",
"japanese": "jf_alpha",
"korean": "kf_sarah",
"portuguese": "pf_dora",
"spanish": "ef_dora",
}
//go:embed all:static/*
var static embed.FS
@@ -67,7 +52,6 @@ type Service struct {
log *slog.Logger
mux *http.ServeMux
server *http.Server
stt *stt.Client
tmpl *template.Template
tools *tool.Registry
}
@@ -88,13 +72,6 @@ func New(cfg *config.Config, log *slog.Logger) (*Service, error) {
}
svc.tools = registry
// Create STT client.
sttClient, err := stt.NewClient(cfg.STT, log)
if err != nil {
return nil, fmt.Errorf("create STT client: %v", err)
}
svc.stt = sttClient
// Create LLM client.
llmClient, err := llm.NewClient(cfg.LLM, registry, log)
if err != nil {
@@ -135,8 +112,8 @@ func New(cfg *config.Config, log *slog.Logger) (*Service, error) {
"/static/", http.FileServer(http.FS(staticFS)),
),
)
svc.mux.HandleFunc("POST /v1/chat/voice", svc.voice)
svc.mux.HandleFunc("POST /v1/chat/voice/stream", svc.voiceStream)
svc.mux.HandleFunc("POST /v1/chat/voice", svc.chat)
svc.mux.HandleFunc("POST /v1/chat/voice/stream", svc.chatStream)
svc.mux.HandleFunc("GET /v1/models", svc.models)
svc.server = &http.Server{
@@ -226,10 +203,6 @@ func (svc *Service) Run(ctx context.Context) error {
"server",
slog.String("address", svc.cfg.Address),
),
slog.Group(
"stt",
slog.String("url", svc.cfg.STT.URL),
),
slog.Group(
"tools",
slog.Int("count", len(svc.tools.List())),
@@ -243,6 +216,10 @@ func (svc *Service) Run(ctx context.Context) error {
slog.String("url", svc.cfg.TTS.URL),
slog.String("default_voice", svc.cfg.TTS.Voice),
),
slog.Group(
"stt_url",
slog.String("url", svc.cfg.STT.URL),
),
slog.Group(
"jobs",
slog.Int("count", svc.JobCount()),
@@ -355,19 +332,6 @@ func (svc *Service) JobCount() int {
return len(svc.jobs)
}
// selectVoice returns the voice ID for the given language.
// It uses the configured voice map, falling back to defaults.
func (svc *Service) selectVoice(lang string) string {
voiceMap := svc.cfg.TTS.VoiceMap
if len(voiceMap) == 0 {
voiceMap = defaultVoices
}
if voice, ok := voiceMap[lang]; ok {
return voice
}
return ""
}
func (svc *Service) home(w http.ResponseWriter, r *http.Request) {
var (
ctx = r.Context()
@@ -383,9 +347,11 @@ func (svc *Service) home(w http.ResponseWriter, r *http.Request) {
w, "index.gohtml", struct {
TTSURL string
TTSDefaultVoice string
STTURL string
}{
TTSURL: svc.cfg.TTS.URL,
TTSDefaultVoice: svc.cfg.TTS.Voice,
STTURL: svc.cfg.STT.URL,
},
); err != nil {
log.ErrorContext(
@@ -404,10 +370,8 @@ func (svc *Service) status(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
// Request is the incoming request format for chat and voice endpoints.
// Request is the incoming request format for the chat endpoints.
type Request struct {
// Audio is base64-encoded audio data (webm) for transcription.
Audio string `json:"audio,omitempty"`
// Messages is the conversation history.
Messages []openai.ChatCompletionMessage `json:"messages"`
// Model is the LLM model ID. If empty, the default model is used.
@@ -420,21 +384,17 @@ type Request struct {
// Response is the response format for chat and voice endpoints.
type Response struct {
// DetectedLanguage is the language detected in the input speech.
DetectedLanguage string `json:"detected_language,omitempty"`
// Messages is the full list of messages generated during the query,
// including tool calls and tool results.
Messages []openai.ChatCompletionMessage `json:"messages,omitempty"`
// Model is the LLM model used for the response.
Model string `json:"used_model,omitempty"`
// Transcription is the transcribed user speech from the input audio.
Transcription string `json:"transcription,omitempty"`
// Voice is the voice used for TTS synthesis.
Voice string `json:"used_voice,omitempty"`
}
// voice processes voice requests with audio input/output.
func (svc *Service) voice(w http.ResponseWriter, r *http.Request) {
// chat processes text chat requests.
func (svc *Service) chat(w http.ResponseWriter, r *http.Request) {
var (
ctx = r.Context()
log = ctx.Value(logKey).(*slog.Logger)
@@ -462,101 +422,12 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) {
slog.Any("data", req.Messages),
)
var (
messages = req.Messages
transcription string
detectedLang string
)
// If audio provided, transcribe and append to last message.
if req.Audio != "" {
last := &messages[len(messages)-1]
if last.Role != openai.ChatMessageRoleUser {
http.Error(
w,
"last message must be role=user when audio is provided",
http.StatusBadRequest,
)
return
}
data, err := base64.StdEncoding.DecodeString(req.Audio)
if err != nil {
log.ErrorContext(
ctx,
"failed to decode audio",
slog.Any("error", err),
)
http.Error(w, "invalid audio", http.StatusBadRequest)
return
}
output, err := svc.stt.Transcribe(ctx, data)
if err != nil {
log.ErrorContext(
ctx,
"STT failed",
slog.Any("error", err),
)
http.Error(
w,
"STT error",
http.StatusInternalServerError,
)
return
}
transcription = strings.TrimSpace(output.Text)
detectedLang = output.DetectedLanguage
if detectedLang == "" {
detectedLang = output.Language
}
log.InfoContext(
ctx,
"transcribed audio",
slog.String("text", transcription),
slog.String("language", detectedLang),
)
// Append transcription to last message's content.
switch {
// Already using MultiContent, append text part.
case len(last.MultiContent) > 0:
last.MultiContent = append(last.MultiContent,
openai.ChatMessagePart{
Type: openai.ChatMessagePartTypeText,
Text: transcription,
},
)
last.Content = ""
// Has string content, convert to MultiContent.
case last.Content != "":
last.MultiContent = []openai.ChatMessagePart{
{
Type: openai.ChatMessagePartTypeText,
Text: last.Content,
},
{
Type: openai.ChatMessagePartTypeText,
Text: transcription,
},
}
last.Content = ""
// Empty message, just set content.
// Clear MultiContent, as required by the API spec.
default:
last.Content = transcription
last.MultiContent = nil
}
}
// Get LLM response.
var model = req.Model
if model == "" {
model = svc.llm.DefaultModel()
}
msgs, err := svc.llm.Query(ctx, messages, model, req.SystemMessage, 0)
msgs, err := svc.llm.Query(ctx, req.Messages, model, req.SystemMessage, 0)
if err != nil {
log.ErrorContext(
ctx,
@@ -582,32 +453,11 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) {
slog.String("model", model),
)
// Determine voice to use.
var voice = req.Voice
if req.Voice == "" && detectedLang != "" {
if autoVoice := svc.selectVoice(
detectedLang,
); autoVoice != "" {
voice = autoVoice
log.InfoContext(ctx, "auto-selected voice",
slog.String("language", detectedLang),
slog.String("voice", voice),
)
}
} else if req.Voice == "" {
log.WarnContext(
ctx,
"auto-voice enabled but no language detected",
)
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(Response{
DetectedLanguage: detectedLang,
Messages: msgs,
Model: model,
Transcription: transcription,
Voice: voice,
Messages: msgs,
Model: model,
Voice: req.Voice,
}); err != nil {
log.ErrorContext(
ctx,
@@ -617,24 +467,20 @@ func (svc *Service) voice(w http.ResponseWriter, r *http.Request) {
}
}
// StreamMessage is the SSE event payload for the streaming voice endpoint.
// StreamMessage is the SSE event payload for the streaming chat endpoint.
type StreamMessage struct {
// DetectedLanguage is the language detected in the input speech.
DetectedLanguage string `json:"detected_language,omitempty"`
// Error is an error message, if any.
Error string `json:"error,omitempty"`
// Message is the chat completion message.
Message openai.ChatCompletionMessage `json:"message"`
// Model is the LLM model used for the response.
Model string `json:"model,omitempty"`
// Transcription is the transcribed user speech from the input audio.
Transcription string `json:"transcription,omitempty"`
// Voice is the voice used for TTS synthesis.
Voice string `json:"voice,omitempty"`
}
// voiceStream processes voice requests with streaming SSE output.
func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) {
// chatStream processes chat requests with streaming SSE output.
func (svc *Service) chatStream(w http.ResponseWriter, r *http.Request) {
var (
ctx = r.Context()
log = ctx.Value(logKey).(*slog.Logger)
@@ -670,90 +516,6 @@ func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) {
return
}
var (
messages = req.Messages
transcription string
detectedLang string
)
// If audio provided, transcribe and append to last message.
if req.Audio != "" {
last := &messages[len(messages)-1]
if last.Role != openai.ChatMessageRoleUser {
http.Error(
w,
"last message must be role=user when audio is provided",
http.StatusBadRequest,
)
return
}
data, err := base64.StdEncoding.DecodeString(req.Audio)
if err != nil {
log.ErrorContext(
ctx,
"failed to decode audio",
slog.Any("error", err),
)
http.Error(w, "invalid audio", http.StatusBadRequest)
return
}
output, err := svc.stt.Transcribe(ctx, data)
if err != nil {
log.ErrorContext(
ctx,
"STT failed",
slog.Any("error", err),
)
http.Error(
w,
"STT error",
http.StatusInternalServerError,
)
return
}
transcription = strings.TrimSpace(output.Text)
detectedLang = output.DetectedLanguage
if detectedLang == "" {
detectedLang = output.Language
}
log.InfoContext(
ctx,
"transcribed audio",
slog.String("text", transcription),
slog.String("language", detectedLang),
)
// Append transcription to last message's content.
switch {
case len(last.MultiContent) > 0:
last.MultiContent = append(last.MultiContent,
openai.ChatMessagePart{
Type: openai.ChatMessagePartTypeText,
Text: transcription,
},
)
last.Content = ""
case last.Content != "":
last.MultiContent = []openai.ChatMessagePart{
{
Type: openai.ChatMessagePartTypeText,
Text: last.Content,
},
{
Type: openai.ChatMessagePartTypeText,
Text: transcription,
},
}
last.Content = ""
default:
last.Content = transcription
last.MultiContent = nil
}
}
// Set SSE headers.
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
@@ -772,45 +534,19 @@ func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) {
flusher.Flush()
}
// If audio was transcribed, send user message with transcription.
if transcription != "" {
send(StreamMessage{
Message: openai.ChatCompletionMessage{
Role: openai.ChatMessageRoleUser,
Content: transcription,
},
Transcription: transcription,
DetectedLanguage: detectedLang,
})
}
// Get model.
var model = req.Model
if model == "" {
model = svc.llm.DefaultModel()
}
// Determine voice to use.
var voice = req.Voice
if req.Voice == "" && detectedLang != "" {
if autoVoice := svc.selectVoice(
detectedLang,
); autoVoice != "" {
voice = autoVoice
log.InfoContext(ctx, "auto-selected voice",
slog.String("language", detectedLang),
slog.String("voice", voice),
)
}
}
// Start streaming LLM query.
var (
events = make(chan llm.StreamEvent)
llmErr error
)
go func() {
llmErr = svc.llm.QueryStream(ctx, messages, model, req.SystemMessage, 0, events)
llmErr = svc.llm.QueryStream(ctx, req.Messages, model, req.SystemMessage, 0, events)
}()
// Consume events and send as SSE.
@@ -844,7 +580,7 @@ func (svc *Service) voiceStream(w http.ResponseWriter, r *http.Request) {
}
last.Model = model
last.Voice = voice
last.Voice = req.Voice
send(last)
}

4
internal/service/static/icons.svg
View File

@@ -57,6 +57,10 @@
<line x1="12" x2="12" y1="19" y2="22"/>
</symbol>
<symbol id="dictate" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<path d="M13 21h8m.174-14.188a1 1 0 0 0-3.986-3.987L3.842 16.174a2 2 0 0 0-.5.83l-1.321 4.352a.5.5 0 0 0 .623.622l4.353-1.32a2 2 0 0 0 .83-.497z"/>
</symbol>
<symbol id="send" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<path d="M14.536 21.686a.5.5 0 0 0 .937-.024l6.5-19a.496.496 0 0 0-.635-.635l-19 6.5a.5.5 0 0 0-.024.937l7.93 3.18a2 2 0 0 1 1.112 1.11z"/>
<path d="m21.854 2.147-10.94 10.939"/>
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 6.7 KiB

After

Width:  |  Height:  |  Size: 7.0 KiB

13
internal/service/static/main.css
View File

@@ -279,6 +279,19 @@ body {
animation: pulse 1s ease-in-out infinite;
}
/* Transcribe button: recording state (same as PTT but for transcribe) */
.compose__action-btn--recording {
color: white;
background: var(--color-recording);
animation: pulse 1s ease-in-out infinite;
}
/* Transcribe button: transcribing state (spinning) */
.compose__action-btn--transcribing {
color: var(--color-primary);
animation: spin 1s linear infinite;
}
.compose__action-btn--send .compose__icon--stop {
display: none;
}

306
internal/service/static/main.js
View File

@@ -6,6 +6,21 @@ const STORAGE_KEY = 'odidere_history';
const SYSTEM_MESSAGE_KEY = 'odidere_system_message';
const VOICE_KEY = 'odidere_voice';
/**
* VOICE_MAP maps whisper language names to default Kokoro voices.
* Used for auto-selecting a voice when the selector is set to "Auto".
*/
const VOICE_MAP = {
chinese: 'zf_xiaobei',
english: 'af_heart',
french: 'ff_siwis',
hindi: 'hf_alpha',
italian: 'if_sara',
japanese: 'jf_alpha',
korean: 'kf_sarah',
portuguese: 'pf_dora',
spanish: 'ef_dora',
};
/**
* Odidere is the main application class for the voice assistant UI.
* It manages audio recording, chat history, and communication with the API.
@@ -25,6 +40,7 @@ class Odidere {
this.currentController = null;
this.isProcessing = false;
this.isRecording = false;
this.isTranscribing = false;
this.isMuted = false;
this.isResetPending = false;
this.isScrollPending = false;
@@ -45,6 +61,9 @@ class Odidere {
this.isPlaying = false;
this.currentMessageId = null;
// STT state
this.sttUrl = document.querySelector('meta[name="stt-url"]')?.content || '';
// Auto-scroll: enabled by default, disabled when user scrolls up,
// re-enabled when they scroll back to bottom or send a new message.
this.isAutoScrollEnabled = true;
@@ -64,6 +83,7 @@ class Odidere {
this.$textInput = document.getElementById('text-input');
this.$voice = document.getElementById('voice');
this.$mute = document.getElementById('mute');
this.$transcribe = document.getElementById('transcribe');
// Settings modal
this.$settings = document.getElementById('settings');
@@ -192,6 +212,35 @@ class Odidere {
this.#handleAttachments(e.target.files),
);
// Transcribe button: hold to record, release to transcribe and populate textarea.
this.$transcribe.addEventListener(
'touchstart',
this.#handleTranscribeTouchStart,
{
passive: false,
},
);
this.$transcribe.addEventListener(
'touchend',
this.#handleTranscribeTouchEnd,
);
this.$transcribe.addEventListener(
'touchcancel',
this.#handleTranscribeTouchEnd,
);
// Prevent context menu on long press.
this.$transcribe.addEventListener('contextmenu', (e) => e.preventDefault());
this.$transcribe.addEventListener(
'mousedown',
this.#handleTranscribeMouseDown,
);
this.$transcribe.addEventListener('mouseup', this.#handleTranscribeMouseUp);
this.$transcribe.addEventListener(
'mouseleave',
this.#handleTranscribeMouseUp,
);
// Save selections on change.
this.$model.addEventListener('change', () => {
localStorage.setItem(MODEL_KEY, this.$model.value);
@@ -830,6 +879,122 @@ class Odidere {
this.#setLoadingState(false);
}
// ====================
// STT: FRONTEND TRANSCRIPTION
// ====================
/**
* #transcribeAudio sends an audio blob to whisper-server directly from
* the browser and returns the transcription result.
* @param {Blob} audioBlob
* @returns {Promise<{text: string, detectedLanguage: string}>}
*/
async #transcribeAudio(audioBlob) {
if (!this.sttUrl) {
throw new Error('STT URL not configured');
}
const formData = new FormData();
formData.append('file', audioBlob, 'audio.webm');
formData.append('response_format', 'verbose_json');
const res = await fetch(`${this.sttUrl}/inference`, {
method: 'POST',
body: formData,
});
if (!res.ok) {
const errText = await res.text().catch(() => '');
throw new Error(`STT error ${res.status}: ${errText}`);
}
const data = await res.json();
const text = (data.text || '').trim();
const detectedLanguage = data.detected_language || data.language || '';
return { text, detectedLanguage };
}
/**
* #handleTranscribeTouchStart handles touch start on the transcribe button.
* Begins recording for transcription (hold-to-record, like PTT).
* @param {TouchEvent} event
*/
#handleTranscribeTouchStart = (event) => {
event.preventDefault();
event.stopPropagation();
this.#startTranscribeRecording();
};
/**
* #handleTranscribeTouchEnd handles touch end on the transcribe button.
* Stops recording; the shared stop handler will populate the textarea.
* @param {TouchEvent} event
*/
#handleTranscribeTouchEnd = (event) => {
event.stopPropagation();
this.#stopTranscribeRecording();
};
/**
* #handleTranscribeMouseDown handles mouse down on the transcribe button.
* Ignores events that originate from touch to avoid double-firing.
* @param {MouseEvent} event
*/
#handleTranscribeMouseDown = (event) => {
if (event.sourceCapabilities?.firesTouchEvents) return;
event.preventDefault();
event.stopPropagation();
this.#startTranscribeRecording();
};
/**
* #handleTranscribeMouseUp handles mouse up on the transcribe button.
* @param {MouseEvent} event
*/
#handleTranscribeMouseUp = (event) => {
if (event.sourceCapabilities?.firesTouchEvents) return;
event.stopPropagation();
this.#stopTranscribeRecording();
};
/**
* #startTranscribeRecording begins recording for transcription mode.
* Unlike PTT, this populates the textarea on stop rather than sending.
*/
async #startTranscribeRecording() {
if (this.isTranscribing) return;
if (this.isRecording) return;
if (this.isProcessing) return;
this.#stopCurrentAudio();
this.#stopTTS();
// Initialize MediaRecorder on first use (shared with PTT).
if (!this.mediaRecorder) {
const success = await this.#initMediaRecorder();
if (!success) return;
}
this.audioChunks = [];
this.mediaRecorder.start();
this.isTranscribing = true;
this.$transcribe.classList.add('compose__action-btn--recording');
}
/**
* #stopTranscribeRecording stops the media recorder for transcription mode.
* The shared 'stop' event handler (set up in #initMediaRecorder) will
* fire and clear UI state since this.isTranscribing is still true.
*/
#stopTranscribeRecording() {
if (!this.isTranscribing) return;
if (!this.mediaRecorder) return;
// Keep isTranscribing=true so the stop handler knows to populate textarea.
// The stop handler is responsible for clearing the recording class,
// so we do not remove it here.
this.mediaRecorder.stop();
}
// ====================
// AUDIO RECORDING
// ====================
@@ -840,6 +1005,7 @@ class Odidere {
*/
async #startRecording() {
if (this.isRecording) return;
if (this.isTranscribing) return;
if (this.isProcessing) return;
this.#stopCurrentAudio();
@@ -854,7 +1020,6 @@ class Odidere {
this.audioChunks = [];
this.mediaRecorder.start();
this.#setRecordingState(true);
this.#acquireWakeLock();
}
/**
@@ -864,13 +1029,17 @@ class Odidere {
if (!this.isRecording) return;
if (!this.mediaRecorder) return;
// Do not clear recording state here. The async 'stop' event handler
// is responsible for clearing it (#setRecordingState(false)) after
// the audio blob has been assembled.
this.mediaRecorder.stop();
this.#setRecordingState(false);
}
/**
* #initMediaRecorder initializes the MediaRecorder with microphone access.
* Prefers opus codec for better compression if available.
* The stop handler branches on this.isTranscribing: when true it populates
* the textarea; when false (PTT flow) it sends the request.
* @returns {Promise<boolean>} true if successful
*/
async #initMediaRecorder() {
@@ -891,22 +1060,87 @@ class Odidere {
this.mediaRecorder.addEventListener('stop', async () => {
if (this.audioChunks.length === 0) return;
// Capture the mode synchronously before any async work.
// The 'stop' event fires asynchronously, so this.isTranscribing
// may have changed by the time we await below.
const wasTranscribing = this.isTranscribing;
const audioBlob = new Blob(this.audioChunks, {
type: this.mediaRecorder.mimeType,
});
this.audioChunks = [];
// Capture and clear text input to send with audio.
const text = this.$textInput.value.trim();
this.$textInput.value = '';
this.$textInput.style.height = 'auto';
// Clear the flag now that the stop event has fired.
if (wasTranscribing) {
this.isTranscribing = false;
this.$transcribe.classList.remove('compose__action-btn--recording');
} else {
this.#setRecordingState(false);
}
await this.#sendRequest({ audio: audioBlob, text });
// Transcribe the audio.
try {
const { text: transcription, detectedLanguage } =
await this.#transcribeAudio(audioBlob);
// Resolve the effective voice for this message.
// If the selector is "Auto", pick a default for the detected language.
// This does NOT change the dropdown - the voice is per-message only.
let voice = this.$voice.value;
if (!voice && detectedLanguage) {
voice = VOICE_MAP[detectedLanguage] || '';
}
// Branch: transcribe-only (populate textarea) vs PTT (send).
if (wasTranscribing) {
// Transcribe mode: populate textarea, do not send.
this.$transcribe.classList.add('compose__action-btn--transcribing');
try {
if (transcription) {
const existing = this.$textInput.value;
if (existing.trim()) {
this.$textInput.value = `${existing}\n${transcription}`;
} else {
this.$textInput.value = transcription;
}
this.#handleTextareaInput();
}
} finally {
this.$transcribe.classList.remove(
'compose__action-btn--transcribing',
);
this.$textInput.focus();
}
} else {
// PTT mode: capture text, clear input, send request.
const text = this.$textInput.value.trim();
this.$textInput.value = '';
this.$textInput.style.height = 'auto';
const combined = text ? `${text}\n${transcription}` : transcription;
await this.#sendRequest({
text: combined,
detectedLanguage,
voice,
});
}
} catch (e) {
console.error('transcription failed:', e);
this.#renderError(`Transcription failed: ${e.message}`);
if (!wasTranscribing) {
this.#setLoadingState(false);
}
}
});
this.mediaRecorder.addEventListener('error', (event) => {
console.error('MediaRecorder error:', event.error);
this.#setRecordingState(false);
if (this.isTranscribing) {
this.isTranscribing = false;
this.$transcribe.classList.remove('compose__action-btn--recording');
} else {
this.#setRecordingState(false);
}
this.audioChunks = [];
this.#renderError(
`Recording error: ${event.error?.message || 'Unknown error'}`,
@@ -1181,25 +1415,24 @@ class Odidere {
*
* For text-only input, renders the user message immediately for
* responsiveness.
* For audio input, waits for the server's transcription event before
* rendering the user message.
*
* Messages are rendered incrementally as SSE events arrive:
* - user (transcription), assistant (tool calls), tool (results),
* - user, assistant (tool calls), tool (results),
* and a final assistant message with content.
*
* @param {Object} options
* @param {Blob} [options.audio] - Recorded audio blob
* @param {string} [options.text] - Text input
* @param {string} [options.detectedLanguage] - Detected language from STT
* @param {string} [options.voice] - Resolved voice for this message (from auto-selection)
*/
async #sendRequest({ audio = null, text = '' }) {
async #sendRequest({ text = '', detectedLanguage = '', voice }) {
this.#setLoadingState(true);
// Re-enable auto-scroll when the user sends a new message.
this.isAutoScrollEnabled = true;
this.currentController = new AbortController();
try {
const hasNewInput = text || this.attachments.length > 0 || audio;
const hasNewInput = text || this.attachments.length > 0;
// Build messages array from active path + current user message.
const activePath = this.getActivePath();
@@ -1221,25 +1454,24 @@ class Odidere {
const payload = {
messages,
voice: this.$voice.value,
voice: voice ?? this.$voice.value,
model: this.$model.value,
};
const systemMessage = localStorage.getItem(SYSTEM_MESSAGE_KEY);
if (systemMessage) {
payload.system_message = systemMessage;
}
if (audio) {
payload.audio = await this.#toBase64(audio);
}
// Clear attachments after building payload (before async operations).
this.#clearAttachments();
// For text-only requests with new input, add to history and render
// immediately. Skip when continuing conversation (no new user message).
if (!audio && hasNewInput) {
const appended = this.#appendHistory([userMessage]);
this.#renderMessages(appended);
if (hasNewInput) {
const meta = {};
if (detectedLanguage) meta.language = detectedLanguage;
const appended = this.#appendHistory([userMessage], meta);
this.#renderMessages(appended, meta);
}
const res = await fetch(STREAM_ENDPOINT, {
@@ -1296,37 +1528,6 @@ class Odidere {
}
const message = event.message;
// Handle user message from server (audio transcription).
if (message.role === 'user' && event.transcription) {
const displayParts = [];
if (text) displayParts.push(text);
if (event.transcription) displayParts.push(event.transcription);
const displayContent = displayParts.join('\n\n');
let historyContent;
if (typeof userMessage.content === 'string') {
historyContent = displayContent;
} else {
historyContent = [...userMessage.content];
if (event.transcription) {
historyContent.push({
type: 'text',
text: event.transcription,
});
}
}
const audioUserMessage = {
id: userMessage.id,
role: 'user',
content: historyContent,
meta: { language: event.detected_language },
};
this.#appendHistory([audioUserMessage]);
this.#renderMessages([audioUserMessage]);
continue;
}
// Stash assistant messages with tool calls; render once all
// tool results have arrived so the output is visible in the UI.
@@ -1488,6 +1689,7 @@ class Odidere {
this.$chatLoading.hidden = !loading;
// Keep the send button enabled during loading so it can act as a stop button.
this.$ptt.disabled = loading;
this.$transcribe.disabled = loading;
// Clear any pending stop state when loading ends.
if (!loading && this.isStopPending) {
this.isStopPending = false;

9
internal/service/templates/static/footer/compose.gohtml
View File

@@ -19,6 +19,15 @@
</button>
<input type="file" id="file-input" multiple hidden />
<div class="compose__actions-right">
<button
type="button"
class="compose__action-btn"
id="transcribe"
aria-label="Transcribe to text"
title="Record and transcribe (does not send)"
>
<svg class="icon"><use href="/static/icons.svg#dictate"></use></svg>
</button>
<button
type="button"
class="compose__action-btn compose__action-btn--record"

1
internal/service/templates/static/head.gohtml
View File

@@ -5,6 +5,7 @@
<meta name="viewport"
content="width=device-width, initial-scale=1.0, shrink-to-fit=no, interactive-widget=resizes-content">
<meta name="tts-url" content="{{ .TTSURL }}">
<meta name="stt-url" content="{{ .STTURL }}">
<meta name="tts-default-voice" content="{{ .TTSDefaultVoice }}">
<title>Odidere</title>
<link rel="stylesheet" href="/static/main.css">

171
internal/stt/stt.go
View File

@@ -1,171 +0,0 @@
// Package stt provides a client for whisper.cpp speech-to-text.
package stt
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"time"
"log/slog"
)
// Config holds the configuration for a STT client.
type Config struct {
// URL is the base URL of the whisper-server.
// For example, "http://localhost:8178".
URL string `yaml:"url"`
// Timeout is the maximum duration for a transcription request.
// Defaults to 30s.
Timeout string `yaml:"timeout"`
}
// Validate checks that required configuration values are present and valid.
func (cfg Config) Validate() error {
if cfg.URL == "" {
return fmt.Errorf("missing URL")
}
if cfg.Timeout != "" {
if _, err := time.ParseDuration(cfg.Timeout); err != nil {
return fmt.Errorf("invalid timeout: %w", err)
}
}
return nil
}
// Client wraps an HTTP client for whisper-server requests.
type Client struct {
client *http.Client
log *slog.Logger
url string
timeout time.Duration
}
// NewClient creates a new STT client with the provided configuration.
func NewClient(cfg Config, log *slog.Logger) (*Client, error) {
if err := cfg.Validate(); err != nil {
return nil, fmt.Errorf("invalid config: %v", err)
}
var timeout = 30 * time.Second
if cfg.Timeout != "" {
d, err := time.ParseDuration(cfg.Timeout)
if err != nil {
return nil, fmt.Errorf("parse timeout: %v", err)
}
timeout = d
}
return &Client{
client: &http.Client{Timeout: timeout},
log: log,
url: cfg.URL,
timeout: timeout,
}, nil
}
// Output represents the JSON response from whisper.cpp (verbose_json format).
type Output struct {
Task string `json:"task,omitempty"`
Language string `json:"language,omitempty"`
Duration float64 `json:"duration,omitempty"`
Text string `json:"text"`
DetectedLanguage string `json:"detected_language,omitempty"`
DetectedLanguageProbability float64 `json:"detected_language_probability,omitempty"`
Segments []Segment `json:"segments,omitempty"`
}
// Segment represents a transcription segment with timing and confidence
// information.
type Segment struct {
ID int `json:"id"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Temperature float64 `json:"temperature,omitempty"`
AvgLogProb float64 `json:"avg_logprob,omitempty"`
NoSpeechProb float64 `json:"no_speech_prob,omitempty"`
Tokens []int `json:"tokens,omitempty"`
}
// Transcribe sends audio to whisper.cpp and returns the transcription output.
func (c *Client) Transcribe(ctx context.Context, audio []byte) (*Output, error) {
ctx, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()
// Build multipart form.
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
fw, err := w.CreateFormFile("file", "audio.webm")
if err != nil {
return nil, fmt.Errorf("create form file: %w", err)
}
if _, err := fw.Write(audio); err != nil {
return nil, fmt.Errorf("write audio: %w", err)
}
// Request verbose JSON response format to get full output including
// detected_language.
if err := w.WriteField("response_format", "verbose_json"); err != nil {
return nil, fmt.Errorf("write response_format: %w", err)
}
if err := w.Close(); err != nil {
return nil, fmt.Errorf("close multipart: %w", err)
}
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
c.url+"/inference",
&buf,
)
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", w.FormDataContentType())
res, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("send request: %w", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
body, err := io.ReadAll(res.Body)
if err != nil {
c.log.ErrorContext(
ctx,
"failed to read response body",
slog.Any("error", err),
)
}
return nil, fmt.Errorf(
"whisper error %d: %s", res.StatusCode, body,
)
}
body, err := io.ReadAll(res.Body)
if err != nil {
return nil, fmt.Errorf("read response: %w", err)
}
var output Output
if err := json.Unmarshal(body, &output); err != nil {
return nil, fmt.Errorf("parse response: %w", err)
}
c.log.DebugContext(ctx, "stt response",
slog.String("text", output.Text),
slog.String("language", output.Language),
slog.String("detected_language", output.DetectedLanguage),
slog.Float64("duration", output.Duration),
)
return &output, nil
}

106
internal/stt/stt_test.go
View File

@@ -1,106 +0,0 @@
package stt
import (
"testing"
)
func TestConfigValidate(t *testing.T) {
tests := []struct {
name string
cfg Config
wantErr bool
}{
{
name: "empty config",
cfg: Config{},
wantErr: true,
},
{
name: "missing URL",
cfg: Config{
Timeout: "30s",
},
wantErr: true,
},
{
name: "invalid timeout",
cfg: Config{
URL: "http://localhost:8178",
Timeout: "not-a-duration",
},
wantErr: true,
},
{
name: "valid minimal config",
cfg: Config{
URL: "http://localhost:8178",
},
wantErr: false,
},
{
name: "valid config with timeout",
cfg: Config{
URL: "http://localhost:8178",
Timeout: "60s",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.cfg.Validate()
if (err != nil) != tt.wantErr {
t.Errorf(
"Validate() error = %v, wantErr %v",
err, tt.wantErr,
)
}
})
}
}
func TestNewClient(t *testing.T) {
tests := []struct {
name string
cfg Config
wantErr bool
}{
{
name: "invalid config",
cfg: Config{},
wantErr: true,
},
{
name: "valid config without timeout",
cfg: Config{
URL: "http://localhost:8178",
},
wantErr: false,
},
{
name: "valid config with timeout",
cfg: Config{
URL: "http://localhost:8178",
Timeout: "45s",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client, err := NewClient(tt.cfg, nil)
if (err != nil) != tt.wantErr {
t.Errorf(
"NewClient() error = %v, wantErr %v",
err, tt.wantErr,
)
return
}
if !tt.wantErr && client == nil {
t.Error("NewClient() returned nil client")
}
})
}
}