chimerical/odidere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for multiple providers

Browse Source
This commit is contained in:
dwrz
2026-06-19 14:31:41 +00:00
parent c71300b1bf
commit f5e72b1c7a
12 changed files with 948 additions and 869 deletions
Download Patch File Download Diff File Expand all files Collapse all files

366
internal/service/service.go
View File

@@ -1,5 +1,5 @@
// Package service orchestrates the odidere voice assistant server.
// It coordinates the HTTP server, LLM client, and handles
// It coordinates the HTTP server, LLM clients, and handles
// graceful shutdown.
package service
@@ -16,6 +16,7 @@ import (
"os/signal"
"runtime/debug"
"strings"
"sync"
"syscall"
"time"
@@ -37,6 +38,8 @@ const (
logKey key = "log"
idKey key = "id"
ipKey key = "ip"
modelsTimeout = 10 * time.Second
)
//go:embed all:static/*
@@ -45,15 +48,16 @@ var static embed.FS
// Service is the main application coordinator.
// It owns the HTTP server and all processing clients.
type Service struct {
cfg *config.Config
cron *cron.Cron
jobs []*job.Job
llm *llm.Client
log *slog.Logger
mux *http.ServeMux
server *http.Server
tmpl *template.Template
tools *tool.Registry
cfg *config.Config
cron *cron.Cron
jobs []*job.Job
llms map[string]*llm.Client
log *slog.Logger
allowedModels map[string]map[string]struct{}
mux *http.ServeMux
server *http.Server
tmpl *template.Template
tools *tool.Registry
}
// New creates a Service from the provided configuration.
@@ -72,19 +76,80 @@ func New(cfg *config.Config, log *slog.Logger) (*Service, error) {
}
svc.tools = registry
// Create LLM client.
llmClient, err := llm.NewClient(cfg.LLM, registry, log)
if err != nil {
return nil, fmt.Errorf("create LLM client: %v", err)
// Create LLM clients for each provider.
svc.llms = make(map[string]*llm.Client, len(cfg.Providers))
for _, pc := range cfg.Providers {
client, err := llm.NewClient(
llm.Config{
Key: pc.Key,
SystemMessage: cfg.SystemMessage,
Timeout: pc.Timeout,
URL: pc.URL,
},
registry,
log,
)
if err != nil {
return nil, fmt.Errorf(
"create LLM client for provider %q: %w",
pc.Name, err,
)
}
svc.llms[pc.Name] = client
log.Info(
"LLM provider registered",
slog.String("provider", pc.Name),
slog.String("url", pc.URL),
)
}
// Build model allowed list map: provider name -> set of base model IDs.
svc.allowedModels = make(
map[string]map[string]struct{}, len(cfg.Providers),
)
for _, pc := range cfg.Providers {
if len(pc.Models) == 0 {
continue
}
allowed := make(map[string]struct{}, len(pc.Models))
for _, m := range pc.Models {
base, _, _ := strings.Cut(m, ":")
allowed[base] = struct{}{}
}
svc.allowedModels[pc.Name] = allowed
}
svc.llm = llmClient
// Convert job configs to jobs.
jobs := make([]*job.Job, 0, len(cfg.Jobs))
for _, jc := range cfg.Jobs {
j, err := job.New(jc, log, svc.llm)
// Resolve the provider and model for this job.
var provider, modelID string
if jc.Model.Provider != "" {
if jc.Model.Model == "" {
return nil, fmt.Errorf(
"job %q: model.provider set but model.model is empty",
jc.Name,
)
}
provider = jc.Model.Provider
modelID = jc.Model.Model
} else {
provider = svc.cfg.DefaultModel.Provider
modelID = svc.cfg.DefaultModel.Model
}
llmc := svc.llms[provider]
if llmc == nil {
return nil, fmt.Errorf(
"create job %q: no LLM client for provider %q",
jc.Name,
provider,
)
}
j, err := job.New(jc, log, llmc, modelID)
if err != nil {
return nil, fmt.Errorf("create job %q: %w", jc.Name, err)
return nil, fmt.Errorf(
"create job %q: %w", jc.Name, err,
)
}
jobs = append(jobs, j)
}
@@ -132,8 +197,9 @@ func (svc *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) {
id = uuid.NewString()
ip = func() string {
if ip := r.Header.Get("X-Forwarded-For"); ip != "" {
if idx := strings.Index(ip, ","); idx != -1 {
return strings.TrimSpace(ip[:idx])
before, _, found := strings.Cut(ip, ",")
if found {
return strings.TrimSpace(before)
}
return ip
}
@@ -196,8 +262,14 @@ func (svc *Service) Run(ctx context.Context) error {
"starting odidere",
slog.Group(
"llm",
slog.String("url", svc.cfg.LLM.URL),
slog.String("model", svc.cfg.LLM.Model),
slog.String(
"default_provider",
svc.cfg.DefaultModel.Provider,
),
slog.String(
"default_model", svc.cfg.DefaultModel.Model,
),
slog.Int("providers", len(svc.cfg.Providers)),
),
slog.Group(
"server",
@@ -237,10 +309,6 @@ func (svc *Service) Run(ctx context.Context) error {
// Register jobs with cron scheduler.
svc.cron = cron.New()
for _, j := range svc.jobs {
if !j.IsEnabled() {
svc.log.Info("job disabled", slog.String("name", j.Name()))
continue
}
entry, err := svc.cron.AddFunc(j.Schedule(), func() {
result := j.Run(ctx)
if result.Error != nil {
@@ -374,6 +442,8 @@ func (svc *Service) status(w http.ResponseWriter, r *http.Request) {
type Request struct {
// Messages is the conversation history.
Messages []openai.ChatCompletionMessage `json:"messages"`
// Provider is the LLM provider name. If empty, the default provider is used.
Provider string `json:"provider,omitempty"`
// Model is the LLM model ID. If empty, the default model is used.
Model string `json:"model,omitempty"`
// SystemMessage overrides the configured system message for this request.
@@ -387,6 +457,8 @@ type Response struct {
// Messages is the full list of messages generated during the query,
// including tool calls and tool results.
Messages []openai.ChatCompletionMessage `json:"messages,omitempty"`
// Provider is the LLM provider used for the response.
Provider string `json:"used_provider,omitempty"`
// Model is the LLM model used for the response.
Model string `json:"used_model,omitempty"`
// Voice is the voice used for TTS synthesis.
@@ -422,12 +494,53 @@ func (svc *Service) chat(w http.ResponseWriter, r *http.Request) {
slog.Any("data", req.Messages),
)
// Get LLM response.
var model = req.Model
if model == "" {
model = svc.llm.DefaultModel()
// Resolve provider and model.
provider := req.Provider
if provider == "" {
provider = svc.cfg.DefaultModel.Provider
}
msgs, err := svc.llm.Query(ctx, req.Messages, model, req.SystemMessage, 0)
llmc := svc.llms[provider]
if llmc == nil {
log.ErrorContext(
ctx,
"unknown provider",
slog.String("provider", provider),
)
http.Error(
w,
fmt.Sprintf("unknown provider: %q", provider),
http.StatusBadRequest,
)
return
}
model := req.Model
if model == "" {
model = svc.cfg.DefaultModel.Model
}
// Validate model against provider allowed list, if configured.
if allowed, ok := svc.allowedModels[provider]; ok {
m, _, _ := strings.Cut(model, ":")
if _, ok := allowed[m]; !ok {
log.ErrorContext(
ctx,
"model not in allowed list",
slog.String("provider", provider),
slog.String("model", model),
)
http.Error(
w,
fmt.Sprintf(
"model %q not allowed for provider %q",
model, provider,
),
http.StatusBadRequest,
)
return
}
}
msgs, err := llmc.Query(ctx, req.Messages, model, req.SystemMessage, 0)
if err != nil {
log.ErrorContext(
ctx,
@@ -450,12 +563,14 @@ func (svc *Service) chat(w http.ResponseWriter, r *http.Request) {
ctx,
"LLM response",
slog.String("text", final.Content),
slog.String("provider", provider),
slog.String("model", model),
)
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(Response{
Messages: msgs,
Provider: provider,
Model: model,
Voice: req.Voice,
}); err != nil {
@@ -473,6 +588,8 @@ type StreamMessage struct {
Error string `json:"error,omitempty"`
// Message is the chat completion message.
Message openai.ChatCompletionMessage `json:"message"`
// Provider is the LLM provider used for the response.
Provider string `json:"provider,omitempty"`
// Model is the LLM model used for the response.
Model string `json:"model,omitempty"`
// Voice is the voice used for TTS synthesis.
@@ -516,6 +633,52 @@ func (svc *Service) chatStream(w http.ResponseWriter, r *http.Request) {
return
}
// Resolve provider and model.
provider := req.Provider
if provider == "" {
provider = svc.cfg.DefaultModel.Provider
}
model := req.Model
if model == "" {
model = svc.cfg.DefaultModel.Model
}
llmc := svc.llms[provider]
if llmc == nil {
log.ErrorContext(
ctx,
"unknown provider",
slog.String("provider", provider),
)
http.Error(
w,
fmt.Sprintf("unknown provider: %q", provider),
http.StatusBadRequest,
)
return
}
// Validate model against provider allowed list, if configured.
if allowed, ok := svc.allowedModels[provider]; ok {
m, _, _ := strings.Cut(model, ":")
if _, ok := allowed[m]; !ok {
log.ErrorContext(
ctx,
"model not in allowed list",
slog.String("provider", provider),
slog.String("model", model),
)
http.Error(
w,
fmt.Sprintf(
"model %q not allowed for provider %q",
model, provider,
),
http.StatusBadRequest,
)
return
}
}
// Set SSE headers.
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
@@ -534,19 +697,15 @@ func (svc *Service) chatStream(w http.ResponseWriter, r *http.Request) {
flusher.Flush()
}
// Get model.
var model = req.Model
if model == "" {
model = svc.llm.DefaultModel()
}
// Start streaming LLM query.
var (
events = make(chan llm.StreamEvent)
llmErr error
)
go func() {
llmErr = svc.llm.QueryStream(ctx, req.Messages, model, req.SystemMessage, 0, events)
llmErr = llmc.QueryStream(
ctx, req.Messages, model, req.SystemMessage, 0, events,
)
}()
// Consume events and send as SSE.
@@ -578,46 +737,127 @@ func (svc *Service) chatStream(w http.ResponseWriter, r *http.Request) {
})
return
}
last.Provider = provider
last.Model = model
last.Voice = req.Voice
send(last)
}
// models returns available LLM models.
// ModelStatus represents the availability status of a model.
type ModelStatus string
const (
// ModelStatusAvailable indicates the model is available for use.
ModelStatusAvailable ModelStatus = "available"
// ModelStatusNotFound indicates the model is configured but not found
// in the provider's model list.
ModelStatusNotFound ModelStatus = "not_found"
// ModelStatusProviderError indicates the provider could not be reached
// to verify the model.
ModelStatusProviderError ModelStatus = "provider_error"
)
// Model represents a model in the /v1/models response.
type Model struct {
Model string `json:"model"`
Status ModelStatus `json:"status"`
}
// Models is the response format for the /v1/models endpoint.
type Models struct {
Providers map[string][]Model `json:"providers"`
DefaultProvider string `json:"default_provider"`
DefaultModel string `json:"default_model"`
}
// models returns available LLM models from all configured providers.
func (svc *Service) models(w http.ResponseWriter, r *http.Request) {
var (
ctx = r.Context()
log = ctx.Value(logKey).(*slog.Logger)
mu sync.Mutex
wg sync.WaitGroup
providers = make(map[string][]Model, len(svc.llms))
)
models, err := svc.llm.ListModels(ctx)
if err != nil {
log.ErrorContext(
ctx,
"failed to list models",
slog.Any("error", err),
)
http.Error(
w,
"failed to list models",
http.StatusInternalServerError,
)
return
cfg := make(map[string]config.ProviderConfig, len(svc.cfg.Providers))
for _, pc := range svc.cfg.Providers {
cfg[pc.Name] = pc
}
for name, client := range svc.llms {
wg.Add(1)
go func(name string, c *llm.Client, pc config.ProviderConfig) {
defer wg.Done()
var result = []Model{}
fetchCtx, cancel := context.WithTimeout(
ctx, modelsTimeout,
)
defer cancel()
models, err := c.ListModels(fetchCtx)
if err != nil {
log.WarnContext(
ctx,
"failed to list models for provider",
slog.String("provider", name),
slog.Any("error", err),
)
for _, m := range pc.Models {
result = append(result, Model{
Model: m,
Status: ModelStatusProviderError,
})
}
mu.Lock()
providers[name] = result
mu.Unlock()
return
}
available := make(map[string]struct{}, len(models))
for _, m := range models {
baseID, _, _ := strings.Cut(m.ID, ":")
available[baseID] = struct{}{}
}
var toCheck []string
if len(pc.Models) > 0 {
toCheck = pc.Models
} else {
toCheck = make([]string, 0, len(models))
for _, m := range models {
toCheck = append(toCheck, m.ID)
}
}
for _, m := range toCheck {
base, _, _ := strings.Cut(m, ":")
mi := Model{Model: m}
if _, ok := available[base]; ok {
mi.Status = ModelStatusAvailable
} else {
mi.Status = ModelStatusNotFound
}
result = append(result, mi)
}
mu.Lock()
providers[name] = result
mu.Unlock()
}(name, client, cfg[name])
}
wg.Wait()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(struct {
Models []openai.Model `json:"models"`
DefaultModel string `json:"default_model"`
}{
Models: models,
DefaultModel: svc.llm.DefaultModel(),
if err := json.NewEncoder(w).Encode(Models{
Providers: providers,
DefaultProvider: svc.cfg.DefaultModel.Provider,
DefaultModel: svc.cfg.DefaultModel.Model,
}); err != nil {
log.ErrorContext(
ctx,
"failed to encode models response",
slog.Any("error", err),
)
log.ErrorContext(ctx, "failed to encode models response",
slog.Any("error", err))
}
}

17
internal/service/static/main.css
View File

@@ -970,3 +970,20 @@ body {
border: none;
}
}
/* Unavailable model indicators in settings select */
.settings-select option.model-unavailable-not-found {
color: #dc2626;
font-style: italic;
}
.settings-select option.model-unavailable-error {
color: #6b7280;
font-style: italic;
}
.settings-select optgroup {
font-weight: 600;
font-family: var(--font-mono);
color: var(--color-text-secondary);
}

108
internal/service/static/main.js
View File

@@ -2,6 +2,7 @@ const STREAM_ENDPOINT = '/v1/chat/voice/stream';
const ICONS_URL = '/static/icons.svg';
const MODELS_ENDPOINT = '/v1/models';
const MODEL_KEY = 'odidere_model';
const PROVIDER_KEY = 'odidere_provider';
const STORAGE_KEY = 'odidere_history';
const SYSTEM_MESSAGE_KEY = 'odidere_system_message';
const VOICE_KEY = 'odidere_voice';
@@ -243,7 +244,9 @@ class Odidere {
// Save selections on change.
this.$model.addEventListener('change', () => {
localStorage.setItem(MODEL_KEY, this.$model.value);
const $opt = this.$model.options[this.$model.selectedIndex];
localStorage.setItem(PROVIDER_KEY, $opt.dataset.provider);
localStorage.setItem(MODEL_KEY, $opt.dataset.model);
});
this.$voice.addEventListener('change', () => {
localStorage.setItem(VOICE_KEY, this.$voice.value);
@@ -1348,13 +1351,18 @@ class Odidere {
// ====================
/**
* #fetchModels fetches available models from the API and populates selectors.
* The response now includes models with availability status and provider grouping.
*/
async #fetchModels() {
try {
const res = await fetch(MODELS_ENDPOINT);
if (!res.ok) throw new Error(`${res.status}`);
const data = await res.json();
this.#populateModels(data.models, data.default_model);
this.#populateModels(
data.providers,
data.default_provider,
data.default_model,
);
} catch (e) {
console.error('failed to fetch models:', e);
this.#populateModelsFallback();
@@ -1452,10 +1460,12 @@ class Odidere {
);
}
const $opt = this.$model.options[this.$model.selectedIndex];
const payload = {
messages,
voice: voice ?? this.$voice.value,
model: this.$model.value,
provider: $opt?.dataset.provider,
model: $opt?.dataset.model,
};
const systemMessage = localStorage.getItem(SYSTEM_MESSAGE_KEY);
if (systemMessage) {
@@ -1795,21 +1805,58 @@ class Odidere {
// RENDER: SELECTS
// ====================
/**
* #populateModels populates the model selector with available options.
* @param {Array<{id: string}>} models
* #populateModels populates the model selector grouped by provider.
* Models are displayed with availability indicators:
* - Available: normal selectable option
* - Not found: red strikethrough, disabled
* - Provider error: grayed out, disabled
* @param {Object<string, Array<{model: string, status: string}>>} providers
* @param {string} defaultProvider
* @param {string} defaultModel
*/
#populateModels(models, defaultModel) {
#populateModels(providers, defaultProvider, defaultModel) {
this.$model.innerHTML = '';
for (const m of models) {
const $opt = this.document.createElement('option');
$opt.value = m.id;
$opt.textContent = m.id;
this.$model.appendChild($opt);
// Sort provider names.
const sortedProviders = Object.keys(providers).sort();
// Create optgroups for each provider.
for (const provider of sortedProviders) {
const $optgroup = this.document.createElement('optgroup');
$optgroup.label = provider;
// Sort models within provider.
for (const m of providers[provider].sort((a, b) =>
a.model.localeCompare(b.model),
)) {
const $opt = this.document.createElement('option');
$opt.value = m.model;
$opt.dataset.provider = provider;
$opt.dataset.model = m.model;
switch (m.status) {
case 'available':
$opt.textContent = m.model;
break;
case 'not_found':
$opt.textContent = `~~${m.model}~~ (not found)`;
$opt.disabled = true;
$opt.classList.add('model-unavailable-not-found');
break;
case 'provider_error':
$opt.textContent = `${m.model} (provider error)`;
$opt.disabled = true;
$opt.classList.add('model-unavailable-error');
break;
}
$optgroup.appendChild($opt);
}
this.$model.appendChild($optgroup);
}
this.#loadModel(defaultModel);
this.#loadModel(defaultProvider, defaultModel);
}
/**
@@ -1829,24 +1876,37 @@ class Odidere {
/**
* #loadModel restores the model selection from localStorage or uses the
* default.
* @param {string} defaultProvider
* @param {string} defaultModel
*/
#loadModel(defaultModel) {
const stored = localStorage.getItem(MODEL_KEY);
const escaped = stored ? CSS.escape(stored) : null;
#loadModel(defaultProvider, defaultModel) {
const storedProvider = localStorage.getItem(PROVIDER_KEY);
const storedModel = localStorage.getItem(MODEL_KEY);
// Try stored value first, then default, then first available option.
let selectedValue;
if (escaped && this.$model.querySelector(`option[value="${escaped}"]`)) {
selectedValue = stored;
} else if (defaultModel) {
selectedValue = defaultModel;
} else if (this.$model.options.length > 0) {
selectedValue = this.$model.options[0].value;
let $opt;
if (storedProvider && storedModel) {
$opt = this.$model.querySelector(
`option[data-provider="${CSS.escape(storedProvider)}"][data-model="${CSS.escape(storedModel)}"]`,
);
}
if (!$opt && defaultProvider && defaultModel) {
$opt = this.$model.querySelector(
`option[data-provider="${CSS.escape(defaultProvider)}"][data-model="${CSS.escape(defaultModel)}"]`,
);
}
if (!$opt && this.$model.options.length > 0) {
// Find first available (non-disabled) option.
for (const opt of this.$model.options) {
if (!opt.disabled) {
$opt = opt;
break;
}
}
}
if (selectedValue) {
this.$model.value = selectedValue;
if ($opt) {
this.$model.value = $opt.value;
}
}