From ec9e0a80f8cfb2877a81fdfc0cbda7851bfcca42 Mon Sep 17 00:00:00 2001 From: dwrz Date: Sat, 27 Jun 2026 12:35:11 +0000 Subject: [PATCH] Add context tracking --- internal/llm/llm.go | 17 +++-- internal/service/service.go | 12 +++- internal/service/static/main.css | 19 ++++++ internal/service/static/main.js | 67 +++++++++++++++++++ .../templates/static/footer/toolbar.gohtml | 1 + 5 files changed, 110 insertions(+), 6 deletions(-) diff --git a/internal/llm/llm.go b/internal/llm/llm.go index b828942..156f97f 100644 --- a/internal/llm/llm.go +++ b/internal/llm/llm.go @@ -249,10 +249,19 @@ func (m *Message) UnmarshalJSON(data []byte) error { // Model represents an available model from the API. type Model struct { - ID string `json:"id"` - Created int64 `json:"created"` - Object string `json:"object"` - OwnedBy string `json:"owned_by"` + ID string `json:"id"` + Created int64 `json:"created"` + Object string `json:"object"` + OwnedBy string `json:"owned_by"` + Meta *ModelMeta `json:"meta,omitempty"` +} + +// ModelMeta contains optional model metadata from the provider. +// llama.cpp includes n_ctx (configured context size) and +// n_ctx_train (model's native training context). +type ModelMeta struct { + NCtx int `json:"n_ctx,omitempty"` + NCtxTrain int `json:"n_ctx_train,omitempty"` } // ModelsResponse is the response for GET /models. diff --git a/internal/service/service.go b/internal/service/service.go index 368be5c..25055be 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -768,8 +768,9 @@ const ( // Model represents a model in the /v1/models response. type Model struct { - Model string `json:"model"` - Status ModelStatus `json:"status"` + Model string `json:"model"` + Status ModelStatus `json:"status"` + ContextSize *int `json:"context_size,omitempty"` } // Models is the response format for the /v1/models endpoint. @@ -828,9 +829,13 @@ func (svc *Service) models(w http.ResponseWriter, r *http.Request) { } available := make(map[string]struct{}, len(models)) + contextSizes := make(map[string]int, len(models)) for _, m := range models { baseID, _, _ := strings.Cut(m.ID, ":") available[baseID] = struct{}{} + if m.Meta != nil && m.Meta.NCtx > 0 { + contextSizes[baseID] = m.Meta.NCtx + } } var toCheck []string @@ -847,6 +852,9 @@ func (svc *Service) models(w http.ResponseWriter, r *http.Request) { mi := Model{Model: m} if _, ok := available[base]; ok { mi.Status = ModelStatusAvailable + if cs, ok := contextSizes[base]; ok { + mi.ContextSize = &cs + } } else { mi.Status = ModelStatusNotFound } diff --git a/internal/service/static/main.css b/internal/service/static/main.css index ce43734..ef3d7c2 100644 --- a/internal/service/static/main.css +++ b/internal/service/static/main.css @@ -991,3 +991,22 @@ body { font-family: var(--font-mono); color: var(--color-text-secondary); } + +/* Context usage indicator in toolbar. */ +.footer__toolbar-context { + display: inline-flex; + align-items: center; + gap: var(--s-1); + padding: 0 var(--s-1); + font-size: 0.7rem; + font-family: var(--font-mono); + color: var(--color-text-muted); + white-space: nowrap; + line-height: 1; +} +.footer__toolbar-context--warning { + color: var(--color-yellow); +} +.footer__toolbar-context--critical { + color: var(--color-red); +} diff --git a/internal/service/static/main.js b/internal/service/static/main.js index ab4c2a7..3dc1a87 100644 --- a/internal/service/static/main.js +++ b/internal/service/static/main.js @@ -69,6 +69,10 @@ class Odidere { this.leafId = null; this.rootId = null; + // Context tracking + this.modelContextSizes = new Map(); // model -> context_size + this.totalTokens = 0; + // TTS state this.ttsURL = document.querySelector('meta[name="tts-url"]')?.content || ''; this.ttsDefaultVoice = @@ -99,6 +103,7 @@ class Odidere { this.$textInput = document.getElementById('text-input'); this.$voice = document.getElementById('voice'); this.$mute = document.getElementById('mute'); + this.$context = document.getElementById('context'); this.$transcribe = document.getElementById('transcribe'); // Settings modal @@ -167,6 +172,8 @@ class Odidere { this.#clearAttachments(); this.$textInput.value = ''; this.$textInput.style.height = 'auto'; + this.totalTokens = 0; + this.#updateContext(); this.#updateSendButtonState(); } @@ -529,6 +536,11 @@ class Odidere { if ($el) $el.remove(); } + // Token count is no longer accurate after deletion; + // it will be refreshed on the next LLM response. + this.totalTokens = 0; + this.#updateContext(); + this.#saveToStorage(); this.#updateSendButtonState(); } @@ -1627,6 +1639,12 @@ class Odidere { if (eventType === StreamEventDone) { if (message.role !== 'assistant') continue; + // Track cumulative token usage. + if (event.usage?.total_tokens) { + this.totalTokens = event.usage.total_tokens; + this.#updateContext(); + } + const meta = streamingMeta(event.usage, event.timings); let finalMessage = message; let $streaming = null; @@ -1946,6 +1964,7 @@ class Odidere { */ #populateModels(providers, defaultProvider, defaultModel) { this.$model.innerHTML = ''; + this.modelContextSizes.clear(); // Sort provider names. const sortedProviders = Object.keys(providers).sort(); @@ -1963,6 +1982,9 @@ class Odidere { $opt.value = m.model; $opt.dataset.provider = provider; $opt.dataset.model = m.model; + if (m.context_size) { + this.modelContextSizes.set(m.model, m.context_size); + } switch (m.status) { case 'available': @@ -2040,6 +2062,51 @@ class Odidere { } } + /** + * #updateContext updates the context usage indicator in the toolbar. + * Shows percentage if context_size is known, token count if only + * usage data is available, or hides the indicator otherwise. + */ + #updateContext() { + if (!this.$context) return; + + const model = this.$model?.value; + const contextSize = model ? this.modelContextSizes.get(model) : null; + + // Hide if no data. + if (this.totalTokens === 0) { + this.$context.hidden = true; + return; + } + this.$context.hidden = false; + + // Remove severity classes. + this.$context.classList.remove( + 'footer__toolbar-context--warning', + 'footer__toolbar-context--critical', + ); + + const tokenLabel = + this.totalTokens >= 1000 + ? `${(this.totalTokens / 1000).toFixed(1)}k tokens` + : `${this.totalTokens} tokens`; + + if (contextSize && contextSize > 0) { + const pct = Math.round((this.totalTokens / contextSize) * 100); + this.$context.textContent = `${pct}%`; + this.$context.title = tokenLabel; + if (pct >= 90) { + this.$context.classList.add('footer__toolbar-context--critical'); + } else if (pct >= 70) { + this.$context.classList.add('footer__toolbar-context--warning'); + } + } else { + // No context_size known; show token count. + this.$context.textContent = tokenLabel; + this.$context.title = ''; + } + } + /** * #populateVoiceSelect populates voice selector grouped by language. * Voice IDs follow the pattern: {lang}{gender}_{name} (e.g., "af_bella"). diff --git a/internal/service/templates/static/footer/toolbar.gohtml b/internal/service/templates/static/footer/toolbar.gohtml index 4d64adb..582bbc6 100644 --- a/internal/service/templates/static/footer/toolbar.gohtml +++ b/internal/service/templates/static/footer/toolbar.gohtml @@ -8,6 +8,7 @@ > +