From f2edd39160afb7e9d9a28bb0c14eb66c594c5a3a Mon Sep 17 00:00:00 2001 From: dwrz Date: Sat, 27 Jun 2026 00:13:16 +0000 Subject: [PATCH] Add debug stats --- internal/job/job.go | 8 +- internal/llm/client_completions.go | 70 ++++++- internal/llm/client_completions_test.go | 255 ++++++++++++++++++++++++ internal/llm/llm.go | 32 +++ internal/llm/llm_integration_test.go | 12 +- internal/service/service.go | 24 ++- internal/service/service_test.go | 85 ++++++++ internal/service/static/main.js | 95 ++++++++- 8 files changed, 553 insertions(+), 28 deletions(-) create mode 100644 internal/llm/client_completions_test.go diff --git a/internal/job/job.go b/internal/job/job.go index 33da9d5..a2c0650 100644 --- a/internal/job/job.go +++ b/internal/job/job.go @@ -156,7 +156,7 @@ func (j *Job) Run(ctx context.Context) Result { defer cancel() } - msgs, err := j.llm.Completions( + res, err := j.llm.Completions( ctx, j.model, j.SystemMessage(), @@ -178,7 +178,7 @@ func (j *Job) Run(ctx context.Context) Result { Error: fmt.Errorf("job %q: %w", j.Name(), err), } } - if len(msgs) == 0 { + if len(res.Messages) == 0 { log.ErrorContext( ctx, "job returned no messages", @@ -197,11 +197,11 @@ func (j *Job) Run(ctx context.Context) Result { ctx, "job completed", slog.Duration("duration", duration), - slog.Int("messages", len(msgs)), + slog.Int("messages", len(res.Messages)), ) return Result{ Name: j.Name(), Duration: duration, - Messages: msgs, + Messages: res.Messages, } } diff --git a/internal/llm/client_completions.go b/internal/llm/client_completions.go index 8e3e3af..74f6c00 100644 --- a/internal/llm/client_completions.go +++ b/internal/llm/client_completions.go @@ -17,19 +17,27 @@ import ( // maximum number of iterations. var ErrMaxIterations = fmt.Errorf("max iterations exceeded") +// CompletionsResult bundles the messages generated during a completions +// request along with usage and timings from the final API response. +type CompletionsResult struct { + Messages []Message + Usage *Usage + Timings *Timings +} + // Completions sends messages to the LLM using the specified model. // If systemMessage is non-empty, it overrides the configured system message. // maxIterations caps the number of agent loop iterations. // A value of 0 means unlimited (no cap). // Returns all messages generated during the completions request, including -// tool calls and tool results. +// tool calls and tool results, along with usage and timings from the final response. func (c *Client) Completions( ctx context.Context, model string, systemMessage string, maxIterations int, messages []Message, -) ([]Message, error) { +) (*CompletionsResult, error) { ctx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() @@ -57,16 +65,29 @@ func (c *Client) Completions( // generated during this call (the suffix of the conversation history). start := len(messages) // Loop for tool calls. maxIterations == 0 means unlimited. + var finalUsage *Usage + var finalTimings *Timings for i := 0; maxIterations == 0 || i < maxIterations; i++ { cr, err := c.completions(ctx, model, messages) if err != nil { return nil, fmt.Errorf("completions: %w", err) } + // Capture usage and timings from this iteration. + if cr.Usage != nil { + finalUsage = cr.Usage + } + if cr.Timings != nil { + finalTimings = cr.Timings + } // Use first choice for tool call loop. message := &cr.Choices[0].Message messages = append(messages, *message) if len(message.ToolCalls) == 0 { - return messages[start:], nil + return &CompletionsResult{ + Messages: messages[start:], + Usage: finalUsage, + Timings: finalTimings, + }, nil } toolResults, err := c.callTools(ctx, *message) if err != nil { @@ -75,7 +96,11 @@ func (c *Client) Completions( messages = append(messages, toolResults...) } - return messages[start:], ErrMaxIterations + return &CompletionsResult{ + Messages: messages[start:], + Usage: finalUsage, + Timings: finalTimings, + }, ErrMaxIterations } // completions sends a single non-streaming request and returns the @@ -185,6 +210,10 @@ type StreamEvent struct { ReasoningDelta string // Message is the complete chat completion message. Message Message + // Usage is token usage info from the final chunk, sent on the done event. + Usage *Usage + // Timings is llama.cpp timing info from the final chunk, sent on the done event. + Timings *Timings } const ( @@ -273,13 +302,15 @@ type streamAccum struct { // completionsStream sends a single streaming request, accumulates the // response, and returns all assistant messages from the response choices. -// Sends each message to the events channel. +// Sends each message to the events channel. Usage and timings are included +// on the done event for the last message. func (c *Client) completionsStream(ctx context.Context, messages []Message, model string, events chan<- StreamEvent) ([]Message, error) { body, err := json.Marshal(ChatRequest{ - Model: model, - Messages: messages, - Stream: true, - Tools: c.tools, + Model: model, + Messages: messages, + Stream: true, + Tools: c.tools, + StreamOptions: &StreamOptions{IncludeUsage: true}, }) if err != nil { return nil, fmt.Errorf( @@ -361,6 +392,8 @@ func (c *Client) completionsStream(ctx context.Context, messages []Message, mode // Accumulate the streamed response per choice index. accums := make(map[int]*streamAccum) + var usage *Usage + var timings *Timings scanner := bufio.NewScanner(res.Body) for scanner.Scan() { @@ -437,6 +470,13 @@ func (c *Client) completionsStream(ctx context.Context, messages []Message, mode } } } + // Capture usage from the final chunk (has null choices). + if chunk.Usage != nil { + usage = chunk.Usage + } + if chunk.Timings != nil { + timings = chunk.Timings + } } res.Body.Close() @@ -454,7 +494,7 @@ func (c *Client) completionsStream(ctx context.Context, messages []Message, mode slices.Sort(indices) msgs := make([]Message, 0, len(accums)) - for _, idx := range indices { + for i, idx := range indices { acc := accums[idx] // Content and refusal are mutually exclusive in the API. @@ -478,9 +518,19 @@ func (c *Client) completionsStream(ctx context.Context, messages []Message, mode ReasoningContent: acc.reasoning.String(), ToolCalls: acc.toolCalls, } + + // Attach usage and timings to the done event of the last message. + var evUsage *Usage + var evTimings *Timings + if i == len(indices)-1 { + evUsage = usage + evTimings = timings + } events <- StreamEvent{ Type: streamEventDone, Message: message, + Usage: evUsage, + Timings: evTimings, } msgs = append(msgs, message) } diff --git a/internal/llm/client_completions_test.go b/internal/llm/client_completions_test.go new file mode 100644 index 0000000..9989af2 --- /dev/null +++ b/internal/llm/client_completions_test.go @@ -0,0 +1,255 @@ +package llm + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +func TestCompletionsStreamStreamsReasoningDeltas(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/completions" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "text/event-stream") + for _, data := range []string{ + `{"choices":[{"index":0,"delta":{"role":"assistant"}}]}`, + `{"choices":[{"index":0,"delta":{"reasoning_content":"thinking"}}]}`, + `{"choices":[{"index":0,"delta":{"reasoning_content":" more"}}]}`, + `{"choices":[{"index":0,"delta":{"content":"answer"}}]}`, + } { + fmt.Fprintf(w, "data: %s\n\n", data) + } + fmt.Fprint(w, "data: [DONE]\n\n") + })) + defer srv.Close() + + client, err := NewClient(Config{URL: srv.URL}, nil, nil) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + + events := make(chan StreamEvent) + errCh := make(chan error, 1) + go func() { + errCh <- client.CompletionsStream( + context.Background(), + "test-model", + "", + 0, + []Message{{ + Role: RoleUser, + ContentParts: []ContentPart{{ + Type: ContentTypeText, + Text: "hello", + }}, + }}, + events, + ) + }() + + var got []StreamEvent + for evt := range events { + got = append(got, evt) + } + if err := <-errCh; err != nil { + t.Fatalf("CompletionsStream: %v", err) + } + + wantTypes := []string{ + streamEventReasoningDelta, + streamEventReasoningDelta, + streamEventDelta, + streamEventDone, + } + if len(got) != len(wantTypes) { + t.Fatalf("got %d events, want %d: %#v", len(got), len(wantTypes), got) + } + for i, want := range wantTypes { + if got[i].Type != want { + t.Fatalf("event %d type = %q, want %q", i, got[i].Type, want) + } + } + if got[0].ReasoningDelta != "thinking" { + t.Errorf("first reasoning delta = %q", got[0].ReasoningDelta) + } + if got[1].ReasoningDelta != " more" { + t.Errorf("second reasoning delta = %q", got[1].ReasoningDelta) + } + if got[2].Delta != "answer" { + t.Errorf("content delta = %q", got[2].Delta) + } + final := got[3].Message + if final.ReasoningContent != "thinking more" { + t.Errorf("final reasoning = %q", final.ReasoningContent) + } + if final.Text() != "answer" { + t.Errorf("final text = %q", final.Text()) + } +} + +func TestCompletionsReturnsUsageAndTimings(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/completions" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{ + "id": "test-123", + "model": "test-model", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "hello"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "prompt_tokens_details": {"cached_tokens": 3}, + "completion_tokens_details": {"reasoning_tokens": 2} + }, + "timings": { + "prompt_n": 10, + "prompt_ms": 50.5, + "prompt_per_token_ms": 5.05, + "prompt_per_second": 198.0, + "predicted_n": 5, + "predicted_ms": 100.0, + "predicted_per_token_ms": 20.0, + "predicted_per_second": 50.0 + } + }`) + })) + defer srv.Close() + + client, err := NewClient(Config{URL: srv.URL}, nil, nil) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + + res, err := client.Completions( + context.Background(), + "test-model", + "", + 0, + []Message{{ + Role: RoleUser, + ContentParts: []ContentPart{{ + Type: ContentTypeText, + Text: "hi", + }}, + }}, + ) + if err != nil { + t.Fatalf("Completions: %v", err) + } + if len(res.Messages) != 1 { + t.Fatalf("got %d messages, want 1", len(res.Messages)) + } + if res.Usage == nil { + t.Fatal("usage is nil") + } + if res.Usage.PromptTokens != 10 { + t.Errorf("prompt_tokens = %d, want 10", res.Usage.PromptTokens) + } + if res.Usage.CompletionTokens != 5 { + t.Errorf("completion_tokens = %d, want 5", res.Usage.CompletionTokens) + } + if res.Usage.TotalTokens != 15 { + t.Errorf("total_tokens = %d, want 15", res.Usage.TotalTokens) + } + if res.Usage.PromptTokensDetails == nil || res.Usage.PromptTokensDetails.CachedTokens != 3 { + t.Errorf("cached_tokens unexpected: %#v", res.Usage.PromptTokensDetails) + } + if res.Usage.CompletionTokensDetails == nil || res.Usage.CompletionTokensDetails.ReasoningTokens != 2 { + t.Errorf("reasoning_tokens unexpected: %#v", res.Usage.CompletionTokensDetails) + } + if res.Timings == nil { + t.Fatal("timings is nil") + } + if res.Timings.PromptMS != 50.5 { + t.Errorf("prompt_ms = %f, want 50.5", res.Timings.PromptMS) + } + if res.Timings.PredictedMS != 100.0 { + t.Errorf("predicted_ms = %f, want 100.0", res.Timings.PredictedMS) + } +} + +func TestCompletionsStreamEmitsUsageEvent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/completions" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "text/event-stream") + // Stream some content + fmt.Fprintf(w, "data: %s\n\n", + `{"choices":[{"index":0,"delta":{"role":"assistant"}}]}`) + fmt.Fprintf(w, "data: %s\n\n", + `{"choices":[{"index":0,"delta":{"content":"hello"}}]}`) + // Final chunk with usage + fmt.Fprintf(w, "data: %s\n\n", + `{"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}`) + fmt.Fprint(w, "data: [DONE]\n\n") + })) + defer srv.Close() + + client, err := NewClient(Config{URL: srv.URL}, nil, nil) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + + events := make(chan StreamEvent) + errCh := make(chan error, 1) + go func() { + errCh <- client.CompletionsStream( + context.Background(), + "test-model", + "", + 0, + []Message{{ + Role: RoleUser, + ContentParts: []ContentPart{{ + Type: ContentTypeText, + Text: "hi", + }}, + }}, + events, + ) + }() + + var got []StreamEvent + for evt := range events { + got = append(got, evt) + } + if err := <-errCh; err != nil { + t.Fatalf("CompletionsStream: %v", err) + } + + // Expect: delta, done events (usage attached to done event) + wantTypes := []string{ + streamEventDelta, + streamEventDone, + } + if len(got) != len(wantTypes) { + t.Fatalf("got %d events, want %d: %#v", len(got), len(wantTypes), got) + } + for i, want := range wantTypes { + if got[i].Type != want { + t.Fatalf("event %d type = %q, want %q", i, got[i].Type, want) + } + } + // Verify usage is on the done event + doneEvt := got[1] + if doneEvt.Usage == nil { + t.Fatal("done event has nil Usage") + } + if doneEvt.Usage.PromptTokens != 10 { + t.Errorf("prompt_tokens = %d, want 10", doneEvt.Usage.PromptTokens) + } + if doneEvt.Usage.TotalTokens != 15 { + t.Errorf("total_tokens = %d, want 15", doneEvt.Usage.TotalTokens) + } +} diff --git a/internal/llm/llm.go b/internal/llm/llm.go index 6903b6d..b828942 100644 --- a/internal/llm/llm.go +++ b/internal/llm/llm.go @@ -59,11 +59,41 @@ type ChatRequest struct { ServiceTier string `json:"service_tier,omitempty"` } +// Usage represents token usage from the API response. +type Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"` + CompletionTokensDetails *TokenDetails `json:"completion_tokens_details,omitempty"` +} + +// TokenDetails breaks down token usage by category. +type TokenDetails struct { + ReasoningTokens int `json:"reasoning_tokens,omitempty"` + CachedTokens int `json:"cached_tokens,omitempty"` +} + +// Timings is a llama.cpp-specific extension to the chat completion +// response. It is not part of the OpenAI API spec. +type Timings struct { + PromptN int `json:"prompt_n"` + PromptMS float64 `json:"prompt_ms"` + PromptPerTokenMS float64 `json:"prompt_per_token_ms"` + PromptPerSecond float64 `json:"prompt_per_second"` + PredictedN int `json:"predicted_n"` + PredictedMS float64 `json:"predicted_ms"` + PredictedPerTokenMS float64 `json:"predicted_per_token_ms"` + PredictedPerSecond float64 `json:"predicted_per_second"` +} + // ChatResponse is the non-streaming response from the API. type ChatResponse struct { ID string `json:"id"` Model string `json:"model"` Choices []Choice `json:"choices"` + Usage *Usage `json:"usage,omitempty"` + Timings *Timings `json:"timings,omitempty"` } // Choice is a single choice in a ChatResponse. @@ -246,6 +276,8 @@ type StreamChunk struct { Delta Delta `json:"delta"` FinishReason string `json:"finish_reason,omitempty"` } `json:"choices"` + Usage *Usage `json:"usage,omitempty"` + Timings *Timings `json:"timings,omitempty"` } // StreamOptions controls streaming behavior. diff --git a/internal/llm/llm_integration_test.go b/internal/llm/llm_integration_test.go index 05a0014..5574ff2 100644 --- a/internal/llm/llm_integration_test.go +++ b/internal/llm/llm_integration_test.go @@ -76,7 +76,7 @@ func TestIntegration_Completions(t *testing.T) { t.Fatalf("NewClient: %v", err) } - msgs, err := client.Completions( + res, err := client.Completions( context.Background(), testModel(t), "", @@ -92,10 +92,10 @@ func TestIntegration_Completions(t *testing.T) { if err != nil { t.Fatalf("Completions: %v", err) } - if len(msgs) == 0 { + if len(res.Messages) == 0 { t.Fatal("expected at least one message") } - text := msgs[len(msgs)-1].Text() + text := res.Messages[len(res.Messages)-1].Text() if text == "" { t.Error("final message has no text") } @@ -181,7 +181,7 @@ func TestIntegration_ToolCalling(t *testing.T) { t.Fatalf("NewClient: %v", err) } - msgs, err := client.Completions( + res, err := client.Completions( context.Background(), testModel(t), "", @@ -197,11 +197,11 @@ func TestIntegration_ToolCalling(t *testing.T) { if err != nil { t.Fatalf("Completions: %v", err) } - if len(msgs) == 0 { + if len(res.Messages) == 0 { t.Fatal("expected at least one message") } // The final message should be a non-tool-call assistant message. - final := msgs[len(msgs)-1] + final := res.Messages[len(res.Messages)-1] if len(final.ToolCalls) > 0 { t.Error("final message still has tool calls") } diff --git a/internal/service/service.go b/internal/service/service.go index 80086b0..368be5c 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -457,6 +457,10 @@ type Response struct { Provider string `json:"used_provider,omitempty"` // Model is the LLM model used for the response. Model string `json:"used_model,omitempty"` + // Usage is token usage from the API response. + Usage *llm.Usage `json:"usage,omitempty"` + // Timings is llama.cpp timing info from the API response. + Timings *llm.Timings `json:"timings,omitempty"` } // chat processes text chat requests. @@ -536,7 +540,7 @@ func (svc *Service) chat(w http.ResponseWriter, r *http.Request) { } } - msgs, err := llmc.Completions( + res, err := llmc.Completions( ctx, model, req.SystemMessage, 0, req.Messages, ) if err != nil { @@ -548,7 +552,7 @@ func (svc *Service) chat(w http.ResponseWriter, r *http.Request) { http.Error(w, "LLM error", http.StatusInternalServerError) return } - if len(msgs) == 0 { + if len(res.Messages) == 0 { http.Error( w, "no response from LLM", @@ -556,7 +560,7 @@ func (svc *Service) chat(w http.ResponseWriter, r *http.Request) { ) return } - final := msgs[len(msgs)-1] + final := res.Messages[len(res.Messages)-1] log.DebugContext( ctx, "LLM response", @@ -567,9 +571,11 @@ func (svc *Service) chat(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(Response{ - Messages: msgs, + Messages: res.Messages, Provider: provider, Model: model, + Usage: res.Usage, + Timings: res.Timings, }); err != nil { log.ErrorContext( ctx, @@ -593,6 +599,10 @@ type StreamMessage struct { Provider string `json:"provider,omitempty"` // Model is the LLM model used for the response. Model string `json:"model,omitempty"` + // Usage is token usage from the API response. + Usage *llm.Usage `json:"usage,omitempty"` + // Timings is llama.cpp timing info from the API response. + Timings *llm.Timings `json:"timings,omitempty"` } // chatStream processes chat requests with streaming SSE output. @@ -719,6 +729,12 @@ func (svc *Service) chatStream(w http.ResponseWriter, r *http.Request) { if evt.Message.Role != "" { msg.Message = &evt.Message } + if evt.Usage != nil { + msg.Usage = evt.Usage + } + if evt.Timings != nil { + msg.Timings = evt.Timings + } send(evt.Type, msg) } if err := <-errs; err != nil { diff --git a/internal/service/service_test.go b/internal/service/service_test.go index e77f653..8f86492 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -1,9 +1,13 @@ package service import ( + "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" + + "code.chimeric.al/chimerical/odidere/internal/llm" ) func TestStatusHandler(t *testing.T) { @@ -22,3 +26,84 @@ func TestStatusHandler(t *testing.T) { ) } } + +func TestResponseIncludesStatsInJSON(t *testing.T) { + resp := Response{ + Messages: []llm.Message{ + {Role: llm.RoleAssistant, ContentParts: []llm.ContentPart{{ + Type: llm.ContentTypeText, + Text: "hello", + }}}, + }, + Provider: "test-provider", + Model: "test-model", + Usage: &llm.Usage{ + PromptTokens: 10, + CompletionTokens: 5, + TotalTokens: 15, + PromptTokensDetails: &llm.TokenDetails{ + CachedTokens: 3, + }, + CompletionTokensDetails: &llm.TokenDetails{ + ReasoningTokens: 2, + }, + }, + Timings: &llm.Timings{ + PromptMS: 50.5, + PromptPerSecond: 198.0, + PredictedMS: 100.0, + PredictedPerSecond: 50.0, + }, + } + + // Verify JSON encoding includes stats fields + var buf strings.Builder + enc := json.NewEncoder(&buf) + if err := enc.Encode(resp); err != nil { + t.Fatalf("encode Response: %v", err) + } + + jsonStr := buf.String() + if !strings.Contains(jsonStr, `"prompt_tokens":10`) { + t.Errorf("JSON missing prompt_tokens: %s", jsonStr) + } + if !strings.Contains(jsonStr, `"completion_tokens":5`) { + t.Errorf("JSON missing completion_tokens: %s", jsonStr) + } + if !strings.Contains(jsonStr, `"total_tokens":15`) { + t.Errorf("JSON missing total_tokens: %s", jsonStr) + } + if !strings.Contains(jsonStr, `"prompt_ms":50.5`) { + t.Errorf("JSON missing prompt_ms: %s", jsonStr) + } + if !strings.Contains(jsonStr, `"predicted_ms":100`) { + t.Errorf("JSON missing predicted_ms: %s", jsonStr) + } +} + +func TestStreamMessageIncludesUsageInJSON(t *testing.T) { + msg := StreamMessage{ + Delta: "hello", + Provider: "test-provider", + Model: "test-model", + Usage: &llm.Usage{ + PromptTokens: 10, + CompletionTokens: 5, + TotalTokens: 15, + }, + } + + var buf strings.Builder + enc := json.NewEncoder(&buf) + if err := enc.Encode(msg); err != nil { + t.Fatalf("encode StreamMessage: %v", err) + } + + jsonStr := buf.String() + if !strings.Contains(jsonStr, `"prompt_tokens":10`) { + t.Errorf("JSON missing prompt_tokens: %s", jsonStr) + } + if !strings.Contains(jsonStr, `"total_tokens":15`) { + t.Errorf("JSON missing total_tokens: %s", jsonStr) + } +} diff --git a/internal/service/static/main.js b/internal/service/static/main.js index f801a44..ab4c2a7 100644 --- a/internal/service/static/main.js +++ b/internal/service/static/main.js @@ -1519,10 +1519,11 @@ class Odidere { // Stash assistant messages with tool_calls until their results arrive. let pendingTools = null; let streamingMessage = null; - - const streamingMeta = () => { + const streamingMeta = (usage, timings) => { const meta = {}; if (voice) meta.voice = voice; + if (usage) meta.usage = usage; + if (timings) meta.timings = timings; return meta; }; @@ -1593,7 +1594,7 @@ class Odidere { role: 'assistant', content: [{ type: ContentTypeText, text: '' }], }; - const meta = streamingMeta(); + const meta = streamingMeta(event.usage, event.timings); const appended = this.#appendHistory([message], meta); const $el = this.#renderStreamingAssistantMessage( appended[0], @@ -1626,7 +1627,7 @@ class Odidere { if (eventType === StreamEventDone) { if (message.role !== 'assistant') continue; - const meta = streamingMeta(); + const meta = streamingMeta(event.usage, event.timings); let finalMessage = message; let $streaming = null; if (streamingMessage) { @@ -2351,6 +2352,49 @@ class Odidere { // Populate debug panel. const $dl = $msg.querySelector('.message__debug-list'); if (meta?.voice) this.#appendDebugRow($dl, 'Voice', meta.voice); + if (meta?.usage) { + const u = meta.usage; + this.#appendDebugRow($dl, 'Prompt tokens', String(u.prompt_tokens)); + this.#appendDebugRow( + $dl, + 'Completion tokens', + String(u.completion_tokens), + ); + this.#appendDebugRow($dl, 'Total tokens', String(u.total_tokens)); + if (u.completion_tokens_details?.reasoning_tokens > 0) + this.#appendDebugRow( + $dl, + 'Reasoning tokens', + String(u.completion_tokens_details.reasoning_tokens), + ); + if (u.prompt_tokens_details?.cached_tokens > 0) + this.#appendDebugRow( + $dl, + 'Cached tokens', + String(u.prompt_tokens_details.cached_tokens), + ); + } + if (meta?.timings) { + const t = meta.timings; + if (t.prompt_ms > 0) + this.#appendDebugRow( + $dl, + 'Prompt eval', + t.prompt_ms.toFixed(1) + + ' ms (' + + t.prompt_per_second.toFixed(0) + + ' tok/s)', + ); + if (t.predicted_ms > 0) + this.#appendDebugRow( + $dl, + 'Predicted eval', + t.predicted_ms.toFixed(1) + + ' ms (' + + t.predicted_per_second.toFixed(0) + + ' tok/s)', + ); + } if ($dl.children.length === 0) this.#appendDebugRow($dl, 'No data', ''); // Bind action buttons. @@ -2450,6 +2494,49 @@ class Odidere { // Populate debug panel. const $dl = $msg.querySelector('.message__debug-list'); if (meta?.voice) this.#appendDebugRow($dl, 'Voice', meta.voice); + if (meta?.usage) { + const u = meta.usage; + this.#appendDebugRow($dl, 'Prompt tokens', String(u.prompt_tokens)); + this.#appendDebugRow( + $dl, + 'Completion tokens', + String(u.completion_tokens), + ); + this.#appendDebugRow($dl, 'Total tokens', String(u.total_tokens)); + if (u.completion_tokens_details?.reasoning_tokens > 0) + this.#appendDebugRow( + $dl, + 'Reasoning tokens', + String(u.completion_tokens_details.reasoning_tokens), + ); + if (u.prompt_tokens_details?.cached_tokens > 0) + this.#appendDebugRow( + $dl, + 'Cached tokens', + String(u.prompt_tokens_details.cached_tokens), + ); + } + if (meta?.timings) { + const t = meta.timings; + if (t.prompt_ms > 0) + this.#appendDebugRow( + $dl, + 'Prompt eval', + t.prompt_ms.toFixed(1) + + ' ms (' + + t.prompt_per_second.toFixed(0) + + ' tok/s)', + ); + if (t.predicted_ms > 0) + this.#appendDebugRow( + $dl, + 'Predicted eval', + t.predicted_ms.toFixed(1) + + ' ms (' + + t.predicted_per_second.toFixed(0) + + ' tok/s)', + ); + } if ($dl.children.length === 0) this.#appendDebugRow($dl, 'No data', ''); // Bind action buttons.