Refactor llm pkg

internal/llm/client_completions.go

@@ -28,14 +28,11 @@ type CompletionsParams struct {
 
 // CompletionResult contains one generated Message and its response metadata.
 type CompletionResult struct {
-	// FinishReason is the endpoint's reason for ending generation.
-	FinishReason string
 	// Message is the generated message.
 	Message Message
-	// Timings contains llama.cpp performance data, when reported.
-	Timings *Timings
-	// Usage contains token counts, when reported.
-	Usage *Usage
+	// Metadata describes this inference's selected choice and endpoint,
+	// separately from the model-visible message.
+	Metadata Metadata
 }
 
 // Completions requests a non-streaming chat completion and returns the first
@@ -53,9 +50,20 @@ func (c *Client) Completions(
 	choice := cr.Choices[0]
 	return &CompletionResult{
 		Message: choice.Message,
+		Metadata: Metadata{
+			ChoiceIndex:       choice.Index,
+			Created:           cr.Created,
 			FinishReason:      choice.FinishReason,
-		Usage:        cr.Usage,
+			ID:                cr.ID,
+			Logprobs:          choice.Logprobs,
+			Model:             cr.Model,
+			Provider:          new(c.provider),
+			RequestedModel:    &p.Model,
+			ServiceTier:       cr.ServiceTier,
+			SystemFingerprint: cr.SystemFingerprint,
 			Timings:           cr.Timings,
+			Usage:             cr.Usage,
+		},
 	}, nil
 }
 
@@ -242,9 +250,11 @@ type CompletionStream struct {
 
 	acc     StreamAccumulator
 	current StreamChunk
-	finishReason  string
-	usage         *Usage
-	timings       *Timings
+	// metadata retains reported scalars and the latest usage/timing
+	// snapshots.
+	// Logprob sequences contain every reported token in arrival order.
+	// Nested metadata values are shared with Current and Result.
+	metadata      Metadata
 	err           error
 	cleanupErr    error
 	terminated    bool
@@ -356,6 +366,7 @@ func (c *Client) CompletionsStream(
 		reader:   bufio.NewReader(res.Body),
 		body:     res.Body,
 		cancel:   cancel,
+		metadata: Metadata{Provider: new(c.provider), RequestedModel: &p.Model},
 	}, nil
 }
 
@@ -412,45 +423,102 @@ func (s *CompletionStream) Next() bool {
 		return false
 	}
 	for _, choice := range chunk.Choices {
-		if choice.Index != 0 {
+		if choice.Index != nil && *choice.Index != 0 {
 			s.fail(fmt.Errorf(
 				"completions stream: unexpected choice index %d",
-				choice.Index,
+				*choice.Index,
 			))
 			return false
 		}
-		if choice.FinishReason != "" {
-			s.finishReason = choice.FinishReason
+		if choice.Index != nil {
+			s.metadata.ChoiceIndex = choice.Index
+		}
+		if choice.FinishReason != nil {
+			s.metadata.FinishReason = choice.FinishReason
+		}
+		if choice.Logprobs != nil {
+			if s.metadata.Logprobs == nil {
+				s.metadata.Logprobs = &Logprobs{}
+			}
+			if choice.Logprobs.Content != nil &&
+				s.metadata.Logprobs.Content == nil {
+				s.metadata.Logprobs.Content = []TokenLogprob{}
+			}
+			if choice.Logprobs.Refusal != nil &&
+				s.metadata.Logprobs.Refusal == nil {
+				s.metadata.Logprobs.Refusal = []TokenLogprob{}
+			}
+			s.metadata.Logprobs.Content = append(
+				s.metadata.Logprobs.Content,
+				choice.Logprobs.Content...,
+			)
+			s.metadata.Logprobs.Refusal = append(
+				s.metadata.Logprobs.Refusal,
+				choice.Logprobs.Refusal...,
+			)
 		}
 		s.acc.Add(choice.Delta)
 	}
+	if chunk.Created != nil {
+		s.metadata.Created = chunk.Created
+	}
+	if chunk.ID != nil {
+		s.metadata.ID = chunk.ID
+	}
+	if chunk.Model != nil {
+		s.metadata.Model = chunk.Model
+	}
+	if chunk.ServiceTier != nil {
+		s.metadata.ServiceTier = chunk.ServiceTier
+	}
+	if chunk.SystemFingerprint != nil {
+		s.metadata.SystemFingerprint = chunk.SystemFingerprint
+	}
 	if chunk.Usage != nil {
-		s.usage = chunk.Usage
+		s.metadata.Usage = chunk.Usage
 	}
 	if chunk.Timings != nil {
-		s.timings = chunk.Timings
+		s.metadata.Timings = chunk.Timings
 	}
 	s.current = chunk
 	return true
 }
 
 // Current returns the most recently decoded chunk. The value is replaced by
-// the next successful call to Next.
+// the next successful call to Next. Previously returned chunks remain valid.
+// Reported metadata shares pointer targets and nested logprob values with
+// the stream and Result. Mutations through those references affect both.
 func (s *CompletionStream) Current() StreamChunk {
 	return s.current
 }
 
-// Result returns a snapshot of the state accumulated so far. Err reports
-// whether the snapshot is a complete response or a partial result.
+// Result returns the message and metadata accumulated so far. Advancing the
+// stream does not change previously returned metadata values. Metadata pointer
+// targets and nested logprob slices are shared with the stream and other
+// results; mutations through those references are visible through the aliases.
+// Err reports failures. Nil Err alone does not establish completion: the
+// stream completes successfully when Next consumes the DONE marker.
 func (s *CompletionStream) Result() CompletionResult {
 	message := s.acc.Message()
 	message.ToolCalls = append([]ToolCall(nil), message.ToolCalls...)
-	return CompletionResult{
-		Message:      message,
-		FinishReason: s.finishReason,
-		Usage:        s.usage,
-		Timings:      s.timings,
+	metadata := s.metadata
+	if metadata.Logprobs != nil {
+		logprobs := &Logprobs{}
+		if metadata.Logprobs.Content != nil {
+			logprobs.Content = make(
+				[]TokenLogprob, len(metadata.Logprobs.Content),
+			)
+			copy(logprobs.Content, metadata.Logprobs.Content)
 		}
+		if metadata.Logprobs.Refusal != nil {
+			logprobs.Refusal = make(
+				[]TokenLogprob, len(metadata.Logprobs.Refusal),
+			)
+			copy(logprobs.Refusal, metadata.Logprobs.Refusal)
+		}
+		metadata.Logprobs = logprobs
+	}
+	return CompletionResult{Message: message, Metadata: metadata}
 }
 
 // Err returns the stream failure, if any. Closing a stream does not create or

internal/llm/client_completions_test.go

@@ -2,12 +2,15 @@ package llm
 
 import (
 	"context"
+	"encoding/json"
+	jsonv2 "encoding/json/v2"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"strings"
 	"testing"
 )
@@ -70,8 +73,8 @@ func TestCompletionsStreamReconstructsReasoning(t *testing.T) {
 	if result.Message.Text() != "answer" {
 		t.Errorf("text = %q", result.Message.Text())
 	}
-	if result.FinishReason != "stop" {
-		t.Errorf("finish reason = %q", result.FinishReason)
+	if result.Metadata.FinishReason == nil || *result.Metadata.FinishReason != "stop" {
+		t.Errorf("finish reason = %v", result.Metadata.FinishReason)
 	}
 	if stream.Next() {
 		t.Error("Next returned true after termination")
@@ -148,76 +151,600 @@ func TestCompletionsStreamReconstructsFragmentedParallelToolCalls(t *testing.T)
 		toolCalls[1].Function.Arguments != `{"format":"%s"}` {
 		t.Errorf("tool call 1 = %#v", toolCalls[1])
 	}
-	if result.FinishReason != "tool_calls" {
-		t.Errorf("finish reason = %q", result.FinishReason)
+	if result.Metadata.FinishReason == nil || *result.Metadata.FinishReason != "tool_calls" {
+		t.Errorf("finish reason = %v", result.Metadata.FinishReason)
 	}
 }
 
 func TestCompletionsReturnsUsageAndTimings(t *testing.T) {
+	tests := []struct {
+		name string
+		body string
+		want string
+	}{
+		{
+			name: "all known metadata",
+			body: `{
+				"id":"body-id","created":123,"model":"reported-model",
+				"service_tier":"default","system_fingerprint":"backend",
+				"choices":[{"index":3,"finish_reason":"stop",
+					"message":{"role":"assistant","content":"hello"},
+					"logprobs":{"content":[{"token":"hello","logprob":0,
+						"bytes":[104,195,169],"top_logprobs":[{
+							"token":"other","logprob":-2,"bytes":[111]}]}],
+						"refusal":[{"token":"no","logprob":-9999}]}}],
+				"usage":{"prompt_tokens":10,"completion_tokens":5,
+					"total_tokens":15,"prompt_tokens_details":{
+						"audio_tokens":1,"cache_write_tokens":2,
+						"cached_tokens":3,"image_tokens":4,"text_tokens":5},
+					"completion_tokens_details":{"accepted_prediction_tokens":1,
+						"audio_tokens":2,"reasoning_tokens":3,
+						"rejected_prediction_tokens":4,"text_tokens":5}},
+				"timings":{"cache_n":1,"draft_n":2,"draft_n_accepted":1,
+					"prompt_n":10,"prompt_ms":50.5,"prompt_per_token_ms":5.05,
+					"prompt_per_second":198,"predicted_n":5,"predicted_ms":100,
+					"predicted_per_token_ms":20,"predicted_per_second":50}
+			}`,
+			want: `{
+				  "choice_index":3,
+				  "created":123,
+				  "finish_reason":"stop",
+				  "id":"body-id",
+				  "logprobs":{
+				    "content":[
+				      {
+				        "bytes":[104,195,169],
+				        "logprob":0,
+				        "token":"hello",
+				        "top_logprobs":[
+				          {
+				            "bytes":[111],
+				            "logprob":-2,
+				            "token":"other"
+				          }
+				        ]
+				      }
+				    ],
+				    "refusal":[
+				      {
+				        "bytes":null,
+				        "logprob":-9999,
+				        "token":"no",
+				        "top_logprobs":null
+				      }
+				    ]
+				  },
+				  "model":"reported-model",
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":"default",
+				  "system_fingerprint":"backend",
+				  "timings":{
+				    "cache_n":1,
+				    "draft_n":2,
+				    "draft_n_accepted":1,
+				    "predicted_ms":100,
+				    "predicted_n":5,
+				    "predicted_per_second":50,
+				    "predicted_per_token_ms":20,
+				    "prompt_ms":50.5,
+				    "prompt_n":10,
+				    "prompt_per_second":198,
+				    "prompt_per_token_ms":5.05
+				  },
+				  "usage":{
+				    "completion_tokens":5,
+				    "completion_tokens_details":{
+				      "accepted_prediction_tokens":1,
+				      "audio_tokens":2,
+				      "reasoning_tokens":3,
+				      "rejected_prediction_tokens":4,
+				      "text_tokens":5
+				    },
+				    "prompt_tokens":10,
+				    "prompt_tokens_details":{
+				      "audio_tokens":1,
+				      "cache_write_tokens":2,
+				      "cached_tokens":3,
+				      "image_tokens":4,
+				      "text_tokens":5
+				    },
+				    "total_tokens":15
+				  }
+				}`,
+		},
+		{
+			name: "omitted metadata",
+			body: `{"choices":[{"message":{"content":"hello"}}]}`,
+			want: `{
+				  "choice_index":null,
+				  "created":null,
+				  "finish_reason":null,
+				  "id":null,
+				  "logprobs":null,
+				  "model":null,
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":null,
+				  "system_fingerprint":null,
+				  "timings":null,
+				  "usage":null
+				}`,
+		},
+		{
+			name: "reported zero and empty values",
+			body: `{"id":"","created":0,"model":"","service_tier":"",
+				"system_fingerprint":"","choices":[{"index":0,
+					"finish_reason":"","message":{"content":"hello"}}],
+				"usage":{"prompt_tokens":0,"completion_tokens":0,"total_tokens":0,
+					"prompt_tokens_details":{"cached_tokens":0},
+					"completion_tokens_details":{"reasoning_tokens":0}},
+				"timings":{"prompt_ms":0,"predicted_n":0}}`,
+			want: `{
+				  "choice_index":0,
+				  "created":0,
+				  "finish_reason":"",
+				  "id":"",
+				  "logprobs":null,
+				  "model":"",
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":"",
+				  "system_fingerprint":"",
+				  "timings":{
+				    "cache_n":null,
+				    "draft_n":null,
+				    "draft_n_accepted":null,
+				    "predicted_ms":null,
+				    "predicted_n":0,
+				    "predicted_per_second":null,
+				    "predicted_per_token_ms":null,
+				    "prompt_ms":0,
+				    "prompt_n":null,
+				    "prompt_per_second":null,
+				    "prompt_per_token_ms":null
+				  },
+				  "usage":{
+				    "completion_tokens":0,
+				    "completion_tokens_details":{
+				      "accepted_prediction_tokens":null,
+				      "audio_tokens":null,
+				      "reasoning_tokens":0,
+				      "rejected_prediction_tokens":null,
+				      "text_tokens":null
+				    },
+				    "prompt_tokens":0,
+				    "prompt_tokens_details":{
+				      "audio_tokens":null,
+				      "cache_write_tokens":null,
+				      "cached_tokens":0,
+				      "image_tokens":null,
+				      "text_tokens":null
+				    },
+				    "total_tokens":0
+				  }
+				}`,
+		},
+		{
+			name: "empty and null logprob arrays",
+			body: `{"choices":[{"message":{"content":"hello"},
+				"logprobs":{"content":[{"bytes":[],"top_logprobs":[]},
+					{"bytes":null,"top_logprobs":null},
+					{"top_logprobs":[{"bytes":[]},{}]}],"refusal":[]}}]}`,
+			want: `{
+				  "choice_index":null,
+				  "created":null,
+				  "finish_reason":null,
+				  "id":null,
+				  "logprobs":{
+				    "content":[
+				      {
+				        "bytes":[],
+				        "logprob":null,
+				        "token":null,
+				        "top_logprobs":[]
+				      },
+				      {
+				        "bytes":null,
+				        "logprob":null,
+				        "token":null,
+				        "top_logprobs":null
+				      },
+				      {
+				        "bytes":null,
+				        "logprob":null,
+				        "token":null,
+				        "top_logprobs":[
+				          {
+				            "bytes":[],
+				            "logprob":null,
+				            "token":null
+				          },
+				          {
+				            "bytes":null,
+				            "logprob":null,
+				            "token":null
+				          }
+				        ]
+				      }
+				    ],
+				    "refusal":[]
+				  },
+				  "model":null,
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":null,
+				  "system_fingerprint":null,
+				  "timings":null,
+				  "usage":null
+				}`,
+		},
+		{
+			name: "empty nested metadata objects",
+			body: `{"choices":[{"message":{"content":"hello"},
+				"logprobs":{}}],"usage":{"prompt_tokens_details":{},
+				"completion_tokens_details":{}},"timings":{}}`,
+			want: `{
+				  "choice_index":null,
+				  "created":null,
+				  "finish_reason":null,
+				  "id":null,
+				  "logprobs":{"content":null,"refusal":null},
+				  "model":null,
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":null,
+				  "system_fingerprint":null,
+				  "timings":{
+				    "cache_n":null,
+				    "draft_n":null,
+				    "draft_n_accepted":null,
+				    "predicted_ms":null,
+				    "predicted_n":null,
+				    "predicted_per_second":null,
+				    "predicted_per_token_ms":null,
+				    "prompt_ms":null,
+				    "prompt_n":null,
+				    "prompt_per_second":null,
+				    "prompt_per_token_ms":null
+				  },
+				  "usage":{
+				    "completion_tokens":null,
+				    "completion_tokens_details":{
+				      "accepted_prediction_tokens":null,
+				      "audio_tokens":null,
+				      "reasoning_tokens":null,
+				      "rejected_prediction_tokens":null,
+				      "text_tokens":null
+				    },
+				    "prompt_tokens":null,
+				    "prompt_tokens_details":{
+				      "audio_tokens":null,
+				      "cache_write_tokens":null,
+				      "cached_tokens":null,
+				      "image_tokens":null,
+				      "text_tokens":null
+				    },
+				    "total_tokens":null
+				  }
+				}`,
+		},
+		{
+			name: "null metadata",
+			body: `{"id":null,"usage":null,"timings":null,
+				"choices":[{"index":null,"logprobs":null,
+					"message":{"content":"hello"}}]}`,
+			want: `{
+				  "choice_index":null,
+				  "created":null,
+				  "finish_reason":null,
+				  "id":null,
+				  "logprobs":null,
+				  "model":null,
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":null,
+				  "system_fingerprint":null,
+				  "timings":null,
+				  "usage":null
+				}`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
 			client := newCompletionTestClient(t, func(w http.ResponseWriter, r *http.Request) {
 				if r.URL.Path != "/chat/completions" {
-			t.Fatalf("unexpected path: %s", r.URL.Path)
+					t.Errorf("unexpected path: %s", r.URL.Path)
 				}
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprint(w, `{
-			"id": "test-123",
-			"model": "test-model",
-			"choices": [{
-				"index": 0,
-				"message": {"role": "assistant", "content": "hello"},
-				"finish_reason": "stop"
-			}],
-			"usage": {
-				"prompt_tokens": 10,
-				"completion_tokens": 5,
-				"total_tokens": 15,
-				"prompt_tokens_details": {"cached_tokens": 3},
-				"completion_tokens_details": {"reasoning_tokens": 2}
-			},
-			"timings": {
-				"prompt_n": 10,
-				"prompt_ms": 50.5,
-				"prompt_per_token_ms": 5.05,
-				"prompt_per_second": 198.0,
-				"predicted_n": 5,
-				"predicted_ms": 100.0,
-				"predicted_per_token_ms": 20.0,
-				"predicted_per_second": 50.0
+				var request ChatRequest
+				if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+					t.Error(err)
 				}
-		}`)
+				if request.LogProbs || request.TopLogProbs != 0 {
+					t.Error("log probabilities requested")
+				}
+				w.Header().Set("Request-ID", "not-the-body-id")
+				fmt.Fprint(w, tt.body)
 			})
-
-	res, err := client.Completions(context.Background(), CompletionsParams{Model: "test-model"})
+			res, err := client.Completions(t.Context(), CompletionsParams{Model: "test-model"})
 			if err != nil {
 				t.Fatalf("Completions: %v", err)
 			}
-	if res.Message.Text() != "hello" || res.Usage == nil || res.Timings == nil {
-		t.Fatalf("unexpected result: %#v", res)
+			if res.Message.Text() != "hello" {
+				t.Errorf("text = %q", res.Message.Text())
 			}
-	if res.Usage.PromptTokens != 10 || res.Usage.CompletionTokens != 5 || res.Usage.TotalTokens != 15 {
-		t.Errorf("usage = %#v", res.Usage)
+			var want Metadata
+			if err := json.Unmarshal([]byte(tt.want), &want); err != nil {
+				t.Fatal(err)
 			}
-	if res.Usage.PromptTokensDetails == nil || res.Usage.PromptTokensDetails.CachedTokens != 3 {
-		t.Errorf("prompt token details = %#v", res.Usage.PromptTokensDetails)
+			if !reflect.DeepEqual(res.Metadata, want) {
+				t.Errorf("metadata = %#v, want %#v", res.Metadata, want)
 			}
-	if res.Usage.CompletionTokensDetails == nil || res.Usage.CompletionTokensDetails.ReasoningTokens != 2 {
-		t.Errorf("completion token details = %#v", res.Usage.CompletionTokensDetails)
+			encoded, err := jsonv2.Marshal(
+				res.Metadata, jsonv2.FormatNilSliceAsNull(true),
+			)
+			if err != nil {
+				t.Fatal(err)
 			}
-	if res.Timings.PromptMS != 50.5 || res.Timings.PredictedMS != 100 {
-		t.Errorf("timings = %#v", res.Timings)
+			var gotJSON, wantJSON map[string]any
+			if err := jsonv2.Unmarshal(encoded, &gotJSON); err != nil {
+				t.Fatal(err)
+			}
+			if err := jsonv2.Unmarshal([]byte(tt.want), &wantJSON); err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(gotJSON, wantJSON) {
+				t.Errorf("JSON fields = %s, want %s", encoded, tt.want)
+			}
+			var roundtrip Metadata
+			if err := json.Unmarshal(encoded, &roundtrip); err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(roundtrip, want) {
+				t.Errorf("roundtrip = %s", encoded)
+			}
+			if strings.Contains(tt.body, `"bytes":[104,195,169]`) &&
+				!strings.Contains(string(encoded), `"bytes":[104,195,169]`) {
+				t.Errorf("token bytes are not integer arrays: %s", encoded)
+			}
+		})
 	}
 }
 
 func TestCompletionsStreamSSEAndMetadata(t *testing.T) {
 	large := strings.Repeat("x", 70*1024)
-	client := newCompletionTestClient(t, func(w http.ResponseWriter, _ *http.Request) {
-		fmt.Fprint(w, ": comment\n")
-		fmt.Fprintf(w, "data:{\"choices\":[{\"index\":0,\"delta\":{\"content\":%q}}],\n", large)
-		fmt.Fprint(w, "data: \"usage\":{\"prompt_tokens\":10,\"total_tokens\":15}}\n\n")
-		fmt.Fprint(w, "data: {\"choices\":[],\"timings\":{\"prompt_ms\":50.5}}\n\n")
-		fmt.Fprint(w, "data:[DONE]\n\n")
-	})
+	tests := []struct {
+		name         string
+		response     string
+		wantText     string
+		wantMetadata string
+		wantChunks   int
+	}{
+		{
+			name: "large multiline event and metadata-only timing",
+			response: ": comment\n" +
+				fmt.Sprintf("data:{\"choices\":[{\"index\":0,\"delta\":{\"content\":%q}}],\n", large) +
+				"data: \"usage\":{\"prompt_tokens\":10,\"total_tokens\":15}}\n\n" +
+				"data: {\"choices\":[],\"timings\":{\"prompt_ms\":50.5}}\n\n",
+			wantText: large,
+			wantMetadata: `{
+				  "choice_index":0,
+				  "created":null,
+				  "finish_reason":null,
+				  "id":null,
+				  "logprobs":null,
+				  "model":null,
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":null,
+				  "system_fingerprint":null,
+				  "timings":{
+				    "cache_n":null,
+				    "draft_n":null,
+				    "draft_n_accepted":null,
+				    "predicted_ms":null,
+				    "predicted_n":null,
+				    "predicted_per_second":null,
+				    "predicted_per_token_ms":null,
+				    "prompt_ms":50.5,
+				    "prompt_n":null,
+				    "prompt_per_second":null,
+				    "prompt_per_token_ms":null
+				  },
+				  "usage":{
+				    "completion_tokens":null,
+				    "completion_tokens_details":null,
+				    "prompt_tokens":10,
+				    "prompt_tokens_details":null,
+				    "total_tokens":15
+				  }
+				}`,
+			wantChunks: 2,
+		},
+		{
+			name: "last scalars and snapshots, ordered logprobs",
+			response: `data: {"id":"first","created":123,"model":"first-model","service_tier":"first-tier","system_fingerprint":"first-backend","choices":[{"index":0,"delta":{"content":"answer"},"finish_reason":"stop","logprobs":{"content":[{"token":"a","logprob":0,"bytes":[97],"top_logprobs":[{"token":"b","logprob":-1,"bytes":[98]}]}],"refusal":[{"token":"r1","logprob":-2}]}}],"usage":{"prompt_tokens":5,"total_tokens":9},"timings":{"prompt_ms":5,"predicted_n":3}}
 
+data: {"choices":[{"index":0,"delta":{},"finish_reason":"","logprobs":{"content":[{"token":"b","logprob":-1}],"refusal":[{"token":"r2","logprob":0}]}}]}
+
+data: {"id":"","created":0,"model":"reported-model","service_tier":"last-tier","system_fingerprint":"last-backend","choices":[],"usage":{"prompt_tokens":0},"timings":{"predicted_n":0}}
+
+data: {"choices":[],"usage":null,"timings":null,"id":null}
+
+`,
+			wantText: "answer",
+			wantMetadata: `{
+				  "choice_index":0,
+				  "created":0,
+				  "finish_reason":"",
+				  "id":"",
+				  "logprobs":{
+				    "content":[
+				      {
+				        "bytes":[97],
+				        "logprob":0,
+				        "token":"a",
+				        "top_logprobs":[
+				          {
+				            "bytes":[98],
+				            "logprob":-1,
+				            "token":"b"
+				          }
+				        ]
+				      },
+				      {
+				        "bytes":null,
+				        "logprob":-1,
+				        "token":"b",
+				        "top_logprobs":null
+				      }
+				    ],
+				    "refusal":[
+				      {
+				        "bytes":null,
+				        "logprob":-2,
+				        "token":"r1",
+				        "top_logprobs":null
+				      },
+				      {
+				        "bytes":null,
+				        "logprob":0,
+				        "token":"r2",
+				        "top_logprobs":null
+				      }
+				    ]
+				  },
+				  "model":"reported-model",
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":"last-tier",
+				  "system_fingerprint":"last-backend",
+				  "timings":{
+				    "cache_n":null,
+				    "draft_n":null,
+				    "draft_n_accepted":null,
+				    "predicted_ms":null,
+				    "predicted_n":0,
+				    "predicted_per_second":null,
+				    "predicted_per_token_ms":null,
+				    "prompt_ms":null,
+				    "prompt_n":null,
+				    "prompt_per_second":null,
+				    "prompt_per_token_ms":null
+				  },
+				  "usage":{
+				    "completion_tokens":null,
+				    "completion_tokens_details":null,
+				    "prompt_tokens":0,
+				    "prompt_tokens_details":null,
+				    "total_tokens":null
+				  }
+				}`,
+			wantChunks: 4,
+		},
+		{
+			name: "empty and null logprob arrays",
+			response: `data: {"choices":[{"index":0,"delta":{},"logprobs":{"content":[],"refusal":null}}]}
+
+data: {"choices":[{"index":0,"delta":{},"logprobs":{"content":null,"refusal":[{"bytes":[],"top_logprobs":[]},{"top_logprobs":[{"bytes":[]}]}]}}]}
+
+`,
+			wantMetadata: `{
+				  "choice_index":0,
+				  "created":null,
+				  "finish_reason":null,
+				  "id":null,
+				  "logprobs":{
+				    "content":[],
+				    "refusal":[
+				      {
+				        "bytes":[],
+				        "logprob":null,
+				        "token":null,
+				        "top_logprobs":[]
+				      },
+				      {
+				        "bytes":null,
+				        "logprob":null,
+				        "token":null,
+				        "top_logprobs":[
+				          {
+				            "bytes":[],
+				            "logprob":null,
+				            "token":null
+				          }
+				        ]
+				      }
+				    ]
+				  },
+				  "model":null,
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":null,
+				  "system_fingerprint":null,
+				  "timings":null,
+				  "usage":null
+				}`,
+			wantChunks: 2,
+		},
+		{
+			name:     "metadata-only without choice",
+			response: "data: {\"created\":0,\"usage\":{\"total_tokens\":0}}\n\n",
+			wantMetadata: `{
+				  "choice_index":null,
+				  "created":0,
+				  "finish_reason":null,
+				  "id":null,
+				  "logprobs":null,
+				  "model":null,
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":null,
+				  "system_fingerprint":null,
+				  "timings":null,
+				  "usage":{
+				    "completion_tokens":null,
+				    "completion_tokens_details":null,
+				    "prompt_tokens":null,
+				    "prompt_tokens_details":null,
+				    "total_tokens":0
+				  }
+				}`,
+			wantChunks: 1,
+		},
+		{
+			name:         "omitted choice index and metadata",
+			response:     "data: {\"choices\":[{\"delta\":{\"content\":\"answer\"}}]}\n\n",
+			wantText:     "answer",
+			wantMetadata: `{
+				  "choice_index":null,
+				  "created":null,
+				  "finish_reason":null,
+				  "id":null,
+				  "logprobs":null,
+				  "model":null,
+				  "provider":"test",
+				  "requested_model":"test-model",
+				  "service_tier":null,
+				  "system_fingerprint":null,
+				  "timings":null,
+				  "usage":null
+				}`,
+			wantChunks:   1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			client := newCompletionTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+				var request ChatRequest
+				if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+					t.Error(err)
+				}
+				if request.LogProbs || request.TopLogProbs != 0 {
+					t.Error("log probabilities requested")
+				}
+				fmt.Fprint(w, tt.response+"data:[DONE]\n\n")
+			})
 			stream := openCompletionTestStream(t, client)
 			var count int
 			for stream.Next() {
@@ -227,11 +754,41 @@ func TestCompletionsStreamSSEAndMetadata(t *testing.T) {
 				t.Fatalf("stream.Err: %v", err)
 			}
 			result := stream.Result()
-	if count != 2 || result.Message.Text() != large {
+			if count != tt.wantChunks || result.Message.Text() != tt.wantText {
 				t.Fatalf("count = %d, text length = %d", count, len(result.Message.Text()))
 			}
-	if result.Usage == nil || result.Usage.PromptTokens != 10 || result.Timings == nil || result.Timings.PromptMS != 50.5 {
-		t.Errorf("metadata = usage %#v, timings %#v", result.Usage, result.Timings)
+			var want Metadata
+			if err := json.Unmarshal([]byte(tt.wantMetadata), &want); err != nil {
+				t.Fatal(err)
+			}
+			encoded, err := jsonv2.Marshal(
+				result.Metadata, jsonv2.FormatNilSliceAsNull(true),
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var gotJSON, wantJSON map[string]any
+			if err := jsonv2.Unmarshal(encoded, &gotJSON); err != nil {
+				t.Fatal(err)
+			}
+			if err := jsonv2.Unmarshal([]byte(tt.wantMetadata), &wantJSON); err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(gotJSON, wantJSON) {
+				t.Errorf("JSON fields = %s, want %s", encoded, tt.wantMetadata)
+			}
+			var roundtrip Metadata
+			if err := json.Unmarshal(encoded, &roundtrip); err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(roundtrip, want) {
+				t.Errorf("metadata roundtrip = %s", encoded)
+			}
+			if !reflect.DeepEqual(result.Metadata, want) {
+				encoded, _ := json.Marshal(result.Metadata)
+				t.Errorf("metadata = %s, want %s", encoded, tt.wantMetadata)
+			}
+		})
 	}
 }
 
@@ -246,6 +803,11 @@ func TestCompletionsStreamFailuresPreservePartialResult(t *testing.T) {
 			response: "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"partial\"}}]}\n\n",
 			wantErr:  io.ErrUnexpectedEOF,
 		},
+		{
+			name:     "finish reason without DONE",
+			response: "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"partial\"},\"finish_reason\":\"stop\"}]}\n\n",
+			wantErr:  io.ErrUnexpectedEOF,
+		},
 		{
 			name:     "malformed JSON",
 			response: "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"partial\"}}]}\n\ndata: {bad}\n\n",

internal/llm/llm.go

@@ -1,6 +1,5 @@
 // Package llm implements an OpenAI-compatible inference client and its
 // protocol types.
-
 package llm
 
 import (
@@ -114,69 +113,229 @@ type ChatRequest struct {
 	TopP *float32 `json:"top_p,omitempty"`
 }
 
-// Usage reports input, output, and total token counts for a completion.
-type Usage struct {
-	// CompletionTokens is the number of tokens generated by the model.
-	CompletionTokens int `json:"completion_tokens"`
-	// CompletionTokensDetails breaks down CompletionTokens by category.
-	CompletionTokensDetails *TokenDetails `json:"completion_tokens_details,omitempty"`
-	// PromptTokens is the number of tokens in the input.
-	PromptTokens int `json:"prompt_tokens"`
-	// PromptTokensDetails breaks down PromptTokens by category.
-	PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"`
-	// TotalTokens is the sum of PromptTokens and CompletionTokens.
-	TotalTokens int `json:"total_tokens"`
+// Metadata describes one selected completion, separately from its message.
+// Nil optional fields mean the endpoint did not report a value. Reported zero
+// counts and empty strings remain distinct from absent values. Only known body
+// fields are retained; response headers are not part of Metadata.
+//
+// See the response and streaming definitions in
+// [OpenAI completion response] and [OpenAI completion chunk].
+//
+// [OpenAI completion response]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
+// [OpenAI completion chunk]: https://developers.openai.com/api/reference/resources/chat/subresources/completions/streaming-events
+type Metadata struct {
+	// ChoiceIndex is the endpoint's zero-based index of the selected
+	// choice.
+	ChoiceIndex *int `json:"choice_index"`
+	// Created is the endpoint's completion creation time in Unix seconds,
+	// not the database message's creation time.
+	Created *int64 `json:"created"`
+	// FinishReason is the endpoint's reason for ending the selected choice.
+	// A reported reason does not establish receipt of the stream's DONE
+	// marker.
+	FinishReason *string `json:"finish_reason"`
+	// ID is the completion ID from the response body, not a request-ID
+	// header or a database message ID.
+	ID *string `json:"id"`
+	// Logprobs contains token probabilities for the selected choice.
+	Logprobs *Logprobs `json:"logprobs"`
+	// Model is the model identifier reported in the response body. It can
+	// differ from RequestedModel, which was sent in the request.
+	Model *string `json:"model"`
+	// Provider is the application-defined label of the configured endpoint.
+	// It is not a provider-reported identity or a response header.
+	Provider *string `json:"provider"`
+	// RequestedModel is the model identifier sent to the endpoint for this
+	// inference, before any endpoint-side model alias resolution.
+	RequestedModel *string `json:"requested_model"`
+	// ServiceTier is the processing tier reported in the response body,
+	// which can differ from a requested tier.
+	ServiceTier *string `json:"service_tier"`
+	// SystemFingerprint identifies the backend configuration reported by
+	// the endpoint; it is not a model identifier.
+	SystemFingerprint *string `json:"system_fingerprint"`
+	// Timings contains performance measurements for the LLM request that
+	// produced this assistant message. During streaming, a newer timing
+	// report replaces the earlier report: the measurements cover the same
+	// request at a later point, not additional work to add together. Fields
+	// missing from the newer report remain nil.
+	Timings *Timings `json:"timings"`
+	// Usage contains token counts for the LLM request that produced this
+	// assistant message. During streaming, a newer usage report replaces
+	// the earlier report rather than adding to its counts. Fields missing
+	// from the newer report remain nil.
+	Usage *Usage `json:"usage"`
 }
 
-// TokenDetails reports token categories within a Usage count. A zero value may
-// mean either that no tokens belong to the category or that it was not
-// reported.
-type TokenDetails struct {
-	// CachedTokens is the number of input tokens served from a cache.
-	CachedTokens int `json:"cached_tokens,omitempty"`
+// Usage reports input, output, and total token counts for one inference.
+// Nil counts mean unreported, not zero. Category counts are endpoint-reported
+// breakdowns, not additional tokens to add to the totals.
+//
+// See [OpenAI completion usage].
+//
+// [OpenAI completion usage]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
+type Usage struct {
+	// CompletionTokens is the number of tokens generated by the model.
+	CompletionTokens *int `json:"completion_tokens"`
+	// CompletionTokensDetails breaks down CompletionTokens by category.
+	CompletionTokensDetails *CompletionTokensDetails `json:"completion_tokens_details"`
+	// PromptTokens is the number of tokens in the input.
+	PromptTokens *int `json:"prompt_tokens"`
+	// PromptTokensDetails breaks down PromptTokens by category.
+	PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details"`
+	// TotalTokens is the reported sum of prompt and completion token
+	// counts.
+	TotalTokens *int `json:"total_tokens"`
+}
+
+// CompletionTokensDetails reports output-token categories in [Usage].
+// Nil counts mean the category was not reported. The categories follow
+// [OpenAI completion usage].
+//
+// [OpenAI completion usage]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
+type CompletionTokensDetails struct {
+	// AcceptedPredictionTokens counts predicted-output tokens that appeared
+	// in the completion.
+	AcceptedPredictionTokens *int `json:"accepted_prediction_tokens"`
+	// AudioTokens counts audio tokens generated by the model.
+	AudioTokens *int `json:"audio_tokens"`
 	// ReasoningTokens is the number of output tokens used for reasoning.
-	ReasoningTokens int `json:"reasoning_tokens,omitempty"`
+	ReasoningTokens *int `json:"reasoning_tokens"`
+	// RejectedPredictionTokens counts predicted-output tokens that did not
+	// appear in the completion. OpenAI includes these in completion counts
+	// for billing, output, and context-window limits.
+	RejectedPredictionTokens *int `json:"rejected_prediction_tokens"`
+	// TextTokens counts text tokens generated by the model.
+	TextTokens *int `json:"text_tokens"`
+}
+
+// PromptTokensDetails reports input-token categories in [Usage].
+// Nil counts mean the category was not reported. The categories follow
+// [OpenAI completion usage].
+//
+// [OpenAI completion usage]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
+type PromptTokensDetails struct {
+	// AudioTokens counts audio input tokens in the prompt.
+	AudioTokens *int `json:"audio_tokens"`
+	// CacheWriteTokens is the unadjusted count of prompt tokens written to
+	// cache, as reported by the endpoint.
+	CacheWriteTokens *int `json:"cache_write_tokens"`
+	// CachedTokens is the number of input tokens served from a cache.
+	CachedTokens *int `json:"cached_tokens"`
+	// ImageTokens counts image input tokens in the prompt.
+	ImageTokens *int `json:"image_tokens"`
+	// TextTokens counts text input tokens in the prompt.
+	TextTokens *int `json:"text_tokens"`
+}
+
+// Logprobs reports token log probabilities for one choice, when returned.
+// Content and refusal tokens are separate ordered sequences. Receiving these
+// values does not cause the client to request log probabilities. Nil slices
+// represent absent or null arrays; non-nil empty slices retain reported [].
+//
+// See [OpenAI token log probabilities] and [OpenAI completion response].
+//
+// [OpenAI token log probabilities]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
+// [OpenAI completion response]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
+type Logprobs struct {
+	// Content contains log probabilities for message content tokens in
+	// order.
+	Content []TokenLogprob `json:"content"`
+	// Refusal contains log probabilities for refusal tokens in order.
+	Refusal []TokenLogprob `json:"refusal"`
+}
+
+// TokenLogprob describes a generated token and its alternatives at one
+// position, using [OpenAI token log probabilities]. Nil optional scalars mean
+// unreported; a zero log probability remains a reported value.
+//
+// [OpenAI token log probabilities]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
+type TokenLogprob struct {
+	// Bytes is the token's UTF-8 byte representation as integers, not
+	// base64.
+	// A token can contain only part of a multibyte character. Nil means no
+	// byte representation was reported.
+	Bytes []int `json:"bytes"`
+	// Logprob is the token's log probability. OpenAI uses -9999 for a token
+	// outside the top 20 most likely tokens.
+	Logprob *float64 `json:"logprob"`
+	// Token is the token text reported by the endpoint.
+	Token *string `json:"token"`
+	// TopLogprobs contains alternatives at this position; the endpoint can
+	// return fewer alternatives than requested.
+	TopLogprobs []TopLogprob `json:"top_logprobs"`
+}
+
+// TopLogprob describes one alternative token at a generated token's position.
+// Alternatives have no nested alternatives in [OpenAI token log probabilities].
+// Nil optional scalars mean unreported, rather than zero or empty text.
+//
+// [OpenAI token log probabilities]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
+type TopLogprob struct {
+	// Bytes is the alternative's UTF-8 byte representation as integers,
+	// not base64. Nil means no byte representation was reported.
+	Bytes []int `json:"bytes"`
+	// Logprob is the alternative's log probability. OpenAI uses -9999 for
+	// a token outside the top 20 most likely tokens.
+	Logprob *float64 `json:"logprob"`
+	// Token is the alternative token text reported by the endpoint.
+	Token *string `json:"token"`
 }
 
 // Timings reports llama.cpp prompt-processing and token-generation performance.
 // The N fields are token counts, MS fields are elapsed milliseconds,
 // PerTokenMS fields are milliseconds per token, and PerSecond fields are tokens
-// per second. Timings is a llama.cpp extension to the OpenAI API.
+// per second. Timings is a llama.cpp extension to the OpenAI API. Nil fields
+// mean unreported. Streaming objects are snapshots, not increments to sum.
 //
 // See [llama.cpp API documentation].
 //
-// [llama.cpp API documentation]: https://llama.app/docs/api
+// [llama.cpp API documentation]: https://llama.app/docs/api#timings-and-usage
 type Timings struct {
+	// CacheN is the number of prompt tokens reused from the cache.
+	CacheN *int `json:"cache_n"`
+	// DraftN is the number of tokens drafted during speculative decoding.
+	DraftN *int `json:"draft_n"`
+	// DraftNAccepted is the number of drafted tokens accepted during
+	// speculative decoding.
+	DraftNAccepted *int `json:"draft_n_accepted"`
 	// PredictedMS is the time spent generating output, in milliseconds.
-	PredictedMS float64 `json:"predicted_ms"`
+	PredictedMS *float64 `json:"predicted_ms"`
 	// PredictedN is the number of generated tokens.
-	PredictedN int `json:"predicted_n"`
+	PredictedN *int `json:"predicted_n"`
 	// PredictedPerSecond is the output-generation rate in tokens per
 	// second.
-	PredictedPerSecond float64 `json:"predicted_per_second"`
+	PredictedPerSecond *float64 `json:"predicted_per_second"`
 	// PredictedPerTokenMS is the mean output-generation time per token, in
 	// milliseconds.
-	PredictedPerTokenMS float64 `json:"predicted_per_token_ms"`
+	PredictedPerTokenMS *float64 `json:"predicted_per_token_ms"`
 	// PromptMS is the time spent processing the prompt, in milliseconds.
-	PromptMS float64 `json:"prompt_ms"`
+	PromptMS *float64 `json:"prompt_ms"`
 	// PromptN is the number of prompt tokens processed.
-	PromptN int `json:"prompt_n"`
+	PromptN *int `json:"prompt_n"`
 	// PromptPerSecond is the prompt-processing rate in tokens per second.
-	PromptPerSecond float64 `json:"prompt_per_second"`
+	PromptPerSecond *float64 `json:"prompt_per_second"`
 	// PromptPerTokenMS is the mean prompt-processing time per token, in
 	// milliseconds.
-	PromptPerTokenMS float64 `json:"prompt_per_token_ms"`
+	PromptPerTokenMS *float64 `json:"prompt_per_token_ms"`
 }
 
 // ChatResponse is the body of a non-streaming chat completion
-// response.
+// response. Nil metadata fields mean unreported values.
 type ChatResponse struct {
 	// Choices contains the generated alternatives.
 	Choices []Choice `json:"choices"`
-	// ID identifies the completion.
-	ID string `json:"id"`
+	// Created is the endpoint's completion creation time in Unix seconds.
+	Created *int64 `json:"created,omitempty"`
+	// ID identifies the completion in the response body, not a request
+	// header.
+	ID *string `json:"id,omitempty"`
 	// Model identifies the model that generated the response.
-	Model string `json:"model"`
+	Model *string `json:"model,omitempty"`
+	// ServiceTier is the processing tier reported by the endpoint.
+	ServiceTier *string `json:"service_tier,omitempty"`
+	// SystemFingerprint identifies the reported backend configuration.
+	SystemFingerprint *string `json:"system_fingerprint,omitempty"`
 	// Timings contains llama.cpp performance data, when reported.
 	Timings *Timings `json:"timings,omitempty"`
 	// Usage contains token counts, when reported.
@@ -185,12 +344,15 @@ type ChatResponse struct {
 
 // Choice is one generated alternative in a non-streaming response. Index is its
 // position among the alternatives, and FinishReason reports why generation
-// ended.
+// ended. Nil metadata fields mean the endpoint did not report those values.
 type Choice struct {
 	// FinishReason is the endpoint's reason for ending generation.
-	FinishReason string `json:"finish_reason"`
-	// Index is the position of this choice in the response.
-	Index int `json:"index"`
+	FinishReason *string `json:"finish_reason,omitempty"`
+	// Index is the zero-based position of this choice in the response.
+	Index *int `json:"index,omitempty"`
+	// Logprobs contains probabilities for this choice's content and
+	// refusal.
+	Logprobs *Logprobs `json:"logprobs,omitempty"`
 	// Message is the generated message.
 	Message Message `json:"message"`
 }
@@ -468,22 +630,35 @@ type ResponseFormat struct {
 
 // StreamChunk is one data event in a streaming response. Delta fields are
 // incremental and must be combined in order to reconstruct a message. A chunk
-// may contain Usage or Timings without containing a choice.
+// may contain Usage or Timings without containing a choice. Nil metadata fields
+// mean unreported values, not reported zero counts or empty strings.
 type StreamChunk struct {
 	// Choices contains incremental updates for generated alternatives.
 	// It may be empty in a metadata-only chunk.
 	Choices []struct {
-		// Index identifies the choice updated by this delta.
-		Index int `json:"index"`
 		// Delta contains the incremental message update.
 		Delta Delta `json:"delta"`
-		// FinishReason is set when generation for the choice ends.
-		FinishReason string `json:"finish_reason,omitempty"`
+		// FinishReason is the reported reason for ending this choice;
+		// it does not establish receipt of DONE.
+		FinishReason *string `json:"finish_reason,omitempty"`
+		// Index identifies the choice updated by this delta. Nil means
+		// unreported and selects the first choice for accumulation.
+		Index *int `json:"index,omitempty"`
+		// Logprobs contains the next content and refusal token
+		// probabilities.
+		Logprobs *Logprobs `json:"logprobs,omitempty"`
 	} `json:"choices"`
-	// ID identifies the completion to which the chunk belongs.
-	ID string `json:"id"`
+	// Created is the endpoint's completion creation time in Unix seconds.
+	Created *int64 `json:"created,omitempty"`
+	// ID identifies the completion in the response body, not a request
+	// header.
+	ID *string `json:"id,omitempty"`
 	// Model identifies the model generating the response.
-	Model string `json:"model"`
+	Model *string `json:"model,omitempty"`
+	// ServiceTier is the processing tier reported by the endpoint.
+	ServiceTier *string `json:"service_tier,omitempty"`
+	// SystemFingerprint identifies the reported backend configuration.
+	SystemFingerprint *string `json:"system_fingerprint,omitempty"`
 	// Timings contains llama.cpp performance data, when reported.
 	Timings *Timings `json:"timings,omitempty"`
 	// Usage contains aggregate token counts, when reported.