chimerical/odidere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactol llm pkg

Browse Source
This commit is contained in:
dwrz
2026-10-30 22:36:15 +00:00
parent 652d41ca8f
commit ceaab03ea2
3 changed files with 239 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/llm/client_completions.go
+12 -4
View File
@@ -72,9 +72,13 @@ func (c *Client) Completions(
func (c *Client) completions( func (c *Client) completions(
ctx context.Context, p CompletionsParams, ctx context.Context, p CompletionsParams,
) (*ChatResponse, error) { ) (*ChatResponse, error) {
kwargs := map[string]any{} // A nil map is omitted from the request; an empty map is
// sent as {}. Keep kwargs nil without a budget.
var kwargs map[string]any
if p.ThinkingBudgetTokens != nil { if p.ThinkingBudgetTokens != nil {
kwargs[chatTemplateEnableThinking] = *p.ThinkingBudgetTokens != 0 kwargs = map[string]any{
chatTemplateEnableThinking: *p.ThinkingBudgetTokens != 0,
}
} }
body, err := json.Marshal(ChatRequest{ body, err := json.Marshal(ChatRequest{
Model: p.Model, Model: p.Model,
@@ -270,9 +274,13 @@ func (c *Client) CompletionsStream(
) (*CompletionStream, error) { ) (*CompletionStream, error) {
ctx, cancel := context.WithTimeout(ctx, c.timeout) ctx, cancel := context.WithTimeout(ctx, c.timeout)
kwargs := map[string]any{} // A nil map is omitted from the request; an empty map is
// sent as {}. Keep kwargs nil without a budget.
var kwargs map[string]any
if p.ThinkingBudgetTokens != nil { if p.ThinkingBudgetTokens != nil {
kwargs[chatTemplateEnableThinking] = *p.ThinkingBudgetTokens != 0 kwargs = map[string]any{
chatTemplateEnableThinking: *p.ThinkingBudgetTokens != 0,
}
} }
body, err := json.Marshal(ChatRequest{ body, err := json.Marshal(ChatRequest{
ChatTemplateKWARGS: kwargs, ChatTemplateKWARGS: kwargs,
internal/llm/llm.go
+117 -75
View File
@@ -39,7 +39,7 @@ func (e *APIError) Error() string {
// Type is ToolTypeFunction. // Type is ToolTypeFunction.
type APITool struct { type APITool struct {
// Function defines the callable function when Type is ToolTypeFunction. // Function defines the callable function when Type is ToolTypeFunction.
Function *FunctionDef `json:"function,omitempty"` Function *FunctionDef `json:"function,omitzero"`
// Type selects the representation of the tool. // Type selects the representation of the tool.
Type ToolType `json:"type"` Type ToolType `json:"type"`
} }
@@ -53,65 +53,65 @@ type ChatRequest struct {
// ChatTemplateKWARGS contains arguments made available to the // ChatTemplateKWARGS contains arguments made available to the
// endpoint's chat template. // endpoint's chat template.
// It is a llama.cpp extension to the OpenAI API. // It is a llama.cpp extension to the OpenAI API.
ChatTemplateKWARGS map[string]any `json:"chat_template_kwargs,omitempty"` ChatTemplateKWARGS map[string]any `json:"chat_template_kwargs,omitzero"`
// FrequencyPenalty adjusts token likelihood according to frequency in // FrequencyPenalty adjusts token likelihood according to frequency in
// the generated text. Nil leaves the choice to the endpoint. // the generated text. Nil leaves the choice to the endpoint.
FrequencyPenalty *float32 `json:"frequency_penalty,omitempty"` FrequencyPenalty *float32 `json:"frequency_penalty,omitzero"`
// LogProbs requests log probabilities for generated tokens. // LogProbs requests log probabilities for generated tokens.
LogProbs bool `json:"logprobs,omitempty"` LogProbs bool `json:"logprobs,omitzero"`
// MaxCompletionTokens bounds generated tokens, including reasoning // MaxCompletionTokens bounds generated tokens, including reasoning
// tokens. // tokens.
MaxCompletionTokens int `json:"max_completion_tokens,omitempty"` MaxCompletionTokens int `json:"max_completion_tokens,omitzero"`
// MaxTokens is the deprecated predecessor of MaxCompletionTokens. // MaxTokens is the deprecated predecessor of MaxCompletionTokens.
MaxTokens int `json:"max_tokens,omitempty"` MaxTokens int `json:"max_tokens,omitzero"`
// Messages is the conversation presented to the model, in order. // Messages is the conversation presented to the model, in order.
Messages []Message `json:"messages"` Messages []Message `json:"messages"`
// Model identifies the model to use according to the endpoint. // Model identifies the model to use according to the endpoint.
Model string `json:"model"` Model string `json:"model"`
// N is the number of alternatives to generate. // N is the number of alternatives to generate.
N int `json:"n,omitempty"` N int `json:"n,omitzero"`
// ParallelToolCalls controls whether the model may request multiple // ParallelToolCalls controls whether the model may request multiple
// function calls in one turn. Nil leaves the choice to the endpoint. // function calls in one turn. Nil leaves the choice to the endpoint.
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` ParallelToolCalls *bool `json:"parallel_tool_calls,omitzero"`
// PresencePenalty adjusts token likelihood according to whether a // PresencePenalty adjusts token likelihood according to whether a
// token has already appeared. Nil leaves the choice to the endpoint. // token has already appeared. Nil leaves the choice to the endpoint.
PresencePenalty *float32 `json:"presence_penalty,omitempty"` PresencePenalty *float32 `json:"presence_penalty,omitzero"`
// ResponseFormat constrains the representation of generated content. // ResponseFormat constrains the representation of generated content.
ResponseFormat *ResponseFormat `json:"response_format,omitempty"` ResponseFormat *ResponseFormat `json:"response_format,omitzero"`
// Seed requests deterministic sampling on endpoints that support it. // Seed requests deterministic sampling on endpoints that support it.
// Equal seeds and parameters do not guarantee equal results. // Equal seeds and parameters do not guarantee equal results.
Seed *int `json:"seed,omitempty"` Seed *int `json:"seed,omitzero"`
// ServiceTier selects the endpoint's request-processing tier. // ServiceTier selects the endpoint's request-processing tier.
ServiceTier string `json:"service_tier,omitempty"` ServiceTier string `json:"service_tier,omitzero"`
// Stop contains sequences that end generation. // Stop contains sequences that end generation.
// The returned text excludes the matched sequence. // The returned text excludes the matched sequence.
Stop []string `json:"stop,omitempty"` Stop []string `json:"stop,omitzero"`
// Stream requests a sequence of StreamChunk values instead of // Stream requests a sequence of StreamChunk values instead of
// ChatResponse. // ChatResponse.
Stream bool `json:"stream,omitempty"` Stream bool `json:"stream,omitzero"`
// StreamOptions configures a streaming response. // StreamOptions configures a streaming response.
// It is meaningful only when Stream is true. // It is meaningful only when Stream is true.
StreamOptions *StreamOptions `json:"stream_options,omitempty"` StreamOptions *StreamOptions `json:"stream_options,omitzero"`
// Temperature controls sampling randomness. // Temperature controls sampling randomness.
// Nil leaves the choice to the endpoint. // Nil leaves the choice to the endpoint.
Temperature *float32 `json:"temperature,omitempty"` Temperature *float32 `json:"temperature,omitzero"`
// ThinkingBudgetTokens is a llama.cpp extension controlling the token // ThinkingBudgetTokens is a llama.cpp extension controlling the token
// budget for reasoning. Nil omits it, a negative value requests an // budget for reasoning. Nil omits it, a negative value requests an
// unrestricted budget, zero requests an immediate end, and a positive // unrestricted budget, zero requests an immediate end, and a positive
// value sets the budget. // value sets the budget.
ThinkingBudgetTokens *int `json:"thinking_budget_tokens,omitempty"` ThinkingBudgetTokens *int `json:"thinking_budget_tokens,omitzero"`
// ToolChoice controls whether the model may or must call a tool. // ToolChoice controls whether the model may or must call a tool.
// It accepts a mode string or an endpoint-defined object selecting a // It accepts a mode string or an endpoint-defined object selecting a
// particular tool. // particular tool.
ToolChoice any `json:"tool_choice,omitempty"` ToolChoice any `json:"tool_choice,omitzero"`
// Tools describes the tools available to the model. // Tools describes the tools available to the model.
Tools []APITool `json:"tools,omitempty"` Tools []APITool `json:"tools,omitzero"`
// TopLogProbs is the number of most likely tokens reported at each // TopLogProbs is the number of most likely tokens reported at each
// position when LogProbs is true. // position when LogProbs is true.
TopLogProbs int `json:"top_logprobs,omitempty"` TopLogProbs int `json:"top_logprobs,omitzero"`
// TopP controls nucleus sampling. // TopP controls nucleus sampling.
// Nil leaves the choice to the endpoint. // Nil leaves the choice to the endpoint.
TopP *float32 `json:"top_p,omitempty"` TopP *float32 `json:"top_p,omitzero"`
} }
// Metadata describes one selected completion, separately from its message. // Metadata describes one selected completion, separately from its message.
@@ -327,20 +327,20 @@ type ChatResponse struct {
// Choices contains the generated alternatives. // Choices contains the generated alternatives.
Choices []Choice `json:"choices"` Choices []Choice `json:"choices"`
// Created is the endpoint's completion creation time in Unix seconds. // Created is the endpoint's completion creation time in Unix seconds.
Created *int64 `json:"created,omitempty"` Created *int64 `json:"created,omitzero"`
// ID identifies the completion in the response body, not a request // ID identifies the completion in the response body, not a request
// header. // header.
ID *string `json:"id,omitempty"` ID *string `json:"id,omitzero"`
// Model identifies the model that generated the response. // Model identifies the model that generated the response.
Model *string `json:"model,omitempty"` Model *string `json:"model,omitzero"`
// ServiceTier is the processing tier reported by the endpoint. // ServiceTier is the processing tier reported by the endpoint.
ServiceTier *string `json:"service_tier,omitempty"` ServiceTier *string `json:"service_tier,omitzero"`
// SystemFingerprint identifies the reported backend configuration. // SystemFingerprint identifies the reported backend configuration.
SystemFingerprint *string `json:"system_fingerprint,omitempty"` SystemFingerprint *string `json:"system_fingerprint,omitzero"`
// Timings contains llama.cpp performance data, when reported. // Timings contains llama.cpp performance data, when reported.
Timings *Timings `json:"timings,omitempty"` Timings *Timings `json:"timings,omitzero"`
// Usage contains token counts, when reported. // Usage contains token counts, when reported.
Usage *Usage `json:"usage,omitempty"` Usage *Usage `json:"usage,omitzero"`
} }
// Choice is one generated alternative in a non-streaming response. Index is its // Choice is one generated alternative in a non-streaming response. Index is its
@@ -348,12 +348,12 @@ type ChatResponse struct {
// ended. Nil metadata fields mean the endpoint did not report those values. // ended. Nil metadata fields mean the endpoint did not report those values.
type Choice struct { type Choice struct {
// FinishReason is the endpoint's reason for ending generation. // FinishReason is the endpoint's reason for ending generation.
FinishReason *string `json:"finish_reason,omitempty"` FinishReason *string `json:"finish_reason,omitzero"`
// Index is the zero-based position of this choice in the response. // Index is the zero-based position of this choice in the response.
Index *int `json:"index,omitempty"` Index *int `json:"index,omitzero"`
// Logprobs contains probabilities for this choice's content and // Logprobs contains probabilities for this choice's content and
// refusal. // refusal.
Logprobs *Logprobs `json:"logprobs,omitempty"` Logprobs *Logprobs `json:"logprobs,omitzero"`
// Message is the generated message. // Message is the generated message.
Message Message `json:"message"` Message Message `json:"message"`
} }
@@ -379,48 +379,90 @@ const (
// Text. The package does not validate that correspondence. // Text. The package does not validate that correspondence.
type ContentPart struct { type ContentPart struct {
// File contains the payload for ContentTypeFile. // File contains the payload for ContentTypeFile.
File *FileInput `json:"file,omitempty"` File *FileInput `json:"file,omitzero"`
// ImageURL contains the payload for ContentTypeImageURL. // ImageURL contains the payload for ContentTypeImageURL.
ImageURL *ImageURL `json:"image_url,omitempty"` ImageURL *ImageURL `json:"image_url,omitzero"`
// InputAudio contains the payload for ContentTypeInputAudio. // InputAudio contains the payload for ContentTypeInputAudio.
InputAudio *InputAudio `json:"input_audio,omitempty"` InputAudio *InputAudio `json:"input_audio,omitzero"`
// Refusal contains the payload for ContentTypeRefusal. // Refusal contains the payload for ContentTypeRefusal.
Refusal string `json:"refusal,omitempty"` Refusal string `json:"refusal,omitzero"`
// Text contains the payload for ContentTypeText. // Text contains the payload for ContentTypeText.
Text string `json:"text,omitempty"` Text string `json:"text,omitzero"`
// Type identifies which payload field is meaningful. // Type identifies which payload field is meaningful.
Type ContentType `json:"type"` Type ContentType `json:"type"`
} }
// MarshalJSON writes the payload field that Type selects, then the type.
// The payload field is always written, even when it is empty. The OpenAI
// API requires the payload field in every content part, and a text part
// without a text key makes llama.cpp reject the request. Struct tags
// cannot express this: a field that is required for one part type must be
// omitted from the other part types. An unknown Type falls back to the
// struct tags.
func (p ContentPart) MarshalJSON() ([]byte, error) {
switch p.Type {
case ContentTypeText:
return json.Marshal(struct {
Text string `json:"text"`
Type ContentType `json:"type"`
}{p.Text, p.Type})
case ContentTypeRefusal:
return json.Marshal(struct {
Refusal string `json:"refusal"`
Type ContentType `json:"type"`
}{p.Refusal, p.Type})
case ContentTypeImageURL:
return json.Marshal(struct {
ImageURL *ImageURL `json:"image_url"`
Type ContentType `json:"type"`
}{p.ImageURL, p.Type})
case ContentTypeInputAudio:
return json.Marshal(struct {
InputAudio *InputAudio `json:"input_audio"`
Type ContentType `json:"type"`
}{p.InputAudio, p.Type})
case ContentTypeFile:
return json.Marshal(struct {
File *FileInput `json:"file"`
Type ContentType `json:"type"`
}{p.File, p.Type})
}
// Unknown types keep the struct-tag encoding. The method must not
// reject them: the endpoint, not this client, defines which types
// are valid.
type alias ContentPart
return json.Marshal(alias(p))
}
// Delta is an incremental message update in a StreamChunk. Reasoning and // Delta is an incremental message update in a StreamChunk. Reasoning and
// ReasoningContent are alternative field names used by compatible endpoints; // ReasoningContent are alternative field names used by compatible endpoints;
// StreamAccumulator preserves fragments from both. // StreamAccumulator preserves fragments from both.
type Delta struct { type Delta struct {
// Content is the next fragment of message text. // Content is the next fragment of message text.
Content string `json:"content,omitempty"` Content string `json:"content,omitzero"`
// Reasoning is the next reasoning fragment under the reasoning field // Reasoning is the next reasoning fragment under the reasoning field
// name. // name.
Reasoning string `json:"reasoning,omitempty"` Reasoning string `json:"reasoning,omitzero"`
// ReasoningContent is the next reasoning fragment under the // ReasoningContent is the next reasoning fragment under the
// reasoning_content field name. // reasoning_content field name.
ReasoningContent string `json:"reasoning_content,omitempty"` ReasoningContent string `json:"reasoning_content,omitzero"`
// Refusal is the next fragment of refusal text. // Refusal is the next fragment of refusal text.
Refusal string `json:"refusal,omitempty"` Refusal string `json:"refusal,omitzero"`
// Role identifies the message role when supplied by the endpoint. // Role identifies the message role when supplied by the endpoint.
Role Role `json:"role,omitempty"` Role Role `json:"role,omitzero"`
// ToolCalls contains the next fragments of one or more tool calls. // ToolCalls contains the next fragments of one or more tool calls.
ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCalls []ToolCall `json:"tool_calls,omitzero"`
} }
// FileInput identifies file content either inline through FileData or by a // FileInput identifies file content either inline through FileData or by a
// server-assigned FileID. // server-assigned FileID.
type FileInput struct { type FileInput struct {
// FileData is the base64-encoded contents of an inline file. // FileData is the base64-encoded contents of an inline file.
FileData string `json:"file_data,omitempty"` FileData string `json:"file_data,omitzero"`
// FileID references a file previously uploaded to the server. // FileID references a file previously uploaded to the server.
FileID string `json:"file_id,omitempty"` FileID string `json:"file_id,omitzero"`
// Filename names an inline file supplied through FileData. // Filename names an inline file supplied through FileData.
Filename string `json:"filename,omitempty"` Filename string `json:"filename,omitzero"`
} }
// Function identifies a called function and carries its arguments as a JSON // Function identifies a called function and carries its arguments as a JSON
@@ -428,15 +470,15 @@ type FileInput struct {
// before invoking the function. // before invoking the function.
type Function struct { type Function struct {
// Arguments is the JSON-encoded argument object generated by the model. // Arguments is the JSON-encoded argument object generated by the model.
Arguments string `json:"arguments,omitempty"` Arguments string `json:"arguments,omitzero"`
// Name identifies the function to call. // Name identifies the function to call.
Name string `json:"name,omitempty"` Name string `json:"name,omitzero"`
} }
// FunctionDef declares a function the model may call. // FunctionDef declares a function the model may call.
type FunctionDef struct { type FunctionDef struct {
// Description states what the function does and when to call it. // Description states what the function does and when to call it.
Description string `json:"description,omitempty"` Description string `json:"description,omitzero"`
// Name identifies the function. // Name identifies the function.
Name string `json:"name"` Name string `json:"name"`
// Parameters is a JSON Schema describing the arguments accepted by the // Parameters is a JSON Schema describing the arguments accepted by the
@@ -444,7 +486,7 @@ type FunctionDef struct {
Parameters any `json:"parameters"` Parameters any `json:"parameters"`
// Strict requests exact conformance to Parameters, where the server // Strict requests exact conformance to Parameters, where the server
// supports it. // supports it.
Strict bool `json:"strict,omitempty"` Strict bool `json:"strict,omitzero"`
} }
// ImageDetail selects the fidelity used when processing an image. Auto lets the // ImageDetail selects the fidelity used when processing an image. Auto lets the
@@ -460,7 +502,7 @@ const (
// ImageURL carries an image content part. // ImageURL carries an image content part.
type ImageURL struct { type ImageURL struct {
// Detail is optional; when omitted the server chooses. // Detail is optional; when omitted the server chooses.
Detail ImageDetail `json:"detail,omitempty"` Detail ImageDetail `json:"detail,omitzero"`
// URL is a public image URL or a base64 data URI. // URL is a public image URL or a base64 data URI.
URL string `json:"url"` URL string `json:"url"`
} }
@@ -494,23 +536,23 @@ type InputAudio struct {
type Message struct { type Message struct {
// ContentParts holds the message content in parts; it is the // ContentParts holds the message content in parts; it is the
// in-memory form of the JSON "content" field. // in-memory form of the JSON "content" field.
ContentParts []ContentPart `json:"content,omitempty"` ContentParts []ContentPart `json:"content,omitzero"`
// Name optionally identifies the sender. // Name optionally identifies the sender.
Name string `json:"name,omitempty"` Name string `json:"name,omitzero"`
// Reasoning and ReasoningContent are alternative wire fields used by // Reasoning and ReasoningContent are alternative wire fields used by
// compatible servers. After unmarshalling, ReasoningContent contains // compatible servers. After unmarshalling, ReasoningContent contains
// either representation, with reasoning_content taking precedence when // either representation, with reasoning_content taking precedence when
// both occur. // both occur.
Reasoning string `json:"reasoning,omitempty"` Reasoning string `json:"reasoning,omitzero"`
ReasoningContent string `json:"reasoning_content,omitempty"` ReasoningContent string `json:"reasoning_content,omitzero"`
// Refusal contains the endpoint's refusal text, when present. // Refusal contains the endpoint's refusal text, when present.
Refusal string `json:"refusal,omitempty"` Refusal string `json:"refusal,omitzero"`
// Role identifies the author or purpose of the message. // Role identifies the author or purpose of the message.
Role Role `json:"role"` Role Role `json:"role"`
// ToolCallID links a tool response to the call it answers. // ToolCallID links a tool response to the call it answers.
ToolCallID string `json:"tool_call_id,omitempty"` ToolCallID string `json:"tool_call_id,omitzero"`
// ToolCalls contains calls requested by an assistant message. // ToolCalls contains calls requested by an assistant message.
ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCalls []ToolCall `json:"tool_calls,omitzero"`
} }
// Text concatenates the Text payload of every text content part in order. It // Text concatenates the Text payload of every text content part in order. It
@@ -628,13 +670,13 @@ func (m *Message) UnmarshalJSON(data []byte) error {
// [OpenAI Models API]: https://platform.openai.com/docs/api-reference/models/list // [OpenAI Models API]: https://platform.openai.com/docs/api-reference/models/list
type Model struct { type Model struct {
// ContextLength is the model's context window in tokens, when reported. // ContextLength is the model's context window in tokens, when reported.
ContextLength int `json:"context_length,omitempty"` ContextLength int `json:"context_length,omitzero"`
// Created is the model's creation time as Unix seconds. // Created is the model's creation time as Unix seconds.
Created int64 `json:"created"` Created int64 `json:"created"`
// ID identifies the model in inference requests. // ID identifies the model in inference requests.
ID string `json:"id"` ID string `json:"id"`
// Meta contains endpoint-specific model metadata, when reported. // Meta contains endpoint-specific model metadata, when reported.
Meta *ModelMeta `json:"meta,omitempty"` Meta *ModelMeta `json:"meta,omitzero"`
// Object is the endpoint's object type for the model entry. // Object is the endpoint's object type for the model entry.
Object string `json:"object"` Object string `json:"object"`
// OwnedBy identifies the organization that owns the model, when // OwnedBy identifies the organization that owns the model, when
@@ -650,9 +692,9 @@ type Model struct {
// [llama.cpp API documentation]: https://llama.app/docs/api // [llama.cpp API documentation]: https://llama.app/docs/api
type ModelMeta struct { type ModelMeta struct {
// NCtx is the context size with which llama.cpp loaded the model. // NCtx is the context size with which llama.cpp loaded the model.
NCtx int `json:"n_ctx,omitempty"` NCtx int `json:"n_ctx,omitzero"`
// NCtxTrain is the model's training context size. // NCtxTrain is the model's training context size.
NCtxTrain int `json:"n_ctx_train,omitempty"` NCtxTrain int `json:"n_ctx_train,omitzero"`
} }
// ModelsResponse is the response body of GET /models. // ModelsResponse is the response body of GET /models.
@@ -678,7 +720,7 @@ const (
type ResponseFormat struct { type ResponseFormat struct {
// JSONSchema defines the required output schema when Type is // JSONSchema defines the required output schema when Type is
// json_schema. // json_schema.
JSONSchema any `json:"json_schema,omitempty"` JSONSchema any `json:"json_schema,omitzero"`
// Type selects text, json_object, or json_schema output. // Type selects text, json_object, or json_schema output.
Type string `json:"type"` Type string `json:"type"`
} }
@@ -695,29 +737,29 @@ type StreamChunk struct {
Delta Delta `json:"delta"` Delta Delta `json:"delta"`
// FinishReason is the reported reason for ending this choice; // FinishReason is the reported reason for ending this choice;
// it does not establish receipt of DONE. // it does not establish receipt of DONE.
FinishReason *string `json:"finish_reason,omitempty"` FinishReason *string `json:"finish_reason,omitzero"`
// Index identifies the choice updated by this delta. Nil means // Index identifies the choice updated by this delta. Nil means
// unreported and selects the first choice for accumulation. // unreported and selects the first choice for accumulation.
Index *int `json:"index,omitempty"` Index *int `json:"index,omitzero"`
// Logprobs contains the next content and refusal token // Logprobs contains the next content and refusal token
// probabilities. // probabilities.
Logprobs *Logprobs `json:"logprobs,omitempty"` Logprobs *Logprobs `json:"logprobs,omitzero"`
} `json:"choices"` } `json:"choices"`
// Created is the endpoint's completion creation time in Unix seconds. // Created is the endpoint's completion creation time in Unix seconds.
Created *int64 `json:"created,omitempty"` Created *int64 `json:"created,omitzero"`
// ID identifies the completion in the response body, not a request // ID identifies the completion in the response body, not a request
// header. // header.
ID *string `json:"id,omitempty"` ID *string `json:"id,omitzero"`
// Model identifies the model generating the response. // Model identifies the model generating the response.
Model *string `json:"model,omitempty"` Model *string `json:"model,omitzero"`
// ServiceTier is the processing tier reported by the endpoint. // ServiceTier is the processing tier reported by the endpoint.
ServiceTier *string `json:"service_tier,omitempty"` ServiceTier *string `json:"service_tier,omitzero"`
// SystemFingerprint identifies the reported backend configuration. // SystemFingerprint identifies the reported backend configuration.
SystemFingerprint *string `json:"system_fingerprint,omitempty"` SystemFingerprint *string `json:"system_fingerprint,omitzero"`
// Timings contains llama.cpp performance data, when reported. // Timings contains llama.cpp performance data, when reported.
Timings *Timings `json:"timings,omitempty"` Timings *Timings `json:"timings,omitzero"`
// Usage contains aggregate token counts, when reported. // Usage contains aggregate token counts, when reported.
Usage *Usage `json:"usage,omitempty"` Usage *Usage `json:"usage,omitzero"`
} }
// StreamOptions configures a streaming response. // StreamOptions configures a streaming response.
@@ -725,7 +767,7 @@ type StreamOptions struct {
// IncludeUsage requests an additional chunk containing aggregate token // IncludeUsage requests an additional chunk containing aggregate token
// usage before the terminal data marker. If the stream is interrupted, // usage before the terminal data marker. If the stream is interrupted,
// that chunk may not arrive. // that chunk may not arrive.
IncludeUsage bool `json:"include_usage,omitempty"` IncludeUsage bool `json:"include_usage,omitzero"`
} }
// ToolCall is one tool call requested by the model. // ToolCall is one tool call requested by the model.
@@ -734,10 +776,10 @@ type ToolCall struct {
Function Function `json:"function"` Function Function `json:"function"`
// ID identifies the call. The corresponding tool response carries the // ID identifies the call. The corresponding tool response carries the
// same value in Message.ToolCallID. // same value in Message.ToolCallID.
ID string `json:"id,omitempty"` ID string `json:"id,omitzero"`
// Index identifies the call whose fragments a streaming delta // Index identifies the call whose fragments a streaming delta
// contains. It is omitted from non-streaming responses. // contains. It is omitted from non-streaming responses.
Index *int `json:"index,omitempty"` Index *int `json:"index,omitzero"`
// Type identifies the kind of tool being called. // Type identifies the kind of tool being called.
Type ToolType `json:"type"` Type ToolType `json:"type"`
} }
internal/llm/llm_test.go
+110
View File
@@ -34,6 +34,17 @@ func TestContentPartMarshalJSON(t *testing.T) {
}, },
want: `{"refusal":"cannot comply","type":"refusal"}`, want: `{"refusal":"cannot comply","type":"refusal"}`,
}, },
{
// The payload field is required, also when it is empty.
name: "empty text",
part: ContentPart{Type: ContentTypeText},
want: `{"text":"","type":"text"}`,
},
{
name: "empty refusal",
part: ContentPart{Type: ContentTypeRefusal},
want: `{"refusal":"","type":"refusal"}`,
},
{ {
name: "image URL", name: "image URL",
part: ContentPart{ part: ContentPart{
@@ -157,3 +168,102 @@ func TestMessageUnmarshalJSONReplacesArrayContent(t *testing.T) {
t.Errorf("Message.Text() = %q, want %q", got, "new") t.Errorf("Message.Text() = %q, want %q", got, "new")
} }
} }
// TestMessageEmptyTextRoundTrip is a regression test. ContentPart.Text used
// the omitzero option, so a text part with empty text marshaled as
// {"type":"text"}, without a text key. The message was stored in that form
// and sent to the model server on the next inference, and the server
// rejected the request: the OpenAI API requires the text field, and
// llama.cpp fails to parse the part. MarshalJSON now always writes the
// payload field. Rows stored in the old form are corrected when they are
// decoded and encoded again.
func TestMessageEmptyTextRoundTrip(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
// Model replies that contain only tool calls can send
// an empty content string.
name: "empty content string",
input: `{"role":"assistant","content":""}`,
want: `{"content":[{"text":"","type":"text"}],"role":"assistant"}`,
},
{
// The UI sends this when the user edits a message's
// text to empty.
name: "empty text part",
input: `{"role":"user","content":[{"type":"text","text":""}]}`,
want: `{"content":[{"text":"","type":"text"}],"role":"user"}`,
},
{
// Rows stored before the fix have this form.
name: "text part without text key",
input: `{"role":"user","content":[{"type":"text"}]}`,
want: `{"content":[{"text":"","type":"text"}],"role":"user"}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var message Message
if err := json.Unmarshal([]byte(tt.input), &message); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
got, err := json.Marshal(message)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if string(got) != tt.want {
t.Errorf("json.Marshal() = %s, want %s", got, tt.want)
}
})
}
}
// TestResponseFormatMarshalJSON records the schema-omission policy of
// ResponseFormat. The omitzero option drops only a nil schema. An empty
// JSON schema object is valid and permits any output, so it must be
// sent, not dropped. The omitempty option would drop it.
func TestResponseFormatMarshalJSON(t *testing.T) {
tests := []struct {
name string
format ResponseFormat
want string
}{
{
name: "nil schema",
format: ResponseFormat{Type: "json_object"},
want: `{"type":"json_object"}`,
},
{
name: "schema object",
format: ResponseFormat{
JSONSchema: map[string]any{"name": "calendar"},
Type: "json_schema",
},
want: `{"json_schema":{"name":"calendar"},"type":"json_schema"}`,
},
{
name: "empty schema object",
format: ResponseFormat{
JSONSchema: map[string]any{},
Type: "json_schema",
},
want: `{"json_schema":{},"type":"json_schema"}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := json.Marshal(tt.format)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if string(got) != tt.want {
t.Errorf("json.Marshal() = %s, want %s", got, tt.want)
}
})
}
}