Refactor llm package
This commit is contained in:
dwrz
+373
-164
@@ -1,16 +1,5 @@
|
||||
// Package llm provides an OpenAI-compatible client for LLM interactions.
|
||||
// It handles chat completions with automatic tool call execution, where the
|
||||
// client iteratively sends messages to the LLM, executes any requested tool
|
||||
// calls, and continues until the LLM produces a final text response.
|
||||
//
|
||||
// The package defines the core types for constructing requests and parsing
|
||||
// responses: Message, ChatRequest, ChatResponse, and streaming types
|
||||
// such as StreamChunk and Delta.
|
||||
//
|
||||
// Tools are defined declaratively in YAML configuration, stored as Tool
|
||||
// structs in a Registry, and executed as subprocesses when the LLM invokes
|
||||
// them. The Registry also converts tool definitions into the API format for
|
||||
// inclusion in ChatRequest bodies.
|
||||
// Package llm implements an OpenAI-compatible inference client and its
|
||||
// protocol types.
|
||||
|
||||
package llm
|
||||
|
||||
@@ -19,195 +8,350 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Role constants for messages.
|
||||
const (
|
||||
RoleSystem = "system"
|
||||
RoleUser = "user"
|
||||
RoleAssistant = "assistant"
|
||||
RoleTool = "tool"
|
||||
RoleDeveloper = "developer"
|
||||
)
|
||||
// APIError contains the structured error details returned by an
|
||||
// OpenAI-compatible API. Error values are defined by the server. Compatible
|
||||
// servers need not use the same set of types or codes. Callers should use
|
||||
// Message for diagnostics and Type and Code for programmatic error handling.
|
||||
//
|
||||
// See [OpenAI API error codes].
|
||||
//
|
||||
// [OpenAI API error codes]: https://developers.openai.com/api/docs/guides/error-codes
|
||||
type APIError struct {
|
||||
// Code identifies the precise error condition reported by the server.
|
||||
// Callers may use it to distinguish errors that have the same Type.
|
||||
Code string `json:"code"`
|
||||
|
||||
// APITool defines a tool available to the LLM in the API request format.
|
||||
// Message explains what went wrong. Its wording is server-defined and
|
||||
// should not be parsed by callers.
|
||||
Message string `json:"message"`
|
||||
|
||||
// Type identifies the general kind of failure reported by the server.
|
||||
// Callers may use it to handle related error conditions as a group.
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// Error returns the server-defined code and diagnostic message.
|
||||
func (e *APIError) Error() string {
|
||||
return e.Code + ": " + e.Message
|
||||
}
|
||||
|
||||
// APITool describes a tool that a model may call. Function must be set when
|
||||
// Type is ToolTypeFunction.
|
||||
type APITool struct {
|
||||
Type string `json:"type"` // "function"
|
||||
// Function defines the callable function when Type is ToolTypeFunction.
|
||||
Function *FunctionDef `json:"function,omitempty"`
|
||||
// Type selects the representation of the tool.
|
||||
Type ToolType `json:"type"`
|
||||
}
|
||||
|
||||
// ChatRequest is the request body for chat completions.
|
||||
// ChatRequest is the request body of POST /chat/completions.
|
||||
//
|
||||
// See [OpenAI Chat Completions API].
|
||||
//
|
||||
// [OpenAI Chat Completions API]: https://platform.openai.com/docs/api-reference/chat/create
|
||||
type ChatRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []Message `json:"messages"`
|
||||
Temperature *float32 `json:"temperature,omitempty"`
|
||||
TopP *float32 `json:"top_p,omitempty"`
|
||||
N int `json:"n,omitempty"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
Stop []string `json:"stop,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
MaxCompletionTokens int `json:"max_completion_tokens,omitempty"`
|
||||
ResponseFormat *ResponseFormat `json:"response_format,omitempty"`
|
||||
Seed *int `json:"seed,omitempty"`
|
||||
PresencePenalty *float32 `json:"presence_penalty,omitempty"`
|
||||
FrequencyPenalty *float32 `json:"frequency_penalty,omitempty"`
|
||||
LogProbs bool `json:"logprobs,omitempty"`
|
||||
TopLogProbs int `json:"top_logprobs,omitempty"`
|
||||
Tools []APITool `json:"tools,omitempty"`
|
||||
ToolChoice any `json:"tool_choice,omitempty"`
|
||||
StreamOptions *StreamOptions `json:"stream_options,omitempty"`
|
||||
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"`
|
||||
ChatTemplateKWARGS map[string]any `json:"chat_template_kwargs,omitempty"`
|
||||
ThinkingBudgetTokens *int `json:"thinking_budget_tokens,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
// ChatTemplateKWARGS contains arguments made available to the
|
||||
// endpoint's chat template.
|
||||
// It is a llama.cpp extension to the OpenAI API.
|
||||
ChatTemplateKWARGS map[string]any `json:"chat_template_kwargs,omitempty"`
|
||||
// FrequencyPenalty adjusts token likelihood according to frequency in
|
||||
// the generated text. Nil leaves the choice to the endpoint.
|
||||
FrequencyPenalty *float32 `json:"frequency_penalty,omitempty"`
|
||||
// LogProbs requests log probabilities for generated tokens.
|
||||
LogProbs bool `json:"logprobs,omitempty"`
|
||||
// MaxCompletionTokens bounds generated tokens, including reasoning
|
||||
// tokens.
|
||||
MaxCompletionTokens int `json:"max_completion_tokens,omitempty"`
|
||||
// MaxTokens is the deprecated predecessor of MaxCompletionTokens.
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
// Messages is the conversation presented to the model, in order.
|
||||
Messages []Message `json:"messages"`
|
||||
// Model identifies the model to use according to the endpoint.
|
||||
Model string `json:"model"`
|
||||
// N is the number of alternatives to generate.
|
||||
N int `json:"n,omitempty"`
|
||||
// ParallelToolCalls controls whether the model may request multiple
|
||||
// function calls in one turn. Nil leaves the choice to the endpoint.
|
||||
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"`
|
||||
// PresencePenalty adjusts token likelihood according to whether a
|
||||
// token has already appeared. Nil leaves the choice to the endpoint.
|
||||
PresencePenalty *float32 `json:"presence_penalty,omitempty"`
|
||||
// ResponseFormat constrains the representation of generated content.
|
||||
ResponseFormat *ResponseFormat `json:"response_format,omitempty"`
|
||||
// Seed requests deterministic sampling on endpoints that support it.
|
||||
// Equal seeds and parameters do not guarantee equal results.
|
||||
Seed *int `json:"seed,omitempty"`
|
||||
// ServiceTier selects the endpoint's request-processing tier.
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
// Stop contains sequences that end generation.
|
||||
// The returned text excludes the matched sequence.
|
||||
Stop []string `json:"stop,omitempty"`
|
||||
// Stream requests a sequence of StreamChunk values instead of
|
||||
// ChatResponse.
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
// StreamOptions configures a streaming response.
|
||||
// It is meaningful only when Stream is true.
|
||||
StreamOptions *StreamOptions `json:"stream_options,omitempty"`
|
||||
// Temperature controls sampling randomness.
|
||||
// Nil leaves the choice to the endpoint.
|
||||
Temperature *float32 `json:"temperature,omitempty"`
|
||||
// ThinkingBudgetTokens is a llama.cpp extension controlling the token
|
||||
// budget for reasoning. Nil omits it, a negative value requests an
|
||||
// unrestricted budget, zero requests an immediate end, and a positive
|
||||
// value sets the budget.
|
||||
ThinkingBudgetTokens *int `json:"thinking_budget_tokens,omitempty"`
|
||||
// ToolChoice controls whether the model may or must call a tool.
|
||||
// It accepts a mode string or an endpoint-defined object selecting a
|
||||
// particular tool.
|
||||
ToolChoice any `json:"tool_choice,omitempty"`
|
||||
// Tools describes the tools available to the model.
|
||||
Tools []APITool `json:"tools,omitempty"`
|
||||
// TopLogProbs is the number of most likely tokens reported at each
|
||||
// position when LogProbs is true.
|
||||
TopLogProbs int `json:"top_logprobs,omitempty"`
|
||||
// TopP controls nucleus sampling.
|
||||
// Nil leaves the choice to the endpoint.
|
||||
TopP *float32 `json:"top_p,omitempty"`
|
||||
}
|
||||
|
||||
// Usage represents token usage from the API response.
|
||||
// Usage reports input, output, and total token counts for a completion.
|
||||
type Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"`
|
||||
// CompletionTokens is the number of tokens generated by the model.
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
// CompletionTokensDetails breaks down CompletionTokens by category.
|
||||
CompletionTokensDetails *TokenDetails `json:"completion_tokens_details,omitempty"`
|
||||
// PromptTokens is the number of tokens in the input.
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// PromptTokensDetails breaks down PromptTokens by category.
|
||||
PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"`
|
||||
// TotalTokens is the sum of PromptTokens and CompletionTokens.
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
// TokenDetails breaks down token usage by category.
|
||||
// TokenDetails reports token categories within a Usage count. A zero value may
|
||||
// mean either that no tokens belong to the category or that it was not
|
||||
// reported.
|
||||
type TokenDetails struct {
|
||||
// CachedTokens is the number of input tokens served from a cache.
|
||||
CachedTokens int `json:"cached_tokens,omitempty"`
|
||||
// ReasoningTokens is the number of output tokens used for reasoning.
|
||||
ReasoningTokens int `json:"reasoning_tokens,omitempty"`
|
||||
CachedTokens int `json:"cached_tokens,omitempty"`
|
||||
}
|
||||
|
||||
// Timings is a llama.cpp-specific extension to the chat completion
|
||||
// response. It is not part of the OpenAI API spec.
|
||||
// Timings reports llama.cpp prompt-processing and token-generation performance.
|
||||
// The N fields are token counts, MS fields are elapsed milliseconds,
|
||||
// PerTokenMS fields are milliseconds per token, and PerSecond fields are tokens
|
||||
// per second. Timings is a llama.cpp extension to the OpenAI API.
|
||||
//
|
||||
// See [llama.cpp API documentation].
|
||||
//
|
||||
// [llama.cpp API documentation]: https://llama.app/docs/api
|
||||
type Timings struct {
|
||||
PromptN int `json:"prompt_n"`
|
||||
PromptMS float64 `json:"prompt_ms"`
|
||||
PromptPerTokenMS float64 `json:"prompt_per_token_ms"`
|
||||
PromptPerSecond float64 `json:"prompt_per_second"`
|
||||
PredictedN int `json:"predicted_n"`
|
||||
PredictedMS float64 `json:"predicted_ms"`
|
||||
// PredictedMS is the time spent generating output, in milliseconds.
|
||||
PredictedMS float64 `json:"predicted_ms"`
|
||||
// PredictedN is the number of generated tokens.
|
||||
PredictedN int `json:"predicted_n"`
|
||||
// PredictedPerSecond is the output-generation rate in tokens per
|
||||
// second.
|
||||
PredictedPerSecond float64 `json:"predicted_per_second"`
|
||||
// PredictedPerTokenMS is the mean output-generation time per token, in
|
||||
// milliseconds.
|
||||
PredictedPerTokenMS float64 `json:"predicted_per_token_ms"`
|
||||
PredictedPerSecond float64 `json:"predicted_per_second"`
|
||||
// PromptMS is the time spent processing the prompt, in milliseconds.
|
||||
PromptMS float64 `json:"prompt_ms"`
|
||||
// PromptN is the number of prompt tokens processed.
|
||||
PromptN int `json:"prompt_n"`
|
||||
// PromptPerSecond is the prompt-processing rate in tokens per second.
|
||||
PromptPerSecond float64 `json:"prompt_per_second"`
|
||||
// PromptPerTokenMS is the mean prompt-processing time per token, in
|
||||
// milliseconds.
|
||||
PromptPerTokenMS float64 `json:"prompt_per_token_ms"`
|
||||
}
|
||||
|
||||
// ChatResponse is the non-streaming response from the API.
|
||||
// ChatResponse is the body of a non-streaming chat completion
|
||||
// response.
|
||||
type ChatResponse struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
// Choices contains the generated alternatives.
|
||||
Choices []Choice `json:"choices"`
|
||||
Usage *Usage `json:"usage,omitempty"`
|
||||
// ID identifies the completion.
|
||||
ID string `json:"id"`
|
||||
// Model identifies the model that generated the response.
|
||||
Model string `json:"model"`
|
||||
// Timings contains llama.cpp performance data, when reported.
|
||||
Timings *Timings `json:"timings,omitempty"`
|
||||
// Usage contains token counts, when reported.
|
||||
Usage *Usage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
// Choice is a single choice in a ChatResponse.
|
||||
// Choice is one generated alternative in a non-streaming response. Index is its
|
||||
// position among the alternatives, and FinishReason reports why generation
|
||||
// ended.
|
||||
type Choice struct {
|
||||
Index int `json:"index"`
|
||||
Message Message `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
// FinishReason is the endpoint's reason for ending generation.
|
||||
FinishReason string `json:"finish_reason"`
|
||||
// Index is the position of this choice in the response.
|
||||
Index int `json:"index"`
|
||||
// Message is the generated message.
|
||||
Message Message `json:"message"`
|
||||
}
|
||||
|
||||
// ContentType identifies the kind of data in a ContentPart.
|
||||
// ContentType identifies the representation of a ContentPart. Which content
|
||||
// types are valid depends on the message role and the endpoint.
|
||||
//
|
||||
// User messages (ChatCompletionContentPart) support: text, image_url,
|
||||
// input_audio, file.
|
||||
// See [OpenAI Chat Completions API].
|
||||
//
|
||||
// Assistant messages (ChatCompletionAssistantMessageParam) support:
|
||||
// text, refusal.
|
||||
//
|
||||
// See: https://platform.openai.com/docs/api-reference/chat/create
|
||||
// [OpenAI Chat Completions API]: https://platform.openai.com/docs/api-reference/chat/create
|
||||
type ContentType string
|
||||
|
||||
const (
|
||||
ContentTypeText ContentType = "text"
|
||||
ContentTypeFile ContentType = "file"
|
||||
ContentTypeImageURL ContentType = "image_url"
|
||||
ContentTypeInputAudio ContentType = "input_audio"
|
||||
ContentTypeFile ContentType = "file"
|
||||
ContentTypeRefusal ContentType = "refusal"
|
||||
ContentTypeText ContentType = "text"
|
||||
)
|
||||
|
||||
// ContentPart represents a single part of a multi-content message.
|
||||
// ContentPart is one element of a message's content array. Type determines
|
||||
// which payload field is meaningful: File, ImageURL, InputAudio, Refusal, or
|
||||
// Text. The package does not validate that correspondence.
|
||||
type ContentPart struct {
|
||||
Type ContentType `json:"type"`
|
||||
Text string `json:"text"`
|
||||
ImageURL *ImageURL `json:"image_url,omitempty"`
|
||||
// File contains the payload for ContentTypeFile.
|
||||
File *FileInput `json:"file,omitempty"`
|
||||
// ImageURL contains the payload for ContentTypeImageURL.
|
||||
ImageURL *ImageURL `json:"image_url,omitempty"`
|
||||
// InputAudio contains the payload for ContentTypeInputAudio.
|
||||
InputAudio *InputAudio `json:"input_audio,omitempty"`
|
||||
File *FileInput `json:"file,omitempty"`
|
||||
Refusal string `json:"refusal"`
|
||||
// Refusal contains the payload for ContentTypeRefusal.
|
||||
Refusal string `json:"refusal,omitempty"`
|
||||
// Text contains the payload for ContentTypeText.
|
||||
Text string `json:"text,omitempty"`
|
||||
// Type identifies which payload field is meaningful.
|
||||
Type ContentType `json:"type"`
|
||||
}
|
||||
|
||||
// Delta is a partial message update in a streaming response.
|
||||
// Delta is an incremental message update in a StreamChunk. Reasoning and
|
||||
// ReasoningContent are alternative field names used by compatible endpoints;
|
||||
// StreamAccumulator preserves fragments from both.
|
||||
type Delta struct {
|
||||
Role string `json:"role,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
Refusal string `json:"refusal,omitempty"`
|
||||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||
Reasoning string `json:"reasoning,omitempty"`
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||
// Content is the next fragment of message text.
|
||||
Content string `json:"content,omitempty"`
|
||||
// Reasoning is the next reasoning fragment under the reasoning field
|
||||
// name.
|
||||
Reasoning string `json:"reasoning,omitempty"`
|
||||
// ReasoningContent is the next reasoning fragment under the
|
||||
// reasoning_content field name.
|
||||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||
// Refusal is the next fragment of refusal text.
|
||||
Refusal string `json:"refusal,omitempty"`
|
||||
// Role identifies the message role when supplied by the endpoint.
|
||||
Role Role `json:"role,omitempty"`
|
||||
// ToolCalls contains the next fragments of one or more tool calls.
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||
}
|
||||
|
||||
// FileInput holds file data for file content parts.
|
||||
// FileInput identifies file content either inline through FileData or by a
|
||||
// server-assigned FileID.
|
||||
type FileInput struct {
|
||||
// FileData is the base64-encoded contents of an inline file.
|
||||
FileData string `json:"file_data,omitempty"`
|
||||
FileID string `json:"file_id,omitempty"`
|
||||
// FileID references a file previously uploaded to the server.
|
||||
FileID string `json:"file_id,omitempty"`
|
||||
// Filename names an inline file supplied through FileData.
|
||||
Filename string `json:"filename,omitempty"`
|
||||
}
|
||||
|
||||
// Function holds the name and arguments of a tool call.
|
||||
// Function identifies a called function and carries its arguments as a JSON
|
||||
// string. The model may emit invalid JSON, so callers must validate Arguments
|
||||
// before invoking the function.
|
||||
type Function struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
// Arguments is the JSON-encoded argument object generated by the model.
|
||||
Arguments string `json:"arguments,omitempty"`
|
||||
// Name identifies the function to call.
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// FunctionDef defines a function that can be called by the LLM.
|
||||
// FunctionDef declares a function the model may call.
|
||||
type FunctionDef struct {
|
||||
Name string `json:"name"`
|
||||
// Description states what the function does and when to call it.
|
||||
Description string `json:"description,omitempty"`
|
||||
Parameters any `json:"parameters"` // JSON schema
|
||||
Strict bool `json:"strict,omitempty"`
|
||||
// Name identifies the function.
|
||||
Name string `json:"name"`
|
||||
// Parameters is a JSON Schema describing the arguments accepted by the
|
||||
// function.
|
||||
Parameters any `json:"parameters"`
|
||||
// Strict requests exact conformance to Parameters, where the server
|
||||
// supports it.
|
||||
Strict bool `json:"strict,omitempty"`
|
||||
}
|
||||
|
||||
// ImageURL holds an image reference for multi-modal messages.
|
||||
// ImageDetail selects the fidelity used when processing an image. Auto lets the
|
||||
// endpoint choose.
|
||||
type ImageDetail string
|
||||
|
||||
const (
|
||||
ImageDetailAuto ImageDetail = "auto"
|
||||
ImageDetailHigh ImageDetail = "high"
|
||||
ImageDetailLow ImageDetail = "low"
|
||||
)
|
||||
|
||||
// ImageURL carries an image content part.
|
||||
type ImageURL struct {
|
||||
URL string `json:"url"`
|
||||
Detail string `json:"detail,omitempty"` // "auto", "low", "high"
|
||||
// Detail is optional; when omitted the server chooses.
|
||||
Detail ImageDetail `json:"detail,omitempty"`
|
||||
// URL is a public image URL or a base64 data URI.
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// InputAudio holds audio data for multi-modal messages.
|
||||
// AudioFormat names an audio encoding accepted in an input_audio
|
||||
// part.
|
||||
type AudioFormat string
|
||||
|
||||
const (
|
||||
AudioFormatWAV AudioFormat = "wav"
|
||||
AudioFormatMP3 AudioFormat = "mp3"
|
||||
)
|
||||
|
||||
// InputAudio carries base64-encoded WAV or MP3 audio.
|
||||
type InputAudio struct {
|
||||
Data string `json:"data"`
|
||||
Format string `json:"format"` // "wav", "mp3"
|
||||
// Data is base64-encoded audio data.
|
||||
Data string `json:"data"`
|
||||
// Format identifies the encoding of Data.
|
||||
Format AudioFormat `json:"format"`
|
||||
}
|
||||
|
||||
// Message represents a chat message in a conversation.
|
||||
// Content is always stored as []ContentPart internally.
|
||||
// A custom UnmarshalJSON handles the JSON polymorphism where content
|
||||
// can arrive as either a string or an array of parts.
|
||||
// Message is one message in a conversation or completion response.
|
||||
//
|
||||
// Refusal handling:
|
||||
// The JSON content field may be a string or an array. UnmarshalJSON normalizes
|
||||
// either representation into ContentParts; marshaling a Message encodes
|
||||
// ContentParts as an array.
|
||||
//
|
||||
// In API responses (ChatCompletionMessage), the model sends refusal as a
|
||||
// top-level field alongside content:
|
||||
// See [OpenAI Chat Completions API].
|
||||
//
|
||||
// "content": "text response"
|
||||
// "refusal": "I can't help with that"
|
||||
//
|
||||
// Only one of content or refusal will be non-empty.
|
||||
//
|
||||
// In API requests (ChatCompletionAssistantMessageParam), the model can
|
||||
// receive refusal as a content part with type "refusal" in the content
|
||||
// array. Our ContentPart type supports this for completeness.
|
||||
//
|
||||
// See: https://platform.openai.com/docs/api-reference/chat/create
|
||||
// [OpenAI Chat Completions API]: https://platform.openai.com/docs/api-reference/chat/create
|
||||
type Message struct {
|
||||
Role string `json:"role"`
|
||||
ContentParts []ContentPart `json:"content,omitempty"`
|
||||
Refusal string `json:"refusal,omitempty"`
|
||||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||
Reasoning string `json:"reasoning,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||
ToolCallID string `json:"tool_call_id,omitempty"`
|
||||
// ContentParts holds the message content in parts; it is the
|
||||
// in-memory form of the JSON "content" field.
|
||||
ContentParts []ContentPart `json:"content,omitempty"`
|
||||
// Name optionally identifies the sender.
|
||||
Name string `json:"name,omitempty"`
|
||||
// Reasoning and ReasoningContent are alternative wire fields used by
|
||||
// compatible servers. After unmarshalling, ReasoningContent contains
|
||||
// either representation, with reasoning_content taking precedence when
|
||||
// both occur.
|
||||
Reasoning string `json:"reasoning,omitempty"`
|
||||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||||
// Refusal contains the endpoint's refusal text, when present.
|
||||
Refusal string `json:"refusal,omitempty"`
|
||||
// Role identifies the author or purpose of the message.
|
||||
Role Role `json:"role"`
|
||||
// ToolCallID links a tool response to the call it answers.
|
||||
ToolCallID string `json:"tool_call_id,omitempty"`
|
||||
// ToolCalls contains calls requested by an assistant message.
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||
}
|
||||
|
||||
// Text returns the concatenated text from all text content parts.
|
||||
// Text concatenates the Text payload of every text content part in order. It
|
||||
// ignores non-text parts and does not fall back to Refusal or reasoning text.
|
||||
func (m *Message) Text() string {
|
||||
var b strings.Builder
|
||||
for _, p := range m.ContentParts {
|
||||
@@ -218,8 +362,10 @@ func (m *Message) Text() string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// UnmarshalJSON handles the polymorphic content field which can be
|
||||
// either a string or an array of ContentPart objects.
|
||||
// UnmarshalJSON decodes a message while normalizing a string-valued content
|
||||
// field to one text ContentPart. An array-valued content field is decoded in
|
||||
// order. If reasoning_content is absent, reasoning is copied into
|
||||
// ReasoningContent so callers can use the latter for either server spelling.
|
||||
func (m *Message) UnmarshalJSON(data []byte) error {
|
||||
type Alias Message
|
||||
aux := &struct {
|
||||
@@ -235,6 +381,8 @@ func (m *Message) UnmarshalJSON(data []byte) error {
|
||||
case string:
|
||||
m.ContentParts = []ContentPart{{Type: ContentTypeText, Text: c}}
|
||||
case []any:
|
||||
// Array content replaces any content already held by the Message.
|
||||
parts := make([]ContentPart, 0, len(c))
|
||||
for _, raw := range c {
|
||||
partBytes, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
@@ -244,68 +392,129 @@ func (m *Message) UnmarshalJSON(data []byte) error {
|
||||
if err := json.Unmarshal(partBytes, &part); err != nil {
|
||||
return err
|
||||
}
|
||||
m.ContentParts = append(m.ContentParts, part)
|
||||
parts = append(parts, part)
|
||||
}
|
||||
m.ContentParts = parts
|
||||
}
|
||||
// Normalize: if ReasoningContent is empty but Reasoning is set, use Reasoning
|
||||
// If ReasoningContent is empty but Reasoning is set, use Reasoning.
|
||||
if m.ReasoningContent == "" && m.Reasoning != "" {
|
||||
m.ReasoningContent = m.Reasoning
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Model represents an available model from the API.
|
||||
// Model describes one model exposed by the server's /models
|
||||
// endpoint.
|
||||
//
|
||||
// See [OpenAI Models API].
|
||||
//
|
||||
// [OpenAI Models API]: https://platform.openai.com/docs/api-reference/models/list
|
||||
type Model struct {
|
||||
ID string `json:"id"`
|
||||
Created int64 `json:"created"`
|
||||
Object string `json:"object"`
|
||||
OwnedBy string `json:"owned_by"`
|
||||
Meta *ModelMeta `json:"meta,omitempty"`
|
||||
ContextLength int `json:"context_length,omitempty"`
|
||||
// ContextLength is the model's context window in tokens, when reported.
|
||||
ContextLength int `json:"context_length,omitempty"`
|
||||
// Created is the model's creation time as Unix seconds.
|
||||
Created int64 `json:"created"`
|
||||
// ID identifies the model in inference requests.
|
||||
ID string `json:"id"`
|
||||
// Meta contains endpoint-specific model metadata, when reported.
|
||||
Meta *ModelMeta `json:"meta,omitempty"`
|
||||
// Object is the endpoint's object type for the model entry.
|
||||
Object string `json:"object"`
|
||||
// OwnedBy identifies the organization that owns the model, when
|
||||
//reported.
|
||||
OwnedBy string `json:"owned_by"`
|
||||
}
|
||||
|
||||
// ModelMeta contains optional model metadata from the provider.
|
||||
// llama.cpp includes n_ctx (configured context size) and
|
||||
// n_ctx_train (model's native training context).
|
||||
// ModelMeta contains llama.cpp model metadata. Context-size fields are measured
|
||||
// in tokens and are zero when omitted.
|
||||
//
|
||||
// See [llama.cpp API documentation].
|
||||
//
|
||||
// [llama.cpp API documentation]: https://llama.app/docs/api
|
||||
type ModelMeta struct {
|
||||
NCtx int `json:"n_ctx,omitempty"`
|
||||
// NCtx is the context size with which llama.cpp loaded the model.
|
||||
NCtx int `json:"n_ctx,omitempty"`
|
||||
// NCtxTrain is the model's training context size.
|
||||
NCtxTrain int `json:"n_ctx_train,omitempty"`
|
||||
}
|
||||
|
||||
// ModelsResponse is the response for GET /models.
|
||||
// ModelsResponse is the response body of GET /models.
|
||||
type ModelsResponse struct {
|
||||
Object string `json:"object"`
|
||||
Data []Model `json:"data"`
|
||||
// Data contains the models exposed by the endpoint.
|
||||
Data []Model `json:"data"`
|
||||
// Object is the endpoint's object type for the response.
|
||||
Object string `json:"object"`
|
||||
}
|
||||
|
||||
// ResponseFormat specifies the format of the response.
|
||||
// Role identifies the author or purpose of a message.
|
||||
type Role string
|
||||
|
||||
const (
|
||||
RoleAssistant Role = "assistant"
|
||||
RoleDeveloper Role = "developer"
|
||||
RoleSystem Role = "system"
|
||||
RoleTool Role = "tool"
|
||||
RoleUser Role = "user"
|
||||
)
|
||||
|
||||
// ResponseFormat constrains the format of the generated response.
|
||||
type ResponseFormat struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema any `json:"json_schema,omitempty"`
|
||||
// JSONSchema defines the required output schema when Type is
|
||||
// json_schema.
|
||||
JSONSchema any `json:"json_schema,omitempty"`
|
||||
// Type selects text, json_object, or json_schema output.
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// StreamChunk is a single SSE chunk from a streaming response.
|
||||
// StreamChunk is one data event in a streaming response. Delta fields are
|
||||
// incremental and must be combined in order to reconstruct a message. A chunk
|
||||
// may contain Usage or Timings without containing a choice.
|
||||
type StreamChunk struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
// Choices contains incremental updates for generated alternatives.
|
||||
// It may be empty in a metadata-only chunk.
|
||||
Choices []struct {
|
||||
Index int `json:"index"`
|
||||
Delta Delta `json:"delta"`
|
||||
// Index identifies the choice updated by this delta.
|
||||
Index int `json:"index"`
|
||||
// Delta contains the incremental message update.
|
||||
Delta Delta `json:"delta"`
|
||||
// FinishReason is set when generation for the choice ends.
|
||||
FinishReason string `json:"finish_reason,omitempty"`
|
||||
} `json:"choices"`
|
||||
Usage *Usage `json:"usage,omitempty"`
|
||||
// ID identifies the completion to which the chunk belongs.
|
||||
ID string `json:"id"`
|
||||
// Model identifies the model generating the response.
|
||||
Model string `json:"model"`
|
||||
// Timings contains llama.cpp performance data, when reported.
|
||||
Timings *Timings `json:"timings,omitempty"`
|
||||
// Usage contains aggregate token counts, when reported.
|
||||
Usage *Usage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
// StreamOptions controls streaming behavior.
|
||||
// StreamOptions configures a streaming response.
|
||||
type StreamOptions struct {
|
||||
// IncludeUsage requests an additional chunk containing aggregate token
|
||||
// usage before the terminal data marker. If the stream is interrupted,
|
||||
// that chunk may not arrive.
|
||||
IncludeUsage bool `json:"include_usage,omitempty"`
|
||||
}
|
||||
|
||||
// ToolCall represents a tool call made by the LLM.
|
||||
// ToolCall is one tool call requested by the model.
|
||||
type ToolCall struct {
|
||||
Index *int `json:"index,omitempty"` // streaming only
|
||||
ID string `json:"id,omitempty"`
|
||||
Type string `json:"type"` // "function"
|
||||
// Function identifies the function and its generated arguments.
|
||||
Function Function `json:"function"`
|
||||
// ID identifies the call. The corresponding tool response carries the
|
||||
// same value in Message.ToolCallID.
|
||||
ID string `json:"id,omitempty"`
|
||||
// Index identifies the call whose fragments a streaming delta
|
||||
// contains. It is omitted from non-streaming responses.
|
||||
Index *int `json:"index,omitempty"`
|
||||
// Type identifies the kind of tool being called.
|
||||
Type ToolType `json:"type"`
|
||||
}
|
||||
|
||||
// ToolType identifies the kind of tool in a declaration or tool call.
|
||||
type ToolType string
|
||||
|
||||
const (
|
||||
ToolTypeFunction ToolType = "function"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user