312 lines
11 KiB
Go
312 lines
11 KiB
Go
// Package llm provides an OpenAI-compatible client for LLM interactions.
|
|
// It handles chat completions with automatic tool call execution, where the
|
|
// client iteratively sends messages to the LLM, executes any requested tool
|
|
// calls, and continues until the LLM produces a final text response.
|
|
//
|
|
// The package defines the core types for constructing requests and parsing
|
|
// responses: Message, ChatRequest, ChatResponse, and streaming types
|
|
// such as StreamChunk and Delta.
|
|
//
|
|
// Tools are defined declaratively in YAML configuration, stored as Tool
|
|
// structs in a Registry, and executed as subprocesses when the LLM invokes
|
|
// them. The Registry also converts tool definitions into the API format for
|
|
// inclusion in ChatRequest bodies.
|
|
|
|
package llm
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
)
|
|
|
|
// Role constants for messages.
|
|
const (
|
|
RoleSystem = "system"
|
|
RoleUser = "user"
|
|
RoleAssistant = "assistant"
|
|
RoleTool = "tool"
|
|
RoleDeveloper = "developer"
|
|
)
|
|
|
|
// APITool defines a tool available to the LLM in the API request format.
|
|
type APITool struct {
|
|
Type string `json:"type"` // "function"
|
|
Function *FunctionDef `json:"function,omitempty"`
|
|
}
|
|
|
|
// ChatRequest is the request body for chat completions.
|
|
type ChatRequest struct {
|
|
Model string `json:"model"`
|
|
Messages []Message `json:"messages"`
|
|
Temperature *float32 `json:"temperature,omitempty"`
|
|
TopP *float32 `json:"top_p,omitempty"`
|
|
N int `json:"n,omitempty"`
|
|
Stream bool `json:"stream,omitempty"`
|
|
Stop []string `json:"stop,omitempty"`
|
|
MaxTokens int `json:"max_tokens,omitempty"`
|
|
MaxCompletionTokens int `json:"max_completion_tokens,omitempty"`
|
|
ResponseFormat *ResponseFormat `json:"response_format,omitempty"`
|
|
Seed *int `json:"seed,omitempty"`
|
|
PresencePenalty *float32 `json:"presence_penalty,omitempty"`
|
|
FrequencyPenalty *float32 `json:"frequency_penalty,omitempty"`
|
|
LogProbs bool `json:"logprobs,omitempty"`
|
|
TopLogProbs int `json:"top_logprobs,omitempty"`
|
|
Tools []APITool `json:"tools,omitempty"`
|
|
ToolChoice any `json:"tool_choice,omitempty"`
|
|
StreamOptions *StreamOptions `json:"stream_options,omitempty"`
|
|
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"`
|
|
ChatTemplateKWARGS map[string]any `json:"chat_template_kwargs,omitempty"`
|
|
ThinkingBudgetTokens *int `json:"thinking_budget_tokens,omitempty"`
|
|
ServiceTier string `json:"service_tier,omitempty"`
|
|
}
|
|
|
|
// Usage represents token usage from the API response.
|
|
type Usage struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"`
|
|
CompletionTokensDetails *TokenDetails `json:"completion_tokens_details,omitempty"`
|
|
}
|
|
|
|
// TokenDetails breaks down token usage by category.
|
|
type TokenDetails struct {
|
|
ReasoningTokens int `json:"reasoning_tokens,omitempty"`
|
|
CachedTokens int `json:"cached_tokens,omitempty"`
|
|
}
|
|
|
|
// Timings is a llama.cpp-specific extension to the chat completion
|
|
// response. It is not part of the OpenAI API spec.
|
|
type Timings struct {
|
|
PromptN int `json:"prompt_n"`
|
|
PromptMS float64 `json:"prompt_ms"`
|
|
PromptPerTokenMS float64 `json:"prompt_per_token_ms"`
|
|
PromptPerSecond float64 `json:"prompt_per_second"`
|
|
PredictedN int `json:"predicted_n"`
|
|
PredictedMS float64 `json:"predicted_ms"`
|
|
PredictedPerTokenMS float64 `json:"predicted_per_token_ms"`
|
|
PredictedPerSecond float64 `json:"predicted_per_second"`
|
|
}
|
|
|
|
// ChatResponse is the non-streaming response from the API.
|
|
type ChatResponse struct {
|
|
ID string `json:"id"`
|
|
Model string `json:"model"`
|
|
Choices []Choice `json:"choices"`
|
|
Usage *Usage `json:"usage,omitempty"`
|
|
Timings *Timings `json:"timings,omitempty"`
|
|
}
|
|
|
|
// Choice is a single choice in a ChatResponse.
|
|
type Choice struct {
|
|
Index int `json:"index"`
|
|
Message Message `json:"message"`
|
|
FinishReason string `json:"finish_reason"`
|
|
}
|
|
|
|
// ContentType identifies the kind of data in a ContentPart.
|
|
//
|
|
// User messages (ChatCompletionContentPart) support: text, image_url,
|
|
// input_audio, file.
|
|
//
|
|
// Assistant messages (ChatCompletionAssistantMessageParam) support:
|
|
// text, refusal.
|
|
//
|
|
// See: https://platform.openai.com/docs/api-reference/chat/create
|
|
type ContentType string
|
|
|
|
const (
|
|
ContentTypeText ContentType = "text"
|
|
ContentTypeImageURL ContentType = "image_url"
|
|
ContentTypeInputAudio ContentType = "input_audio"
|
|
ContentTypeFile ContentType = "file"
|
|
ContentTypeRefusal ContentType = "refusal"
|
|
)
|
|
|
|
// ContentPart represents a single part of a multi-content message.
|
|
type ContentPart struct {
|
|
Type ContentType `json:"type"`
|
|
Text string `json:"text"`
|
|
ImageURL *ImageURL `json:"image_url,omitempty"`
|
|
InputAudio *InputAudio `json:"input_audio,omitempty"`
|
|
File *FileInput `json:"file,omitempty"`
|
|
Refusal string `json:"refusal"`
|
|
}
|
|
|
|
// Delta is a partial message update in a streaming response.
|
|
type Delta struct {
|
|
Role string `json:"role,omitempty"`
|
|
Content string `json:"content,omitempty"`
|
|
Refusal string `json:"refusal,omitempty"`
|
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
|
Reasoning string `json:"reasoning,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
}
|
|
|
|
// FileInput holds file data for file content parts.
|
|
type FileInput struct {
|
|
FileData string `json:"file_data,omitempty"`
|
|
FileID string `json:"file_id,omitempty"`
|
|
Filename string `json:"filename,omitempty"`
|
|
}
|
|
|
|
// Function holds the name and arguments of a tool call.
|
|
type Function struct {
|
|
Name string `json:"name,omitempty"`
|
|
Arguments string `json:"arguments,omitempty"`
|
|
}
|
|
|
|
// FunctionDef defines a function that can be called by the LLM.
|
|
type FunctionDef struct {
|
|
Name string `json:"name"`
|
|
Description string `json:"description,omitempty"`
|
|
Parameters any `json:"parameters"` // JSON schema
|
|
Strict bool `json:"strict,omitempty"`
|
|
}
|
|
|
|
// ImageURL holds an image reference for multi-modal messages.
|
|
type ImageURL struct {
|
|
URL string `json:"url"`
|
|
Detail string `json:"detail,omitempty"` // "auto", "low", "high"
|
|
}
|
|
|
|
// InputAudio holds audio data for multi-modal messages.
|
|
type InputAudio struct {
|
|
Data string `json:"data"`
|
|
Format string `json:"format"` // "wav", "mp3"
|
|
}
|
|
|
|
// Message represents a chat message in a conversation.
|
|
// Content is always stored as []ContentPart internally.
|
|
// A custom UnmarshalJSON handles the JSON polymorphism where content
|
|
// can arrive as either a string or an array of parts.
|
|
//
|
|
// Refusal handling:
|
|
//
|
|
// In API responses (ChatCompletionMessage), the model sends refusal as a
|
|
// top-level field alongside content:
|
|
//
|
|
// "content": "text response"
|
|
// "refusal": "I can't help with that"
|
|
//
|
|
// Only one of content or refusal will be non-empty.
|
|
//
|
|
// In API requests (ChatCompletionAssistantMessageParam), the model can
|
|
// receive refusal as a content part with type "refusal" in the content
|
|
// array. Our ContentPart type supports this for completeness.
|
|
//
|
|
// See: https://platform.openai.com/docs/api-reference/chat/create
|
|
type Message struct {
|
|
Role string `json:"role"`
|
|
ContentParts []ContentPart `json:"content,omitempty"`
|
|
Refusal string `json:"refusal,omitempty"`
|
|
ReasoningContent string `json:"reasoning_content,omitempty"`
|
|
Reasoning string `json:"reasoning,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
ToolCallID string `json:"tool_call_id,omitempty"`
|
|
}
|
|
|
|
// Text returns the concatenated text from all text content parts.
|
|
func (m *Message) Text() string {
|
|
var b strings.Builder
|
|
for _, p := range m.ContentParts {
|
|
if p.Type == ContentTypeText {
|
|
b.WriteString(p.Text)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// UnmarshalJSON handles the polymorphic content field which can be
|
|
// either a string or an array of ContentPart objects.
|
|
func (m *Message) UnmarshalJSON(data []byte) error {
|
|
type Alias Message
|
|
aux := &struct {
|
|
Content any `json:"content"`
|
|
*Alias
|
|
}{
|
|
Alias: (*Alias)(m),
|
|
}
|
|
if err := json.Unmarshal(data, &aux); err != nil {
|
|
return err
|
|
}
|
|
switch c := aux.Content.(type) {
|
|
case string:
|
|
m.ContentParts = []ContentPart{{Type: ContentTypeText, Text: c}}
|
|
case []any:
|
|
for _, raw := range c {
|
|
partBytes, err := json.Marshal(raw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var part ContentPart
|
|
if err := json.Unmarshal(partBytes, &part); err != nil {
|
|
return err
|
|
}
|
|
m.ContentParts = append(m.ContentParts, part)
|
|
}
|
|
}
|
|
// Normalize: if ReasoningContent is empty but Reasoning is set, use Reasoning
|
|
if m.ReasoningContent == "" && m.Reasoning != "" {
|
|
m.ReasoningContent = m.Reasoning
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Model represents an available model from the API.
|
|
type Model struct {
|
|
ID string `json:"id"`
|
|
Created int64 `json:"created"`
|
|
Object string `json:"object"`
|
|
OwnedBy string `json:"owned_by"`
|
|
Meta *ModelMeta `json:"meta,omitempty"`
|
|
ContextLength int `json:"context_length,omitempty"`
|
|
}
|
|
|
|
// ModelMeta contains optional model metadata from the provider.
|
|
// llama.cpp includes n_ctx (configured context size) and
|
|
// n_ctx_train (model's native training context).
|
|
type ModelMeta struct {
|
|
NCtx int `json:"n_ctx,omitempty"`
|
|
NCtxTrain int `json:"n_ctx_train,omitempty"`
|
|
}
|
|
|
|
// ModelsResponse is the response for GET /models.
|
|
type ModelsResponse struct {
|
|
Object string `json:"object"`
|
|
Data []Model `json:"data"`
|
|
}
|
|
|
|
// ResponseFormat specifies the format of the response.
|
|
type ResponseFormat struct {
|
|
Type string `json:"type"`
|
|
JSONSchema any `json:"json_schema,omitempty"`
|
|
}
|
|
|
|
// StreamChunk is a single SSE chunk from a streaming response.
|
|
type StreamChunk struct {
|
|
ID string `json:"id"`
|
|
Model string `json:"model"`
|
|
Choices []struct {
|
|
Index int `json:"index"`
|
|
Delta Delta `json:"delta"`
|
|
FinishReason string `json:"finish_reason,omitempty"`
|
|
} `json:"choices"`
|
|
Usage *Usage `json:"usage,omitempty"`
|
|
Timings *Timings `json:"timings,omitempty"`
|
|
}
|
|
|
|
// StreamOptions controls streaming behavior.
|
|
type StreamOptions struct {
|
|
IncludeUsage bool `json:"include_usage,omitempty"`
|
|
}
|
|
|
|
// ToolCall represents a tool call made by the LLM.
|
|
type ToolCall struct {
|
|
Index *int `json:"index,omitempty"` // streaming only
|
|
ID string `json:"id,omitempty"`
|
|
Type string `json:"type"` // "function"
|
|
Function Function `json:"function"`
|
|
}
|