Refactor llm pkg
This commit is contained in:
dwrz
+221
-46
@@ -1,6 +1,5 @@
|
||||
// Package llm implements an OpenAI-compatible inference client and its
|
||||
// protocol types.
|
||||
|
||||
package llm
|
||||
|
||||
import (
|
||||
@@ -114,69 +113,229 @@ type ChatRequest struct {
|
||||
TopP *float32 `json:"top_p,omitempty"`
|
||||
}
|
||||
|
||||
// Usage reports input, output, and total token counts for a completion.
|
||||
type Usage struct {
|
||||
// CompletionTokens is the number of tokens generated by the model.
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
// CompletionTokensDetails breaks down CompletionTokens by category.
|
||||
CompletionTokensDetails *TokenDetails `json:"completion_tokens_details,omitempty"`
|
||||
// PromptTokens is the number of tokens in the input.
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// PromptTokensDetails breaks down PromptTokens by category.
|
||||
PromptTokensDetails *TokenDetails `json:"prompt_tokens_details,omitempty"`
|
||||
// TotalTokens is the sum of PromptTokens and CompletionTokens.
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
// Metadata describes one selected completion, separately from its message.
|
||||
// Nil optional fields mean the endpoint did not report a value. Reported zero
|
||||
// counts and empty strings remain distinct from absent values. Only known body
|
||||
// fields are retained; response headers are not part of Metadata.
|
||||
//
|
||||
// See the response and streaming definitions in
|
||||
// [OpenAI completion response] and [OpenAI completion chunk].
|
||||
//
|
||||
// [OpenAI completion response]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
|
||||
// [OpenAI completion chunk]: https://developers.openai.com/api/reference/resources/chat/subresources/completions/streaming-events
|
||||
type Metadata struct {
|
||||
// ChoiceIndex is the endpoint's zero-based index of the selected
|
||||
// choice.
|
||||
ChoiceIndex *int `json:"choice_index"`
|
||||
// Created is the endpoint's completion creation time in Unix seconds,
|
||||
// not the database message's creation time.
|
||||
Created *int64 `json:"created"`
|
||||
// FinishReason is the endpoint's reason for ending the selected choice.
|
||||
// A reported reason does not establish receipt of the stream's DONE
|
||||
// marker.
|
||||
FinishReason *string `json:"finish_reason"`
|
||||
// ID is the completion ID from the response body, not a request-ID
|
||||
// header or a database message ID.
|
||||
ID *string `json:"id"`
|
||||
// Logprobs contains token probabilities for the selected choice.
|
||||
Logprobs *Logprobs `json:"logprobs"`
|
||||
// Model is the model identifier reported in the response body. It can
|
||||
// differ from RequestedModel, which was sent in the request.
|
||||
Model *string `json:"model"`
|
||||
// Provider is the application-defined label of the configured endpoint.
|
||||
// It is not a provider-reported identity or a response header.
|
||||
Provider *string `json:"provider"`
|
||||
// RequestedModel is the model identifier sent to the endpoint for this
|
||||
// inference, before any endpoint-side model alias resolution.
|
||||
RequestedModel *string `json:"requested_model"`
|
||||
// ServiceTier is the processing tier reported in the response body,
|
||||
// which can differ from a requested tier.
|
||||
ServiceTier *string `json:"service_tier"`
|
||||
// SystemFingerprint identifies the backend configuration reported by
|
||||
// the endpoint; it is not a model identifier.
|
||||
SystemFingerprint *string `json:"system_fingerprint"`
|
||||
// Timings contains performance measurements for the LLM request that
|
||||
// produced this assistant message. During streaming, a newer timing
|
||||
// report replaces the earlier report: the measurements cover the same
|
||||
// request at a later point, not additional work to add together. Fields
|
||||
// missing from the newer report remain nil.
|
||||
Timings *Timings `json:"timings"`
|
||||
// Usage contains token counts for the LLM request that produced this
|
||||
// assistant message. During streaming, a newer usage report replaces
|
||||
// the earlier report rather than adding to its counts. Fields missing
|
||||
// from the newer report remain nil.
|
||||
Usage *Usage `json:"usage"`
|
||||
}
|
||||
|
||||
// TokenDetails reports token categories within a Usage count. A zero value may
|
||||
// mean either that no tokens belong to the category or that it was not
|
||||
// reported.
|
||||
type TokenDetails struct {
|
||||
// CachedTokens is the number of input tokens served from a cache.
|
||||
CachedTokens int `json:"cached_tokens,omitempty"`
|
||||
// Usage reports input, output, and total token counts for one inference.
|
||||
// Nil counts mean unreported, not zero. Category counts are endpoint-reported
|
||||
// breakdowns, not additional tokens to add to the totals.
|
||||
//
|
||||
// See [OpenAI completion usage].
|
||||
//
|
||||
// [OpenAI completion usage]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
|
||||
type Usage struct {
|
||||
// CompletionTokens is the number of tokens generated by the model.
|
||||
CompletionTokens *int `json:"completion_tokens"`
|
||||
// CompletionTokensDetails breaks down CompletionTokens by category.
|
||||
CompletionTokensDetails *CompletionTokensDetails `json:"completion_tokens_details"`
|
||||
// PromptTokens is the number of tokens in the input.
|
||||
PromptTokens *int `json:"prompt_tokens"`
|
||||
// PromptTokensDetails breaks down PromptTokens by category.
|
||||
PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details"`
|
||||
// TotalTokens is the reported sum of prompt and completion token
|
||||
// counts.
|
||||
TotalTokens *int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
// CompletionTokensDetails reports output-token categories in [Usage].
|
||||
// Nil counts mean the category was not reported. The categories follow
|
||||
// [OpenAI completion usage].
|
||||
//
|
||||
// [OpenAI completion usage]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
|
||||
type CompletionTokensDetails struct {
|
||||
// AcceptedPredictionTokens counts predicted-output tokens that appeared
|
||||
// in the completion.
|
||||
AcceptedPredictionTokens *int `json:"accepted_prediction_tokens"`
|
||||
// AudioTokens counts audio tokens generated by the model.
|
||||
AudioTokens *int `json:"audio_tokens"`
|
||||
// ReasoningTokens is the number of output tokens used for reasoning.
|
||||
ReasoningTokens int `json:"reasoning_tokens,omitempty"`
|
||||
ReasoningTokens *int `json:"reasoning_tokens"`
|
||||
// RejectedPredictionTokens counts predicted-output tokens that did not
|
||||
// appear in the completion. OpenAI includes these in completion counts
|
||||
// for billing, output, and context-window limits.
|
||||
RejectedPredictionTokens *int `json:"rejected_prediction_tokens"`
|
||||
// TextTokens counts text tokens generated by the model.
|
||||
TextTokens *int `json:"text_tokens"`
|
||||
}
|
||||
|
||||
// PromptTokensDetails reports input-token categories in [Usage].
|
||||
// Nil counts mean the category was not reported. The categories follow
|
||||
// [OpenAI completion usage].
|
||||
//
|
||||
// [OpenAI completion usage]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
|
||||
type PromptTokensDetails struct {
|
||||
// AudioTokens counts audio input tokens in the prompt.
|
||||
AudioTokens *int `json:"audio_tokens"`
|
||||
// CacheWriteTokens is the unadjusted count of prompt tokens written to
|
||||
// cache, as reported by the endpoint.
|
||||
CacheWriteTokens *int `json:"cache_write_tokens"`
|
||||
// CachedTokens is the number of input tokens served from a cache.
|
||||
CachedTokens *int `json:"cached_tokens"`
|
||||
// ImageTokens counts image input tokens in the prompt.
|
||||
ImageTokens *int `json:"image_tokens"`
|
||||
// TextTokens counts text input tokens in the prompt.
|
||||
TextTokens *int `json:"text_tokens"`
|
||||
}
|
||||
|
||||
// Logprobs reports token log probabilities for one choice, when returned.
|
||||
// Content and refusal tokens are separate ordered sequences. Receiving these
|
||||
// values does not cause the client to request log probabilities. Nil slices
|
||||
// represent absent or null arrays; non-nil empty slices retain reported [].
|
||||
//
|
||||
// See [OpenAI token log probabilities] and [OpenAI completion response].
|
||||
//
|
||||
// [OpenAI token log probabilities]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
|
||||
// [OpenAI completion response]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
|
||||
type Logprobs struct {
|
||||
// Content contains log probabilities for message content tokens in
|
||||
// order.
|
||||
Content []TokenLogprob `json:"content"`
|
||||
// Refusal contains log probabilities for refusal tokens in order.
|
||||
Refusal []TokenLogprob `json:"refusal"`
|
||||
}
|
||||
|
||||
// TokenLogprob describes a generated token and its alternatives at one
|
||||
// position, using [OpenAI token log probabilities]. Nil optional scalars mean
|
||||
// unreported; a zero log probability remains a reported value.
|
||||
//
|
||||
// [OpenAI token log probabilities]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
|
||||
type TokenLogprob struct {
|
||||
// Bytes is the token's UTF-8 byte representation as integers, not
|
||||
// base64.
|
||||
// A token can contain only part of a multibyte character. Nil means no
|
||||
// byte representation was reported.
|
||||
Bytes []int `json:"bytes"`
|
||||
// Logprob is the token's log probability. OpenAI uses -9999 for a token
|
||||
// outside the top 20 most likely tokens.
|
||||
Logprob *float64 `json:"logprob"`
|
||||
// Token is the token text reported by the endpoint.
|
||||
Token *string `json:"token"`
|
||||
// TopLogprobs contains alternatives at this position; the endpoint can
|
||||
// return fewer alternatives than requested.
|
||||
TopLogprobs []TopLogprob `json:"top_logprobs"`
|
||||
}
|
||||
|
||||
// TopLogprob describes one alternative token at a generated token's position.
|
||||
// Alternatives have no nested alternatives in [OpenAI token log probabilities].
|
||||
// Nil optional scalars mean unreported, rather than zero or empty text.
|
||||
//
|
||||
// [OpenAI token log probabilities]: https://developers.openai.com/api/reference/resources/chat/subresources/completions
|
||||
type TopLogprob struct {
|
||||
// Bytes is the alternative's UTF-8 byte representation as integers,
|
||||
// not base64. Nil means no byte representation was reported.
|
||||
Bytes []int `json:"bytes"`
|
||||
// Logprob is the alternative's log probability. OpenAI uses -9999 for
|
||||
// a token outside the top 20 most likely tokens.
|
||||
Logprob *float64 `json:"logprob"`
|
||||
// Token is the alternative token text reported by the endpoint.
|
||||
Token *string `json:"token"`
|
||||
}
|
||||
|
||||
// Timings reports llama.cpp prompt-processing and token-generation performance.
|
||||
// The N fields are token counts, MS fields are elapsed milliseconds,
|
||||
// PerTokenMS fields are milliseconds per token, and PerSecond fields are tokens
|
||||
// per second. Timings is a llama.cpp extension to the OpenAI API.
|
||||
// per second. Timings is a llama.cpp extension to the OpenAI API. Nil fields
|
||||
// mean unreported. Streaming objects are snapshots, not increments to sum.
|
||||
//
|
||||
// See [llama.cpp API documentation].
|
||||
//
|
||||
// [llama.cpp API documentation]: https://llama.app/docs/api
|
||||
// [llama.cpp API documentation]: https://llama.app/docs/api#timings-and-usage
|
||||
type Timings struct {
|
||||
// CacheN is the number of prompt tokens reused from the cache.
|
||||
CacheN *int `json:"cache_n"`
|
||||
// DraftN is the number of tokens drafted during speculative decoding.
|
||||
DraftN *int `json:"draft_n"`
|
||||
// DraftNAccepted is the number of drafted tokens accepted during
|
||||
// speculative decoding.
|
||||
DraftNAccepted *int `json:"draft_n_accepted"`
|
||||
// PredictedMS is the time spent generating output, in milliseconds.
|
||||
PredictedMS float64 `json:"predicted_ms"`
|
||||
PredictedMS *float64 `json:"predicted_ms"`
|
||||
// PredictedN is the number of generated tokens.
|
||||
PredictedN int `json:"predicted_n"`
|
||||
PredictedN *int `json:"predicted_n"`
|
||||
// PredictedPerSecond is the output-generation rate in tokens per
|
||||
// second.
|
||||
PredictedPerSecond float64 `json:"predicted_per_second"`
|
||||
PredictedPerSecond *float64 `json:"predicted_per_second"`
|
||||
// PredictedPerTokenMS is the mean output-generation time per token, in
|
||||
// milliseconds.
|
||||
PredictedPerTokenMS float64 `json:"predicted_per_token_ms"`
|
||||
PredictedPerTokenMS *float64 `json:"predicted_per_token_ms"`
|
||||
// PromptMS is the time spent processing the prompt, in milliseconds.
|
||||
PromptMS float64 `json:"prompt_ms"`
|
||||
PromptMS *float64 `json:"prompt_ms"`
|
||||
// PromptN is the number of prompt tokens processed.
|
||||
PromptN int `json:"prompt_n"`
|
||||
PromptN *int `json:"prompt_n"`
|
||||
// PromptPerSecond is the prompt-processing rate in tokens per second.
|
||||
PromptPerSecond float64 `json:"prompt_per_second"`
|
||||
PromptPerSecond *float64 `json:"prompt_per_second"`
|
||||
// PromptPerTokenMS is the mean prompt-processing time per token, in
|
||||
// milliseconds.
|
||||
PromptPerTokenMS float64 `json:"prompt_per_token_ms"`
|
||||
PromptPerTokenMS *float64 `json:"prompt_per_token_ms"`
|
||||
}
|
||||
|
||||
// ChatResponse is the body of a non-streaming chat completion
|
||||
// response.
|
||||
// response. Nil metadata fields mean unreported values.
|
||||
type ChatResponse struct {
|
||||
// Choices contains the generated alternatives.
|
||||
Choices []Choice `json:"choices"`
|
||||
// ID identifies the completion.
|
||||
ID string `json:"id"`
|
||||
// Created is the endpoint's completion creation time in Unix seconds.
|
||||
Created *int64 `json:"created,omitempty"`
|
||||
// ID identifies the completion in the response body, not a request
|
||||
// header.
|
||||
ID *string `json:"id,omitempty"`
|
||||
// Model identifies the model that generated the response.
|
||||
Model string `json:"model"`
|
||||
Model *string `json:"model,omitempty"`
|
||||
// ServiceTier is the processing tier reported by the endpoint.
|
||||
ServiceTier *string `json:"service_tier,omitempty"`
|
||||
// SystemFingerprint identifies the reported backend configuration.
|
||||
SystemFingerprint *string `json:"system_fingerprint,omitempty"`
|
||||
// Timings contains llama.cpp performance data, when reported.
|
||||
Timings *Timings `json:"timings,omitempty"`
|
||||
// Usage contains token counts, when reported.
|
||||
@@ -185,12 +344,15 @@ type ChatResponse struct {
|
||||
|
||||
// Choice is one generated alternative in a non-streaming response. Index is its
|
||||
// position among the alternatives, and FinishReason reports why generation
|
||||
// ended.
|
||||
// ended. Nil metadata fields mean the endpoint did not report those values.
|
||||
type Choice struct {
|
||||
// FinishReason is the endpoint's reason for ending generation.
|
||||
FinishReason string `json:"finish_reason"`
|
||||
// Index is the position of this choice in the response.
|
||||
Index int `json:"index"`
|
||||
FinishReason *string `json:"finish_reason,omitempty"`
|
||||
// Index is the zero-based position of this choice in the response.
|
||||
Index *int `json:"index,omitempty"`
|
||||
// Logprobs contains probabilities for this choice's content and
|
||||
// refusal.
|
||||
Logprobs *Logprobs `json:"logprobs,omitempty"`
|
||||
// Message is the generated message.
|
||||
Message Message `json:"message"`
|
||||
}
|
||||
@@ -468,22 +630,35 @@ type ResponseFormat struct {
|
||||
|
||||
// StreamChunk is one data event in a streaming response. Delta fields are
|
||||
// incremental and must be combined in order to reconstruct a message. A chunk
|
||||
// may contain Usage or Timings without containing a choice.
|
||||
// may contain Usage or Timings without containing a choice. Nil metadata fields
|
||||
// mean unreported values, not reported zero counts or empty strings.
|
||||
type StreamChunk struct {
|
||||
// Choices contains incremental updates for generated alternatives.
|
||||
// It may be empty in a metadata-only chunk.
|
||||
Choices []struct {
|
||||
// Index identifies the choice updated by this delta.
|
||||
Index int `json:"index"`
|
||||
// Delta contains the incremental message update.
|
||||
Delta Delta `json:"delta"`
|
||||
// FinishReason is set when generation for the choice ends.
|
||||
FinishReason string `json:"finish_reason,omitempty"`
|
||||
// FinishReason is the reported reason for ending this choice;
|
||||
// it does not establish receipt of DONE.
|
||||
FinishReason *string `json:"finish_reason,omitempty"`
|
||||
// Index identifies the choice updated by this delta. Nil means
|
||||
// unreported and selects the first choice for accumulation.
|
||||
Index *int `json:"index,omitempty"`
|
||||
// Logprobs contains the next content and refusal token
|
||||
// probabilities.
|
||||
Logprobs *Logprobs `json:"logprobs,omitempty"`
|
||||
} `json:"choices"`
|
||||
// ID identifies the completion to which the chunk belongs.
|
||||
ID string `json:"id"`
|
||||
// Created is the endpoint's completion creation time in Unix seconds.
|
||||
Created *int64 `json:"created,omitempty"`
|
||||
// ID identifies the completion in the response body, not a request
|
||||
// header.
|
||||
ID *string `json:"id,omitempty"`
|
||||
// Model identifies the model generating the response.
|
||||
Model string `json:"model"`
|
||||
Model *string `json:"model,omitempty"`
|
||||
// ServiceTier is the processing tier reported by the endpoint.
|
||||
ServiceTier *string `json:"service_tier,omitempty"`
|
||||
// SystemFingerprint identifies the reported backend configuration.
|
||||
SystemFingerprint *string `json:"system_fingerprint,omitempty"`
|
||||
// Timings contains llama.cpp performance data, when reported.
|
||||
Timings *Timings `json:"timings,omitempty"`
|
||||
// Usage contains aggregate token counts, when reported.
|
||||
|
||||
Reference in New Issue
Block a user