Add context tracking
This commit is contained in:
@@ -768,8 +768,9 @@ const (
|
||||
|
||||
// Model represents a model in the /v1/models response.
|
||||
type Model struct {
|
||||
Model string `json:"model"`
|
||||
Status ModelStatus `json:"status"`
|
||||
Model string `json:"model"`
|
||||
Status ModelStatus `json:"status"`
|
||||
ContextSize *int `json:"context_size,omitempty"`
|
||||
}
|
||||
|
||||
// Models is the response format for the /v1/models endpoint.
|
||||
@@ -828,9 +829,13 @@ func (svc *Service) models(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
available := make(map[string]struct{}, len(models))
|
||||
contextSizes := make(map[string]int, len(models))
|
||||
for _, m := range models {
|
||||
baseID, _, _ := strings.Cut(m.ID, ":")
|
||||
available[baseID] = struct{}{}
|
||||
if m.Meta != nil && m.Meta.NCtx > 0 {
|
||||
contextSizes[baseID] = m.Meta.NCtx
|
||||
}
|
||||
}
|
||||
|
||||
var toCheck []string
|
||||
@@ -847,6 +852,9 @@ func (svc *Service) models(w http.ResponseWriter, r *http.Request) {
|
||||
mi := Model{Model: m}
|
||||
if _, ok := available[base]; ok {
|
||||
mi.Status = ModelStatusAvailable
|
||||
if cs, ok := contextSizes[base]; ok {
|
||||
mi.ContextSize = &cs
|
||||
}
|
||||
} else {
|
||||
mi.Status = ModelStatusNotFound
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user