Benson Wong
cc77139ff8
Add a new configuration parameter globalTTL that all models will inherit. The default value is 0 which matches the currently functionality to never automatically unload a model. The model.ttl's default has changed to -1, which means use the global TTL value. Any model.ttl >=0 is now value with 0 meaning never unload. This allows a model to override a globalTTL > 0 and be configured to never unload. Fixes #459 Closes #512
113 lines
2.9 KiB
Go
113 lines
2.9 KiB
Go
package config
|
|
|
|
import (
|
|
"errors"
|
|
"runtime"
|
|
)
|
|
|
|
const (
|
|
MODEL_CONFIG_DEFAULT_TTL = -1
|
|
)
|
|
|
|
type ModelConfig struct {
|
|
Cmd string `yaml:"cmd"`
|
|
CmdStop string `yaml:"cmdStop"`
|
|
Proxy string `yaml:"proxy"`
|
|
Aliases []string `yaml:"aliases"`
|
|
Env []string `yaml:"env"`
|
|
CheckEndpoint string `yaml:"checkEndpoint"`
|
|
UnloadAfter int `yaml:"ttl"`
|
|
Unlisted bool `yaml:"unlisted"`
|
|
UseModelName string `yaml:"useModelName"`
|
|
|
|
// #179 for /v1/models
|
|
Name string `yaml:"name"`
|
|
Description string `yaml:"description"`
|
|
|
|
// Limit concurrency of HTTP requests to process
|
|
ConcurrencyLimit int `yaml:"concurrencyLimit"`
|
|
|
|
// Model filters see issue #174
|
|
Filters ModelFilters `yaml:"filters"`
|
|
|
|
// Macros: see #264
|
|
// Model level macros take precedence over the global macros
|
|
Macros MacroList `yaml:"macros"`
|
|
|
|
// Metadata: see #264
|
|
// Arbitrary metadata that can be exposed through the API
|
|
Metadata map[string]any `yaml:"metadata"`
|
|
|
|
// override global setting
|
|
SendLoadingState *bool `yaml:"sendLoadingState"`
|
|
}
|
|
|
|
func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
|
type rawModelConfig ModelConfig
|
|
defaults := rawModelConfig{
|
|
Cmd: "",
|
|
CmdStop: "",
|
|
Proxy: "http://localhost:${PORT}",
|
|
Aliases: []string{},
|
|
Env: []string{},
|
|
CheckEndpoint: "/health",
|
|
UnloadAfter: MODEL_CONFIG_DEFAULT_TTL, // use GlobalTTL
|
|
Unlisted: false,
|
|
UseModelName: "",
|
|
ConcurrencyLimit: 0,
|
|
Name: "",
|
|
Description: "",
|
|
}
|
|
|
|
// the default cmdStop to taskkill /f /t /pid ${PID}
|
|
if runtime.GOOS == "windows" {
|
|
defaults.CmdStop = "taskkill /f /t /pid ${PID}"
|
|
}
|
|
|
|
if err := unmarshal(&defaults); err != nil {
|
|
return err
|
|
}
|
|
|
|
*m = ModelConfig(defaults)
|
|
return nil
|
|
}
|
|
|
|
func (m *ModelConfig) SanitizedCommand() ([]string, error) {
|
|
return SanitizeCommand(m.Cmd)
|
|
}
|
|
|
|
// ModelFilters embeds Filters and adds legacy support for strip_params field
|
|
// See issue #174
|
|
type ModelFilters struct {
|
|
Filters `yaml:",inline"`
|
|
}
|
|
|
|
func (m *ModelFilters) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
|
type rawModelFilters ModelFilters
|
|
defaults := rawModelFilters{}
|
|
|
|
if err := unmarshal(&defaults); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Try to unmarshal with the old field name for backwards compatibility
|
|
if defaults.StripParams == "" {
|
|
var legacy struct {
|
|
StripParams string `yaml:"strip_params"`
|
|
}
|
|
if legacyErr := unmarshal(&legacy); legacyErr != nil {
|
|
return errors.New("failed to unmarshal legacy filters.strip_params: " + legacyErr.Error())
|
|
}
|
|
defaults.StripParams = legacy.StripParams
|
|
}
|
|
|
|
*m = ModelFilters(defaults)
|
|
return nil
|
|
}
|
|
|
|
// SanitizedStripParams wraps Filters.SanitizedStripParams for backwards compatibility
|
|
// Returns ([]string, error) to match existing API
|
|
func (f ModelFilters) SanitizedStripParams() ([]string, error) {
|
|
return f.Filters.SanitizedStripParams(), nil
|
|
}
|