Benson Wong
22e098ac8b
This PR allows a single llama-swap to be the central proxy for models served by other inference servers. The peer servers can be another llama-swap or any API that supports the /v1/* inference endpoint. Updates: #433, #299 Closes: #296
48 lines
1010 B
Go
48 lines
1010 B
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"net/url"
|
|
)
|
|
|
|
type PeerDictionaryConfig map[string]PeerConfig
|
|
type PeerConfig struct {
|
|
Proxy string `yaml:"proxy"`
|
|
ProxyURL *url.URL `yaml:"-"`
|
|
ApiKey string `yaml:"apiKey"`
|
|
Models []string `yaml:"models"`
|
|
}
|
|
|
|
func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
|
type rawPeerConfig PeerConfig
|
|
defaults := rawPeerConfig{
|
|
Proxy: "",
|
|
ApiKey: "",
|
|
Models: []string{},
|
|
}
|
|
|
|
if err := unmarshal(&defaults); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Validate proxy is not empty
|
|
if defaults.Proxy == "" {
|
|
return fmt.Errorf("proxy is required")
|
|
}
|
|
|
|
// Validate proxy is a valid URL and store the parsed value
|
|
parsedURL, err := url.Parse(defaults.Proxy)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid peer proxy URL (%s): %w", defaults.Proxy, err)
|
|
}
|
|
defaults.ProxyURL = parsedURL
|
|
|
|
// Validate models is not empty
|
|
if len(defaults.Models) == 0 {
|
|
return fmt.Errorf("peer models can not be empty")
|
|
}
|
|
|
|
*c = PeerConfig(defaults)
|
|
return nil
|
|
}
|