steve/llama-swap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

proxy: add configurable HTTP timeouts for models and peers (#619)

Browse Source 
Add configurable HTTP timeout settings to both models and peers to support installations that requires longer timeouts than the current hardcoded defaults.

Closes #618
This commit is contained in:
Ron M
2026-04-06 04:30:27 -07:00
committed by GitHub
parent 981910d734
commit a37b4866d8
13 changed files with 437 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
proxy/config/config_posix_test.go
+28
View File
@@ -187,6 +187,13 @@ groups:
Name: "Model 1",
Description: "This is model 1",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
},
"model2": {
Cmd: "path/to/server --arg1 one",
@@ -195,6 +202,13 @@ groups:
Env: []string{},
CheckEndpoint: "/",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
},
"model3": {
Cmd: "path/to/cmd --arg1 one",
@@ -203,6 +217,13 @@ groups:
Env: []string{},
CheckEndpoint: "/",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
},
"model4": {
Cmd: "path/to/cmd --arg1 one",
@@ -211,6 +232,13 @@ groups:
Aliases: []string{},
Env: []string{},
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
},
},
HealthCheckTimeout: 15,
proxy/config/config_test.go
+106
View File
@@ -6,6 +6,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConfig_GroupMemberIsUnique(t *testing.T) {
@@ -1438,3 +1439,108 @@ models:
})
}
func TestConfig_TimeoutsParsing(t *testing.T) {
configYaml := `
models:
model1:
cmd: test-server --port ${PORT}
timeouts:
connect: 45
responseHeader: 120
`
config, err := LoadConfigFromReader(strings.NewReader(configYaml))
require.NoError(t, err)
modelConfig, found := config.Models["model1"]
require.True(t, found, "model1 should exist in config")
assert.Equal(t, 45, modelConfig.Timeouts.Connect)
assert.Equal(t, 120, modelConfig.Timeouts.ResponseHeader)
}
func TestConfig_TimeoutsDefaults(t *testing.T) {
configYaml := `
models:
model1:
cmd: test-server --port ${PORT}
`
config, err := LoadConfigFromReader(strings.NewReader(configYaml))
require.NoError(t, err)
modelConfig, found := config.Models["model1"]
require.True(t, found, "model1 should exist in config")
// Default values should be set during unmarshaling
assert.Equal(t, 30, modelConfig.Timeouts.Connect)
assert.Equal(t, 60, modelConfig.Timeouts.ResponseHeader)
assert.Equal(t, 10, modelConfig.Timeouts.TLSHandshake)
assert.Equal(t, 1, modelConfig.Timeouts.ExpectContinue)
assert.Equal(t, 90, modelConfig.Timeouts.IdleConn)
}
func TestConfig_TimeoutsZeroAllowed(t *testing.T) {
configYaml := `
models:
model1:
cmd: test-server --port ${PORT}
timeouts:
connect: 0
responseHeader: 0
`
config, err := LoadConfigFromReader(strings.NewReader(configYaml))
require.NoError(t, err)
modelConfig, found := config.Models["model1"]
require.True(t, found, "model1 should exist in config")
// Explicit 0 should be preserved (disables timeout)
assert.Equal(t, 0, modelConfig.Timeouts.Connect)
assert.Equal(t, 0, modelConfig.Timeouts.ResponseHeader)
}
func TestConfig_PeerTimeoutsParsing(t *testing.T) {
configYaml := `
peers:
peer1:
proxy: http://example.com
models: [model1]
timeouts:
connect: 45
responseHeader: 120
`
config, err := LoadConfigFromReader(strings.NewReader(configYaml))
require.NoError(t, err)
peerConfig, found := config.Peers["peer1"]
require.True(t, found, "peer1 should exist in config")
assert.Equal(t, 45, peerConfig.Timeouts.Connect)
assert.Equal(t, 120, peerConfig.Timeouts.ResponseHeader)
}
func TestConfig_PeerTimeoutsDefaults(t *testing.T) {
configYaml := `
peers:
peer1:
proxy: http://example.com
models: [model1]
`
config, err := LoadConfigFromReader(strings.NewReader(configYaml))
require.NoError(t, err)
peerConfig, found := config.Peers["peer1"]
require.True(t, found, "peer1 should exist in config")
// Default values should be set during unmarshaling
assert.Equal(t, 30, peerConfig.Timeouts.Connect)
assert.Equal(t, 60, peerConfig.Timeouts.ResponseHeader)
assert.Equal(t, 10, peerConfig.Timeouts.TLSHandshake)
assert.Equal(t, 1, peerConfig.Timeouts.ExpectContinue)
assert.Equal(t, 90, peerConfig.Timeouts.IdleConn)
}
proxy/config/config_windows_test.go
+28
View File
@@ -173,6 +173,13 @@ groups:
Env: []string{"VAR1=value1", "VAR2=value2"},
CheckEndpoint: "/health",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
},
"model2": {
Cmd: "path/to/server --arg1 one",
@@ -182,6 +189,13 @@ groups:
Env: []string{},
CheckEndpoint: "/",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
},
"model3": {
Cmd: "path/to/cmd --arg1 one",
@@ -191,6 +205,13 @@ groups:
Env: []string{},
CheckEndpoint: "/",
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
},
"model4": {
Cmd: "path/to/cmd --arg1 one",
@@ -200,6 +221,13 @@ groups:
Aliases: []string{},
Env: []string{},
SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
},
},
HealthCheckTimeout: 15,
proxy/config/model_config.go
+19
View File
@@ -9,6 +9,15 @@ const (
MODEL_CONFIG_DEFAULT_TTL = -1
)
// TimeoutsConfig holds timeout settings for proxy connections
type TimeoutsConfig struct {
Connect int `yaml:"connect"` // seconds, 0 = no timeout (not recommended)
ResponseHeader int `yaml:"responseHeader"` // seconds, 0 = no timeout (not recommended)
TLSHandshake int `yaml:"tlsHandshake"` // seconds, 0 = no timeout (not recommended)
ExpectContinue int `yaml:"expectContinue"` // seconds, 0 = no timeout (not recommended)
IdleConn int `yaml:"idleConn"` // seconds, 0 = no timeout (not recommended)
}
type ModelConfig struct {
Cmd string `yaml:"cmd"`
CmdStop string `yaml:"cmdStop"`
@@ -40,6 +49,9 @@ type ModelConfig struct {
// override global setting
SendLoadingState *bool `yaml:"sendLoadingState"`
// Timeout settings for proxy connections
Timeouts TimeoutsConfig `yaml:"timeouts"`
}
func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -57,6 +69,13 @@ func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
ConcurrencyLimit: 0,
Name: "",
Description: "",
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}
// the default cmdStop to taskkill /f /t /pid ${PID}
proxy/config/peer.go
+10
View File
@@ -12,6 +12,9 @@ type PeerConfig struct {
ApiKey string `yaml:"apiKey"`
Models []string `yaml:"models"`
Filters Filters `yaml:"filters"`
// Timeout settings for proxy connections
Timeouts TimeoutsConfig `yaml:"timeouts"`
}
func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -21,6 +24,13 @@ func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
ApiKey: "",
Models: []string{},
Filters: Filters{},
Timeouts: TimeoutsConfig{
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}
if err := unmarshal(&defaults); err != nil {