steve/llama-swap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

proxy,proxy/config: restore timeouts to pre PR 619 (#648)

Browse Source 
Reset the default ResponseHeader timeout to 0 (no timeout) which was set
to 60 seconds in PR #619.

Fixes #647
This commit is contained in:
Benson Wong
2026-04-11 20:42:13 -07:00
committed by GitHub
parent 7b2b82777f
commit a9d840ffd7
9 changed files with 74 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config-schema.json
+23 -11
View File
@@ -47,31 +47,37 @@
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 30, "default": 30,
"description": "TCP connection timeout in seconds. Set to 0 to disable (not recommended)." "description": "TCP connection timeout in seconds. Set to 0 to disable."
},
"keepalive": {
"type": "integer",
"minimum": 0,
"default": 30,
"description": "TCP keepalive timeout in seconds. Set to 0 to disable."
}, },
"responseHeader": { "responseHeader": {
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 60, "default": 0,
"description": "Time to wait for response headers in seconds. Set to 0 to disable (not recommended)." "description": "Time to wait for response headers in seconds. Set to 0 to disable."
}, },
"tlsHandshake": { "tlsHandshake": {
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 10, "default": 10,
"description": "TLS handshake timeout in seconds. Set to 0 to disable (not recommended)." "description": "TLS handshake timeout in seconds. Set to 0 to disable."
}, },
"expectContinue": { "expectContinue": {
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 1, "default": 1,
"description": "Expect-Continue timeout in seconds. Set to 0 to disable (not recommended)." "description": "Expect-Continue timeout in seconds. Set to 0 to disable."
}, },
"idleConn": { "idleConn": {
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 90, "default": 90,
"description": "Idle connection timeout in seconds. Set to 0 to disable (not recommended)." "description": "Idle connection timeout in seconds. Set to 0 to disable."
} }
}, },
"additionalProperties": false, "additionalProperties": false,
@@ -413,25 +419,31 @@
"properties": { "properties": {
"connect": { "connect": {
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 0,
"default": 30, "default": 30,
"description": "TCP connection timeout in seconds." "description": "TCP connection timeout in seconds."
}, },
"keepalive": {
"type": "integer",
"minimum": 0,
"default": 30,
"description": "TCP keepalive connection timeout in seconds."
},
"responseHeader": { "responseHeader": {
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 0,
"default": 60, "default": 0,
"description": "Time to wait for response headers in seconds." "description": "Time to wait for response headers in seconds."
}, },
"tlsHandshake": { "tlsHandshake": {
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 0,
"default": 10, "default": 10,
"description": "TLS handshake timeout in seconds." "description": "TLS handshake timeout in seconds."
}, },
"idleConn": { "idleConn": {
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 0,
"default": 90, "default": 90,
"description": "Idle connection timeout in seconds." "description": "Idle connection timeout in seconds."
} }
config.example.yaml
+7 -5
View File
@@ -287,14 +287,15 @@ models:
# timeouts: configure proxy connection timeouts for this model # timeouts: configure proxy connection timeouts for this model
# - optional, defaults shown below # - optional, defaults shown below
# - useful for models running on slower hardware that need longer timeouts # - useful for models running on slower hardware that need longer timeouts
# - connect: TCP connection timeout in seconds # - connect: TCP dial connection timeout in seconds, default: 30 seconds
# - responseHeader: time to wait for response headers in seconds # - keepalive: TCP connection keepalive timeout, default: 30 seconds
# (increasing this helps avoid 502 errors on slow hardware) # - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
# - tlsHandshake: TLS handshake timeout in seconds # - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
# - idleConn: idle connection timeout in seconds # - idleConn: idle connection timeout in seconds, default: 90 seconds
# - set any value to 0 to disable that timeout (not recommended) # - set any value to 0 to disable that timeout (not recommended)
timeouts: timeouts:
connect: 30 connect: 30
keepalive: 0
responseHeader: 60 responseHeader: 60
tlsHandshake: 10 tlsHandshake: 10
idleConn: 90 idleConn: 90
@@ -447,6 +448,7 @@ peers:
# - set any value to 0 to disable that timeout (not recommended) # - set any value to 0 to disable that timeout (not recommended)
timeouts: timeouts:
connect: 30 connect: 30
keepalive: 30
responseHeader: 60 responseHeader: 60
tlsHandshake: 10 tlsHandshake: 10
idleConn: 90 idleConn: 90
proxy/config/config_posix_test.go
+13 -28
View File
@@ -163,6 +163,15 @@ groups:
modelLoadingState := false modelLoadingState := false
defaultTimeout := TimeoutsConfig{
Connect: 30,
KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
}
expected := Config{ expected := Config{
LogLevel: "info", LogLevel: "info",
LogTimeFormat: "", LogTimeFormat: "",
@@ -187,13 +196,7 @@ groups:
Name: "Model 1", Name: "Model 1",
Description: "This is model 1", Description: "This is model 1",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model2": { "model2": {
Cmd: "path/to/server --arg1 one", Cmd: "path/to/server --arg1 one",
@@ -202,13 +205,7 @@ groups:
Env: []string{}, Env: []string{},
CheckEndpoint: "/", CheckEndpoint: "/",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model3": { "model3": {
Cmd: "path/to/cmd --arg1 one", Cmd: "path/to/cmd --arg1 one",
@@ -217,13 +214,7 @@ groups:
Env: []string{}, Env: []string{},
CheckEndpoint: "/", CheckEndpoint: "/",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model4": { "model4": {
Cmd: "path/to/cmd --arg1 one", Cmd: "path/to/cmd --arg1 one",
@@ -232,13 +223,7 @@ groups:
Aliases: []string{}, Aliases: []string{},
Env: []string{}, Env: []string{},
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
}, },
HealthCheckTimeout: 15, HealthCheckTimeout: 15,
proxy/config/config_test.go
+1 -1
View File
@@ -1475,7 +1475,7 @@ models:
// Default values should be set during unmarshaling // Default values should be set during unmarshaling
assert.Equal(t, 30, modelConfig.Timeouts.Connect) assert.Equal(t, 30, modelConfig.Timeouts.Connect)
assert.Equal(t, 60, modelConfig.Timeouts.ResponseHeader) assert.Equal(t, 0, modelConfig.Timeouts.ResponseHeader)
assert.Equal(t, 10, modelConfig.Timeouts.TLSHandshake) assert.Equal(t, 10, modelConfig.Timeouts.TLSHandshake)
assert.Equal(t, 1, modelConfig.Timeouts.ExpectContinue) assert.Equal(t, 1, modelConfig.Timeouts.ExpectContinue)
assert.Equal(t, 90, modelConfig.Timeouts.IdleConn) assert.Equal(t, 90, modelConfig.Timeouts.IdleConn)
proxy/config/config_windows_test.go
+13 -28
View File
@@ -155,6 +155,15 @@ groups:
modelLoadingState := false modelLoadingState := false
defaultTimeout := TimeoutsConfig{
Connect: 30,
KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
}
expected := Config{ expected := Config{
LogLevel: "info", LogLevel: "info",
LogTimeFormat: "", LogTimeFormat: "",
@@ -173,13 +182,7 @@ groups:
Env: []string{"VAR1=value1", "VAR2=value2"}, Env: []string{"VAR1=value1", "VAR2=value2"},
CheckEndpoint: "/health", CheckEndpoint: "/health",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model2": { "model2": {
Cmd: "path/to/server --arg1 one", Cmd: "path/to/server --arg1 one",
@@ -189,13 +192,7 @@ groups:
Env: []string{}, Env: []string{},
CheckEndpoint: "/", CheckEndpoint: "/",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model3": { "model3": {
Cmd: "path/to/cmd --arg1 one", Cmd: "path/to/cmd --arg1 one",
@@ -205,13 +202,7 @@ groups:
Env: []string{}, Env: []string{},
CheckEndpoint: "/", CheckEndpoint: "/",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model4": { "model4": {
Cmd: "path/to/cmd --arg1 one", Cmd: "path/to/cmd --arg1 one",
@@ -221,13 +212,7 @@ groups:
Aliases: []string{}, Aliases: []string{},
Env: []string{}, Env: []string{},
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
}, },
HealthCheckTimeout: 15, HealthCheckTimeout: 15,
proxy/config/model_config.go
+11 -6
View File
@@ -10,12 +10,14 @@ const (
) )
// TimeoutsConfig holds timeout settings for proxy connections // TimeoutsConfig holds timeout settings for proxy connections
// 0 = no timeout
type TimeoutsConfig struct { type TimeoutsConfig struct {
Connect int `yaml:"connect"` // seconds, 0 = no timeout (not recommended) Connect int `yaml:"connect"`
ResponseHeader int `yaml:"responseHeader"` // seconds, 0 = no timeout (not recommended) KeepAlive int `yaml:"keepalive"`
TLSHandshake int `yaml:"tlsHandshake"` // seconds, 0 = no timeout (not recommended) ResponseHeader int `yaml:"responseHeader"`
ExpectContinue int `yaml:"expectContinue"` // seconds, 0 = no timeout (not recommended) TLSHandshake int `yaml:"tlsHandshake"`
IdleConn int `yaml:"idleConn"` // seconds, 0 = no timeout (not recommended) ExpectContinue int `yaml:"expectContinue"`
IdleConn int `yaml:"idleConn"`
} }
type ModelConfig struct { type ModelConfig struct {
@@ -69,9 +71,12 @@ func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
ConcurrencyLimit: 0, ConcurrencyLimit: 0,
Name: "", Name: "",
Description: "", Description: "",
// matches http.DefaultTransport
Timeouts: TimeoutsConfig{ Timeouts: TimeoutsConfig{
Connect: 30, Connect: 30,
ResponseHeader: 60, KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10, TLSHandshake: 10,
ExpectContinue: 1, ExpectContinue: 1,
IdleConn: 90, IdleConn: 90,
proxy/config/peer.go
+4
View File
@@ -24,8 +24,12 @@ func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
ApiKey: "", ApiKey: "",
Models: []string{}, Models: []string{},
Filters: Filters{}, Filters: Filters{},
// mostly matches http.DefaultTransport but with a 60s ResponseHeader timeout
// to match the pre PR #619 functionality
Timeouts: TimeoutsConfig{ Timeouts: TimeoutsConfig{
Connect: 30, Connect: 30,
KeepAlive: 30,
ResponseHeader: 60, ResponseHeader: 60,
TLSHandshake: 10, TLSHandshake: 10,
ExpectContinue: 1, ExpectContinue: 1,
proxy/peerproxy.go
+1 -1
View File
@@ -42,7 +42,7 @@ func NewPeerProxy(peers config.PeerDictionaryConfig, proxyLogger *LogMonitor) (*
Proxy: http.ProxyFromEnvironment, Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{ DialContext: (&net.Dialer{
Timeout: time.Duration(peer.Timeouts.Connect) * time.Second, Timeout: time.Duration(peer.Timeouts.Connect) * time.Second,
KeepAlive: 30 * time.Second, KeepAlive: time.Duration(peer.Timeouts.KeepAlive) * time.Second,
}).DialContext, }).DialContext,
TLSHandshakeTimeout: time.Duration(peer.Timeouts.TLSHandshake) * time.Second, TLSHandshakeTimeout: time.Duration(peer.Timeouts.TLSHandshake) * time.Second,
ResponseHeaderTimeout: time.Duration(peer.Timeouts.ResponseHeader) * time.Second, ResponseHeaderTimeout: time.Duration(peer.Timeouts.ResponseHeader) * time.Second,
proxy/process.go
+1 -1
View File
@@ -102,7 +102,7 @@ func NewProcess(ID string, healthCheckTimeout int, config config.ModelConfig, pr
Proxy: http.ProxyFromEnvironment, Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{ DialContext: (&net.Dialer{
Timeout: time.Duration(config.Timeouts.Connect) * time.Second, Timeout: time.Duration(config.Timeouts.Connect) * time.Second,
KeepAlive: 30 * time.Second, KeepAlive: time.Duration(config.Timeouts.KeepAlive) * time.Second,
}).DialContext, }).DialContext,
TLSHandshakeTimeout: time.Duration(config.Timeouts.TLSHandshake) * time.Second, TLSHandshakeTimeout: time.Duration(config.Timeouts.TLSHandshake) * time.Second,
ResponseHeaderTimeout: time.Duration(config.Timeouts.ResponseHeader) * time.Second, ResponseHeaderTimeout: time.Duration(config.Timeouts.ResponseHeader) * time.Second,