feat(failover): package-level default observer for transparently-built chains
CI / Build, Test & Lint (push) Successful in 10m43s
CI / Build, Test & Lint (push) Successful in 10m43s
The transparent comma-Parse path builds failover chains via NewFailoverModel with no options, so defaultFailoverConfig() left the observer nil and observers only fired when a caller passed WithFailoverObserver explicitly. Add a package-level default observer (SetFailoverObserver / DefaultFailoverObserver), guarded by the existing defaultsMu, and seed it in defaultFailoverConfig() so chains built transparently still notify it. An explicit WithFailoverObserver still overrides the default per-chain. mort sets this at boot to persist failover events. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -241,6 +241,134 @@ func TestFailover_Observer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailover_DefaultObserverFiresOnTransparentChain verifies that a chain
|
||||
// built via NewFailoverModel with NO options still notifies a package-level
|
||||
// default observer set via SetFailoverObserver. This is the transparent
|
||||
// comma-Parse path: defaultFailoverConfig() must seed the default observer.
|
||||
func TestFailover_DefaultObserverFiresOnTransparentChain(t *testing.T) {
|
||||
resetHealthForTest()
|
||||
t.Cleanup(func() {
|
||||
SetFailoverObserver(nil)
|
||||
resetHealthForTest()
|
||||
})
|
||||
|
||||
var mu sync.Mutex
|
||||
var events []FailoverEvent
|
||||
SetFailoverObserver(func(ctx context.Context, ev FailoverEvent) {
|
||||
mu.Lock()
|
||||
events = append(events, ev)
|
||||
mu.Unlock()
|
||||
})
|
||||
|
||||
a := newMockProviderFunc(func(ctx context.Context, req provider.Request) (provider.Response, error) {
|
||||
return provider.Response{}, &openai.Error{StatusCode: 401}
|
||||
})
|
||||
b := newMockProvider(provider.Response{Text: "from-b"})
|
||||
|
||||
// NO options: the only way the observer can fire is via the package default.
|
||||
fo := NewFailoverModel([]*Model{modelFor(a, "p/a"), modelFor(b, "p/b")})
|
||||
_, err := fo.Complete(context.Background(), []Message{{Role: RoleUser, Content: Content{Text: "hi"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(events) == 0 {
|
||||
t.Fatal("expected default observer to fire on a transparently-built chain")
|
||||
}
|
||||
if events[0].Model != "p/a" || events[0].Kind != ErrAuthDead {
|
||||
t.Errorf("unexpected event: %+v", events[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailover_DefaultObserverFiresOnParseChain verifies the comma-Parse seam:
|
||||
// a chain built through the registry's ParseChain (no per-call observer) fires
|
||||
// the package-level default observer.
|
||||
func TestFailover_DefaultObserverFiresOnParseChain(t *testing.T) {
|
||||
resetHealthForTest()
|
||||
t.Cleanup(func() {
|
||||
SetFailoverObserver(nil)
|
||||
resetHealthForTest()
|
||||
})
|
||||
|
||||
var mu sync.Mutex
|
||||
var events []FailoverEvent
|
||||
SetFailoverObserver(func(ctx context.Context, ev FailoverEvent) {
|
||||
mu.Lock()
|
||||
events = append(events, ev)
|
||||
mu.Unlock()
|
||||
})
|
||||
|
||||
r, alpha, _ := testRegistry(nil)
|
||||
// alpha returns a 401 (auth-dead) so the chain fails over and emits an event.
|
||||
alpha.err = &openai.Error{StatusCode: 401}
|
||||
|
||||
m, err := r.Parse("alpha/model-a,beta/model-b")
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if _, ok := m.provider.(*failoverProvider); !ok {
|
||||
t.Fatalf("expected a failover provider, got %T", m.provider)
|
||||
}
|
||||
|
||||
_, err = m.Complete(context.Background(), []Message{{Role: RoleUser, Content: Content{Text: "hi"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(events) == 0 {
|
||||
t.Fatal("expected default observer to fire on a comma-Parse'd chain")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailover_ExplicitObserverOverridesDefault verifies WithFailoverObserver
|
||||
// still wins: when both a package default and an explicit observer are present,
|
||||
// only the explicit one fires for that chain.
|
||||
func TestFailover_ExplicitObserverOverridesDefault(t *testing.T) {
|
||||
resetHealthForTest()
|
||||
t.Cleanup(func() {
|
||||
SetFailoverObserver(nil)
|
||||
resetHealthForTest()
|
||||
})
|
||||
|
||||
var mu sync.Mutex
|
||||
var defaultCalls, explicitCalls int
|
||||
SetFailoverObserver(func(ctx context.Context, ev FailoverEvent) {
|
||||
mu.Lock()
|
||||
defaultCalls++
|
||||
mu.Unlock()
|
||||
})
|
||||
explicit := func(ctx context.Context, ev FailoverEvent) {
|
||||
mu.Lock()
|
||||
explicitCalls++
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
a := newMockProviderFunc(func(ctx context.Context, req provider.Request) (provider.Response, error) {
|
||||
return provider.Response{}, &openai.Error{StatusCode: 401}
|
||||
})
|
||||
b := newMockProvider(provider.Response{Text: "from-b"})
|
||||
|
||||
fo := NewFailoverModel([]*Model{modelFor(a, "p/a"), modelFor(b, "p/b")},
|
||||
WithFailoverObserver(explicit))
|
||||
_, err := fo.Complete(context.Background(), []Message{{Role: RoleUser, Content: Content{Text: "hi"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if explicitCalls == 0 {
|
||||
t.Error("explicit observer should fire")
|
||||
}
|
||||
if defaultCalls != 0 {
|
||||
t.Errorf("default observer must NOT fire when an explicit one is set; got %d calls", defaultCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailover_ControlAPI(t *testing.T) {
|
||||
resetHealthForTest()
|
||||
if IsBenched("x/y") {
|
||||
|
||||
Reference in New Issue
Block a user