feat(v2): add WithPromptCaching() request option
Introduces an opt-in RequestOption that callers can pass to enable automatic prompt-caching markers. The option populates a cacheConfig on requestConfig but has no effect yet — plumbing through to provider.Request and on to the Anthropic provider lands in subsequent commits. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,12 @@ type requestConfig struct {
|
||||
maxTokens *int
|
||||
topP *float64
|
||||
stop []string
|
||||
cacheConfig *cacheConfig
|
||||
}
|
||||
|
||||
// cacheConfig holds prompt-caching settings. nil = disabled.
|
||||
type cacheConfig struct {
|
||||
enabled bool
|
||||
}
|
||||
|
||||
// WithTools attaches a toolbox to the request.
|
||||
@@ -35,3 +41,28 @@ func WithTopP(p float64) RequestOption {
|
||||
func WithStop(sequences ...string) RequestOption {
|
||||
return func(c *requestConfig) { c.stop = sequences }
|
||||
}
|
||||
|
||||
// WithPromptCaching enables automatic prompt-caching markers on providers
|
||||
// that support it (currently Anthropic). On providers that don't support
|
||||
// explicit cache markers (OpenAI, Google), this is a no-op.
|
||||
//
|
||||
// When enabled, the library places cache breakpoints at natural seams:
|
||||
// - the last tool definition (caches all tools)
|
||||
// - the last system message (caches tools + system)
|
||||
// - the last non-system message in the history (caches tools + system +
|
||||
// conversation so far)
|
||||
//
|
||||
// Breakpoints are placed only when the corresponding section is non-empty.
|
||||
// Up to 3 markers are emitted per request, leaving one of Anthropic's 4
|
||||
// marker slots for future use.
|
||||
//
|
||||
// Cache hits give a 90% discount on cached input tokens (5-minute ephemeral
|
||||
// tier). Cache writes cost 25% more than normal input tokens, so this option
|
||||
// is only worth enabling for prompts whose cacheable prefix exceeds the
|
||||
// minimum (1024 tokens on Opus/Sonnet, 2048 tokens on Haiku) AND is re-sent
|
||||
// at least twice within the 5-minute TTL.
|
||||
func WithPromptCaching() RequestOption {
|
||||
return func(c *requestConfig) {
|
||||
c.cacheConfig = &cacheConfig{enabled: true}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,3 +135,22 @@ func TestBuildProviderRequest_EmptyConfig(t *testing.T) {
|
||||
t.Errorf("expected no tools, got %d", len(req.Tools))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithPromptCaching(t *testing.T) {
|
||||
cfg := &requestConfig{}
|
||||
WithPromptCaching()(cfg)
|
||||
if cfg.cacheConfig == nil {
|
||||
t.Fatal("expected cacheConfig to be set after WithPromptCaching()")
|
||||
}
|
||||
if !cfg.cacheConfig.enabled {
|
||||
t.Error("expected cacheConfig.enabled to be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutPromptCaching(t *testing.T) {
|
||||
cfg := &requestConfig{}
|
||||
// No option applied
|
||||
if cfg.cacheConfig != nil {
|
||||
t.Error("expected cacheConfig to be nil when option not applied")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user