feat(v2): add WithPromptCaching() request option
Introduces an opt-in RequestOption that callers can pass to enable automatic prompt-caching markers. The option populates a cacheConfig on requestConfig but has no effect yet — plumbing through to provider.Request and on to the Anthropic provider lands in subsequent commits. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,12 @@ type requestConfig struct {
|
|||||||
maxTokens *int
|
maxTokens *int
|
||||||
topP *float64
|
topP *float64
|
||||||
stop []string
|
stop []string
|
||||||
|
cacheConfig *cacheConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// cacheConfig holds prompt-caching settings. nil = disabled.
|
||||||
|
type cacheConfig struct {
|
||||||
|
enabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithTools attaches a toolbox to the request.
|
// WithTools attaches a toolbox to the request.
|
||||||
@@ -35,3 +41,28 @@ func WithTopP(p float64) RequestOption {
|
|||||||
func WithStop(sequences ...string) RequestOption {
|
func WithStop(sequences ...string) RequestOption {
|
||||||
return func(c *requestConfig) { c.stop = sequences }
|
return func(c *requestConfig) { c.stop = sequences }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithPromptCaching enables automatic prompt-caching markers on providers
|
||||||
|
// that support it (currently Anthropic). On providers that don't support
|
||||||
|
// explicit cache markers (OpenAI, Google), this is a no-op.
|
||||||
|
//
|
||||||
|
// When enabled, the library places cache breakpoints at natural seams:
|
||||||
|
// - the last tool definition (caches all tools)
|
||||||
|
// - the last system message (caches tools + system)
|
||||||
|
// - the last non-system message in the history (caches tools + system +
|
||||||
|
// conversation so far)
|
||||||
|
//
|
||||||
|
// Breakpoints are placed only when the corresponding section is non-empty.
|
||||||
|
// Up to 3 markers are emitted per request, leaving one of Anthropic's 4
|
||||||
|
// marker slots for future use.
|
||||||
|
//
|
||||||
|
// Cache hits give a 90% discount on cached input tokens (5-minute ephemeral
|
||||||
|
// tier). Cache writes cost 25% more than normal input tokens, so this option
|
||||||
|
// is only worth enabling for prompts whose cacheable prefix exceeds the
|
||||||
|
// minimum (1024 tokens on Opus/Sonnet, 2048 tokens on Haiku) AND is re-sent
|
||||||
|
// at least twice within the 5-minute TTL.
|
||||||
|
func WithPromptCaching() RequestOption {
|
||||||
|
return func(c *requestConfig) {
|
||||||
|
c.cacheConfig = &cacheConfig{enabled: true}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -135,3 +135,22 @@ func TestBuildProviderRequest_EmptyConfig(t *testing.T) {
|
|||||||
t.Errorf("expected no tools, got %d", len(req.Tools))
|
t.Errorf("expected no tools, got %d", len(req.Tools))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWithPromptCaching(t *testing.T) {
|
||||||
|
cfg := &requestConfig{}
|
||||||
|
WithPromptCaching()(cfg)
|
||||||
|
if cfg.cacheConfig == nil {
|
||||||
|
t.Fatal("expected cacheConfig to be set after WithPromptCaching()")
|
||||||
|
}
|
||||||
|
if !cfg.cacheConfig.enabled {
|
||||||
|
t.Error("expected cacheConfig.enabled to be true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithoutPromptCaching(t *testing.T) {
|
||||||
|
cfg := &requestConfig{}
|
||||||
|
// No option applied
|
||||||
|
if cfg.cacheConfig != nil {
|
||||||
|
t.Error("expected cacheConfig to be nil when option not applied")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user