steve/go-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(v2/anthropic): apply cache_control markers from CacheHints

Browse Source 
buildRequest now tracks a source-index → built-message-index mapping
during the role-merge pass, then uses the mapping to attach
cache_control: {type: ephemeral} markers at the positions indicated by
Request.CacheHints. The last tool, the last system part, and the last
non-system message each get a marker when the corresponding hint is set.

Covers the merge-induced index drift that would otherwise cause the
breakpoint to land on the wrong content block when consecutive same-role
source messages are combined into a single Anthropic message with
multiple content blocks.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Steve Dudenhoeffer
2026-04-09 19:33:25 +00:00
parent 4c6dfb9058
commit 34b2e29019
2 changed files with 328 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
v2/anthropic/cache_test.go
View File

@@ -3,6 +3,8 @@ package anthropic
import (
"testing"
anth "github.com/liushuangls/go-anthropic/v2"
"gitea.stevedudenhoeffer.com/steve/go-llm/v2/provider"
)
@@ -79,3 +81,269 @@ func TestBuildRequest_NoSystemMessage(t *testing.T) {
t.Errorf("expected empty System string, got %q", anthReq.System)
}
}
// TestBuildRequest_CacheHints_Tools verifies that the last tool definition
// gets a cache_control marker when CacheHints.CacheTools is set.
func TestBuildRequest_CacheHints_Tools(t *testing.T) {
p := New("test-key")
req := provider.Request{
Model: "claude-sonnet-4-6",
Tools: []provider.ToolDef{
{Name: "a", Description: "tool a", Schema: map[string]any{}},
{Name: "b", Description: "tool b", Schema: map[string]any{}},
},
Messages: []provider.Message{{Role: "user", Content: "hi"}},
CacheHints: &provider.CacheHints{
CacheTools: true,
LastCacheableMessageIndex: -1,
},
}
anthReq := p.buildRequest(req)
if len(anthReq.Tools) != 2 {
t.Fatalf("expected 2 tools, got %d", len(anthReq.Tools))
}
if anthReq.Tools[0].CacheControl != nil {
t.Error("expected first tool to have no CacheControl")
}
if anthReq.Tools[1].CacheControl == nil {
t.Fatal("expected last tool to have CacheControl")
}
if anthReq.Tools[1].CacheControl.Type != anth.CacheControlTypeEphemeral {
t.Errorf("expected last tool CacheControl type ephemeral, got %q", anthReq.Tools[1].CacheControl.Type)
}
}
// TestBuildRequest_CacheHints_System verifies that the final system part
// gets a cache_control marker when CacheHints.CacheSystem is set.
func TestBuildRequest_CacheHints_System(t *testing.T) {
p := New("test-key")
req := provider.Request{
Model: "claude-sonnet-4-6",
Messages: []provider.Message{
{Role: "system", Content: "you are helpful"},
{Role: "user", Content: "hi"},
},
CacheHints: &provider.CacheHints{
CacheSystem: true,
LastCacheableMessageIndex: -1,
},
}
anthReq := p.buildRequest(req)
if len(anthReq.MultiSystem) != 1 {
t.Fatalf("expected 1 MultiSystem part, got %d", len(anthReq.MultiSystem))
}
if anthReq.MultiSystem[0].CacheControl == nil {
t.Fatal("expected system part to have CacheControl")
}
if anthReq.MultiSystem[0].CacheControl.Type != anth.CacheControlTypeEphemeral {
t.Errorf("expected system CacheControl type ephemeral, got %q", anthReq.MultiSystem[0].CacheControl.Type)
}
}
// TestBuildRequest_CacheHints_LastMessage verifies that the last content
// block of the target message receives a cache_control marker.
func TestBuildRequest_CacheHints_LastMessage(t *testing.T) {
p := New("test-key")
req := provider.Request{
Model: "claude-sonnet-4-6",
Messages: []provider.Message{
{Role: "user", Content: "first turn"},
{Role: "assistant", Content: "ok"},
{Role: "user", Content: "second turn"},
},
CacheHints: &provider.CacheHints{
LastCacheableMessageIndex: 2,
},
}
anthReq := p.buildRequest(req)
if len(anthReq.Messages) != 3 {
t.Fatalf("expected 3 messages, got %d", len(anthReq.Messages))
}
last := anthReq.Messages[2]
if len(last.Content) != 1 {
t.Fatalf("expected 1 content block on last message, got %d", len(last.Content))
}
if last.Content[0].CacheControl == nil {
t.Fatal("expected CacheControl on last content block of last message")
}
if anthReq.Messages[0].Content[0].CacheControl != nil {
t.Error("expected no CacheControl on first message")
}
}
// TestBuildRequest_CacheHints_IndexDriftFromMerge verifies that when the
// provider merges consecutive same-role messages, the cache breakpoint
// lands on the correct merged output message.
//
// Scenario: source [user:"a", user:"b", assistant:"c", user:"d"].
// After merging: msgs[0]=user[a,b], msgs[1]=assistant[c], msgs[2]=user[d].
// With LastCacheableMessageIndex=3 (source idx of "d"), marker should land on
// msgs[2].Content[0].
func TestBuildRequest_CacheHints_IndexDriftFromMerge(t *testing.T) {
p := New("test-key")
req := provider.Request{
Model: "claude-sonnet-4-6",
Messages: []provider.Message{
{Role: "user", Content: "a"},
{Role: "user", Content: "b"},
{Role: "assistant", Content: "c"},
{Role: "user", Content: "d"},
},
CacheHints: &provider.CacheHints{
LastCacheableMessageIndex: 3,
},
}
anthReq := p.buildRequest(req)
if len(anthReq.Messages) != 3 {
t.Fatalf("expected 3 merged messages, got %d", len(anthReq.Messages))
}
if len(anthReq.Messages[2].Content) != 1 {
t.Fatalf("expected 1 content block on merged user, got %d", len(anthReq.Messages[2].Content))
}
if anthReq.Messages[2].Content[0].CacheControl == nil {
t.Fatal("expected CacheControl on last content block of merged user message")
}
for i, cb := range anthReq.Messages[0].Content {
if cb.CacheControl != nil {
t.Errorf("expected no CacheControl on anthReq.Messages[0].Content[%d]", i)
}
}
}
// TestBuildRequest_CacheHints_IndexDriftTargetIsMerged: scenario where
// LastCacheableMessageIndex points into a message that was merged.
// Source [user:"a", user:"b"] → msgs[0]=user[a,b]. Index 1 should land on
// msgs[0].Content[1] ("b").
func TestBuildRequest_CacheHints_IndexDriftTargetIsMerged(t *testing.T) {
p := New("test-key")
req := provider.Request{
Model: "claude-sonnet-4-6",
Messages: []provider.Message{
{Role: "user", Content: "a"},
{Role: "user", Content: "b"},
},
CacheHints: &provider.CacheHints{
LastCacheableMessageIndex: 1,
},
}
anthReq := p.buildRequest(req)
if len(anthReq.Messages) != 1 {
t.Fatalf("expected 1 merged message, got %d", len(anthReq.Messages))
}
if len(anthReq.Messages[0].Content) != 2 {
t.Fatalf("expected 2 content blocks after merge, got %d", len(anthReq.Messages[0].Content))
}
if anthReq.Messages[0].Content[0].CacheControl != nil {
t.Error("expected no CacheControl on first content block (source index 0)")
}
if anthReq.Messages[0].Content[1].CacheControl == nil {
t.Fatal("expected CacheControl on second content block (source index 1)")
}
}
// TestBuildRequest_CacheHints_AllThree: tools + system + last-message all
// receive markers in a single request.
func TestBuildRequest_CacheHints_AllThree(t *testing.T) {
p := New("test-key")
req := provider.Request{
Model: "claude-sonnet-4-6",
Tools: []provider.ToolDef{
{Name: "t1", Description: "tool", Schema: map[string]any{}},
},
Messages: []provider.Message{
{Role: "system", Content: "be helpful"},
{Role: "user", Content: "hi"},
},
CacheHints: &provider.CacheHints{
CacheTools: true,
CacheSystem: true,
LastCacheableMessageIndex: 1,
},
}
anthReq := p.buildRequest(req)
if anthReq.Tools[0].CacheControl == nil {
t.Error("expected tool CacheControl")
}
if anthReq.MultiSystem[0].CacheControl == nil {
t.Error("expected system CacheControl")
}
if len(anthReq.Messages) != 1 {
t.Fatalf("expected 1 message, got %d", len(anthReq.Messages))
}
if anthReq.Messages[0].Content[0].CacheControl == nil {
t.Error("expected user-message CacheControl")
}
}
// TestBuildRequest_CacheHints_Nil: nil CacheHints → no markers.
func TestBuildRequest_CacheHints_Nil(t *testing.T) {
p := New("test-key")
req := provider.Request{
Model: "claude-sonnet-4-6",
Tools: []provider.ToolDef{
{Name: "t", Description: "d", Schema: map[string]any{}},
},
Messages: []provider.Message{
{Role: "system", Content: "s"},
{Role: "user", Content: "hi"},
},
}
anthReq := p.buildRequest(req)
if anthReq.Tools[0].CacheControl != nil {
t.Error("expected no tool CacheControl with nil hints")
}
if len(anthReq.MultiSystem) == 1 && anthReq.MultiSystem[0].CacheControl != nil {
t.Error("expected no system CacheControl with nil hints")
}
for i, m := range anthReq.Messages {
for j, cb := range m.Content {
if cb.CacheControl != nil {
t.Errorf("expected no CacheControl on anthReq.Messages[%d].Content[%d]", i, j)
}
}
}
}
// TestBuildRequest_CacheHints_ToolResultMessage: marker lands on the right
// tool-result message.
func TestBuildRequest_CacheHints_ToolResultMessage(t *testing.T) {
p := New("test-key")
req := provider.Request{
Model: "claude-sonnet-4-6",
Messages: []provider.Message{
{Role: "user", Content: "use the tool"},
{Role: "assistant", Content: "", ToolCalls: []provider.ToolCall{
{ID: "tc1", Name: "x", Arguments: "{}"},
}},
{Role: "tool", Content: "result data", ToolCallID: "tc1"},
},
CacheHints: &provider.CacheHints{
LastCacheableMessageIndex: 2,
},
}
anthReq := p.buildRequest(req)
var toolMsg *anth.Message
for i := range anthReq.Messages {
for _, cb := range anthReq.Messages[i].Content {
if cb.MessageContentToolResult != nil {
toolMsg = &anthReq.Messages[i]
break
}
}
}
if toolMsg == nil {
t.Fatal("expected to find tool-result message")
}
lastBlock := toolMsg.Content[len(toolMsg.Content)-1]
if lastBlock.CacheControl == nil {
t.Error("expected CacheControl on last content block of tool-result message")
}
}