From 34b2e290199d112cbab3977e9519c218447e9d39 Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Thu, 9 Apr 2026 19:33:25 +0000 Subject: [PATCH] feat(v2/anthropic): apply cache_control markers from CacheHints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildRequest now tracks a source-index → built-message-index mapping during the role-merge pass, then uses the mapping to attach cache_control: {type: ephemeral} markers at the positions indicated by Request.CacheHints. The last tool, the last system part, and the last non-system message each get a marker when the corresponding hint is set. Covers the merge-induced index drift that would otherwise cause the breakpoint to land on the wrong content block when consecutive same-role source messages are combined into a single Anthropic message with multiple content blocks. Co-Authored-By: Claude Opus 4.6 --- v2/anthropic/anthropic.go | 64 ++++++++- v2/anthropic/cache_test.go | 268 +++++++++++++++++++++++++++++++++++++ 2 files changed, 328 insertions(+), 4 deletions(-) diff --git a/v2/anthropic/anthropic.go b/v2/anthropic/anthropic.go index 9dc713f..138ffbf 100644 --- a/v2/anthropic/anthropic.go +++ b/v2/anthropic/anthropic.go @@ -84,7 +84,16 @@ func (p *Provider) buildRequest(req provider.Request) anth.MessagesRequest { var msgs []anth.Message var systemText string - for _, msg := range req.Messages { + // sourceLanding[srcIdx] = (msgIndex, blockIndex) of the LAST content block + // corresponding to that source message, after merging. System-role source + // messages map to {-1, -1} since they don't appear in msgs. + type landing struct{ msg, block int } + sourceLanding := make([]landing, len(req.Messages)) + for i := range sourceLanding { + sourceLanding[i] = landing{-1, -1} + } + + for srcIdx, msg := range req.Messages { if msg.Role == "system" { if len(systemText) > 0 { systemText += "\n" @@ -98,7 +107,7 @@ func (p *Provider) buildRequest(req provider.Request) anth.MessagesRequest { toolUseID := msg.ToolCallID content := msg.Content isError := false - msgs = append(msgs, anth.Message{ + newMsg := anth.Message{ Role: anth.RoleUser, Content: []anth.MessageContent{ { @@ -115,7 +124,14 @@ func (p *Provider) buildRequest(req provider.Request) anth.MessagesRequest { }, }, }, - }) + } + // Tool-result messages bypass the role-merge logic — they always + // create a new msgs entry. Preserve that. + msgs = append(msgs, newMsg) + sourceLanding[srcIdx] = landing{ + msg: len(msgs) - 1, + block: len(newMsg.Content) - 1, + } continue } @@ -207,11 +223,23 @@ func (p *Provider) buildRequest(req provider.Request) anth.MessagesRequest { // Audio is not supported by Anthropic — skip silently. - // Merge consecutive same-role messages (Anthropic requires alternating) + // Merge consecutive same-role messages (Anthropic requires alternating). if len(msgs) > 0 && msgs[len(msgs)-1].Role == role { + // Track the landing BEFORE mutating: the source message lands in + // the existing last msgs entry, and its last block is at the + // current end-of-content plus len(m.Content)-1. + existingEnd := len(msgs[len(msgs)-1].Content) msgs[len(msgs)-1].Content = append(msgs[len(msgs)-1].Content, m.Content...) + sourceLanding[srcIdx] = landing{ + msg: len(msgs) - 1, + block: existingEnd + len(m.Content) - 1, + } } else { msgs = append(msgs, m) + sourceLanding[srcIdx] = landing{ + msg: len(msgs) - 1, + block: len(m.Content) - 1, + } } } @@ -245,6 +273,34 @@ func (p *Provider) buildRequest(req provider.Request) anth.MessagesRequest { anthReq.StopSequences = req.Stop } + // Apply cache_control markers from hints. + if req.CacheHints != nil { + h := req.CacheHints + + if h.CacheTools && len(anthReq.Tools) > 0 { + anthReq.Tools[len(anthReq.Tools)-1].CacheControl = &anth.MessageCacheControl{ + Type: anth.CacheControlTypeEphemeral, + } + } + if h.CacheSystem && len(anthReq.MultiSystem) > 0 { + anthReq.MultiSystem[len(anthReq.MultiSystem)-1].CacheControl = &anth.MessageCacheControl{ + Type: anth.CacheControlTypeEphemeral, + } + } + if h.LastCacheableMessageIndex >= 0 && h.LastCacheableMessageIndex < len(sourceLanding) { + land := sourceLanding[h.LastCacheableMessageIndex] + if land.msg >= 0 && land.msg < len(anthReq.Messages) { + blocks := anthReq.Messages[land.msg].Content + if land.block >= 0 && land.block < len(blocks) { + blocks[land.block].CacheControl = &anth.MessageCacheControl{ + Type: anth.CacheControlTypeEphemeral, + } + anthReq.Messages[land.msg].Content = blocks + } + } + } + } + return anthReq } diff --git a/v2/anthropic/cache_test.go b/v2/anthropic/cache_test.go index c44c5cb..6f9eda7 100644 --- a/v2/anthropic/cache_test.go +++ b/v2/anthropic/cache_test.go @@ -3,6 +3,8 @@ package anthropic import ( "testing" + anth "github.com/liushuangls/go-anthropic/v2" + "gitea.stevedudenhoeffer.com/steve/go-llm/v2/provider" ) @@ -79,3 +81,269 @@ func TestBuildRequest_NoSystemMessage(t *testing.T) { t.Errorf("expected empty System string, got %q", anthReq.System) } } + +// TestBuildRequest_CacheHints_Tools verifies that the last tool definition +// gets a cache_control marker when CacheHints.CacheTools is set. +func TestBuildRequest_CacheHints_Tools(t *testing.T) { + p := New("test-key") + req := provider.Request{ + Model: "claude-sonnet-4-6", + Tools: []provider.ToolDef{ + {Name: "a", Description: "tool a", Schema: map[string]any{}}, + {Name: "b", Description: "tool b", Schema: map[string]any{}}, + }, + Messages: []provider.Message{{Role: "user", Content: "hi"}}, + CacheHints: &provider.CacheHints{ + CacheTools: true, + LastCacheableMessageIndex: -1, + }, + } + anthReq := p.buildRequest(req) + + if len(anthReq.Tools) != 2 { + t.Fatalf("expected 2 tools, got %d", len(anthReq.Tools)) + } + if anthReq.Tools[0].CacheControl != nil { + t.Error("expected first tool to have no CacheControl") + } + if anthReq.Tools[1].CacheControl == nil { + t.Fatal("expected last tool to have CacheControl") + } + if anthReq.Tools[1].CacheControl.Type != anth.CacheControlTypeEphemeral { + t.Errorf("expected last tool CacheControl type ephemeral, got %q", anthReq.Tools[1].CacheControl.Type) + } +} + +// TestBuildRequest_CacheHints_System verifies that the final system part +// gets a cache_control marker when CacheHints.CacheSystem is set. +func TestBuildRequest_CacheHints_System(t *testing.T) { + p := New("test-key") + req := provider.Request{ + Model: "claude-sonnet-4-6", + Messages: []provider.Message{ + {Role: "system", Content: "you are helpful"}, + {Role: "user", Content: "hi"}, + }, + CacheHints: &provider.CacheHints{ + CacheSystem: true, + LastCacheableMessageIndex: -1, + }, + } + anthReq := p.buildRequest(req) + + if len(anthReq.MultiSystem) != 1 { + t.Fatalf("expected 1 MultiSystem part, got %d", len(anthReq.MultiSystem)) + } + if anthReq.MultiSystem[0].CacheControl == nil { + t.Fatal("expected system part to have CacheControl") + } + if anthReq.MultiSystem[0].CacheControl.Type != anth.CacheControlTypeEphemeral { + t.Errorf("expected system CacheControl type ephemeral, got %q", anthReq.MultiSystem[0].CacheControl.Type) + } +} + +// TestBuildRequest_CacheHints_LastMessage verifies that the last content +// block of the target message receives a cache_control marker. +func TestBuildRequest_CacheHints_LastMessage(t *testing.T) { + p := New("test-key") + req := provider.Request{ + Model: "claude-sonnet-4-6", + Messages: []provider.Message{ + {Role: "user", Content: "first turn"}, + {Role: "assistant", Content: "ok"}, + {Role: "user", Content: "second turn"}, + }, + CacheHints: &provider.CacheHints{ + LastCacheableMessageIndex: 2, + }, + } + anthReq := p.buildRequest(req) + + if len(anthReq.Messages) != 3 { + t.Fatalf("expected 3 messages, got %d", len(anthReq.Messages)) + } + last := anthReq.Messages[2] + if len(last.Content) != 1 { + t.Fatalf("expected 1 content block on last message, got %d", len(last.Content)) + } + if last.Content[0].CacheControl == nil { + t.Fatal("expected CacheControl on last content block of last message") + } + if anthReq.Messages[0].Content[0].CacheControl != nil { + t.Error("expected no CacheControl on first message") + } +} + +// TestBuildRequest_CacheHints_IndexDriftFromMerge verifies that when the +// provider merges consecutive same-role messages, the cache breakpoint +// lands on the correct merged output message. +// +// Scenario: source [user:"a", user:"b", assistant:"c", user:"d"]. +// After merging: msgs[0]=user[a,b], msgs[1]=assistant[c], msgs[2]=user[d]. +// With LastCacheableMessageIndex=3 (source idx of "d"), marker should land on +// msgs[2].Content[0]. +func TestBuildRequest_CacheHints_IndexDriftFromMerge(t *testing.T) { + p := New("test-key") + req := provider.Request{ + Model: "claude-sonnet-4-6", + Messages: []provider.Message{ + {Role: "user", Content: "a"}, + {Role: "user", Content: "b"}, + {Role: "assistant", Content: "c"}, + {Role: "user", Content: "d"}, + }, + CacheHints: &provider.CacheHints{ + LastCacheableMessageIndex: 3, + }, + } + anthReq := p.buildRequest(req) + + if len(anthReq.Messages) != 3 { + t.Fatalf("expected 3 merged messages, got %d", len(anthReq.Messages)) + } + if len(anthReq.Messages[2].Content) != 1 { + t.Fatalf("expected 1 content block on merged user, got %d", len(anthReq.Messages[2].Content)) + } + if anthReq.Messages[2].Content[0].CacheControl == nil { + t.Fatal("expected CacheControl on last content block of merged user message") + } + for i, cb := range anthReq.Messages[0].Content { + if cb.CacheControl != nil { + t.Errorf("expected no CacheControl on anthReq.Messages[0].Content[%d]", i) + } + } +} + +// TestBuildRequest_CacheHints_IndexDriftTargetIsMerged: scenario where +// LastCacheableMessageIndex points into a message that was merged. +// Source [user:"a", user:"b"] → msgs[0]=user[a,b]. Index 1 should land on +// msgs[0].Content[1] ("b"). +func TestBuildRequest_CacheHints_IndexDriftTargetIsMerged(t *testing.T) { + p := New("test-key") + req := provider.Request{ + Model: "claude-sonnet-4-6", + Messages: []provider.Message{ + {Role: "user", Content: "a"}, + {Role: "user", Content: "b"}, + }, + CacheHints: &provider.CacheHints{ + LastCacheableMessageIndex: 1, + }, + } + anthReq := p.buildRequest(req) + + if len(anthReq.Messages) != 1 { + t.Fatalf("expected 1 merged message, got %d", len(anthReq.Messages)) + } + if len(anthReq.Messages[0].Content) != 2 { + t.Fatalf("expected 2 content blocks after merge, got %d", len(anthReq.Messages[0].Content)) + } + if anthReq.Messages[0].Content[0].CacheControl != nil { + t.Error("expected no CacheControl on first content block (source index 0)") + } + if anthReq.Messages[0].Content[1].CacheControl == nil { + t.Fatal("expected CacheControl on second content block (source index 1)") + } +} + +// TestBuildRequest_CacheHints_AllThree: tools + system + last-message all +// receive markers in a single request. +func TestBuildRequest_CacheHints_AllThree(t *testing.T) { + p := New("test-key") + req := provider.Request{ + Model: "claude-sonnet-4-6", + Tools: []provider.ToolDef{ + {Name: "t1", Description: "tool", Schema: map[string]any{}}, + }, + Messages: []provider.Message{ + {Role: "system", Content: "be helpful"}, + {Role: "user", Content: "hi"}, + }, + CacheHints: &provider.CacheHints{ + CacheTools: true, + CacheSystem: true, + LastCacheableMessageIndex: 1, + }, + } + anthReq := p.buildRequest(req) + + if anthReq.Tools[0].CacheControl == nil { + t.Error("expected tool CacheControl") + } + if anthReq.MultiSystem[0].CacheControl == nil { + t.Error("expected system CacheControl") + } + if len(anthReq.Messages) != 1 { + t.Fatalf("expected 1 message, got %d", len(anthReq.Messages)) + } + if anthReq.Messages[0].Content[0].CacheControl == nil { + t.Error("expected user-message CacheControl") + } +} + +// TestBuildRequest_CacheHints_Nil: nil CacheHints → no markers. +func TestBuildRequest_CacheHints_Nil(t *testing.T) { + p := New("test-key") + req := provider.Request{ + Model: "claude-sonnet-4-6", + Tools: []provider.ToolDef{ + {Name: "t", Description: "d", Schema: map[string]any{}}, + }, + Messages: []provider.Message{ + {Role: "system", Content: "s"}, + {Role: "user", Content: "hi"}, + }, + } + anthReq := p.buildRequest(req) + + if anthReq.Tools[0].CacheControl != nil { + t.Error("expected no tool CacheControl with nil hints") + } + if len(anthReq.MultiSystem) == 1 && anthReq.MultiSystem[0].CacheControl != nil { + t.Error("expected no system CacheControl with nil hints") + } + for i, m := range anthReq.Messages { + for j, cb := range m.Content { + if cb.CacheControl != nil { + t.Errorf("expected no CacheControl on anthReq.Messages[%d].Content[%d]", i, j) + } + } + } +} + +// TestBuildRequest_CacheHints_ToolResultMessage: marker lands on the right +// tool-result message. +func TestBuildRequest_CacheHints_ToolResultMessage(t *testing.T) { + p := New("test-key") + req := provider.Request{ + Model: "claude-sonnet-4-6", + Messages: []provider.Message{ + {Role: "user", Content: "use the tool"}, + {Role: "assistant", Content: "", ToolCalls: []provider.ToolCall{ + {ID: "tc1", Name: "x", Arguments: "{}"}, + }}, + {Role: "tool", Content: "result data", ToolCallID: "tc1"}, + }, + CacheHints: &provider.CacheHints{ + LastCacheableMessageIndex: 2, + }, + } + anthReq := p.buildRequest(req) + + var toolMsg *anth.Message + for i := range anthReq.Messages { + for _, cb := range anthReq.Messages[i].Content { + if cb.MessageContentToolResult != nil { + toolMsg = &anthReq.Messages[i] + break + } + } + } + if toolMsg == nil { + t.Fatal("expected to find tool-result message") + } + lastBlock := toolMsg.Content[len(toolMsg.Content)-1] + if lastBlock.CacheControl == nil { + t.Error("expected CacheControl on last content block of tool-result message") + } +}