Fix role setting for assistant-sent images in Anthropic API

Anthropic API does not support assistants sending images directly, so the role is adjusted to "user" for such messages. This ensures compatibility and prevents errors when processing image messages.

anthropic.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"log/slog"
 	"net/http"
 
 	anth "github.com/liushuangls/go-anthropic/v2"
@@ -25,6 +26,13 @@ func (a anthropic) ModelVersion(modelVersion string) (ChatCompletion, error) {
 	return a, nil
 }
 
+func deferClose(c io.Closer) {
+	err := c.Close()
+	if err != nil {
+		slog.Error("error closing", "error", err)
+	}
+}
+
 func (a anthropic) requestToAnthropicRequest(req Request) anth.MessagesRequest {
 	res := anth.MessagesRequest{
 		Model:     anth.Model(a.model),
@@ -63,6 +71,11 @@ func (a anthropic) requestToAnthropicRequest(req Request) anth.MessagesRequest {
 			}
 
 			for _, img := range msg.Images {
+				// anthropic doesn't allow the assistant to send images, so we need to say it's from the user
+				if m.Role == anth.RoleAssistant {
+					m.Role = anth.RoleUser
+				}
+
 				if img.Base64 != "" {
 					m.Content = append(m.Content, anth.NewImageMessageContent(
 						anth.NewMessageContentSource(
@@ -85,7 +98,7 @@ func (a anthropic) requestToAnthropicRequest(req Request) anth.MessagesRequest {
 						continue
 					}
 
-					defer resp.Body.Close()
+					defer deferClose(resp.Body)
 
 					img.ContentType = resp.Header.Get("Content-Type")
 
@@ -121,12 +134,14 @@ func (a anthropic) requestToAnthropicRequest(req Request) anth.MessagesRequest {
 		}
 	}
 
-	for _, tool := range req.Toolbox.funcs {
-		res.Tools = append(res.Tools, anth.ToolDefinition{
-			Name:        tool.Name,
-			Description: tool.Description,
-			InputSchema: tool.Parameters,
-		})
+	if req.Toolbox != nil {
+		for _, tool := range req.Toolbox.funcs {
+			res.Tools = append(res.Tools, anth.ToolDefinition{
+				Name:        tool.Name,
+				Description: tool.Description,
+				InputSchema: tool.Parameters,
+			})
+		}
 	}
 
 	res.Messages = msgs

go.mod

@@ -5,7 +5,8 @@ go 1.23.1
 require (
 	github.com/google/generative-ai-go v0.19.0
 	github.com/liushuangls/go-anthropic/v2 v2.13.0
-	github.com/sashabaranov/go-openai v1.36.0
+	github.com/sashabaranov/go-openai v1.36.1
+	golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67
 	google.golang.org/api v0.214.0
 )
 

go.sum

@@ -39,6 +39,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sashabaranov/go-openai v1.36.0 h1:fcSrn8uGuorzPWCBp8L0aCR95Zjb/Dd+ZSML0YZy9EI=
 github.com/sashabaranov/go-openai v1.36.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/sashabaranov/go-openai v1.36.1 h1:EVfRXwIlW2rUzpx6vR+aeIKCK/xylSrVYAx1TMTSX3g=
+github.com/sashabaranov/go-openai v1.36.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
@@ -59,6 +61,8 @@ go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qq
 go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo=
+golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
 golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=

openai.go

@@ -3,8 +3,9 @@ package go_llm
 import (
 	"context"
 	"fmt"
-	oai "github.com/sashabaranov/go-openai"
 	"strings"
+
+	oai "github.com/sashabaranov/go-openai"
 )
 
 type openaiImpl struct {
@@ -57,18 +58,20 @@ func (o openaiImpl) requestToOpenAIRequest(request Request) oai.ChatCompletionRe
 		res.Messages = append(res.Messages, m)
 	}
 
-	for _, tool := range request.Toolbox.funcs {
-		res.Tools = append(res.Tools, oai.Tool{
-			Type: "function",
-			Function: &oai.FunctionDefinition{
-				Name:        tool.Name,
-				Description: tool.Description,
-				Strict:      tool.Strict,
-				Parameters:  tool.Parameters.Definition(),
-			},
-		})
+	if request.Toolbox != nil {
+		for _, tool := range request.Toolbox.funcs {
+			res.Tools = append(res.Tools, oai.Tool{
+				Type: "function",
+				Function: &oai.FunctionDefinition{
+					Name:        tool.Name,
+					Description: tool.Description,
+					Strict:      tool.Strict,
+					Parameters:  tool.Parameters.Definition(),
+				},
+			})
 
-		fmt.Println("tool:", tool.Name, tool.Description, tool.Strict, tool.Parameters.Definition())
+			fmt.Println("tool:", tool.Name, tool.Description, tool.Strict, tool.Parameters.Definition())
+		}
 	}
 
 	if request.Temperature != nil {