changed the way images are passed

anthropic.go

@@ -58,12 +58,20 @@ func (a anthropic) requestToAnthropicRequest(req Request) anth.MessagesRequest {
 				})
 			}
 
-			if msg.ImageBase64 != "" {
-				m.Content = append(m.Content, anth.NewImageMessageContent(anth.MessageContentImageSource{
-					Type:      "base64",
-					MediaType: "image/png",
-					Data:      msg.ImageBase64,
-				}))
+			for _, img := range msg.Images {
+				if img.Base64 != "" {
+					m.Content = append(m.Content, anth.NewImageMessageContent(anth.MessageContentImageSource{
+						Type:      "base64",
+						MediaType: img.ContentType,
+						Data:      img.Base64,
+					}))
+				} else if img.Url != "" {
+					m.Content = append(m.Content, anth.NewImageMessageContent(anth.MessageContentImageSource{
+						Type:      "url",
+						MediaType: img.ContentType,
+						Data:      img.Url,
+					}))
+				}
 			}
 
 			// if this has the same role as the previous message, we can append it to the previous message

llm.go

@@ -12,11 +12,17 @@ const (
 	RoleAssistant Role = "assistant"
 )
 
+type Image struct {
+	Base64      string
+	ContentType string
+	Url         string
+}
+
 type Message struct {
-	Role        Role
-	Name        string
-	Text        string
-	ImageBase64 string // ImageBase64 is the base64 string if the message contains an image, empty string otherwise.
+	Role   Role
+	Name   string
+	Text   string
+	Images []Image
 }
 
 type Request struct {

openai.go

@@ -25,15 +25,22 @@ func (o openai) requestToOpenAIRequest(request Request) oai.ChatCompletionReques
 			Name:    msg.Name,
 		}
 
-		if msg.ImageBase64 != "" {
-			part := oai.ChatMessagePart{
-				Type: "image_url",
-				ImageURL: &oai.ChatMessageImageURL{
-					URL: fmt.Sprintf("data:image/jpeg;base64,%s", msg.ImageBase64),
-				},
+		for _, img := range msg.Images {
+			if img.Base64 != "" {
+				m.MultiContent = append(m.MultiContent, oai.ChatMessagePart{
+					Type: "image_url",
+					ImageURL: &oai.ChatMessageImageURL{
+						URL: fmt.Sprintf("data:%s;base64,%s", img.ContentType, img.Base64),
+					},
+				})
+			} else if img.Url != "" {
+				m.MultiContent = append(m.MultiContent, oai.ChatMessagePart{
+					Type: "image_url",
+					ImageURL: &oai.ChatMessageImageURL{
+						URL: img.Url,
+					},
+				})
 			}
-
-			m.MultiContent = append(m.MultiContent, part)
 		}
 
 		res.Messages = append(res.Messages, m)