From b8dc1d1a962a074e77af156077916a1c0148ee77 Mon Sep 17 00:00:00 2001
From: Steve Dudenhoeffer <steve@stevedudenhoeffer.com>
Date: Mon, 7 Oct 2024 16:33:57 -0400
Subject: [PATCH] changed the way images are passed

---
 anthropic.go | 20 ++++++++++++++------
 llm.go       | 14 ++++++++++----
 openai.go    | 23 +++++++++++++++--------
 3 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/anthropic.go b/anthropic.go
index c361553..0543b1c 100644
--- a/anthropic.go
+++ b/anthropic.go
@@ -58,12 +58,20 @@ func (a anthropic) requestToAnthropicRequest(req Request) anth.MessagesRequest {
 				})
 			}
 
-			if msg.ImageBase64 != "" {
-				m.Content = append(m.Content, anth.NewImageMessageContent(anth.MessageContentImageSource{
-					Type:      "base64",
-					MediaType: "image/png",
-					Data:      msg.ImageBase64,
-				}))
+			for _, img := range msg.Images {
+				if img.Base64 != "" {
+					m.Content = append(m.Content, anth.NewImageMessageContent(anth.MessageContentImageSource{
+						Type:      "base64",
+						MediaType: img.ContentType,
+						Data:      img.Base64,
+					}))
+				} else if img.Url != "" {
+					m.Content = append(m.Content, anth.NewImageMessageContent(anth.MessageContentImageSource{
+						Type:      "url",
+						MediaType: img.ContentType,
+						Data:      img.Url,
+					}))
+				}
 			}
 
 			// if this has the same role as the previous message, we can append it to the previous message
diff --git a/llm.go b/llm.go
index 00c26d2..87d435b 100644
--- a/llm.go
+++ b/llm.go
@@ -12,11 +12,17 @@ const (
 	RoleAssistant Role = "assistant"
 )
 
+type Image struct {
+	Base64      string
+	ContentType string
+	Url         string
+}
+
 type Message struct {
-	Role        Role
-	Name        string
-	Text        string
-	ImageBase64 string // ImageBase64 is the base64 string if the message contains an image, empty string otherwise.
+	Role   Role
+	Name   string
+	Text   string
+	Images []Image
 }
 
 type Request struct {
diff --git a/openai.go b/openai.go
index 484940f..224b9f5 100644
--- a/openai.go
+++ b/openai.go
@@ -25,15 +25,22 @@ func (o openai) requestToOpenAIRequest(request Request) oai.ChatCompletionReques
 			Name:    msg.Name,
 		}
 
-		if msg.ImageBase64 != "" {
-			part := oai.ChatMessagePart{
-				Type: "image_url",
-				ImageURL: &oai.ChatMessageImageURL{
-					URL: fmt.Sprintf("data:image/jpeg;base64,%s", msg.ImageBase64),
-				},
+		for _, img := range msg.Images {
+			if img.Base64 != "" {
+				m.MultiContent = append(m.MultiContent, oai.ChatMessagePart{
+					Type: "image_url",
+					ImageURL: &oai.ChatMessageImageURL{
+						URL: fmt.Sprintf("data:%s;base64,%s", img.ContentType, img.Base64),
+					},
+				})
+			} else if img.Url != "" {
+				m.MultiContent = append(m.MultiContent, oai.ChatMessagePart{
+					Type: "image_url",
+					ImageURL: &oai.ChatMessageImageURL{
+						URL: img.Url,
+					},
+				})
 			}
-
-			m.MultiContent = append(m.MultiContent, part)
 		}
 
 		res.Messages = append(res.Messages, m)