feat: OpenAI, Anthropic, and native-Ollama providers + media pipeline
Phase 3: - provider/openai: Chat Completions for OpenAI + compat endpoints (SSE streaming with by-index tool-call assembly, response_format json_schema, legacy max_tokens option, reasoning_effort) - provider/anthropic: Messages API (tool_use/tool_result, GA structured output via output_config.format, full SSE event parser, 529 transient) - provider/ollama: one native /api/chat client behind the ollama, ollama-cloud, and foreman built-ins (presets; NDJSON streaming tolerant of foreman's buffered single-object responses; object tool arguments; format-schema structured output; think mapping) - media/: capability normalization (sniff, downscale, transcode, byte ladder, ErrUnsupported), wired into the chain executor per target with penalty-free advance past incapable elements - registry: real provider + scheme wiring, WithHTTPClient option, required env-foreman TLS chat round-trip test - ADR-0009 multimodal strategy, ADR-0010 tools/structured mapping; README matrix + CLAUDE.md synced Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,168 @@
|
||||
// Package ollama implements majordomo's provider contract over Ollama's
|
||||
// native chat API (POST {base}/api/chat), targeted at three backends that
|
||||
// share one wire protocol:
|
||||
//
|
||||
// - a local Ollama instance (preset Local: OLLAMA_HOST or
|
||||
// http://localhost:11434, no auth),
|
||||
// - Ollama Cloud (preset Cloud: https://ollama.com, bearer key from
|
||||
// OLLAMA_API_KEY), and
|
||||
// - foreman, Steve's native-Ollama queue daemon (preset Foreman: explicit
|
||||
// base URL + bearer token).
|
||||
//
|
||||
// Wire surface verified against docs.ollama.com and ollama/ollama
|
||||
// docs/api.md + api/types.go (June 2026): NDJSON streaming (stream defaults
|
||||
// true server-side — Generate always sends stream:false explicitly);
|
||||
// tool_calls carry arguments as a JSON OBJECT (not a string); tool results
|
||||
// return as {"role":"tool","content",...,"tool_name"}; structured output
|
||||
// via "format" (a full JSON-schema object); thinking via the bool-or-string
|
||||
// "think" field; errors as {"error":"message"} with a non-2xx status.
|
||||
//
|
||||
// foreman deviation (verified in its source): sync /api/chat does not
|
||||
// stream — a stream:true request yields ONE buffered application/json
|
||||
// object. The NDJSON reader here handles that transparently (a single JSON
|
||||
// line parses as the final chunk).
|
||||
package ollama
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// DefaultLocalBaseURL is the default base URL for a locally-running Ollama.
|
||||
const DefaultLocalBaseURL = "http://localhost:11434"
|
||||
|
||||
// DefaultCloudBaseURL is the base URL for Ollama Cloud.
|
||||
const DefaultCloudBaseURL = "https://ollama.com"
|
||||
|
||||
// defaultCapabilities is the conservative provider-wide default; individual
|
||||
// models (e.g. high-resolution vision tags) override via llm.WithCapabilities.
|
||||
var defaultCapabilities = llm.Capabilities{
|
||||
SupportsTools: true,
|
||||
SupportsStructured: true,
|
||||
SupportsStreaming: true,
|
||||
MaxImagesPerReq: 8,
|
||||
MaxImageBytes: 20 << 20,
|
||||
MaxImageDimension: 2048,
|
||||
AllowedImageMIME: []string{"image/jpeg", "image/png"},
|
||||
}
|
||||
|
||||
// Provider is a native-Ollama chat client bound to one base URL.
|
||||
type Provider struct {
|
||||
name string
|
||||
baseURL string
|
||||
token string
|
||||
client *http.Client
|
||||
caps llm.Capabilities
|
||||
}
|
||||
|
||||
// Option configures the provider.
|
||||
type Option func(*Provider)
|
||||
|
||||
// WithName overrides the registry name (default "ollama").
|
||||
func WithName(name string) Option { return func(p *Provider) { p.name = name } }
|
||||
|
||||
// WithBaseURL sets the backend base URL (scheme://host[:port][/path]).
|
||||
func WithBaseURL(u string) Option {
|
||||
return func(p *Provider) { p.baseURL = strings.TrimRight(u, "/") }
|
||||
}
|
||||
|
||||
// WithToken sets the bearer token (Ollama Cloud key / foreman token).
|
||||
// Empty means no Authorization header (local mode).
|
||||
func WithToken(token string) Option { return func(p *Provider) { p.token = token } }
|
||||
|
||||
// WithHTTPClient overrides the HTTP client (proxies, test TLS, timeouts —
|
||||
// note foreman sync chat long-polls; prefer context deadlines over client
|
||||
// timeouts).
|
||||
func WithHTTPClient(c *http.Client) Option { return func(p *Provider) { p.client = c } }
|
||||
|
||||
// WithDefaultCapabilities overrides the provider-wide default capabilities.
|
||||
func WithDefaultCapabilities(caps llm.Capabilities) Option {
|
||||
return func(p *Provider) { p.caps = caps }
|
||||
}
|
||||
|
||||
// New creates a generic native-Ollama provider. Most callers want one of
|
||||
// the presets (Local, Cloud, Foreman) or an LLM_* env DSN instead.
|
||||
// Construction never fails; a missing base URL surfaces at request time.
|
||||
func New(opts ...Option) *Provider {
|
||||
p := &Provider{
|
||||
name: "ollama",
|
||||
client: &http.Client{},
|
||||
caps: defaultCapabilities,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(p)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Local returns the local-Ollama preset: name "ollama", base URL from
|
||||
// OLLAMA_HOST (normalized per Ollama conventions) or localhost:11434.
|
||||
func Local(opts ...Option) *Provider {
|
||||
base := DefaultLocalBaseURL
|
||||
if h := os.Getenv("OLLAMA_HOST"); h != "" {
|
||||
base = NormalizeHost(h)
|
||||
}
|
||||
return New(append([]Option{WithBaseURL(base)}, opts...)...)
|
||||
}
|
||||
|
||||
// Cloud returns the Ollama Cloud preset: name "ollama-cloud",
|
||||
// https://ollama.com, bearer key from OLLAMA_API_KEY.
|
||||
func Cloud(opts ...Option) *Provider {
|
||||
return New(append([]Option{
|
||||
WithName("ollama-cloud"),
|
||||
WithBaseURL(DefaultCloudBaseURL),
|
||||
WithToken(os.Getenv("OLLAMA_API_KEY")),
|
||||
}, opts...)...)
|
||||
}
|
||||
|
||||
// Foreman returns a foreman preset bound to the given daemon.
|
||||
func Foreman(baseURL, token string, opts ...Option) *Provider {
|
||||
return New(append([]Option{
|
||||
WithName("foreman"),
|
||||
WithBaseURL(baseURL),
|
||||
WithToken(token),
|
||||
}, opts...)...)
|
||||
}
|
||||
|
||||
// NormalizeHost turns an OLLAMA_HOST-style value into a base URL:
|
||||
// "host" → http://host:11434, "host:port" → http://host:port, full URLs
|
||||
// pass through (trailing slash trimmed).
|
||||
func NormalizeHost(h string) string {
|
||||
h = strings.TrimRight(strings.TrimSpace(h), "/")
|
||||
if strings.Contains(h, "://") {
|
||||
return h
|
||||
}
|
||||
if !strings.Contains(h, ":") {
|
||||
h += ":11434"
|
||||
}
|
||||
return "http://" + h
|
||||
}
|
||||
|
||||
// Name implements llm.Provider.
|
||||
func (p *Provider) Name() string { return p.name }
|
||||
|
||||
// BaseURL reports the configured backend base URL (diagnostics).
|
||||
func (p *Provider) BaseURL() string { return p.baseURL }
|
||||
|
||||
// Model implements llm.Provider; the id passes through verbatim.
|
||||
func (p *Provider) Model(id string, opts ...llm.ModelOption) (llm.Model, error) {
|
||||
cfg := llm.ApplyModelOptions(opts)
|
||||
caps := p.caps
|
||||
if cfg.Capabilities != nil {
|
||||
caps = *cfg.Capabilities
|
||||
}
|
||||
return &model{provider: p, id: id, caps: caps}, nil
|
||||
}
|
||||
|
||||
// checkReady reports a usable configuration (a base URL is the only hard
|
||||
// requirement; auth problems surface as 401s from the backend).
|
||||
func (p *Provider) checkReady() error {
|
||||
if p.baseURL == "" {
|
||||
return fmt.Errorf("ollama provider %q: no base URL configured (set one via the preset, WithBaseURL, or an LLM_* env DSN)", p.name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,492 @@
|
||||
package ollama
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// capture spins up an httptest server that records the request and replies
|
||||
// with the given handler.
|
||||
type captured struct {
|
||||
auth string
|
||||
contentType string
|
||||
path string
|
||||
body map[string]any
|
||||
raw []byte
|
||||
}
|
||||
|
||||
func serve(t *testing.T, status int, respond func(w http.ResponseWriter)) (*Provider, *captured) {
|
||||
t.Helper()
|
||||
cap := &captured{}
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
cap.auth = r.Header.Get("Authorization")
|
||||
cap.contentType = r.Header.Get("Content-Type")
|
||||
cap.path = r.URL.Path
|
||||
cap.raw, _ = io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(cap.raw, &cap.body)
|
||||
w.WriteHeader(status)
|
||||
respond(w)
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
return New(WithBaseURL(ts.URL), WithToken("test-token")), cap
|
||||
}
|
||||
|
||||
func jsonReply(obj string) func(w http.ResponseWriter) {
|
||||
return func(w http.ResponseWriter) { _, _ = io.WriteString(w, obj) }
|
||||
}
|
||||
|
||||
func basicRequest() llm.Request {
|
||||
return llm.Request{Messages: []llm.Message{llm.UserText("hi")}}
|
||||
}
|
||||
|
||||
func TestGenerateRoundTrip(t *testing.T) {
|
||||
p, cap := serve(t, 200, jsonReply(`{
|
||||
"model":"qwen3:30b",
|
||||
"message":{"role":"assistant","content":"hello there"},
|
||||
"done":true,"done_reason":"stop",
|
||||
"prompt_eval_count":12,"eval_count":7
|
||||
}`))
|
||||
|
||||
m, _ := p.Model("qwen3:30b")
|
||||
temp := 0.2
|
||||
resp, err := m.Generate(context.Background(), llm.Request{
|
||||
System: "be terse",
|
||||
Messages: []llm.Message{llm.SystemText("extra sys"), llm.UserText("hi")},
|
||||
Temperature: &temp,
|
||||
MaxTokens: 64,
|
||||
StopSequences: []string{"END"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
|
||||
// Wire assertions.
|
||||
if cap.path != "/api/chat" {
|
||||
t.Errorf("path = %q", cap.path)
|
||||
}
|
||||
if cap.auth != "Bearer test-token" {
|
||||
t.Errorf("auth = %q", cap.auth)
|
||||
}
|
||||
if cap.body["model"] != "qwen3:30b" {
|
||||
t.Errorf("model = %v", cap.body["model"])
|
||||
}
|
||||
if stream, ok := cap.body["stream"].(bool); !ok || stream {
|
||||
t.Errorf("stream must be explicit false, got %v", cap.body["stream"])
|
||||
}
|
||||
msgs := cap.body["messages"].([]any)
|
||||
first := msgs[0].(map[string]any)
|
||||
if first["role"] != "system" || first["content"] != "be terse\n\nextra sys" {
|
||||
t.Errorf("system fold = %v", first)
|
||||
}
|
||||
second := msgs[1].(map[string]any)
|
||||
if second["role"] != "user" || second["content"] != "hi" {
|
||||
t.Errorf("user msg = %v", second)
|
||||
}
|
||||
opts := cap.body["options"].(map[string]any)
|
||||
if opts["temperature"] != 0.2 || opts["num_predict"] != float64(64) {
|
||||
t.Errorf("options = %v", opts)
|
||||
}
|
||||
|
||||
// Response assertions.
|
||||
if resp.Text() != "hello there" {
|
||||
t.Errorf("text = %q", resp.Text())
|
||||
}
|
||||
if resp.FinishReason != llm.FinishStop {
|
||||
t.Errorf("finish = %v", resp.FinishReason)
|
||||
}
|
||||
if resp.Usage.InputTokens != 12 || resp.Usage.OutputTokens != 7 {
|
||||
t.Errorf("usage = %+v", resp.Usage)
|
||||
}
|
||||
if resp.Model != "ollama/qwen3:30b" {
|
||||
t.Errorf("resp.Model = %q", resp.Model)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImagesEncodeAsBase64(t *testing.T) {
|
||||
p, cap := serve(t, 200, jsonReply(`{"message":{"role":"assistant","content":"a cat"},"done":true,"done_reason":"stop"}`))
|
||||
imgBytes := []byte{0xFF, 0xD8, 0xFF, 0xE0, 1, 2, 3}
|
||||
|
||||
m, _ := p.Model("llava")
|
||||
_, err := m.Generate(context.Background(), llm.Request{
|
||||
Messages: []llm.Message{llm.UserParts(llm.Text("describe"), llm.Image("image/jpeg", imgBytes))},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
msgs := cap.body["messages"].([]any)
|
||||
user := msgs[0].(map[string]any)
|
||||
images := user["images"].([]any)
|
||||
if len(images) != 1 || images[0] != base64.StdEncoding.EncodeToString(imgBytes) {
|
||||
t.Errorf("images = %v", images)
|
||||
}
|
||||
if strings.Contains(images[0].(string), "data:") {
|
||||
t.Error("images must be raw base64 without data: prefix")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolsAndToolCallRoundTrip(t *testing.T) {
|
||||
p, cap := serve(t, 200, jsonReply(`{
|
||||
"message":{"role":"assistant","content":"","tool_calls":[
|
||||
{"function":{"index":0,"name":"get_weather","arguments":{"city":"Tokyo"}}}
|
||||
]},
|
||||
"done":true,"done_reason":"stop"
|
||||
}`))
|
||||
|
||||
tool := llm.Tool{
|
||||
Name: "get_weather", Description: "weather",
|
||||
Parameters: json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}`),
|
||||
}
|
||||
m, _ := p.Model("qwen3")
|
||||
resp, err := m.Generate(context.Background(), basicRequest(), llm.WithTools(tool))
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
|
||||
// Tools serialize with parameters as an object.
|
||||
tools := cap.body["tools"].([]any)
|
||||
fn := tools[0].(map[string]any)["function"].(map[string]any)
|
||||
if fn["name"] != "get_weather" {
|
||||
t.Errorf("tool fn = %v", fn)
|
||||
}
|
||||
if _, ok := fn["parameters"].(map[string]any); !ok {
|
||||
t.Errorf("parameters must be an object, got %T", fn["parameters"])
|
||||
}
|
||||
|
||||
// Tool call comes back with arguments as a JSON object → RawMessage.
|
||||
if len(resp.ToolCalls) != 1 {
|
||||
t.Fatalf("tool calls = %v", resp.ToolCalls)
|
||||
}
|
||||
tc := resp.ToolCalls[0]
|
||||
if tc.Name != "get_weather" || tc.ID == "" {
|
||||
t.Errorf("call = %+v (id must be synthesized)", tc)
|
||||
}
|
||||
var args struct {
|
||||
City string `json:"city"`
|
||||
}
|
||||
if err := json.Unmarshal(tc.Arguments, &args); err != nil || args.City != "Tokyo" {
|
||||
t.Errorf("arguments = %s (%v)", tc.Arguments, err)
|
||||
}
|
||||
if resp.FinishReason != llm.FinishToolCalls {
|
||||
t.Errorf("finish = %v, want tool_calls", resp.FinishReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolResultsAndHistoryToolCalls(t *testing.T) {
|
||||
p, cap := serve(t, 200, jsonReply(`{"message":{"role":"assistant","content":"21C"},"done":true,"done_reason":"stop"}`))
|
||||
|
||||
m, _ := p.Model("qwen3")
|
||||
_, err := m.Generate(context.Background(), llm.Request{
|
||||
Messages: []llm.Message{
|
||||
llm.UserText("weather?"),
|
||||
{Role: llm.RoleAssistant, ToolCalls: []llm.ToolCall{
|
||||
{ID: "call_0", Name: "get_weather", Arguments: json.RawMessage(`{"city":"Tokyo"}`)},
|
||||
}},
|
||||
llm.ToolResultsMessage(
|
||||
llm.ToolResult{ID: "call_0", Name: "get_weather", Content: `{"temp":21}`},
|
||||
llm.ToolResult{ID: "call_1", Name: "broken_tool", Content: "boom", IsError: true},
|
||||
),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
msgs := cap.body["messages"].([]any)
|
||||
if len(msgs) != 4 {
|
||||
t.Fatalf("messages = %d, want 4 (user, assistant, 2 tool results)", len(msgs))
|
||||
}
|
||||
asst := msgs[1].(map[string]any)
|
||||
calls := asst["tool_calls"].([]any)
|
||||
args := calls[0].(map[string]any)["function"].(map[string]any)["arguments"]
|
||||
if _, ok := args.(map[string]any); !ok {
|
||||
t.Errorf("history tool-call arguments must be an object, got %T", args)
|
||||
}
|
||||
tr1 := msgs[2].(map[string]any)
|
||||
if tr1["role"] != "tool" || tr1["tool_name"] != "get_weather" || tr1["content"] != `{"temp":21}` {
|
||||
t.Errorf("tool result 1 = %v", tr1)
|
||||
}
|
||||
tr2 := msgs[3].(map[string]any)
|
||||
if tr2["content"] != "ERROR: boom" {
|
||||
t.Errorf("error result content = %v", tr2["content"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredOutputFormat(t *testing.T) {
|
||||
p, cap := serve(t, 200, jsonReply(`{"message":{"role":"assistant","content":"{\"name\":\"Ada\"}"},"done":true,"done_reason":"stop"}`))
|
||||
schema := json.RawMessage(`{"type":"object","properties":{"name":{"type":"string"}},"required":["name"]}`)
|
||||
|
||||
m, _ := p.Model("qwen3")
|
||||
resp, err := m.Generate(context.Background(), basicRequest(), llm.WithSchema(schema, "person"))
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
format, ok := cap.body["format"].(map[string]any)
|
||||
if !ok || format["type"] != "object" {
|
||||
t.Errorf("format = %v, want the schema object", cap.body["format"])
|
||||
}
|
||||
if resp.Text() != `{"name":"Ada"}` {
|
||||
t.Errorf("text = %q", resp.Text())
|
||||
}
|
||||
}
|
||||
|
||||
func TestThinkMapping(t *testing.T) {
|
||||
p, cap := serve(t, 200, jsonReply(`{"message":{"role":"assistant","content":"ok"},"done":true,"done_reason":"stop"}`))
|
||||
m, _ := p.Model("gpt-oss:120b")
|
||||
_, err := m.Generate(context.Background(), basicRequest(), llm.WithReasoningEffort("high"))
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if cap.body["think"] != "high" {
|
||||
t.Errorf("think = %v", cap.body["think"])
|
||||
}
|
||||
|
||||
if _, err := m.Generate(context.Background(), basicRequest(), llm.WithReasoningEffort("max")); err == nil {
|
||||
t.Error("invalid reasoning effort should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolChoiceNoneDropsTools(t *testing.T) {
|
||||
p, cap := serve(t, 200, jsonReply(`{"message":{"role":"assistant","content":"ok"},"done":true,"done_reason":"stop"}`))
|
||||
m, _ := p.Model("qwen3")
|
||||
_, err := m.Generate(context.Background(), basicRequest(),
|
||||
llm.WithTools(llm.Tool{Name: "t"}), llm.WithToolChoice("none"))
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if _, present := cap.body["tools"]; present {
|
||||
t.Error("tool_choice none must omit tools")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingNDJSON(t *testing.T) {
|
||||
p, _ := serve(t, 200, func(w http.ResponseWriter) {
|
||||
w.Header().Set("Content-Type", "application/x-ndjson")
|
||||
_, _ = io.WriteString(w, `{"message":{"role":"assistant","content":"Hel"},"done":false}
|
||||
{"message":{"role":"assistant","content":"lo"},"done":false}
|
||||
{"message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"ping","arguments":{}}}]},"done":false}
|
||||
{"message":{"role":"assistant","content":""},"done":true,"done_reason":"stop","prompt_eval_count":5,"eval_count":9}
|
||||
`)
|
||||
})
|
||||
|
||||
m, _ := p.Model("qwen3")
|
||||
s, err := m.Stream(context.Background(), basicRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Stream: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
var text strings.Builder
|
||||
var toolCalls []llm.ToolCall
|
||||
var final *llm.Response
|
||||
for {
|
||||
ev, err := s.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Next: %v", err)
|
||||
}
|
||||
text.WriteString(ev.TextDelta)
|
||||
if ev.ToolCall != nil {
|
||||
toolCalls = append(toolCalls, *ev.ToolCall)
|
||||
}
|
||||
if ev.Response != nil {
|
||||
final = ev.Response
|
||||
}
|
||||
}
|
||||
if text.String() != "Hello" {
|
||||
t.Errorf("text = %q", text.String())
|
||||
}
|
||||
if len(toolCalls) != 1 || toolCalls[0].Name != "ping" {
|
||||
t.Errorf("tool calls = %+v", toolCalls)
|
||||
}
|
||||
if final == nil {
|
||||
t.Fatal("no final response event")
|
||||
}
|
||||
if final.Usage.InputTokens != 5 || final.Usage.OutputTokens != 9 {
|
||||
t.Errorf("final usage = %+v", final.Usage)
|
||||
}
|
||||
if final.FinishReason != llm.FinishToolCalls {
|
||||
t.Errorf("final finish = %v", final.FinishReason)
|
||||
}
|
||||
if final.Text() != "Hello" {
|
||||
t.Errorf("final text = %q", final.Text())
|
||||
}
|
||||
}
|
||||
|
||||
// TestStreamingForemanSingleObject: foreman returns one buffered JSON
|
||||
// object to a stream:true request; the stream must still deliver the text
|
||||
// and a final response.
|
||||
func TestStreamingForemanSingleObject(t *testing.T) {
|
||||
p, cap := serve(t, 200, func(w http.ResponseWriter) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"message":{"role":"assistant","content":"queued answer"},"done":true,"done_reason":"stop","prompt_eval_count":3,"eval_count":4}`)
|
||||
})
|
||||
|
||||
m, _ := p.Model("qwen3:30b")
|
||||
s, err := m.Stream(context.Background(), basicRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Stream: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
if stream, ok := cap.body["stream"].(bool); !ok || !stream {
|
||||
t.Errorf("stream flag = %v, want true", cap.body["stream"])
|
||||
}
|
||||
|
||||
var text strings.Builder
|
||||
var final *llm.Response
|
||||
for {
|
||||
ev, err := s.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Next: %v", err)
|
||||
}
|
||||
text.WriteString(ev.TextDelta)
|
||||
if ev.Response != nil {
|
||||
final = ev.Response
|
||||
}
|
||||
}
|
||||
if text.String() != "queued answer" || final == nil || final.Usage.OutputTokens != 4 {
|
||||
t.Errorf("text=%q final=%+v", text.String(), final)
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorMapping(t *testing.T) {
|
||||
t.Run("404 is model-not-found", func(t *testing.T) {
|
||||
p, _ := serve(t, 404, jsonReply(`{"error":"model not found"}`))
|
||||
m, _ := p.Model("nope")
|
||||
_, err := m.Generate(context.Background(), basicRequest())
|
||||
if !errors.Is(err, llm.ErrModelNotFound) {
|
||||
t.Errorf("error = %v, want ErrModelNotFound", err)
|
||||
}
|
||||
if llm.Classify(err) != llm.ClassPermanent {
|
||||
t.Error("404 must classify permanent")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("503 transient with message", func(t *testing.T) {
|
||||
p, _ := serve(t, 503, jsonReply(`{"error":"request cancelled while waiting"}`))
|
||||
m, _ := p.Model("qwen3")
|
||||
_, err := m.Generate(context.Background(), basicRequest())
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) || apiErr.Status != 503 || !strings.Contains(apiErr.Message, "cancelled") {
|
||||
t.Errorf("error = %v", err)
|
||||
}
|
||||
if llm.Classify(err) != llm.ClassTransient {
|
||||
t.Error("503 must classify transient")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-JSON error body", func(t *testing.T) {
|
||||
p, _ := serve(t, 500, jsonReply(`upstream exploded`))
|
||||
m, _ := p.Model("qwen3")
|
||||
_, err := m.Generate(context.Background(), basicRequest())
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) || !strings.Contains(apiErr.Message, "upstream exploded") {
|
||||
t.Errorf("error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCapabilityEnforcement(t *testing.T) {
|
||||
p, _ := serve(t, 200, jsonReply(`{"message":{"content":"x"},"done":true}`))
|
||||
|
||||
t.Run("too many images", func(t *testing.T) {
|
||||
m, _ := p.Model("llava", llm.WithCapabilities(llm.Capabilities{MaxImagesPerReq: 1, AllowedImageMIME: []string{"image/png"}}))
|
||||
_, err := m.Generate(context.Background(), llm.Request{Messages: []llm.Message{
|
||||
llm.UserParts(llm.Image("image/png", []byte{1}), llm.Image("image/png", []byte{2})),
|
||||
}})
|
||||
if !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("error = %v, want ErrUnsupported", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("images on text-only model", func(t *testing.T) {
|
||||
m, _ := p.Model("qwen3", llm.WithCapabilities(llm.Capabilities{}))
|
||||
_, err := m.Generate(context.Background(), llm.Request{Messages: []llm.Message{
|
||||
llm.UserParts(llm.Image("image/png", []byte{1})),
|
||||
}})
|
||||
if !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("error = %v, want ErrUnsupported", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("disallowed mime", func(t *testing.T) {
|
||||
m, _ := p.Model("llava") // default caps: jpeg/png only
|
||||
_, err := m.Generate(context.Background(), llm.Request{Messages: []llm.Message{
|
||||
llm.UserParts(llm.Image("image/tiff", []byte{1})),
|
||||
}})
|
||||
if !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("error = %v, want ErrUnsupported", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNoBaseURL(t *testing.T) {
|
||||
p := New(WithBaseURL(""))
|
||||
m, _ := p.Model("x")
|
||||
if _, err := m.Generate(context.Background(), basicRequest()); err == nil ||
|
||||
!strings.Contains(err.Error(), "no base URL") {
|
||||
t.Errorf("error = %v, want a clear no-base-URL message", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeHost(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"myhost": "http://myhost:11434",
|
||||
"myhost:8080": "http://myhost:8080",
|
||||
"http://myhost:8080/": "http://myhost:8080",
|
||||
"https://ollama.com": "https://ollama.com",
|
||||
" 127.0.0.1:11434 ": "http://127.0.0.1:11434",
|
||||
} {
|
||||
if got := NormalizeHost(in); got != want {
|
||||
t.Errorf("NormalizeHost(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresets(t *testing.T) {
|
||||
t.Run("cloud", func(t *testing.T) {
|
||||
t.Setenv("OLLAMA_API_KEY", "cloud-key")
|
||||
p := Cloud()
|
||||
if p.Name() != "ollama-cloud" || p.baseURL != DefaultCloudBaseURL || p.token != "cloud-key" {
|
||||
t.Errorf("cloud preset = %+v", p)
|
||||
}
|
||||
})
|
||||
t.Run("local respects OLLAMA_HOST", func(t *testing.T) {
|
||||
t.Setenv("OLLAMA_HOST", "box.lan:9999")
|
||||
p := Local()
|
||||
if p.Name() != "ollama" || p.baseURL != "http://box.lan:9999" || p.token != "" {
|
||||
t.Errorf("local preset = %+v", p)
|
||||
}
|
||||
})
|
||||
t.Run("foreman", func(t *testing.T) {
|
||||
p := Foreman("http://foreman-m1:8080", "tok")
|
||||
if p.Name() != "foreman" || p.baseURL != "http://foreman-m1:8080" || p.token != "tok" {
|
||||
t.Errorf("foreman preset = %+v", p)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocalNoAuthHeader(t *testing.T) {
|
||||
p, cap := serve(t, 200, jsonReply(`{"message":{"content":"x"},"done":true}`))
|
||||
p.token = "" // simulate local mode on the test server
|
||||
m, _ := p.Model("llama3")
|
||||
if _, err := m.Generate(context.Background(), basicRequest()); err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if cap.auth != "" {
|
||||
t.Errorf("auth header = %q, want none in local mode", cap.auth)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package ollama
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// Stream implements llm.Model over Ollama's NDJSON streaming. It also
|
||||
// transparently handles foreman's non-streaming degradation (a single
|
||||
// buffered JSON object): one JSON line parses as the final chunk.
|
||||
func (m *model) Stream(ctx context.Context, req llm.Request, opts ...llm.Option) (llm.Stream, error) {
|
||||
req = req.Apply(opts...)
|
||||
if err := m.enforceCapabilities(req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wireReq, err := m.buildRequest(req, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := m.do(ctx, wireReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sc := bufio.NewScanner(resp.Body)
|
||||
// Single NDJSON lines can far exceed the 64KB default (thinking dumps,
|
||||
// tool payloads, foreman's whole-response-as-one-line degradation).
|
||||
sc.Buffer(make([]byte, 64<<10), 16<<20)
|
||||
|
||||
return &stream{model: m, body: resp.Body, scanner: sc}, nil
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
model *model
|
||||
body io.Closer
|
||||
scanner *bufio.Scanner
|
||||
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
finished bool
|
||||
toolCalls []llm.ToolCall
|
||||
text []byte
|
||||
pending []llm.StreamEvent
|
||||
usage llm.Usage
|
||||
doneReason string
|
||||
}
|
||||
|
||||
func (s *stream) Next() (llm.StreamEvent, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
for {
|
||||
if len(s.pending) > 0 {
|
||||
ev := s.pending[0]
|
||||
s.pending = s.pending[1:]
|
||||
return ev, nil
|
||||
}
|
||||
if s.finished {
|
||||
return llm.StreamEvent{}, io.EOF
|
||||
}
|
||||
if !s.scanner.Scan() {
|
||||
if err := s.scanner.Err(); err != nil {
|
||||
return llm.StreamEvent{}, fmt.Errorf("ollama %s: read stream: %w", s.model.qualified(), err)
|
||||
}
|
||||
// EOF without a done chunk: synthesize the final response from
|
||||
// what we accumulated rather than losing it.
|
||||
s.queueFinal()
|
||||
continue
|
||||
}
|
||||
line := s.scanner.Bytes()
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var chunk chatResponse
|
||||
if err := json.Unmarshal(line, &chunk); err != nil {
|
||||
return llm.StreamEvent{}, fmt.Errorf("ollama %s: decode stream chunk: %w", s.model.qualified(), err)
|
||||
}
|
||||
|
||||
if chunk.Message.Content != "" {
|
||||
s.text = append(s.text, chunk.Message.Content...)
|
||||
s.pending = append(s.pending, llm.StreamEvent{TextDelta: chunk.Message.Content})
|
||||
}
|
||||
// Tool calls arrive complete per chunk (no partial-argument deltas
|
||||
// in the native protocol).
|
||||
base := len(s.toolCalls)
|
||||
for i, tc := range chunk.Message.ToolCalls {
|
||||
id := tc.ID
|
||||
if id == "" {
|
||||
id = "call_" + strconv.Itoa(base+i)
|
||||
}
|
||||
args := tc.Function.Arguments
|
||||
if len(args) == 0 {
|
||||
args = json.RawMessage("{}")
|
||||
}
|
||||
call := llm.ToolCall{ID: id, Name: tc.Function.Name, Arguments: args}
|
||||
s.toolCalls = append(s.toolCalls, call)
|
||||
s.pending = append(s.pending, llm.StreamEvent{ToolCall: &s.toolCalls[len(s.toolCalls)-1]})
|
||||
}
|
||||
if chunk.Done {
|
||||
s.usage = llm.Usage{InputTokens: chunk.PromptEvalCount, OutputTokens: chunk.EvalCount}
|
||||
s.doneReason = chunk.DoneReason
|
||||
s.queueFinal()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// queueFinal appends the final Response event and marks the stream done.
|
||||
func (s *stream) queueFinal() {
|
||||
resp := &llm.Response{
|
||||
Model: s.model.qualified(),
|
||||
Usage: s.usage,
|
||||
FinishReason: finishReason(s.doneReason, len(s.toolCalls) > 0),
|
||||
}
|
||||
if len(s.text) > 0 {
|
||||
resp.Parts = append(resp.Parts, llm.Text(string(s.text)))
|
||||
}
|
||||
if len(s.toolCalls) > 0 {
|
||||
resp.ToolCalls = append([]llm.ToolCall(nil), s.toolCalls...)
|
||||
}
|
||||
s.pending = append(s.pending, llm.StreamEvent{Response: resp})
|
||||
s.finished = true
|
||||
}
|
||||
|
||||
func (s *stream) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
return nil
|
||||
}
|
||||
s.closed = true
|
||||
return s.body.Close()
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
package ollama
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// ---- wire types (field names per ollama api/types.go) ----
|
||||
|
||||
type chatRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []chatMessage `json:"messages"`
|
||||
Tools []toolDef `json:"tools,omitempty"`
|
||||
Format json.RawMessage `json:"format,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
// Stream has no omitempty on purpose: the server default is true, so
|
||||
// Generate must send an explicit false.
|
||||
Stream bool `json:"stream"`
|
||||
// Think is bool-or-string on the wire ("low"/"medium"/"high" or a bool).
|
||||
Think json.RawMessage `json:"think,omitempty"`
|
||||
}
|
||||
|
||||
type chatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Images []string `json:"images,omitempty"` // raw base64, no data: prefix
|
||||
ToolCalls []toolCall `json:"tool_calls,omitempty"`
|
||||
ToolName string `json:"tool_name,omitempty"` // on role:"tool" results
|
||||
}
|
||||
|
||||
type toolDef struct {
|
||||
Type string `json:"type"`
|
||||
Function toolDefFunc `json:"function"`
|
||||
}
|
||||
|
||||
type toolDefFunc struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Parameters json.RawMessage `json:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
type toolCall struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Function toolCallFunc `json:"function"`
|
||||
}
|
||||
|
||||
type toolCallFunc struct {
|
||||
Index int `json:"index,omitempty"`
|
||||
Name string `json:"name"`
|
||||
// Arguments is a JSON OBJECT on the wire (unlike OpenAI's string).
|
||||
Arguments json.RawMessage `json:"arguments"`
|
||||
}
|
||||
|
||||
type chatResponse struct {
|
||||
Model string `json:"model"`
|
||||
Message respMessage `json:"message"`
|
||||
Done bool `json:"done"`
|
||||
DoneReason string `json:"done_reason"`
|
||||
PromptEvalCount int `json:"prompt_eval_count"`
|
||||
EvalCount int `json:"eval_count"`
|
||||
}
|
||||
|
||||
type respMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Thinking string `json:"thinking"`
|
||||
ToolCalls []toolCall `json:"tool_calls"`
|
||||
}
|
||||
|
||||
type errorBody struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// ---- model ----
|
||||
|
||||
type model struct {
|
||||
provider *Provider
|
||||
id string
|
||||
caps llm.Capabilities
|
||||
}
|
||||
|
||||
func (m *model) Capabilities() llm.Capabilities { return m.caps }
|
||||
|
||||
func (m *model) qualified() string { return m.provider.name + "/" + m.id }
|
||||
|
||||
// enforceCapabilities is the backstop check (the media layer normalizes
|
||||
// before requests get here; see ADR-0009).
|
||||
func (m *model) enforceCapabilities(req llm.Request) error {
|
||||
count := 0
|
||||
for _, msg := range req.Messages {
|
||||
for _, part := range msg.Parts {
|
||||
img, ok := part.(llm.ImagePart)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
if !m.caps.SupportsImages() {
|
||||
return fmt.Errorf("%w: %s does not accept image input", llm.ErrUnsupported, m.qualified())
|
||||
}
|
||||
if !m.caps.MIMEAllowed(img.MIME) {
|
||||
return fmt.Errorf("%w: %s does not accept %s images", llm.ErrUnsupported, m.qualified(), img.MIME)
|
||||
}
|
||||
if m.caps.MaxImageBytes > 0 && len(img.Data) > m.caps.MaxImageBytes {
|
||||
return fmt.Errorf("%w: image of %d bytes exceeds %s limit of %d",
|
||||
llm.ErrUnsupported, len(img.Data), m.qualified(), m.caps.MaxImageBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
if count > 0 && m.caps.MaxImagesPerReq > 0 && count > m.caps.MaxImagesPerReq {
|
||||
return fmt.Errorf("%w: %d images exceed %s limit of %d",
|
||||
llm.ErrUnsupported, count, m.qualified(), m.caps.MaxImagesPerReq)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildRequest maps the canonical request onto the wire shape.
|
||||
func (m *model) buildRequest(req llm.Request, stream bool) (*chatRequest, error) {
|
||||
out := &chatRequest{Model: m.id, Stream: stream}
|
||||
|
||||
// System prompt: dedicated field first, then folded RoleSystem messages.
|
||||
var sys []string
|
||||
if req.System != "" {
|
||||
sys = append(sys, req.System)
|
||||
}
|
||||
for _, msg := range req.Messages {
|
||||
if msg.Role == llm.RoleSystem {
|
||||
if t := msg.Text(); t != "" {
|
||||
sys = append(sys, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(sys) > 0 {
|
||||
out.Messages = append(out.Messages, chatMessage{
|
||||
Role: "system", Content: strings.Join(sys, "\n\n"),
|
||||
})
|
||||
}
|
||||
|
||||
for _, msg := range req.Messages {
|
||||
switch msg.Role {
|
||||
case llm.RoleSystem:
|
||||
// Already folded above.
|
||||
case llm.RoleTool:
|
||||
for _, res := range msg.ToolResults {
|
||||
content := res.Content
|
||||
if res.IsError {
|
||||
content = "ERROR: " + content
|
||||
}
|
||||
out.Messages = append(out.Messages, chatMessage{
|
||||
Role: "tool", Content: content, ToolName: res.Name,
|
||||
})
|
||||
}
|
||||
default:
|
||||
cm := chatMessage{Role: string(msg.Role), Content: msg.Text()}
|
||||
for _, part := range msg.Parts {
|
||||
if img, ok := part.(llm.ImagePart); ok {
|
||||
cm.Images = append(cm.Images, base64.StdEncoding.EncodeToString(img.Data))
|
||||
}
|
||||
}
|
||||
for _, tc := range msg.ToolCalls {
|
||||
args := tc.Arguments
|
||||
if len(args) == 0 {
|
||||
args = json.RawMessage("{}")
|
||||
}
|
||||
cm.ToolCalls = append(cm.ToolCalls, toolCall{
|
||||
ID: tc.ID,
|
||||
Function: toolCallFunc{Name: tc.Name, Arguments: args},
|
||||
})
|
||||
}
|
||||
out.Messages = append(out.Messages, cm)
|
||||
}
|
||||
}
|
||||
|
||||
// Tools. Ollama has no tool_choice: "none" maps to omitting the tools;
|
||||
// "required"/named choices have no wire equivalent and are best-effort
|
||||
// ignored (documented in the README support matrix).
|
||||
if req.ToolChoice != "none" {
|
||||
for _, t := range req.Tools {
|
||||
params := t.Parameters
|
||||
if len(params) == 0 {
|
||||
params = json.RawMessage(`{"type":"object","properties":{}}`)
|
||||
}
|
||||
out.Tools = append(out.Tools, toolDef{
|
||||
Type: "function",
|
||||
Function: toolDefFunc{Name: t.Name, Description: t.Description, Parameters: params},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if len(req.Schema) > 0 {
|
||||
out.Format = req.Schema
|
||||
}
|
||||
|
||||
opts := make(map[string]any)
|
||||
if req.Temperature != nil {
|
||||
opts["temperature"] = *req.Temperature
|
||||
}
|
||||
if req.TopP != nil {
|
||||
opts["top_p"] = *req.TopP
|
||||
}
|
||||
if req.MaxTokens > 0 {
|
||||
opts["num_predict"] = req.MaxTokens
|
||||
}
|
||||
if len(req.StopSequences) > 0 {
|
||||
opts["stop"] = req.StopSequences
|
||||
}
|
||||
if len(opts) > 0 {
|
||||
out.Options = opts
|
||||
}
|
||||
|
||||
switch req.ReasoningEffort {
|
||||
case "":
|
||||
case "low", "medium", "high":
|
||||
out.Think = json.RawMessage(strconv.Quote(req.ReasoningEffort))
|
||||
default:
|
||||
return nil, fmt.Errorf("ollama: invalid reasoning effort %q (want low/medium/high)", req.ReasoningEffort)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// do POSTs /api/chat and returns the response body on 2xx, or a classified
|
||||
// error.
|
||||
func (m *model) do(ctx context.Context, wireReq *chatRequest) (*http.Response, error) {
|
||||
p := m.provider
|
||||
if err := p.checkReady(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(wireReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ollama: encode request: %w", err)
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, p.baseURL+"/api/chat", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ollama: build request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if p.token != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+p.token)
|
||||
}
|
||||
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ollama %s: do request: %w", m.qualified(), err)
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<10))
|
||||
var eb errorBody
|
||||
_ = json.Unmarshal(raw, &eb)
|
||||
msg := eb.Error
|
||||
if msg == "" {
|
||||
msg = strings.TrimSpace(string(raw))
|
||||
}
|
||||
return nil, &llm.APIError{
|
||||
Provider: p.name, Model: m.id,
|
||||
Status: resp.StatusCode, Message: msg,
|
||||
}
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Generate implements llm.Model.
|
||||
func (m *model) Generate(ctx context.Context, req llm.Request, opts ...llm.Option) (*llm.Response, error) {
|
||||
req = req.Apply(opts...)
|
||||
if err := m.enforceCapabilities(req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wireReq, err := m.buildRequest(req, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := m.do(ctx, wireReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var cr chatResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil {
|
||||
return nil, fmt.Errorf("ollama %s: decode response: %w", m.qualified(), err)
|
||||
}
|
||||
return m.toResponse(&cr), nil
|
||||
}
|
||||
|
||||
// toResponse converts a final wire chunk into the canonical response.
|
||||
func (m *model) toResponse(cr *chatResponse) *llm.Response {
|
||||
out := &llm.Response{
|
||||
Model: m.qualified(),
|
||||
Usage: llm.Usage{InputTokens: cr.PromptEvalCount, OutputTokens: cr.EvalCount},
|
||||
Raw: cr,
|
||||
}
|
||||
if cr.Message.Content != "" {
|
||||
out.Parts = append(out.Parts, llm.Text(cr.Message.Content))
|
||||
}
|
||||
out.ToolCalls = convertToolCalls(cr.Message.ToolCalls)
|
||||
out.FinishReason = finishReason(cr.DoneReason, len(out.ToolCalls) > 0)
|
||||
return out
|
||||
}
|
||||
|
||||
// convertToolCalls maps wire tool calls, synthesizing ids where the model
|
||||
// omitted them (ids are optional in Ollama's shape but required by our
|
||||
// agent loop to match results to calls).
|
||||
func convertToolCalls(calls []toolCall) []llm.ToolCall {
|
||||
out := make([]llm.ToolCall, 0, len(calls))
|
||||
for i, tc := range calls {
|
||||
id := tc.ID
|
||||
if id == "" {
|
||||
id = "call_" + strconv.Itoa(i)
|
||||
}
|
||||
args := tc.Function.Arguments
|
||||
if len(args) == 0 {
|
||||
args = json.RawMessage("{}")
|
||||
}
|
||||
out = append(out, llm.ToolCall{ID: id, Name: tc.Function.Name, Arguments: args})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func finishReason(doneReason string, hasToolCalls bool) llm.FinishReason {
|
||||
if hasToolCalls {
|
||||
return llm.FinishToolCalls
|
||||
}
|
||||
switch doneReason {
|
||||
case "stop", "":
|
||||
return llm.FinishStop
|
||||
case "length":
|
||||
return llm.FinishLength
|
||||
default:
|
||||
return llm.FinishOther
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user