feat(llamaswap): add llama-swap provider + canonical imagegen interface
Add provider/llamaswap, a tailored provider for llama-swap (the model-swapping
proxy over llama.cpp / stable-diffusion.cpp). Its chat path delegates to
provider/openai at {base}/v1 — no duplicated wire client (ADR-0007) — with
legacy max_tokens, a Bearer no-key placeholder for keyless local instances, and
a timeout-free client so cold model swaps rely on context deadlines. The
"tailored" surface is concrete management methods (ListModels / Running /
Unload) that don't belong on the canonical llm.Provider interface. The
llama-swap:// DSN scheme builds an http base URL (local-first); a no-URL
built-in errors clearly on use, mirroring foreman.
Add imagegen, a new canonical text-to-image interface separate from llm
(Request/Result/Model/Provider; Image = llm.ImagePart so generated images feed
straight back into chat). First backend is llama-swap via OpenAI
/v1/images/generations (b64_json, bytes-only). Re-exported from the root. v1 is
txt2img only.
Hermetic httptest coverage for chat delegation, management endpoints, image
decode, and scheme wiring. ADR-0015 + ADR-0016, README support matrix +
image-gen section, CLAUDE.md package map, and progress.md updated in the same
commit.
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,241 @@
|
||||
// Package llamaswap implements majordomo's provider contract for llama-swap
|
||||
// (https://github.com/mostlygeek/llama-swap), an on-demand model-swapping
|
||||
// proxy that fronts llama.cpp (and stable-diffusion.cpp) servers, loading and
|
||||
// hot-swapping the requested model per request.
|
||||
//
|
||||
// Chat is OpenAI Chat Completions, byte-for-byte: this package does NOT carry
|
||||
// its own chat wire client. Provider.Model delegates to provider/openai
|
||||
// pointed at {baseURL}/v1 (ADR-0007: reuse, don't duplicate). What this
|
||||
// package adds beyond a bare OpenAI-compat endpoint is the "tailored" surface:
|
||||
//
|
||||
// - llama-swap management endpoints exposed as concrete methods — ListModels
|
||||
// (GET /v1/models), Running (GET /running), Unload (POST /api/models/unload)
|
||||
// — which have no place on the canonical llm.Provider interface;
|
||||
// - image generation via the imagegen interface (see image.go); and
|
||||
// - swap-aware defaults: the HTTP client carries NO timeout, because the
|
||||
// first request to an unloaded model blocks while llama-swap spawns the
|
||||
// upstream (its healthCheckTimeout is at least 15s). Bound a call with a
|
||||
// context deadline, never a client timeout.
|
||||
//
|
||||
// DSN form (registered as the "llama-swap" scheme): llama-swap://token@host:port
|
||||
// builds an http:// base URL (llama-swap is local-first; a TLS-fronted instance
|
||||
// can use the openai:// scheme for chat instead).
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/openai"
|
||||
)
|
||||
|
||||
// DefaultName is the registry name used when WithName is not given.
|
||||
const DefaultName = "llama-swap"
|
||||
|
||||
// Provider is a llama-swap client. It satisfies llm.Provider (chat, delegated
|
||||
// to provider/openai) and imagegen.Provider (image generation), and exposes
|
||||
// llama-swap's management endpoints as concrete methods.
|
||||
type Provider struct {
|
||||
name string
|
||||
baseURL string // no trailing slash, no /v1 suffix; e.g. "http://host:port"
|
||||
token string // bearer credential; empty = no auth (local)
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Option configures the provider.
|
||||
type Option func(*Provider)
|
||||
|
||||
// WithName overrides the registry name (default "llama-swap").
|
||||
func WithName(name string) Option { return func(p *Provider) { p.name = name } }
|
||||
|
||||
// WithBaseURL sets the llama-swap base URL (scheme://host[:port]); the /v1 and
|
||||
// management paths are appended internally. A trailing slash is trimmed.
|
||||
func WithBaseURL(u string) Option {
|
||||
return func(p *Provider) { p.baseURL = strings.TrimRight(u, "/") }
|
||||
}
|
||||
|
||||
// WithToken sets the bearer token (llama-swap API key). Empty means no
|
||||
// Authorization header.
|
||||
func WithToken(token string) Option { return func(p *Provider) { p.token = token } }
|
||||
|
||||
// WithHTTPClient overrides the HTTP client. Prefer context deadlines over a
|
||||
// client timeout: a cold model swap can legitimately take many seconds.
|
||||
func WithHTTPClient(c *http.Client) Option {
|
||||
return func(p *Provider) {
|
||||
if c != nil {
|
||||
p.client = c
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// New creates a llama-swap provider. Construction never fails; a missing base
|
||||
// URL surfaces at request time. The default client has no timeout (swap cold
|
||||
// starts); bound calls with a context deadline.
|
||||
func New(opts ...Option) *Provider {
|
||||
p := &Provider{name: DefaultName, client: &http.Client{}}
|
||||
for _, opt := range opts {
|
||||
opt(p)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Name implements llm.Provider and imagegen.Provider.
|
||||
func (p *Provider) Name() string { return p.name }
|
||||
|
||||
// BaseURL reports the configured base URL (diagnostics).
|
||||
func (p *Provider) BaseURL() string { return p.baseURL }
|
||||
|
||||
// Model implements llm.Provider via llama-swap's OpenAI-compatible chat
|
||||
// endpoint, delegating to provider/openai. The id is passed through verbatim
|
||||
// and selects which upstream llama-swap loads.
|
||||
func (p *Provider) Model(id string, opts ...llm.ModelOption) (llm.Model, error) {
|
||||
if p.baseURL == "" {
|
||||
return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name)
|
||||
}
|
||||
return p.chatProvider().Model(id, opts...)
|
||||
}
|
||||
|
||||
// chatProvider builds the OpenAI-compatible client for llama-swap's chat API.
|
||||
// Why a placeholder key when token is empty: the openai client treats a blank
|
||||
// key as a synthetic 401, but a local llama-swap may require no auth at all —
|
||||
// a bearer it ignores is harmless. Why legacy max_tokens: llama.cpp's OpenAI
|
||||
// shim honors "max_tokens", not "max_completion_tokens".
|
||||
func (p *Provider) chatProvider() *openai.Provider {
|
||||
key := p.token
|
||||
if key == "" {
|
||||
key = "no-key"
|
||||
}
|
||||
return openai.New(
|
||||
openai.WithName(p.name),
|
||||
openai.WithBaseURL(p.baseURL+"/v1"),
|
||||
openai.WithAPIKey(key),
|
||||
openai.WithLegacyMaxTokens(),
|
||||
openai.WithHTTPClient(p.client),
|
||||
)
|
||||
}
|
||||
|
||||
// --- management endpoints ---
|
||||
|
||||
// ModelInfo is one entry from llama-swap's GET /v1/models (the OpenAI model
|
||||
// list shape). Fields llama-swap adds beyond these are ignored.
|
||||
type ModelInfo struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
OwnedBy string `json:"owned_by"`
|
||||
}
|
||||
|
||||
// ListModels returns the models llama-swap is configured to serve (GET
|
||||
// /v1/models). Unlisted models are excluded by llama-swap itself.
|
||||
func (p *Provider) ListModels(ctx context.Context) ([]ModelInfo, error) {
|
||||
var out struct {
|
||||
Data []ModelInfo `json:"data"`
|
||||
}
|
||||
if err := p.doJSON(ctx, http.MethodGet, "/v1/models", nil, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Data, nil
|
||||
}
|
||||
|
||||
// Running returns llama-swap's currently-loaded models as the raw GET /running
|
||||
// payload. Why raw: llama-swap's /running shape is not a stable, OpenAI-style
|
||||
// contract, so this exposes the endpoint without pinning a schema this package
|
||||
// would have to guess.
|
||||
func (p *Provider) Running(ctx context.Context) (json.RawMessage, error) {
|
||||
var out json.RawMessage
|
||||
if err := p.doJSON(ctx, http.MethodGet, "/running", nil, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Unload unloads a running model to free its resources (POST
|
||||
// /api/models/unload/:model). An empty model unloads all running models (POST
|
||||
// /api/models/unload).
|
||||
func (p *Provider) Unload(ctx context.Context, model string) error {
|
||||
path := "/api/models/unload"
|
||||
if model != "" {
|
||||
path += "/" + model
|
||||
}
|
||||
return p.doJSON(ctx, http.MethodPost, path, nil, nil)
|
||||
}
|
||||
|
||||
// --- shared HTTP helper for management + image endpoints ---
|
||||
|
||||
// doJSON performs a request to a llama-swap endpoint relative to baseURL,
|
||||
// optionally encoding body and decoding into out (either may be nil). Transport
|
||||
// failures are wrapped raw so llm.Classify still sees the underlying net error;
|
||||
// non-2xx responses become *llm.APIError.
|
||||
func (p *Provider) doJSON(ctx context.Context, method, path string, body, out any) error {
|
||||
var rdr io.Reader
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llama-swap: encode request: %w", err)
|
||||
}
|
||||
rdr = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, rdr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llama-swap: build request: %w", err)
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if p.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+p.token)
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llama-swap: do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return p.apiError(resp, "")
|
||||
}
|
||||
if out != nil {
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return fmt.Errorf("llama-swap: decode response: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// apiError converts a non-2xx response into *llm.APIError, tolerating the
|
||||
// OpenAI {"error":{"message",...}} envelope, the Ollama-style {"error":"..."}
|
||||
// string form, and a raw body.
|
||||
func (p *Provider) apiError(resp *http.Response, model string) error {
|
||||
e := &llm.APIError{Provider: p.name, Model: model, Status: resp.StatusCode}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
|
||||
var env struct {
|
||||
Error json.RawMessage `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(body, &env) == nil && len(env.Error) > 0 {
|
||||
var obj struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Code string `json:"code"`
|
||||
}
|
||||
if json.Unmarshal(env.Error, &obj) == nil && (obj.Message != "" || obj.Code != "" || obj.Type != "") {
|
||||
e.Message = obj.Message
|
||||
e.Code = obj.Code
|
||||
if e.Code == "" {
|
||||
e.Code = obj.Type
|
||||
}
|
||||
return e
|
||||
}
|
||||
var msg string
|
||||
if json.Unmarshal(env.Error, &msg) == nil && msg != "" {
|
||||
e.Message = msg
|
||||
return e
|
||||
}
|
||||
}
|
||||
e.Message = strings.TrimSpace(string(body))
|
||||
return e
|
||||
}
|
||||
Reference in New Issue
Block a user