majordomo/provider/llamaswap/image.go

package llamaswap

import (
	"context"
	"encoding/base64"
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)

// ImageModel implements imagegen.Provider, binding an image-generation model
// served by llama-swap (routed to a stable-diffusion.cpp upstream). The id is
// passed through verbatim and selects which upstream llama-swap loads.
func (p *Provider) ImageModel(id string, opts ...imagegen.ModelOption) (imagegen.Model, error) {
	if err := p.requireBaseURL(); err != nil {
		return nil, err
	}
	_ = imagegen.ApplyModelOptions(opts)
	return &imageModel{p: p, id: id}, nil
}

type imageModel struct {
	p  *Provider
	id string
}

// txt2imgRequest is the stable-diffusion.cpp sd-server A1111 request shape
// (POST /sdapi/v1/txt2img). We use this endpoint rather than the OpenAI
// /v1/images/generations one because that endpoint IGNORES `seed` on this
// sd-server build — every render of a given prompt comes back byte-identical,
// so a batch of N collapses to one image. /sdapi/v1/txt2img honours `seed`,
// giving real variety. llama-swap still routes by the `model` field in the
// body. Optional fields are pointers/omitempty so an unset value falls back to
// the model's baked default (the per-model --steps/--cfg-scale/etc. flags).
type txt2imgRequest struct {
	Model          string   `json:"model"`
	Prompt         string   `json:"prompt"`
	NegativePrompt string   `json:"negative_prompt,omitempty"`
	Seed           *int64   `json:"seed,omitempty"`
	Steps          *int     `json:"steps,omitempty"`
	CFGScale       *float64 `json:"cfg_scale,omitempty"`
	Width          *int     `json:"width,omitempty"`
	Height         *int     `json:"height,omitempty"`
	SampleMethod   string   `json:"sample_method,omitempty"`
	BatchCount     int      `json:"batch_count,omitempty"`
}

type txt2imgResponse struct {
	Images []string `json:"images"`
}

// sdWire validates the generation knobs shared by Generate and Edit and
// builds the common txt2img wire fields. verb labels validation errors.
func (m *imageModel) sdWire(verb, prompt, negativePrompt, sampler, size string, seed *int64, steps *int, cfgScale *float64, n int) (txt2imgRequest, error) {
	if strings.TrimSpace(prompt) == "" {
		return txt2imgRequest{}, fmt.Errorf("%w: image %s requires a prompt", llm.ErrUnsupported, verb)
	}
	if n < 0 {
		return txt2imgRequest{}, fmt.Errorf("%w: image count N must be >= 0, got %d", llm.ErrUnsupported, n)
	}
	width, height, err := parseSize(size)
	if err != nil {
		return txt2imgRequest{}, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
	}
	return txt2imgRequest{
		Model:          m.id,
		Prompt:         prompt,
		NegativePrompt: negativePrompt,
		Seed:           seed,
		Steps:          steps,
		CFGScale:       cfgScale,
		Width:          width,
		Height:         height,
		SampleMethod:   sampler,
		BatchCount:     n,
	}, nil
}

// Generate implements imagegen.Model via POST {base}/sdapi/v1/txt2img.
func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts ...imagegen.Option) (*imagegen.Result, error) {
	req = req.Apply(opts...)
	wire, err := m.sdWire("generation", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N)
	if err != nil {
		return nil, err
	}

	var resp txt2imgResponse
	if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/txt2img", m.id, &wire, &resp); err != nil {
		return nil, err
	}
	return decodeImages(m.p.name, m.id, &resp)
}

// decodeImages converts an SDAPI response's base64 images into an
// imagegen.Result, erroring when nothing decodable came back.
func decodeImages(provider, model string, resp *txt2imgResponse) (*imagegen.Result, error) {
	out := &imagegen.Result{Raw: resp}
	for i, b64 := range resp.Images {
		if b64 == "" {
			continue
		}
		raw, err := base64.StdEncoding.DecodeString(b64)
		if err != nil {
			return nil, fmt.Errorf("llama-swap: decode image %d: %w", i, err)
		}
		out.Images = append(out.Images, llm.ImagePart{MIME: sniffImageMIME(raw), Data: raw})
	}
	if len(out.Images) == 0 {
		return nil, &llm.APIError{
			Provider: provider,
			Model:    model,
			Message:  "image response contained no images",
		}
	}
	return out, nil
}

// img2imgRequest is the stable-diffusion.cpp sd-server A1111 request shape
// (POST /sdapi/v1/img2img): txt2img's fields plus the init image(s) and
// denoising strength. Same endpoint-family choice as txt2img — the OpenAI
// /v1/images/edits route is multipart and drops `seed` on this sd-server
// build, while the SDAPI shape reuses doJSON and keeps seed parity.
type img2imgRequest struct {
	txt2imgRequest
	InitImages        []string `json:"init_images"`
	DenoisingStrength *float64 `json:"denoising_strength,omitempty"`
	// Mask enables inpainting: base64 image, white = repaint, black = keep
	// (sd-server also accepts a data URL; plain base64 keeps symmetry with
	// init_images). sd-server has no mask_blur/inpaint_full_res — callers
	// wanting soft edges pre-feather the mask.
	Mask string `json:"mask,omitempty"`
}

// refEditRequest is the wire shape for an INSTRUCTION-EDIT model. sd-server
// exposes reference images as `extra_images` on the shared img-gen request
// builder (routes_sdapi.cpp lands them in gen_params.ref_images — the same
// place the CLI's -r/--ref-image goes), and that field is read on BOTH
// /txt2img and /img2img.
//
// It posts to /txt2img because there is no init latent to denoise: the
// reference IS the conditioning, so an init image plus a denoising strength
// would only add noise to a pipeline that does not want any. Output
// resolution follows the reference image.
type refEditRequest struct {
	txt2imgRequest
	ExtraImages []string `json:"extra_images"`
}

// Edit implements imagegen.Editor. Two different pipelines live behind it,
// selected by the request: RefImages routes to an instruction-edit model via
// /sdapi/v1/txt2img + extra_images, everything else is img2img. See
// imagegen.EditRequest.RefImages for why they are not interchangeable.
func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ...imagegen.EditOption) (*imagegen.Result, error) {
	req = req.Apply(opts...)
	if len(req.RefImages) > 0 {
		return m.editByReference(ctx, req)
	}
	if len(req.Init.Data) == 0 {
		return nil, fmt.Errorf("%w: image edit requires an init image", llm.ErrUnsupported)
	}
	if req.Strength != nil && (*req.Strength < 0 || *req.Strength > 1) {
		return nil, fmt.Errorf("%w: edit strength must be in [0,1], got %g", llm.ErrUnsupported, *req.Strength)
	}
	base, err := m.sdWire("edit", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N)
	if err != nil {
		return nil, err
	}
	wire := img2imgRequest{
		txt2imgRequest:    base,
		InitImages:        []string{base64.StdEncoding.EncodeToString(req.Init.Data)},
		DenoisingStrength: req.Strength,
	}
	if len(req.Mask.Data) > 0 {
		wire.Mask = base64.StdEncoding.EncodeToString(req.Mask.Data)
	}

	var resp txt2imgResponse
	if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/img2img", m.id, &wire, &resp); err != nil {
		return nil, err
	}
	return decodeImages(m.p.name, m.id, &resp)
}

// editByReference runs the instruction-edit path. Mask and Strength are
// deliberately NOT rejected when set: a caller that hands the same
// EditRequest to whichever model is configured should get the better result
// on a Kontext-class model, not an error, and both fields describe a
// pipeline that simply does not exist here.
func (m *imageModel) editByReference(ctx context.Context, req imagegen.EditRequest) (*imagegen.Result, error) {
	base, err := m.sdWire("reference edit", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N)
	if err != nil {
		return nil, err
	}
	wire := refEditRequest{txt2imgRequest: base}
	for _, ref := range req.RefImages {
		if len(ref.Data) == 0 {
			continue
		}
		wire.ExtraImages = append(wire.ExtraImages, base64.StdEncoding.EncodeToString(ref.Data))
	}
	if len(wire.ExtraImages) == 0 {
		return nil, fmt.Errorf("%w: reference edit requires at least one non-empty reference image", llm.ErrUnsupported)
	}

	var resp txt2imgResponse
	if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/txt2img", m.id, &wire, &resp); err != nil {
		return nil, err
	}
	return decodeImages(m.p.name, m.id, &resp)
}

// parseSize splits a "WxH" string into width/height pointers. "" yields
// (nil, nil) so the model's own default resolution applies.
func parseSize(size string) (*int, *int, error) {
	size = strings.TrimSpace(size)
	if size == "" {
		return nil, nil, nil
	}
	parts := strings.SplitN(strings.ToLower(size), "x", 2)
	if len(parts) != 2 {
		return nil, nil, fmt.Errorf("invalid size %q (want WxH)", size)
	}
	w, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
	h, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
	if err1 != nil || err2 != nil || w <= 0 || h <= 0 {
		return nil, nil, fmt.Errorf("invalid size %q (want WxH)", size)
	}
	return &w, &h, nil
}

// sniffImageMIME identifies the image format from its leading bytes, defaulting
// to image/png (stable-diffusion.cpp emits PNG) when detection is inconclusive.
func sniffImageMIME(data []byte) string {
	mime := http.DetectContentType(data)
	if !strings.HasPrefix(mime, "image/") {
		return "image/png"
	}
	return mime
}