diff --git a/imagegen/edit.go b/imagegen/edit.go index 6f16b3d..71eb91b 100644 --- a/imagegen/edit.go +++ b/imagegen/edit.go @@ -9,9 +9,32 @@ type EditRequest struct { // Prompt is the text description of the desired edit. Prompt string - // Init is the initial image the edit starts from. Required. + // Init is the initial image the edit starts from. Required, EXCEPT when + // RefImages is set — see there. Init Image + // RefImages carries reference images for INSTRUCTION-EDIT models + // (FLUX.1 Kontext, Qwen-Image-Edit), which are a different kind of edit + // from img2img and reach the model by a different path. + // + // img2img noises Init and denoises it back under the prompt: the prompt + // describes the DESIRED IMAGE, and how much of the original survives is a + // function of Strength. An instruction-edit model instead takes the + // picture as conditioning and the prompt as an INSTRUCTION about it + // ("change the sign to read OPEN"), leaving everything it was not asked + // to touch bit-for-bit intact — no mask, no strength, no compositing. + // + // Sending one of these models an Init instead of a RefImage does not + // degrade gracefully, it silently does the wrong thing: measured against + // FLUX.1-Kontext on 2026-07-30, "change the blue rectangle to green" via + // init_images left the rectangle blue and drifted every other region, + // while the same prompt via a reference image turned it green and left + // the rest of the frame numerically unchanged. + // + // When RefImages is non-empty, Init/Mask/Strength are IGNORED: they + // describe a pipeline this model does not run. + RefImages []Image + // Mask restricts the edit to a region (inpainting): a single-channel or // RGB image the same size as Init where WHITE pixels are repainted and // BLACK pixels are kept. Empty = whole-image edit. Backends without mask @@ -54,6 +77,13 @@ type EditOption func(*EditRequest) // WithEditMask restricts the edit to a region (white = repaint, black = keep). func WithEditMask(m Image) EditOption { return func(r *EditRequest) { r.Mask = m } } +// WithEditRefImages supplies reference images for an instruction-edit model +// (Kontext / Qwen-Image-Edit). See EditRequest.RefImages — this selects a +// different edit path, not a variation on img2img. +func WithEditRefImages(imgs ...Image) EditOption { + return func(r *EditRequest) { r.RefImages = imgs } +} + // WithEditStrength sets the denoising strength in [0,1]. func WithEditStrength(s float64) EditOption { return func(r *EditRequest) { r.Strength = &s } } diff --git a/provider/llamaswap/edit_test.go b/provider/llamaswap/edit_test.go index 5c5b3e0..5c4d59e 100644 --- a/provider/llamaswap/edit_test.go +++ b/provider/llamaswap/edit_test.go @@ -152,3 +152,76 @@ func TestImageEditWithoutMaskOmitsField(t *testing.T) { t.Error("mask field sent for unmasked edit; want omitted") } } + +// TestImageEditByReferenceUsesTxt2ImgExtraImages pins the instruction-edit +// wire shape. It is a DIFFERENT endpoint and a DIFFERENT field from img2img, +// and the difference is not cosmetic: measured against FLUX.1-Kontext on +// 2026-07-30, the same prompt sent as init_images left the thing it was told +// to change untouched and drifted everything else, while extra_images changed +// exactly what was asked and left the rest of the frame numerically +// unchanged. Routing a reference edit down the img2img path would look like +// a working call and silently produce the wrong picture. +func TestImageEditByReferenceUsesTxt2ImgExtraImages(t *testing.T) { + var gotPath string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + im, _ := p.ImageModel("imagegen-flux-kontext") + ed := im.(imagegen.Editor) + + ref := editInit(t) + if _, err := ed.Edit(context.Background(), imagegen.EditRequest{ + Prompt: "make the sign read OPEN", + // Init/Mask/Strength are set and must be IGNORED — they describe a + // pipeline this model does not run. + Init: ref, + Mask: ref, + Strength: func() *float64 { s := 0.75; return &s }(), + }, imagegen.WithEditRefImages(ref)); err != nil { + t.Fatalf("reference edit: %v", err) + } + + if gotPath != "/sdapi/v1/txt2img" { + t.Errorf("path = %q, want /sdapi/v1/txt2img (there is no init latent to denoise)", gotPath) + } + extra, ok := gotBody["extra_images"].([]any) + if !ok || len(extra) != 1 { + t.Fatalf("extra_images = %v, want the one reference image", gotBody["extra_images"]) + } + if _, present := gotBody["init_images"]; present { + t.Error("init_images must NOT be sent on the reference path — it re-noises the picture") + } + if _, present := gotBody["denoising_strength"]; present { + t.Error("denoising_strength must NOT be sent on the reference path") + } + if _, present := gotBody["mask"]; present { + t.Error("mask must NOT be sent on the reference path") + } +} + +// TestImageEditByReferenceRejectsEmptyRefs guards the case that would +// otherwise silently become a plain txt2img: a reference edit whose only +// reference carries no bytes has nothing to condition on, and rendering the +// prompt from scratch is not what the caller asked for. +func TestImageEditByReferenceRejectsEmptyRefs(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + im, _ := p.ImageModel("imagegen-flux-kontext") + ed := im.(imagegen.Editor) + + _, err := ed.Edit(context.Background(), imagegen.EditRequest{Prompt: "anything"}, + imagegen.WithEditRefImages(imagegen.Image{MIME: "image/png"})) + if !errors.Is(err, llm.ErrUnsupported) { + t.Fatalf("err = %v, want ErrUnsupported for an all-empty reference set", err) + } +} diff --git a/provider/llamaswap/image.go b/provider/llamaswap/image.go index a6248f4..fadeb7f 100644 --- a/provider/llamaswap/image.go +++ b/provider/llamaswap/image.go @@ -135,9 +135,30 @@ type img2imgRequest struct { Mask string `json:"mask,omitempty"` } -// Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img. +// refEditRequest is the wire shape for an INSTRUCTION-EDIT model. sd-server +// exposes reference images as `extra_images` on the shared img-gen request +// builder (routes_sdapi.cpp lands them in gen_params.ref_images — the same +// place the CLI's -r/--ref-image goes), and that field is read on BOTH +// /txt2img and /img2img. +// +// It posts to /txt2img because there is no init latent to denoise: the +// reference IS the conditioning, so an init image plus a denoising strength +// would only add noise to a pipeline that does not want any. Output +// resolution follows the reference image. +type refEditRequest struct { + txt2imgRequest + ExtraImages []string `json:"extra_images"` +} + +// Edit implements imagegen.Editor. Two different pipelines live behind it, +// selected by the request: RefImages routes to an instruction-edit model via +// /sdapi/v1/txt2img + extra_images, everything else is img2img. See +// imagegen.EditRequest.RefImages for why they are not interchangeable. func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ...imagegen.EditOption) (*imagegen.Result, error) { req = req.Apply(opts...) + if len(req.RefImages) > 0 { + return m.editByReference(ctx, req) + } if len(req.Init.Data) == 0 { return nil, fmt.Errorf("%w: image edit requires an init image", llm.ErrUnsupported) } @@ -164,6 +185,34 @@ func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts .. return decodeImages(m.p.name, m.id, &resp) } +// editByReference runs the instruction-edit path. Mask and Strength are +// deliberately NOT rejected when set: a caller that hands the same +// EditRequest to whichever model is configured should get the better result +// on a Kontext-class model, not an error, and both fields describe a +// pipeline that simply does not exist here. +func (m *imageModel) editByReference(ctx context.Context, req imagegen.EditRequest) (*imagegen.Result, error) { + base, err := m.sdWire("reference edit", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N) + if err != nil { + return nil, err + } + wire := refEditRequest{txt2imgRequest: base} + for _, ref := range req.RefImages { + if len(ref.Data) == 0 { + continue + } + wire.ExtraImages = append(wire.ExtraImages, base64.StdEncoding.EncodeToString(ref.Data)) + } + if len(wire.ExtraImages) == 0 { + return nil, fmt.Errorf("%w: reference edit requires at least one non-empty reference image", llm.ErrUnsupported) + } + + var resp txt2imgResponse + if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/txt2img", m.id, &wire, &resp); err != nil { + return nil, err + } + return decodeImages(m.p.name, m.id, &resp) +} + // parseSize splits a "WxH" string into width/height pointers. "" yields // (nil, nil) so the model's own default resolution applies. func parseSize(size string) (*int, *int, error) {