feat(imagegen): reference-image editing for instruction-edit models

FLUX.1 Kontext and Qwen-Image-Edit are a different kind of edit from img2img
and reach sd-server by a different path, and nothing in imagegen could
express it: EditRequest only had Init, which is noised and denoised back
under the prompt.

Measured against FLUX.1-Kontext on the netherstorm host 2026-07-30, on a
synthetic scene with a red rectangle, a blue rectangle and a flat background,
prompted "change the blue rectangle on the right to bright green, keep
everything else exactly the same":

  via init_images (the only path that existed)
      right rect (60,60,200) -> (47,82,228)   still blue, instruction ignored
      left rect  (200,60,60) -> (229,43,50)   drifted
      background (150,200,240) -> (154,211,229) drifted

  via extra_images (this change)
      right rect (60,60,200) -> (70,254,4)    green, as asked
      left rect  (200,60,60) -> (204,57,57)   intact
      background (150,200,240) -> (151,202,247) intact

No mask, no strength, no compositing — the model is handed the picture as
conditioning and the prompt as an instruction about it.

EditRequest.RefImages selects the path; when set, Init/Mask/Strength are
ignored rather than rejected, so a caller handing the same request to
whichever model is configured gets the better result on a Kontext-class model
instead of an error. The provider posts /sdapi/v1/txt2img with extra_images
(sd-server reads that field on both routes into gen_params.ref_images, where
the CLI's -r/--ref-image also lands); there is no init latent to denoise, so
sending one would only add noise to a pipeline that does not want any.

An all-empty reference set is refused: it would otherwise degrade into a
plain txt2img and render the prompt from scratch, which is not the request.

imagegen/edit.go

@@ -9,9 +9,32 @@ type EditRequest struct {
 	// Prompt is the text description of the desired edit.
 	Prompt string
 
-	// Init is the initial image the edit starts from. Required.
+	// Init is the initial image the edit starts from. Required, EXCEPT when
+	// RefImages is set — see there.
 	Init Image
 
+	// RefImages carries reference images for INSTRUCTION-EDIT models
+	// (FLUX.1 Kontext, Qwen-Image-Edit), which are a different kind of edit
+	// from img2img and reach the model by a different path.
+	//
+	// img2img noises Init and denoises it back under the prompt: the prompt
+	// describes the DESIRED IMAGE, and how much of the original survives is a
+	// function of Strength. An instruction-edit model instead takes the
+	// picture as conditioning and the prompt as an INSTRUCTION about it
+	// ("change the sign to read OPEN"), leaving everything it was not asked
+	// to touch bit-for-bit intact — no mask, no strength, no compositing.
+	//
+	// Sending one of these models an Init instead of a RefImage does not
+	// degrade gracefully, it silently does the wrong thing: measured against
+	// FLUX.1-Kontext on 2026-07-30, "change the blue rectangle to green" via
+	// init_images left the rectangle blue and drifted every other region,
+	// while the same prompt via a reference image turned it green and left
+	// the rest of the frame numerically unchanged.
+	//
+	// When RefImages is non-empty, Init/Mask/Strength are IGNORED: they
+	// describe a pipeline this model does not run.
+	RefImages []Image
+
 	// Mask restricts the edit to a region (inpainting): a single-channel or
 	// RGB image the same size as Init where WHITE pixels are repainted and
 	// BLACK pixels are kept. Empty = whole-image edit. Backends without mask
@@ -54,6 +77,13 @@ type EditOption func(*EditRequest)
 // WithEditMask restricts the edit to a region (white = repaint, black = keep).
 func WithEditMask(m Image) EditOption { return func(r *EditRequest) { r.Mask = m } }
 
+// WithEditRefImages supplies reference images for an instruction-edit model
+// (Kontext / Qwen-Image-Edit). See EditRequest.RefImages — this selects a
+// different edit path, not a variation on img2img.
+func WithEditRefImages(imgs ...Image) EditOption {
+	return func(r *EditRequest) { r.RefImages = imgs }
+}
+
 // WithEditStrength sets the denoising strength in [0,1].
 func WithEditStrength(s float64) EditOption { return func(r *EditRequest) { r.Strength = &s } }
 

provider/llamaswap/edit_test.go

@@ -152,3 +152,76 @@ func TestImageEditWithoutMaskOmitsField(t *testing.T) {
 		t.Error("mask field sent for unmasked edit; want omitted")
 	}
 }
+
+// TestImageEditByReferenceUsesTxt2ImgExtraImages pins the instruction-edit
+// wire shape. It is a DIFFERENT endpoint and a DIFFERENT field from img2img,
+// and the difference is not cosmetic: measured against FLUX.1-Kontext on
+// 2026-07-30, the same prompt sent as init_images left the thing it was told
+// to change untouched and drifted everything else, while extra_images changed
+// exactly what was asked and left the rest of the frame numerically
+// unchanged. Routing a reference edit down the img2img path would look like
+// a working call and silently produce the wrong picture.
+func TestImageEditByReferenceUsesTxt2ImgExtraImages(t *testing.T) {
+	var gotPath string
+	var gotBody map[string]any
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		_ = json.NewDecoder(r.Body).Decode(&gotBody)
+		_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	im, _ := p.ImageModel("imagegen-flux-kontext")
+	ed := im.(imagegen.Editor)
+
+	ref := editInit(t)
+	if _, err := ed.Edit(context.Background(), imagegen.EditRequest{
+		Prompt: "make the sign read OPEN",
+		// Init/Mask/Strength are set and must be IGNORED — they describe a
+		// pipeline this model does not run.
+		Init:     ref,
+		Mask:     ref,
+		Strength: func() *float64 { s := 0.75; return &s }(),
+	}, imagegen.WithEditRefImages(ref)); err != nil {
+		t.Fatalf("reference edit: %v", err)
+	}
+
+	if gotPath != "/sdapi/v1/txt2img" {
+		t.Errorf("path = %q, want /sdapi/v1/txt2img (there is no init latent to denoise)", gotPath)
+	}
+	extra, ok := gotBody["extra_images"].([]any)
+	if !ok || len(extra) != 1 {
+		t.Fatalf("extra_images = %v, want the one reference image", gotBody["extra_images"])
+	}
+	if _, present := gotBody["init_images"]; present {
+		t.Error("init_images must NOT be sent on the reference path — it re-noises the picture")
+	}
+	if _, present := gotBody["denoising_strength"]; present {
+		t.Error("denoising_strength must NOT be sent on the reference path")
+	}
+	if _, present := gotBody["mask"]; present {
+		t.Error("mask must NOT be sent on the reference path")
+	}
+}
+
+// TestImageEditByReferenceRejectsEmptyRefs guards the case that would
+// otherwise silently become a plain txt2img: a reference edit whose only
+// reference carries no bytes has nothing to condition on, and rendering the
+// prompt from scratch is not what the caller asked for.
+func TestImageEditByReferenceRejectsEmptyRefs(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	im, _ := p.ImageModel("imagegen-flux-kontext")
+	ed := im.(imagegen.Editor)
+
+	_, err := ed.Edit(context.Background(), imagegen.EditRequest{Prompt: "anything"},
+		imagegen.WithEditRefImages(imagegen.Image{MIME: "image/png"}))
+	if !errors.Is(err, llm.ErrUnsupported) {
+		t.Fatalf("err = %v, want ErrUnsupported for an all-empty reference set", err)
+	}
+}

provider/llamaswap/image.go

@@ -135,9 +135,30 @@ type img2imgRequest struct {
 	Mask string `json:"mask,omitempty"`
 }
 
-// Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img.
+// refEditRequest is the wire shape for an INSTRUCTION-EDIT model. sd-server
+// exposes reference images as `extra_images` on the shared img-gen request
+// builder (routes_sdapi.cpp lands them in gen_params.ref_images — the same
+// place the CLI's -r/--ref-image goes), and that field is read on BOTH
+// /txt2img and /img2img.
+//
+// It posts to /txt2img because there is no init latent to denoise: the
+// reference IS the conditioning, so an init image plus a denoising strength
+// would only add noise to a pipeline that does not want any. Output
+// resolution follows the reference image.
+type refEditRequest struct {
+	txt2imgRequest
+	ExtraImages []string `json:"extra_images"`
+}
+
+// Edit implements imagegen.Editor. Two different pipelines live behind it,
+// selected by the request: RefImages routes to an instruction-edit model via
+// /sdapi/v1/txt2img + extra_images, everything else is img2img. See
+// imagegen.EditRequest.RefImages for why they are not interchangeable.
 func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ...imagegen.EditOption) (*imagegen.Result, error) {
 	req = req.Apply(opts...)
+	if len(req.RefImages) > 0 {
+		return m.editByReference(ctx, req)
+	}
 	if len(req.Init.Data) == 0 {
 		return nil, fmt.Errorf("%w: image edit requires an init image", llm.ErrUnsupported)
 	}
@@ -164,6 +185,34 @@ func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ..
 	return decodeImages(m.p.name, m.id, &resp)
 }
 
+// editByReference runs the instruction-edit path. Mask and Strength are
+// deliberately NOT rejected when set: a caller that hands the same
+// EditRequest to whichever model is configured should get the better result
+// on a Kontext-class model, not an error, and both fields describe a
+// pipeline that simply does not exist here.
+func (m *imageModel) editByReference(ctx context.Context, req imagegen.EditRequest) (*imagegen.Result, error) {
+	base, err := m.sdWire("reference edit", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N)
+	if err != nil {
+		return nil, err
+	}
+	wire := refEditRequest{txt2imgRequest: base}
+	for _, ref := range req.RefImages {
+		if len(ref.Data) == 0 {
+			continue
+		}
+		wire.ExtraImages = append(wire.ExtraImages, base64.StdEncoding.EncodeToString(ref.Data))
+	}
+	if len(wire.ExtraImages) == 0 {
+		return nil, fmt.Errorf("%w: reference edit requires at least one non-empty reference image", llm.ErrUnsupported)
+	}
+
+	var resp txt2imgResponse
+	if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/txt2img", m.id, &wire, &resp); err != nil {
+		return nil, err
+	}
+	return decodeImages(m.p.name, m.id, &resp)
+}
+
 // parseSize splits a "WxH" string into width/height pointers. "" yields
 // (nil, nil) so the model's own default resolution applies.
 func parseSize(size string) (*int, *int, error) {