Merge pull request 'feat(faceswap): report whether the likeness actually transferred' (#24) from feat/imagegen-faceswap into main

imagegen/faceswap.go

@@ -55,6 +55,14 @@ type DetectedFace struct {
 	Box [4]int
 	// Score is the detector's confidence, 0-1.
 	Score float64
+
+	// Yaw is how far the head is turned from camera, in degrees, or nil when
+	// the provider does not report pose. Exposed on ENUMERATION, not just
+	// after the fact, because it is how a caller picks a face a swap will
+	// actually work on: past roughly ±45° the features carrying identity are
+	// edge-on, and the result reads as a generic person however good the
+	// transfer is. A bounding box cannot show this.
+	Yaw *float64
 }
 
 // Size returns the box dimensions. Derived rather than stored: carrying
@@ -64,6 +72,57 @@ func (f DetectedFace) Size() (w, h int) {
 	return f.Box[2] - f.Box[0], f.Box[3] - f.Box[1]
 }
 
+// SwappedFace is the MEASURED outcome for one face the provider replaced.
+//
+// It exists because "the call returned an image" and "the likeness
+// transferred" are different claims that look identical from outside, and a
+// caller that cannot tell them apart will go looking for another way to
+// check. The one it reaches for — asking a vision model who the result looks
+// like — is wrong in exactly the cases that matter: a VLM shown a jogger in a
+// Georgetown cap holding McDonald's cups answers "Bill Clinton" whoever's
+// face is on him, so it reports failure on a correct swap.
+type SwappedFace struct {
+	// Index is the face's position in the provider's left-to-right ordering.
+	Index int
+
+	// Width, Height are the replaced face's pixel size in the TARGET.
+	// Meaningful only against ImageWidth/ImageHeight: a 138px face is large
+	// in a 400px picture and nearly invisible in a 2000px one, and it is the
+	// ratio, not the absolute size, that decides whether a person notices.
+	Width, Height int
+
+	// ImageWidth, ImageHeight are the target image's dimensions, repeated on
+	// every entry so a single face is self-describing without the caller
+	// holding onto the rest of the response.
+	ImageWidth, ImageHeight int
+
+	// Yaw is how far the head is turned from camera, in degrees, or nil when
+	// the provider does not report pose. The best single predictor of whether
+	// a swap will READ as the source person: past roughly ±45° the features
+	// carrying identity are edge-on and the result looks like a generic
+	// person rather than a specific one.
+	Yaw *float64
+
+	// IdentitySimilarity is cosine similarity between the source face and the
+	// face actually present in the result, 0-1, or nil when the provider
+	// could not measure it. Above ~0.5 the identity transferred; a LOW value
+	// is the only evidence that a swap genuinely failed.
+	IdentitySimilarity *float64
+}
+
+// FractionOfImage is the swapped face's width as a share of the image's, 0-1.
+// The number that predicts whether a person will SEE the change: the swap
+// that prompted all this replaced a 138px face in a 1010px-wide photo — 14%,
+// correct by every measure and invisible at a glance — while the same code on
+// a 168px face in a 385px picture (44%) is unmistakable. Returns 0 when the
+// dimensions are unknown.
+func (f SwappedFace) FractionOfImage() float64 {
+	if f.ImageWidth <= 0 || f.Width <= 0 {
+		return 0
+	}
+	return float64(f.Width) / float64(f.ImageWidth)
+}
+
 // FaceSwapper is the optional face-transfer surface. Separate interface so
 // existing providers keep compiling; callers type-assert.
 type FaceSwapper interface {

imagegen/imagegen.go

@@ -68,6 +68,15 @@ type Result struct {
 	// Images are the generated images, in the order the backend returned them.
 	Images []Image
 
+	// SwappedFaces is the measured outcome of a FaceSwap, one entry per face
+	// replaced. Empty for every other operation, and empty from a provider
+	// that does not measure. TYPED rather than tucked into Raw: a caller has
+	// to act on this — it is the only way to distinguish a swap that
+	// transferred the likeness from one that returned an image and nothing
+	// more — and a value reachable solely by type-asserting an `any` is one
+	// nobody discovers in time to use.
+	SwappedFaces []SwappedFace
+
 	// Raw is the provider-native response object, an escape hatch for
 	// provider-specific fields. May be nil; never required for normal use.
 	Raw any

provider/llamaswap/audio.go

@@ -338,27 +338,41 @@ func parseVoices(raw []byte) ([]string, error) {
 // varies. contentType sets the request Content-Type when body is non-nil.
 // A response larger than maxBytes is an error, never a silent truncation.
 func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, string, error) {
-	if err := p.requireBaseURL(); err != nil {
+	data, hdr, err := p.doRawHeaders(ctx, method, path, model, contentType, body, maxBytes)
+	if err != nil {
 		return nil, "", err
 	}
+	return data, hdr.Get("Content-Type"), nil
+}
+
+// doRawHeaders is doRaw with the WHOLE response header rather than just
+// Content-Type. Only the face swap needs it — the shim reports whether the
+// likeness actually transferred in X-Swap-Report, and that answer would be
+// thrown away by a function that keeps one header — so doRaw stays the
+// signature 25 other call sites use and delegates here. Two bodies would be
+// two places for the size cap and the status check to drift apart.
+func (p *Provider) doRawHeaders(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, http.Header, error) {
+	if err := p.requireBaseURL(); err != nil {
+		return nil, nil, err
+	}
 	req, err := p.newRequest(ctx, method, path, contentType, body)
 	if err != nil {
-		return nil, "", err
+		return nil, nil, err
 	}
 	resp, err := p.client.Do(req)
 	if err != nil {
-		return nil, "", fmt.Errorf("llama-swap: do request: %w", err)
+		return nil, nil, fmt.Errorf("llama-swap: do request: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode/100 != 2 {
-		return nil, "", p.apiError(resp, model)
+		return nil, nil, p.apiError(resp, model)
 	}
 	data, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
 	if err != nil {
-		return nil, "", fmt.Errorf("llama-swap: read response: %w", err)
+		return nil, nil, fmt.Errorf("llama-swap: read response: %w", err)
 	}
 	if int64(len(data)) > maxBytes {
-		return nil, "", fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes)
+		return nil, nil, fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes)
 	}
-	return data, resp.Header.Get("Content-Type"), nil
+	return data, resp.Header, nil
 }

provider/llamaswap/faceswap.go

@@ -41,11 +41,12 @@ type faceSwapModel struct {
 type facesResponse struct {
 	Count int `json:"count"`
 	Faces []struct {
-		Index  int     `json:"index"`
+		Index  int      `json:"index"`
-		Box    []int   `json:"box"`
+		Box    []int    `json:"box"`
-		Score  float64 `json:"score"`
+		Score  float64  `json:"score"`
-		Width  int     `json:"width"`
+		Width  int      `json:"width"`
-		Height int     `json:"height"`
+		Height int      `json:"height"`
+		Yaw    *float64 `json:"yaw"`
 	} `json:"faces"`
 }
 
@@ -74,7 +75,7 @@ func (m *faceSwapModel) ListFaces(ctx context.Context, img imagegen.Image) ([]im
 	}
 	out := make([]imagegen.DetectedFace, 0, len(parsed.Faces))
 	for _, f := range parsed.Faces {
-		df := imagegen.DetectedFace{Index: f.Index, Score: f.Score}
+		df := imagegen.DetectedFace{Index: f.Index, Score: f.Score, Yaw: f.Yaw}
 		// A short box would silently index out of range below; treat a
 		// malformed entry as a protocol error rather than zero-filling it,
 		// because a wrong box sends the caller at the wrong face.
@@ -126,10 +127,11 @@ func (m *faceSwapModel) FaceSwap(ctx context.Context, req imagegen.FaceSwapReque
 	if err != nil {
 		return nil, err
 	}
-	raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes)
+	raw, respHdr, err := m.p.doRawHeaders(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes)
 	if err != nil {
 		return nil, err
 	}
+	respType := respHdr.Get("Content-Type")
 	if len(raw) == 0 {
 		return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "face swap response contained no image"}
 	}
@@ -151,7 +153,55 @@ func (m *faceSwapModel) FaceSwap(ctx context.Context, req imagegen.FaceSwapReque
 	if hdr := mimeFromContentType(respType, "image/"); hdr != "" {
 		mimeType = hdr
 	}
-	return &imagegen.Result{Images: []llm.ImagePart{{MIME: mimeType, Data: raw}}}, nil
+	return &imagegen.Result{
+		Images:       []llm.ImagePart{{MIME: mimeType, Data: raw}},
+		SwappedFaces: parseSwapReport(respHdr.Get("X-Swap-Report")),
+	}, nil
+}
+
+// swapReport mirrors the shim's X-Swap-Report header.
+type swapReport struct {
+	Image []int `json:"image"`
+	Faces []struct {
+		Index              int      `json:"index"`
+		Size               []int    `json:"size"`
+		Yaw                *float64 `json:"yaw"`
+		IdentitySimilarity *float64 `json:"identity_similarity"`
+	} `json:"faces"`
+}
+
+// parseSwapReport decodes the measured outcome. A missing or malformed header
+// yields nil rather than an error: an older shim does not send it, and a swap
+// that produced a good image must not fail because the diagnostics alongside
+// it were unreadable.
+func parseSwapReport(header string) []imagegen.SwappedFace {
+	header = strings.TrimSpace(header)
+	if header == "" {
+		return nil
+	}
+	var rep swapReport
+	if err := json.Unmarshal([]byte(header), &rep); err != nil {
+		return nil
+	}
+	out := make([]imagegen.SwappedFace, 0, len(rep.Faces))
+	for _, f := range rep.Faces {
+		sf := imagegen.SwappedFace{
+			Index:              f.Index,
+			Yaw:                f.Yaw,
+			IdentitySimilarity: f.IdentitySimilarity,
+		}
+		if len(f.Size) == 2 {
+			sf.Width, sf.Height = f.Size[0], f.Size[1]
+		}
+		if len(rep.Image) == 2 {
+			sf.ImageWidth, sf.ImageHeight = rep.Image[0], rep.Image[1]
+		}
+		out = append(out, sf)
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
 }
 
 // imageFilename picks a multipart filename for an image part. The shim reads

provider/llamaswap/faceswap_test.go

@@ -259,3 +259,86 @@ func TestFaceSwapAllowsNegativeIndexUnderAll(t *testing.T) {
 		t.Error("negative index accepted when it would actually be sent")
 	}
 }
+
+// TestFaceSwapParsesSwapReport: the measured outcome is the whole reason the
+// header exists — a caller that cannot tell "the likeness transferred" from
+// "an image came back" goes and asks a vision model, which is wrong in
+// exactly the cases that matter.
+func TestFaceSwapParsesSwapReport(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "image/png")
+		w.Header().Set("X-Swap-Report",
+			`{"image":[1010,1200],"faces":[{"index":2,"size":[138,172],"yaw":-82.2,"identity_similarity":0.791}]}`)
+		raw, _ := base64.StdEncoding.DecodeString(onePixelPNG)
+		_, _ = w.Write(raw)
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	m, err := p.FaceSwapModel("faceswap")
+	if err != nil {
+		t.Fatalf("model: %v", err)
+	}
+	img := editInit(t)
+	res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img})
+	if err != nil {
+		t.Fatalf("FaceSwap: %v", err)
+	}
+	if len(res.SwappedFaces) != 1 {
+		t.Fatalf("SwappedFaces = %d, want 1 — the measurement was dropped", len(res.SwappedFaces))
+	}
+	f := res.SwappedFaces[0]
+	if f.Index != 2 || f.Width != 138 || f.Height != 172 {
+		t.Errorf("face = %+v, want index 2 at 138x172", f)
+	}
+	if f.Yaw == nil || *f.Yaw != -82.2 {
+		t.Errorf("yaw = %v, want -82.2 — the pose signal is how a caller knows a profile swap will not read", f.Yaw)
+	}
+	if f.IdentitySimilarity == nil || *f.IdentitySimilarity != 0.791 {
+		t.Errorf("identity_similarity = %v, want 0.791", f.IdentitySimilarity)
+	}
+	// 138/1010 — the number that says "correct, and invisible at a glance".
+	if got := f.FractionOfImage(); got < 0.13 || got > 0.14 {
+		t.Errorf("FractionOfImage = %.3f, want ~0.137", got)
+	}
+}
+
+// TestFaceSwapSurvivesMissingReport: an older shim sends no header at all. A
+// swap that produced a good image must not fail because the diagnostics
+// beside it were absent or malformed.
+func TestFaceSwapSurvivesMissingReport(t *testing.T) {
+	for name, hdr := range map[string]string{
+		"absent":    "",
+		"garbage":   "not json at all",
+		"wrongtype": `{"faces":"nope"}`,
+	} {
+		t.Run(name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "image/png")
+				if hdr != "" {
+					w.Header().Set("X-Swap-Report", hdr)
+				}
+				raw, _ := base64.StdEncoding.DecodeString(onePixelPNG)
+				_, _ = w.Write(raw)
+			}))
+			defer srv.Close()
+
+			p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+			m, err := p.FaceSwapModel("faceswap")
+			if err != nil {
+				t.Fatalf("model: %v", err)
+			}
+			img := editInit(t)
+			res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img})
+			if err != nil {
+				t.Fatalf("a %s report failed the whole swap: %v", name, err)
+			}
+			if len(res.Images) != 1 {
+				t.Fatal("image lost")
+			}
+			if res.SwappedFaces != nil {
+				t.Errorf("SwappedFaces = %+v, want nil for a %s report", res.SwappedFaces, name)
+			}
+		})
+	}
+}