From ae2615ca6846e7b6fba772620409a3b759f686a0 Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Fri, 31 Jul 2026 17:49:27 -0400 Subject: [PATCH 1/2] feat(faceswap): carry the measured outcome, not just the image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A face swap always returns an image and always looks like success. Whether the likeness actually transferred is a different question, and until now nothing in the response answered it — so a caller wanting to know went and asked a vision model instead. That is wrong in precisely the cases that matter: shown a jogger in a Georgetown cap holding McDonald's cups, a VLM answers "Bill Clinton" whoever's face is on him. In the run that prompted this it reported failure on six consecutive CORRECT swaps (measured afterwards at 0.79-0.84 cosine), and the caller burned 21 minutes chasing a problem that did not exist. Result.SwappedFaces now carries, per replaced face: pixel size, the target image's dimensions, head yaw, and cosine similarity between the source face and the face actually present in the output. Yaw and FractionOfImage are the two that explain the complaint. The swap in question replaced a 138px face in a 1010px-wide photo — 14% of the width, correct and invisible at a glance — and elsewhere a face turned -82 degrees, where the features carrying identity are edge-on and any swap reads as a generic person. Same code on a 168px face in a 385px picture (44%, yaw 2) is unmistakable. None of that was inferable from a bounding box. Typed on Result rather than stuffed into Raw: a caller has to act on this, and a value reachable only by type-asserting an `any` is one nobody finds in time. doRawHeaders is doRaw with the whole header instead of only Content-Type; doRaw delegates to it, so the other 25 call sites are untouched and there is still one place where the status check and the size cap live. A missing or malformed header yields nil, not an error — an older shim sends no header, and a swap that produced a good image must not fail because the diagnostics beside it were unreadable. Covered for absent/garbage/wrong-type, and the parse is break-checked. --- imagegen/faceswap.go | 51 ++++++++++++++++++ imagegen/imagegen.go | 9 ++++ provider/llamaswap/audio.go | 28 +++++++--- provider/llamaswap/faceswap.go | 53 +++++++++++++++++- provider/llamaswap/faceswap_test.go | 83 +++++++++++++++++++++++++++++ 5 files changed, 215 insertions(+), 9 deletions(-) diff --git a/imagegen/faceswap.go b/imagegen/faceswap.go index d7b09e8..024fec2 100644 --- a/imagegen/faceswap.go +++ b/imagegen/faceswap.go @@ -64,6 +64,57 @@ func (f DetectedFace) Size() (w, h int) { return f.Box[2] - f.Box[0], f.Box[3] - f.Box[1] } +// SwappedFace is the MEASURED outcome for one face the provider replaced. +// +// It exists because "the call returned an image" and "the likeness +// transferred" are different claims that look identical from outside, and a +// caller that cannot tell them apart will go looking for another way to +// check. The one it reaches for — asking a vision model who the result looks +// like — is wrong in exactly the cases that matter: a VLM shown a jogger in a +// Georgetown cap holding McDonald's cups answers "Bill Clinton" whoever's +// face is on him, so it reports failure on a correct swap. +type SwappedFace struct { + // Index is the face's position in the provider's left-to-right ordering. + Index int + + // Width, Height are the replaced face's pixel size in the TARGET. + // Meaningful only against ImageWidth/ImageHeight: a 138px face is large + // in a 400px picture and nearly invisible in a 2000px one, and it is the + // ratio, not the absolute size, that decides whether a person notices. + Width, Height int + + // ImageWidth, ImageHeight are the target image's dimensions, repeated on + // every entry so a single face is self-describing without the caller + // holding onto the rest of the response. + ImageWidth, ImageHeight int + + // Yaw is how far the head is turned from camera, in degrees, or nil when + // the provider does not report pose. The best single predictor of whether + // a swap will READ as the source person: past roughly ±45° the features + // carrying identity are edge-on and the result looks like a generic + // person rather than a specific one. + Yaw *float64 + + // IdentitySimilarity is cosine similarity between the source face and the + // face actually present in the result, 0-1, or nil when the provider + // could not measure it. Above ~0.5 the identity transferred; a LOW value + // is the only evidence that a swap genuinely failed. + IdentitySimilarity *float64 +} + +// FractionOfImage is the swapped face's width as a share of the image's, 0-1. +// The number that predicts whether a person will SEE the change: the swap +// that prompted all this replaced a 138px face in a 1010px-wide photo — 14%, +// correct by every measure and invisible at a glance — while the same code on +// a 168px face in a 385px picture (44%) is unmistakable. Returns 0 when the +// dimensions are unknown. +func (f SwappedFace) FractionOfImage() float64 { + if f.ImageWidth <= 0 || f.Width <= 0 { + return 0 + } + return float64(f.Width) / float64(f.ImageWidth) +} + // FaceSwapper is the optional face-transfer surface. Separate interface so // existing providers keep compiling; callers type-assert. type FaceSwapper interface { diff --git a/imagegen/imagegen.go b/imagegen/imagegen.go index 090f89b..7cd301f 100644 --- a/imagegen/imagegen.go +++ b/imagegen/imagegen.go @@ -68,6 +68,15 @@ type Result struct { // Images are the generated images, in the order the backend returned them. Images []Image + // SwappedFaces is the measured outcome of a FaceSwap, one entry per face + // replaced. Empty for every other operation, and empty from a provider + // that does not measure. TYPED rather than tucked into Raw: a caller has + // to act on this — it is the only way to distinguish a swap that + // transferred the likeness from one that returned an image and nothing + // more — and a value reachable solely by type-asserting an `any` is one + // nobody discovers in time to use. + SwappedFaces []SwappedFace + // Raw is the provider-native response object, an escape hatch for // provider-specific fields. May be nil; never required for normal use. Raw any diff --git a/provider/llamaswap/audio.go b/provider/llamaswap/audio.go index 717a4ec..ff864de 100644 --- a/provider/llamaswap/audio.go +++ b/provider/llamaswap/audio.go @@ -338,27 +338,41 @@ func parseVoices(raw []byte) ([]string, error) { // varies. contentType sets the request Content-Type when body is non-nil. // A response larger than maxBytes is an error, never a silent truncation. func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, string, error) { - if err := p.requireBaseURL(); err != nil { + data, hdr, err := p.doRawHeaders(ctx, method, path, model, contentType, body, maxBytes) + if err != nil { return nil, "", err } + return data, hdr.Get("Content-Type"), nil +} + +// doRawHeaders is doRaw with the WHOLE response header rather than just +// Content-Type. Only the face swap needs it — the shim reports whether the +// likeness actually transferred in X-Swap-Report, and that answer would be +// thrown away by a function that keeps one header — so doRaw stays the +// signature 25 other call sites use and delegates here. Two bodies would be +// two places for the size cap and the status check to drift apart. +func (p *Provider) doRawHeaders(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, http.Header, error) { + if err := p.requireBaseURL(); err != nil { + return nil, nil, err + } req, err := p.newRequest(ctx, method, path, contentType, body) if err != nil { - return nil, "", err + return nil, nil, err } resp, err := p.client.Do(req) if err != nil { - return nil, "", fmt.Errorf("llama-swap: do request: %w", err) + return nil, nil, fmt.Errorf("llama-swap: do request: %w", err) } defer resp.Body.Close() if resp.StatusCode/100 != 2 { - return nil, "", p.apiError(resp, model) + return nil, nil, p.apiError(resp, model) } data, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) if err != nil { - return nil, "", fmt.Errorf("llama-swap: read response: %w", err) + return nil, nil, fmt.Errorf("llama-swap: read response: %w", err) } if int64(len(data)) > maxBytes { - return nil, "", fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes) + return nil, nil, fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes) } - return data, resp.Header.Get("Content-Type"), nil + return data, resp.Header, nil } diff --git a/provider/llamaswap/faceswap.go b/provider/llamaswap/faceswap.go index a69fc9a..2f7fb57 100644 --- a/provider/llamaswap/faceswap.go +++ b/provider/llamaswap/faceswap.go @@ -126,10 +126,11 @@ func (m *faceSwapModel) FaceSwap(ctx context.Context, req imagegen.FaceSwapReque if err != nil { return nil, err } - raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes) + raw, respHdr, err := m.p.doRawHeaders(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes) if err != nil { return nil, err } + respType := respHdr.Get("Content-Type") if len(raw) == 0 { return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "face swap response contained no image"} } @@ -151,7 +152,55 @@ func (m *faceSwapModel) FaceSwap(ctx context.Context, req imagegen.FaceSwapReque if hdr := mimeFromContentType(respType, "image/"); hdr != "" { mimeType = hdr } - return &imagegen.Result{Images: []llm.ImagePart{{MIME: mimeType, Data: raw}}}, nil + return &imagegen.Result{ + Images: []llm.ImagePart{{MIME: mimeType, Data: raw}}, + SwappedFaces: parseSwapReport(respHdr.Get("X-Swap-Report")), + }, nil +} + +// swapReport mirrors the shim's X-Swap-Report header. +type swapReport struct { + Image []int `json:"image"` + Faces []struct { + Index int `json:"index"` + Size []int `json:"size"` + Yaw *float64 `json:"yaw"` + IdentitySimilarity *float64 `json:"identity_similarity"` + } `json:"faces"` +} + +// parseSwapReport decodes the measured outcome. A missing or malformed header +// yields nil rather than an error: an older shim does not send it, and a swap +// that produced a good image must not fail because the diagnostics alongside +// it were unreadable. +func parseSwapReport(header string) []imagegen.SwappedFace { + header = strings.TrimSpace(header) + if header == "" { + return nil + } + var rep swapReport + if err := json.Unmarshal([]byte(header), &rep); err != nil { + return nil + } + out := make([]imagegen.SwappedFace, 0, len(rep.Faces)) + for _, f := range rep.Faces { + sf := imagegen.SwappedFace{ + Index: f.Index, + Yaw: f.Yaw, + IdentitySimilarity: f.IdentitySimilarity, + } + if len(f.Size) == 2 { + sf.Width, sf.Height = f.Size[0], f.Size[1] + } + if len(rep.Image) == 2 { + sf.ImageWidth, sf.ImageHeight = rep.Image[0], rep.Image[1] + } + out = append(out, sf) + } + if len(out) == 0 { + return nil + } + return out } // imageFilename picks a multipart filename for an image part. The shim reads diff --git a/provider/llamaswap/faceswap_test.go b/provider/llamaswap/faceswap_test.go index ca41de9..66aa4a0 100644 --- a/provider/llamaswap/faceswap_test.go +++ b/provider/llamaswap/faceswap_test.go @@ -259,3 +259,86 @@ func TestFaceSwapAllowsNegativeIndexUnderAll(t *testing.T) { t.Error("negative index accepted when it would actually be sent") } } + +// TestFaceSwapParsesSwapReport: the measured outcome is the whole reason the +// header exists — a caller that cannot tell "the likeness transferred" from +// "an image came back" goes and asks a vision model, which is wrong in +// exactly the cases that matter. +func TestFaceSwapParsesSwapReport(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.Header().Set("X-Swap-Report", + `{"image":[1010,1200],"faces":[{"index":2,"size":[138,172],"yaw":-82.2,"identity_similarity":0.791}]}`) + raw, _ := base64.StdEncoding.DecodeString(onePixelPNG) + _, _ = w.Write(raw) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + m, err := p.FaceSwapModel("faceswap") + if err != nil { + t.Fatalf("model: %v", err) + } + img := editInit(t) + res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img}) + if err != nil { + t.Fatalf("FaceSwap: %v", err) + } + if len(res.SwappedFaces) != 1 { + t.Fatalf("SwappedFaces = %d, want 1 — the measurement was dropped", len(res.SwappedFaces)) + } + f := res.SwappedFaces[0] + if f.Index != 2 || f.Width != 138 || f.Height != 172 { + t.Errorf("face = %+v, want index 2 at 138x172", f) + } + if f.Yaw == nil || *f.Yaw != -82.2 { + t.Errorf("yaw = %v, want -82.2 — the pose signal is how a caller knows a profile swap will not read", f.Yaw) + } + if f.IdentitySimilarity == nil || *f.IdentitySimilarity != 0.791 { + t.Errorf("identity_similarity = %v, want 0.791", f.IdentitySimilarity) + } + // 138/1010 — the number that says "correct, and invisible at a glance". + if got := f.FractionOfImage(); got < 0.13 || got > 0.14 { + t.Errorf("FractionOfImage = %.3f, want ~0.137", got) + } +} + +// TestFaceSwapSurvivesMissingReport: an older shim sends no header at all. A +// swap that produced a good image must not fail because the diagnostics +// beside it were absent or malformed. +func TestFaceSwapSurvivesMissingReport(t *testing.T) { + for name, hdr := range map[string]string{ + "absent": "", + "garbage": "not json at all", + "wrongtype": `{"faces":"nope"}`, + } { + t.Run(name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + if hdr != "" { + w.Header().Set("X-Swap-Report", hdr) + } + raw, _ := base64.StdEncoding.DecodeString(onePixelPNG) + _, _ = w.Write(raw) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + m, err := p.FaceSwapModel("faceswap") + if err != nil { + t.Fatalf("model: %v", err) + } + img := editInit(t) + res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img}) + if err != nil { + t.Fatalf("a %s report failed the whole swap: %v", name, err) + } + if len(res.Images) != 1 { + t.Fatal("image lost") + } + if res.SwappedFaces != nil { + t.Errorf("SwappedFaces = %+v, want nil for a %s report", res.SwappedFaces, name) + } + }) + } +} From 6995a8dee162ec3dd6b6631fad1a449a156833bd Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Fri, 31 Jul 2026 17:50:27 -0400 Subject: [PATCH 2/2] feat(faceswap): expose yaw on enumeration too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ListFaces now carries head yaw, so a caller choosing WHICH face to swap can see the thing that decides whether the swap will read — not only learn it afterwards from the swap report. In the run that prompted this the target's three faces sat at -82, -8 and -11 degrees; only the first was hopeless, and nothing in a bounding box said so. --- imagegen/faceswap.go | 8 ++++++++ provider/llamaswap/faceswap.go | 13 +++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/imagegen/faceswap.go b/imagegen/faceswap.go index 024fec2..4755505 100644 --- a/imagegen/faceswap.go +++ b/imagegen/faceswap.go @@ -55,6 +55,14 @@ type DetectedFace struct { Box [4]int // Score is the detector's confidence, 0-1. Score float64 + + // Yaw is how far the head is turned from camera, in degrees, or nil when + // the provider does not report pose. Exposed on ENUMERATION, not just + // after the fact, because it is how a caller picks a face a swap will + // actually work on: past roughly ±45° the features carrying identity are + // edge-on, and the result reads as a generic person however good the + // transfer is. A bounding box cannot show this. + Yaw *float64 } // Size returns the box dimensions. Derived rather than stored: carrying diff --git a/provider/llamaswap/faceswap.go b/provider/llamaswap/faceswap.go index 2f7fb57..8434ce8 100644 --- a/provider/llamaswap/faceswap.go +++ b/provider/llamaswap/faceswap.go @@ -41,11 +41,12 @@ type faceSwapModel struct { type facesResponse struct { Count int `json:"count"` Faces []struct { - Index int `json:"index"` - Box []int `json:"box"` - Score float64 `json:"score"` - Width int `json:"width"` - Height int `json:"height"` + Index int `json:"index"` + Box []int `json:"box"` + Score float64 `json:"score"` + Width int `json:"width"` + Height int `json:"height"` + Yaw *float64 `json:"yaw"` } `json:"faces"` } @@ -74,7 +75,7 @@ func (m *faceSwapModel) ListFaces(ctx context.Context, img imagegen.Image) ([]im } out := make([]imagegen.DetectedFace, 0, len(parsed.Faces)) for _, f := range parsed.Faces { - df := imagegen.DetectedFace{Index: f.Index, Score: f.Score} + df := imagegen.DetectedFace{Index: f.Index, Score: f.Score, Yaw: f.Yaw} // A short box would silently index out of range below; treat a // malformed entry as a protocol error rather than zero-filling it, // because a wrong box sends the caller at the wrong face.