diff --git a/imagegen/faceswap.go b/imagegen/faceswap.go index d7b09e8..4755505 100644 --- a/imagegen/faceswap.go +++ b/imagegen/faceswap.go @@ -55,6 +55,14 @@ type DetectedFace struct { Box [4]int // Score is the detector's confidence, 0-1. Score float64 + + // Yaw is how far the head is turned from camera, in degrees, or nil when + // the provider does not report pose. Exposed on ENUMERATION, not just + // after the fact, because it is how a caller picks a face a swap will + // actually work on: past roughly ±45° the features carrying identity are + // edge-on, and the result reads as a generic person however good the + // transfer is. A bounding box cannot show this. + Yaw *float64 } // Size returns the box dimensions. Derived rather than stored: carrying @@ -64,6 +72,57 @@ func (f DetectedFace) Size() (w, h int) { return f.Box[2] - f.Box[0], f.Box[3] - f.Box[1] } +// SwappedFace is the MEASURED outcome for one face the provider replaced. +// +// It exists because "the call returned an image" and "the likeness +// transferred" are different claims that look identical from outside, and a +// caller that cannot tell them apart will go looking for another way to +// check. The one it reaches for — asking a vision model who the result looks +// like — is wrong in exactly the cases that matter: a VLM shown a jogger in a +// Georgetown cap holding McDonald's cups answers "Bill Clinton" whoever's +// face is on him, so it reports failure on a correct swap. +type SwappedFace struct { + // Index is the face's position in the provider's left-to-right ordering. + Index int + + // Width, Height are the replaced face's pixel size in the TARGET. + // Meaningful only against ImageWidth/ImageHeight: a 138px face is large + // in a 400px picture and nearly invisible in a 2000px one, and it is the + // ratio, not the absolute size, that decides whether a person notices. + Width, Height int + + // ImageWidth, ImageHeight are the target image's dimensions, repeated on + // every entry so a single face is self-describing without the caller + // holding onto the rest of the response. + ImageWidth, ImageHeight int + + // Yaw is how far the head is turned from camera, in degrees, or nil when + // the provider does not report pose. The best single predictor of whether + // a swap will READ as the source person: past roughly ±45° the features + // carrying identity are edge-on and the result looks like a generic + // person rather than a specific one. + Yaw *float64 + + // IdentitySimilarity is cosine similarity between the source face and the + // face actually present in the result, 0-1, or nil when the provider + // could not measure it. Above ~0.5 the identity transferred; a LOW value + // is the only evidence that a swap genuinely failed. + IdentitySimilarity *float64 +} + +// FractionOfImage is the swapped face's width as a share of the image's, 0-1. +// The number that predicts whether a person will SEE the change: the swap +// that prompted all this replaced a 138px face in a 1010px-wide photo — 14%, +// correct by every measure and invisible at a glance — while the same code on +// a 168px face in a 385px picture (44%) is unmistakable. Returns 0 when the +// dimensions are unknown. +func (f SwappedFace) FractionOfImage() float64 { + if f.ImageWidth <= 0 || f.Width <= 0 { + return 0 + } + return float64(f.Width) / float64(f.ImageWidth) +} + // FaceSwapper is the optional face-transfer surface. Separate interface so // existing providers keep compiling; callers type-assert. type FaceSwapper interface { diff --git a/imagegen/imagegen.go b/imagegen/imagegen.go index 090f89b..7cd301f 100644 --- a/imagegen/imagegen.go +++ b/imagegen/imagegen.go @@ -68,6 +68,15 @@ type Result struct { // Images are the generated images, in the order the backend returned them. Images []Image + // SwappedFaces is the measured outcome of a FaceSwap, one entry per face + // replaced. Empty for every other operation, and empty from a provider + // that does not measure. TYPED rather than tucked into Raw: a caller has + // to act on this — it is the only way to distinguish a swap that + // transferred the likeness from one that returned an image and nothing + // more — and a value reachable solely by type-asserting an `any` is one + // nobody discovers in time to use. + SwappedFaces []SwappedFace + // Raw is the provider-native response object, an escape hatch for // provider-specific fields. May be nil; never required for normal use. Raw any diff --git a/provider/llamaswap/audio.go b/provider/llamaswap/audio.go index 717a4ec..ff864de 100644 --- a/provider/llamaswap/audio.go +++ b/provider/llamaswap/audio.go @@ -338,27 +338,41 @@ func parseVoices(raw []byte) ([]string, error) { // varies. contentType sets the request Content-Type when body is non-nil. // A response larger than maxBytes is an error, never a silent truncation. func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, string, error) { - if err := p.requireBaseURL(); err != nil { + data, hdr, err := p.doRawHeaders(ctx, method, path, model, contentType, body, maxBytes) + if err != nil { return nil, "", err } + return data, hdr.Get("Content-Type"), nil +} + +// doRawHeaders is doRaw with the WHOLE response header rather than just +// Content-Type. Only the face swap needs it — the shim reports whether the +// likeness actually transferred in X-Swap-Report, and that answer would be +// thrown away by a function that keeps one header — so doRaw stays the +// signature 25 other call sites use and delegates here. Two bodies would be +// two places for the size cap and the status check to drift apart. +func (p *Provider) doRawHeaders(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, http.Header, error) { + if err := p.requireBaseURL(); err != nil { + return nil, nil, err + } req, err := p.newRequest(ctx, method, path, contentType, body) if err != nil { - return nil, "", err + return nil, nil, err } resp, err := p.client.Do(req) if err != nil { - return nil, "", fmt.Errorf("llama-swap: do request: %w", err) + return nil, nil, fmt.Errorf("llama-swap: do request: %w", err) } defer resp.Body.Close() if resp.StatusCode/100 != 2 { - return nil, "", p.apiError(resp, model) + return nil, nil, p.apiError(resp, model) } data, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) if err != nil { - return nil, "", fmt.Errorf("llama-swap: read response: %w", err) + return nil, nil, fmt.Errorf("llama-swap: read response: %w", err) } if int64(len(data)) > maxBytes { - return nil, "", fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes) + return nil, nil, fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes) } - return data, resp.Header.Get("Content-Type"), nil + return data, resp.Header, nil } diff --git a/provider/llamaswap/faceswap.go b/provider/llamaswap/faceswap.go index a69fc9a..8434ce8 100644 --- a/provider/llamaswap/faceswap.go +++ b/provider/llamaswap/faceswap.go @@ -41,11 +41,12 @@ type faceSwapModel struct { type facesResponse struct { Count int `json:"count"` Faces []struct { - Index int `json:"index"` - Box []int `json:"box"` - Score float64 `json:"score"` - Width int `json:"width"` - Height int `json:"height"` + Index int `json:"index"` + Box []int `json:"box"` + Score float64 `json:"score"` + Width int `json:"width"` + Height int `json:"height"` + Yaw *float64 `json:"yaw"` } `json:"faces"` } @@ -74,7 +75,7 @@ func (m *faceSwapModel) ListFaces(ctx context.Context, img imagegen.Image) ([]im } out := make([]imagegen.DetectedFace, 0, len(parsed.Faces)) for _, f := range parsed.Faces { - df := imagegen.DetectedFace{Index: f.Index, Score: f.Score} + df := imagegen.DetectedFace{Index: f.Index, Score: f.Score, Yaw: f.Yaw} // A short box would silently index out of range below; treat a // malformed entry as a protocol error rather than zero-filling it, // because a wrong box sends the caller at the wrong face. @@ -126,10 +127,11 @@ func (m *faceSwapModel) FaceSwap(ctx context.Context, req imagegen.FaceSwapReque if err != nil { return nil, err } - raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes) + raw, respHdr, err := m.p.doRawHeaders(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes) if err != nil { return nil, err } + respType := respHdr.Get("Content-Type") if len(raw) == 0 { return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "face swap response contained no image"} } @@ -151,7 +153,55 @@ func (m *faceSwapModel) FaceSwap(ctx context.Context, req imagegen.FaceSwapReque if hdr := mimeFromContentType(respType, "image/"); hdr != "" { mimeType = hdr } - return &imagegen.Result{Images: []llm.ImagePart{{MIME: mimeType, Data: raw}}}, nil + return &imagegen.Result{ + Images: []llm.ImagePart{{MIME: mimeType, Data: raw}}, + SwappedFaces: parseSwapReport(respHdr.Get("X-Swap-Report")), + }, nil +} + +// swapReport mirrors the shim's X-Swap-Report header. +type swapReport struct { + Image []int `json:"image"` + Faces []struct { + Index int `json:"index"` + Size []int `json:"size"` + Yaw *float64 `json:"yaw"` + IdentitySimilarity *float64 `json:"identity_similarity"` + } `json:"faces"` +} + +// parseSwapReport decodes the measured outcome. A missing or malformed header +// yields nil rather than an error: an older shim does not send it, and a swap +// that produced a good image must not fail because the diagnostics alongside +// it were unreadable. +func parseSwapReport(header string) []imagegen.SwappedFace { + header = strings.TrimSpace(header) + if header == "" { + return nil + } + var rep swapReport + if err := json.Unmarshal([]byte(header), &rep); err != nil { + return nil + } + out := make([]imagegen.SwappedFace, 0, len(rep.Faces)) + for _, f := range rep.Faces { + sf := imagegen.SwappedFace{ + Index: f.Index, + Yaw: f.Yaw, + IdentitySimilarity: f.IdentitySimilarity, + } + if len(f.Size) == 2 { + sf.Width, sf.Height = f.Size[0], f.Size[1] + } + if len(rep.Image) == 2 { + sf.ImageWidth, sf.ImageHeight = rep.Image[0], rep.Image[1] + } + out = append(out, sf) + } + if len(out) == 0 { + return nil + } + return out } // imageFilename picks a multipart filename for an image part. The shim reads diff --git a/provider/llamaswap/faceswap_test.go b/provider/llamaswap/faceswap_test.go index ca41de9..66aa4a0 100644 --- a/provider/llamaswap/faceswap_test.go +++ b/provider/llamaswap/faceswap_test.go @@ -259,3 +259,86 @@ func TestFaceSwapAllowsNegativeIndexUnderAll(t *testing.T) { t.Error("negative index accepted when it would actually be sent") } } + +// TestFaceSwapParsesSwapReport: the measured outcome is the whole reason the +// header exists — a caller that cannot tell "the likeness transferred" from +// "an image came back" goes and asks a vision model, which is wrong in +// exactly the cases that matter. +func TestFaceSwapParsesSwapReport(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.Header().Set("X-Swap-Report", + `{"image":[1010,1200],"faces":[{"index":2,"size":[138,172],"yaw":-82.2,"identity_similarity":0.791}]}`) + raw, _ := base64.StdEncoding.DecodeString(onePixelPNG) + _, _ = w.Write(raw) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + m, err := p.FaceSwapModel("faceswap") + if err != nil { + t.Fatalf("model: %v", err) + } + img := editInit(t) + res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img}) + if err != nil { + t.Fatalf("FaceSwap: %v", err) + } + if len(res.SwappedFaces) != 1 { + t.Fatalf("SwappedFaces = %d, want 1 — the measurement was dropped", len(res.SwappedFaces)) + } + f := res.SwappedFaces[0] + if f.Index != 2 || f.Width != 138 || f.Height != 172 { + t.Errorf("face = %+v, want index 2 at 138x172", f) + } + if f.Yaw == nil || *f.Yaw != -82.2 { + t.Errorf("yaw = %v, want -82.2 — the pose signal is how a caller knows a profile swap will not read", f.Yaw) + } + if f.IdentitySimilarity == nil || *f.IdentitySimilarity != 0.791 { + t.Errorf("identity_similarity = %v, want 0.791", f.IdentitySimilarity) + } + // 138/1010 — the number that says "correct, and invisible at a glance". + if got := f.FractionOfImage(); got < 0.13 || got > 0.14 { + t.Errorf("FractionOfImage = %.3f, want ~0.137", got) + } +} + +// TestFaceSwapSurvivesMissingReport: an older shim sends no header at all. A +// swap that produced a good image must not fail because the diagnostics +// beside it were absent or malformed. +func TestFaceSwapSurvivesMissingReport(t *testing.T) { + for name, hdr := range map[string]string{ + "absent": "", + "garbage": "not json at all", + "wrongtype": `{"faces":"nope"}`, + } { + t.Run(name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + if hdr != "" { + w.Header().Set("X-Swap-Report", hdr) + } + raw, _ := base64.StdEncoding.DecodeString(onePixelPNG) + _, _ = w.Write(raw) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + m, err := p.FaceSwapModel("faceswap") + if err != nil { + t.Fatalf("model: %v", err) + } + img := editInit(t) + res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img}) + if err != nil { + t.Fatalf("a %s report failed the whole swap: %v", name, err) + } + if len(res.Images) != 1 { + t.Fatal("image lost") + } + if res.SwappedFaces != nil { + t.Errorf("SwappedFaces = %+v, want nil for a %s report", res.SwappedFaces, name) + } + }) + } +}