steve
6995a8dee1
ListFaces now carries head yaw, so a caller choosing WHICH face to swap can see the thing that decides whether the swap will read — not only learn it afterwards from the swap report. In the run that prompted this the target's three faces sat at -82, -8 and -11 degrees; only the first was hopeless, and nothing in a bounding box said so.
237 lines
8.2 KiB
Go
237 lines
8.2 KiB
Go
// faceswap.go implements imagegen.FaceSwapper against the InsightFace shim
|
|
// (buffalo_l + inswapper_128) reached through llama-swap's /upstream
|
|
// passthrough (ADR-0024):
|
|
//
|
|
// POST /upstream/<id>/v1/faces multipart file -> JSON
|
|
// POST /upstream/<id>/v1/faceswap multipart target, source -> PNG
|
|
package llamaswap
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"mime"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
|
)
|
|
|
|
// maxFaceSwapResponseBytes bounds the returned PNG. Generous: the shim echoes
|
|
// the target's dimensions, and a 4K photo round-trips as a large lossless PNG.
|
|
const maxFaceSwapResponseBytes = 64 << 20
|
|
|
|
// FaceSwapModel implements the face-transfer surface. The id selects which
|
|
// upstream llama-swap loads.
|
|
func (p *Provider) FaceSwapModel(id string) (imagegen.FaceSwapper, error) {
|
|
if err := p.requireBaseURL(); err != nil {
|
|
return nil, err
|
|
}
|
|
return &faceSwapModel{p: p, id: id}, nil
|
|
}
|
|
|
|
type faceSwapModel struct {
|
|
p *Provider
|
|
id string
|
|
}
|
|
|
|
// facesResponse mirrors the shim's /v1/faces body.
|
|
type facesResponse struct {
|
|
Count int `json:"count"`
|
|
Faces []struct {
|
|
Index int `json:"index"`
|
|
Box []int `json:"box"`
|
|
Score float64 `json:"score"`
|
|
Width int `json:"width"`
|
|
Height int `json:"height"`
|
|
Yaw *float64 `json:"yaw"`
|
|
} `json:"faces"`
|
|
}
|
|
|
|
// ListFaces implements imagegen.FaceSwapper.
|
|
func (m *faceSwapModel) ListFaces(ctx context.Context, img imagegen.Image) ([]imagegen.DetectedFace, error) {
|
|
if len(img.Data) == 0 {
|
|
return nil, fmt.Errorf("%w: face detection requires image bytes", llm.ErrUnsupported)
|
|
}
|
|
path, err := upstreamPath(m.id, "/v1/faces")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
body, contentType, err := buildMultipart("build faces form",
|
|
filePart{field: "file", filename: imageFilename(img.MIME, "image"), data: img.Data}, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var parsed facesResponse
|
|
if err := json.Unmarshal(raw, &parsed); err != nil {
|
|
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
|
Message: fmt.Sprintf("faces response is not JSON: %s", truncateForError(raw))}
|
|
}
|
|
out := make([]imagegen.DetectedFace, 0, len(parsed.Faces))
|
|
for _, f := range parsed.Faces {
|
|
df := imagegen.DetectedFace{Index: f.Index, Score: f.Score, Yaw: f.Yaw}
|
|
// A short box would silently index out of range below; treat a
|
|
// malformed entry as a protocol error rather than zero-filling it,
|
|
// because a wrong box sends the caller at the wrong face.
|
|
if len(f.Box) != 4 {
|
|
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
|
Message: fmt.Sprintf("face %d has a %d-element box, want 4", f.Index, len(f.Box))}
|
|
}
|
|
copy(df.Box[:], f.Box)
|
|
out = append(out, df)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// FaceSwap implements imagegen.FaceSwapper. The endpoint always answers PNG.
|
|
func (m *faceSwapModel) FaceSwap(ctx context.Context, req imagegen.FaceSwapRequest, opts ...imagegen.FaceSwapOption) (*imagegen.Result, error) {
|
|
req = req.Apply(opts...)
|
|
if len(req.Target.Data) == 0 {
|
|
return nil, fmt.Errorf("%w: face swap requires a target image", llm.ErrUnsupported)
|
|
}
|
|
if len(req.Source.Data) == 0 {
|
|
return nil, fmt.Errorf("%w: face swap requires a source image", llm.ErrUnsupported)
|
|
}
|
|
// Only when it will actually be sent: under All the index is documented
|
|
// as ignored, so rejecting a negative one there would fail a request that
|
|
// is perfectly well formed.
|
|
if !req.All && req.Index != nil && *req.Index < 0 {
|
|
return nil, fmt.Errorf("%w: face index must be >= 0, got %d", llm.ErrUnsupported, *req.Index)
|
|
}
|
|
path, err := upstreamPath(m.id, "/v1/faceswap")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var fields []formField
|
|
if req.All {
|
|
fields = append(fields, formField{key: "all", value: "true", required: true})
|
|
} else if req.Index != nil {
|
|
// Only sent when NOT swapping all: the shim ignores index under
|
|
// all=true, and sending both would imply a precedence the caller
|
|
// cannot see.
|
|
fields = append(fields, formField{key: "index", value: strconv.Itoa(*req.Index), required: true})
|
|
}
|
|
|
|
body, contentType, err := buildMultipartFiles("build faceswap form",
|
|
[]filePart{
|
|
{field: "target", filename: imageFilename(req.Target.MIME, "target"), data: req.Target.Data},
|
|
{field: "source", filename: imageFilename(req.Source.MIME, "source"), data: req.Source.Data},
|
|
}, fields)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
raw, respHdr, err := m.p.doRawHeaders(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
respType := respHdr.Get("Content-Type")
|
|
if len(raw) == 0 {
|
|
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "face swap response contained no image"}
|
|
}
|
|
// Validate the BYTES, not the header. sniffImageMIME falls back to
|
|
// image/png when detection is inconclusive, so trusting it here would
|
|
// label a JSON error body as a PNG and return it as a successful image —
|
|
// and a header check alone misses the case where the response carries no
|
|
// Content-Type at all. The shim answers JSON on a semantic miss (no face
|
|
// found), which is exactly the body that would sail through.
|
|
detected := http.DetectContentType(raw)
|
|
if !strings.HasPrefix(detected, "image/") {
|
|
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
|
Message: fmt.Sprintf("face swap response is not an image (sniffed %q, Content-Type %q): %s",
|
|
detected, respType, truncateForError(raw))}
|
|
}
|
|
// Prefer the server's own label when it is an image type (it knows
|
|
// subtypes the sniffer does not), else what the bytes actually are.
|
|
mimeType := detected
|
|
if hdr := mimeFromContentType(respType, "image/"); hdr != "" {
|
|
mimeType = hdr
|
|
}
|
|
return &imagegen.Result{
|
|
Images: []llm.ImagePart{{MIME: mimeType, Data: raw}},
|
|
SwappedFaces: parseSwapReport(respHdr.Get("X-Swap-Report")),
|
|
}, nil
|
|
}
|
|
|
|
// swapReport mirrors the shim's X-Swap-Report header.
|
|
type swapReport struct {
|
|
Image []int `json:"image"`
|
|
Faces []struct {
|
|
Index int `json:"index"`
|
|
Size []int `json:"size"`
|
|
Yaw *float64 `json:"yaw"`
|
|
IdentitySimilarity *float64 `json:"identity_similarity"`
|
|
} `json:"faces"`
|
|
}
|
|
|
|
// parseSwapReport decodes the measured outcome. A missing or malformed header
|
|
// yields nil rather than an error: an older shim does not send it, and a swap
|
|
// that produced a good image must not fail because the diagnostics alongside
|
|
// it were unreadable.
|
|
func parseSwapReport(header string) []imagegen.SwappedFace {
|
|
header = strings.TrimSpace(header)
|
|
if header == "" {
|
|
return nil
|
|
}
|
|
var rep swapReport
|
|
if err := json.Unmarshal([]byte(header), &rep); err != nil {
|
|
return nil
|
|
}
|
|
out := make([]imagegen.SwappedFace, 0, len(rep.Faces))
|
|
for _, f := range rep.Faces {
|
|
sf := imagegen.SwappedFace{
|
|
Index: f.Index,
|
|
Yaw: f.Yaw,
|
|
IdentitySimilarity: f.IdentitySimilarity,
|
|
}
|
|
if len(f.Size) == 2 {
|
|
sf.Width, sf.Height = f.Size[0], f.Size[1]
|
|
}
|
|
if len(rep.Image) == 2 {
|
|
sf.ImageWidth, sf.ImageHeight = rep.Image[0], rep.Image[1]
|
|
}
|
|
out = append(out, sf)
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
|
|
// imageFilename picks a multipart filename for an image part. The shim reads
|
|
// bytes, not names, but a plausible extension keeps server-side sniffing and
|
|
// request logs honest. base distinguishes the parts of a multi-file form
|
|
// ("target"/"source") so a log line says which one was malformed.
|
|
//
|
|
// initImageFilename (video.go) is this function with base fixed to "frame"
|
|
// and delegates here — two copies of one extension table is how they drift.
|
|
func imageFilename(mimeType, base string) string {
|
|
if base == "" {
|
|
base = "image"
|
|
}
|
|
mt := strings.ToLower(strings.TrimSpace(mimeType))
|
|
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
|
mt = parsed
|
|
}
|
|
switch mt {
|
|
case "image/jpeg", "image/jpg":
|
|
return base + ".jpg"
|
|
case "image/webp":
|
|
return base + ".webp"
|
|
case "image/gif":
|
|
return base + ".gif"
|
|
case "image/bmp":
|
|
return base + ".bmp"
|
|
default:
|
|
// PNG is the safe default: every caller in this repo either sends PNG
|
|
// or sends something the decoder identifies by magic bytes anyway.
|
|
return base + ".png"
|
|
}
|
|
}
|