steveandClaude Opus 5
5994d96921
The 2/4 finding is right: after writeImagePart started passing an explicit filename stem, initImageFilename had no caller in video.go, and its "conditioning frame" doc no longer described its one remaining user (lipsync.go's avatar image). A one-line wrapper that survives only to be misdescribed is not indirection worth keeping. lipsync now calls imageFilename(mime, "frame") directly, and imageFilename's doc lists the real bases — including WHY the video keyframes need distinct ones: a backend that stages uploads by filename would otherwise have the second overwrite the first. Not taken: consolidating the first/last-frame rationale to a single canonical site. The copies address different readers — the wire encoding (provider), the contract's undetectable-support caveat (videogen), and the mode table (README) — and last round's finding was a doc pointing at a note that did not exist. Trading duplication for cross-references is what produced that. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01PLjgrxvHjm1sJgUu9zBPH9
98 lines
3.2 KiB
Go
98 lines
3.2 KiB
Go
// lipsync.go implements videogen.LipsyncProvider against a SadTalker shim
|
|
// reached through llama-swap's /upstream passthrough (ADR-0025):
|
|
//
|
|
// POST /upstream/<id>/v1/talking_head multipart image,audio[,still,enhance,preprocess]
|
|
//
|
|
// The response body IS the encoded clip (same contract as /v1/videos/sync),
|
|
// hence the video-sized response cap and the same MIME resolution rules.
|
|
// Generation is sync and slow (minutes) — bound calls with a context
|
|
// deadline (Hunyuan precedent).
|
|
package llamaswap
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"mime/multipart"
|
|
"net/http"
|
|
|
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
|
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
|
)
|
|
|
|
// LipsyncModel implements videogen.LipsyncProvider. The id selects which
|
|
// upstream llama-swap loads.
|
|
func (p *Provider) LipsyncModel(id string, opts ...videogen.LipsyncModelOption) (videogen.Lipsyncer, error) {
|
|
if err := p.requireBaseURL(); err != nil {
|
|
return nil, err
|
|
}
|
|
_ = videogen.ApplyLipsyncModelOptions(opts)
|
|
return &lipsyncModel{p: p, id: id}, nil
|
|
}
|
|
|
|
type lipsyncModel struct {
|
|
p *Provider
|
|
id string
|
|
}
|
|
|
|
// Lipsync implements videogen.Lipsyncer.
|
|
func (m *lipsyncModel) Lipsync(ctx context.Context, req videogen.LipsyncRequest, opts ...videogen.LipsyncOption) (*videogen.Result, error) {
|
|
req = req.Apply(opts...)
|
|
if len(req.Image.Data) == 0 {
|
|
return nil, fmt.Errorf("%w: lipsync requires a portrait image", llm.ErrUnsupported)
|
|
}
|
|
if len(req.Audio) == 0 {
|
|
return nil, fmt.Errorf("%w: lipsync requires audio bytes", llm.ErrUnsupported)
|
|
}
|
|
if req.Preprocess != "" && req.Preprocess != "crop" && req.Preprocess != "full" {
|
|
return nil, fmt.Errorf("%w: lipsync preprocess must be \"crop\" or \"full\", got %q", llm.ErrUnsupported, req.Preprocess)
|
|
}
|
|
path, err := upstreamPath(m.id, "/v1/talking_head")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Two file parts — buildMultipart handles exactly one, so assemble by
|
|
// hand (mirrors videoModel.Generate).
|
|
var buf bytes.Buffer
|
|
w := multipart.NewWriter(&buf)
|
|
fw, err := w.CreateFormFile("image", imageFilename(req.Image.MIME, "frame"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
if _, err := fw.Write(req.Image.Data); err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
fw, err = w.CreateFormFile("audio", transcriptionFilename(req.AudioFilename, req.AudioMIME))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
if _, err := fw.Write(req.Audio); err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
still := ""
|
|
if req.Still {
|
|
still = "true"
|
|
}
|
|
enhance := ""
|
|
if req.Enhance {
|
|
enhance = "true"
|
|
}
|
|
if err := writeFormFields(w, "build lipsync form", []formField{
|
|
{"still", still, false},
|
|
{"enhance", enhance, false},
|
|
{"preprocess", req.Preprocess, false},
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := w.Close(); err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
|
|
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return singleVideoResult(m.p.name, m.id, "lipsync", raw, respType)
|
|
}
|