// lipsync.go implements videogen.LipsyncProvider against a SadTalker shim // reached through llama-swap's /upstream passthrough (ADR-0025): // // POST /upstream//v1/talking_head multipart image,audio[,still,enhance,preprocess] // // The response body IS the encoded clip (same contract as /v1/videos/sync), // hence the video-sized response cap and the same MIME resolution rules. // Generation is sync and slow (minutes) — bound calls with a context // deadline (Hunyuan precedent). package llamaswap import ( "bytes" "context" "fmt" "mime/multipart" "net/http" "gitea.stevedudenhoeffer.com/steve/majordomo/llm" "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" ) // LipsyncModel implements videogen.LipsyncProvider. The id selects which // upstream llama-swap loads. func (p *Provider) LipsyncModel(id string, opts ...videogen.LipsyncModelOption) (videogen.Lipsyncer, error) { if err := p.requireBaseURL(); err != nil { return nil, err } _ = videogen.ApplyLipsyncModelOptions(opts) return &lipsyncModel{p: p, id: id}, nil } type lipsyncModel struct { p *Provider id string } // Lipsync implements videogen.Lipsyncer. func (m *lipsyncModel) Lipsync(ctx context.Context, req videogen.LipsyncRequest, opts ...videogen.LipsyncOption) (*videogen.Result, error) { req = req.Apply(opts...) if len(req.Image.Data) == 0 { return nil, fmt.Errorf("%w: lipsync requires a portrait image", llm.ErrUnsupported) } if len(req.Audio) == 0 { return nil, fmt.Errorf("%w: lipsync requires audio bytes", llm.ErrUnsupported) } if req.Preprocess != "" && req.Preprocess != "crop" && req.Preprocess != "full" { return nil, fmt.Errorf("%w: lipsync preprocess must be \"crop\" or \"full\", got %q", llm.ErrUnsupported, req.Preprocess) } path, err := upstreamPath(m.id, "/v1/talking_head") if err != nil { return nil, err } // Two file parts — buildMultipart handles exactly one, so assemble by // hand (mirrors videoModel.Generate). var buf bytes.Buffer w := multipart.NewWriter(&buf) fw, err := w.CreateFormFile("image", initImageFilename(req.Image.MIME)) if err != nil { return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) } if _, err := fw.Write(req.Image.Data); err != nil { return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) } fw, err = w.CreateFormFile("audio", transcriptionFilename(req.AudioFilename, req.AudioMIME)) if err != nil { return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) } if _, err := fw.Write(req.Audio); err != nil { return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) } still := "" if req.Still { still = "true" } enhance := "" if req.Enhance { enhance = "true" } if err := writeFormFields(w, "build lipsync form", []formField{ {"still", still, false}, {"enhance", enhance, false}, {"preprocess", req.Preprocess, false}, }); err != nil { return nil, err } if err := w.Close(); err != nil { return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) } raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes) if err != nil { return nil, err } return singleVideoResult(m.p.name, m.id, "lipsync", raw, respType) }