steveandClaude Fable 5
56b5b000a6
- SubmitChain rejects NaN/±Inf segment seconds with ErrUnsupported (previously an obscure json.Marshal error; NaN fails every comparison and +Inf passed the >= 0 check). - ChainStatus skips segment entries with no usable id — JSON null (which no-op-unmarshals into a string, previously appending ""), empty strings, and id-less objects; the unfiltered list survives in Raw. ChainJob.SegmentIDs doc now also says ChainSegmentResult takes the segment index, not an id string. - jobPath rejects '%' in job ids — %2F/%2E%2E percent-escapes decode back into path structure server-side, bypassing the literal check on this upstream-echoed value. - singleVideoResult moves to video.go next to videoMIME, and the two remaining hand-rolled copies of the video-result validation (videoModel.Generate, Interpolate) now use it — one validation, one message shape. - videogen.LipSyncer renamed to videogen.Lipsyncer for consistency with the rest of the surface's Lipsync* naming (LipsyncProvider, LipsyncModel, LipsyncRequest); not yet consumed downstream, so the rename is free now and never again. Co-Authored-By: Claude Fable 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01WWCQcYStWXBYUy5sZWnbLT
98 lines
3.2 KiB
Go
98 lines
3.2 KiB
Go
// lipsync.go implements videogen.LipsyncProvider against a SadTalker shim
|
|
// reached through llama-swap's /upstream passthrough (ADR-0025):
|
|
//
|
|
// POST /upstream/<id>/v1/talking_head multipart image,audio[,still,enhance,preprocess]
|
|
//
|
|
// The response body IS the encoded clip (same contract as /v1/videos/sync),
|
|
// hence the video-sized response cap and the same MIME resolution rules.
|
|
// Generation is sync and slow (minutes) — bound calls with a context
|
|
// deadline (Hunyuan precedent).
|
|
package llamaswap
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"mime/multipart"
|
|
"net/http"
|
|
|
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
|
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
|
)
|
|
|
|
// LipsyncModel implements videogen.LipsyncProvider. The id selects which
|
|
// upstream llama-swap loads.
|
|
func (p *Provider) LipsyncModel(id string, opts ...videogen.LipsyncModelOption) (videogen.Lipsyncer, error) {
|
|
if err := p.requireBaseURL(); err != nil {
|
|
return nil, err
|
|
}
|
|
_ = videogen.ApplyLipsyncModelOptions(opts)
|
|
return &lipsyncModel{p: p, id: id}, nil
|
|
}
|
|
|
|
type lipsyncModel struct {
|
|
p *Provider
|
|
id string
|
|
}
|
|
|
|
// Lipsync implements videogen.Lipsyncer.
|
|
func (m *lipsyncModel) Lipsync(ctx context.Context, req videogen.LipsyncRequest, opts ...videogen.LipsyncOption) (*videogen.Result, error) {
|
|
req = req.Apply(opts...)
|
|
if len(req.Image.Data) == 0 {
|
|
return nil, fmt.Errorf("%w: lipsync requires a portrait image", llm.ErrUnsupported)
|
|
}
|
|
if len(req.Audio) == 0 {
|
|
return nil, fmt.Errorf("%w: lipsync requires audio bytes", llm.ErrUnsupported)
|
|
}
|
|
if req.Preprocess != "" && req.Preprocess != "crop" && req.Preprocess != "full" {
|
|
return nil, fmt.Errorf("%w: lipsync preprocess must be \"crop\" or \"full\", got %q", llm.ErrUnsupported, req.Preprocess)
|
|
}
|
|
path, err := upstreamPath(m.id, "/v1/talking_head")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Two file parts — buildMultipart handles exactly one, so assemble by
|
|
// hand (mirrors videoModel.Generate).
|
|
var buf bytes.Buffer
|
|
w := multipart.NewWriter(&buf)
|
|
fw, err := w.CreateFormFile("image", initImageFilename(req.Image.MIME))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
if _, err := fw.Write(req.Image.Data); err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
fw, err = w.CreateFormFile("audio", transcriptionFilename(req.AudioFilename, req.AudioMIME))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
if _, err := fw.Write(req.Audio); err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
still := ""
|
|
if req.Still {
|
|
still = "true"
|
|
}
|
|
enhance := ""
|
|
if req.Enhance {
|
|
enhance = "true"
|
|
}
|
|
if err := writeFormFields(w, "build lipsync form", []formField{
|
|
{"still", still, false},
|
|
{"enhance", enhance, false},
|
|
{"preprocess", req.Preprocess, false},
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := w.Close(); err != nil {
|
|
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
|
}
|
|
|
|
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return singleVideoResult(m.p.name, m.id, "lipsync", raw, respType)
|
|
}
|