majordomo/provider/llamaswap/lipsync.go

// lipsync.go implements videogen.LipsyncProvider against a SadTalker shim
// reached through llama-swap's /upstream passthrough (ADR-0025):
//
//	POST /upstream/<id>/v1/talking_head   multipart image,audio[,still,enhance,preprocess]
//
// The response body IS the encoded clip (same contract as /v1/videos/sync),
// hence the video-sized response cap and the same MIME resolution rules.
// Generation is sync and slow (minutes) — bound calls with a context
// deadline (Hunyuan precedent).
package llamaswap

import (
	"bytes"
	"context"
	"fmt"
	"mime/multipart"
	"net/http"

	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
	"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
)

// LipsyncModel implements videogen.LipsyncProvider. The id selects which
// upstream llama-swap loads.
func (p *Provider) LipsyncModel(id string, opts ...videogen.LipsyncModelOption) (videogen.LipSyncer, error) {
	if err := p.requireBaseURL(); err != nil {
		return nil, err
	}
	_ = videogen.ApplyLipsyncModelOptions(opts)
	return &lipsyncModel{p: p, id: id}, nil
}

type lipsyncModel struct {
	p  *Provider
	id string
}

// Lipsync implements videogen.LipSyncer.
func (m *lipsyncModel) Lipsync(ctx context.Context, req videogen.LipsyncRequest, opts ...videogen.LipsyncOption) (*videogen.Result, error) {
	req = req.Apply(opts...)
	if len(req.Image.Data) == 0 {
		return nil, fmt.Errorf("%w: lipsync requires a portrait image", llm.ErrUnsupported)
	}
	if len(req.Audio) == 0 {
		return nil, fmt.Errorf("%w: lipsync requires audio bytes", llm.ErrUnsupported)
	}
	if req.Preprocess != "" && req.Preprocess != "crop" && req.Preprocess != "full" {
		return nil, fmt.Errorf("%w: lipsync preprocess must be \"crop\" or \"full\", got %q", llm.ErrUnsupported, req.Preprocess)
	}
	path, err := upstreamPath(m.id, "/v1/talking_head")
	if err != nil {
		return nil, err
	}

	// Two file parts — buildMultipart handles exactly one, so assemble by
	// hand (mirrors videoModel.Generate).
	var buf bytes.Buffer
	w := multipart.NewWriter(&buf)
	fw, err := w.CreateFormFile("image", initImageFilename(req.Image.MIME))
	if err != nil {
		return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
	}
	if _, err := fw.Write(req.Image.Data); err != nil {
		return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
	}
	fw, err = w.CreateFormFile("audio", transcriptionFilename(req.AudioFilename, req.AudioMIME))
	if err != nil {
		return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
	}
	if _, err := fw.Write(req.Audio); err != nil {
		return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
	}
	still := ""
	if req.Still {
		still = "true"
	}
	enhance := ""
	if req.Enhance {
		enhance = "true"
	}
	if err := writeFormFields(w, "build lipsync form", []formField{
		{"still", still, false},
		{"enhance", enhance, false},
		{"preprocess", req.Preprocess, false},
	}); err != nil {
		return nil, err
	}
	if err := w.Close(); err != nil {
		return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
	}

	raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes)
	if err != nil {
		return nil, err
	}
	return singleVideoResult(m.p.name, m.id, "lipsync", raw, respType)
}

// singleVideoResult wraps one raw video body into a videogen.Result,
// requiring positive evidence of video-ness (declared video/* Content-Type
// or sniffed mp4/webm magic) so a proxy error page can never become "the
// clip" — the video sibling of singleImageResult.
func singleVideoResult(provider, model, verb string, raw []byte, contentType string) (*videogen.Result, error) {
	if len(raw) == 0 {
		return nil, &llm.APIError{Provider: provider, Model: model, Message: verb + " response contained no video"}
	}
	mimeType := videoMIME(contentType, raw)
	if mimeType == "" {
		return nil, &llm.APIError{Provider: provider, Model: model,
			Message: fmt.Sprintf("%s response is not a video (Content-Type %q)", verb, contentType)}
	}
	return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil
}