Merge pull request 'feat: wave-3 video surfaces — lipsync, video matte, video upscale, chain jobs (ADR-0025)' (#19) from feat/wave3-video-surfaces into main
CI / Tidy (push) Successful in 9m24s
CI / Build & Test (push) Successful in 9m46s

This commit was merged in pull request #19.
This commit is contained in:
2026-07-16 23:25:18 +00:00
12 changed files with 1507 additions and 24 deletions
+97
View File
@@ -0,0 +1,97 @@
// lipsync.go implements videogen.LipsyncProvider against a SadTalker shim
// reached through llama-swap's /upstream passthrough (ADR-0025):
//
// POST /upstream/<id>/v1/talking_head multipart image,audio[,still,enhance,preprocess]
//
// The response body IS the encoded clip (same contract as /v1/videos/sync),
// hence the video-sized response cap and the same MIME resolution rules.
// Generation is sync and slow (minutes) — bound calls with a context
// deadline (Hunyuan precedent).
package llamaswap
import (
"bytes"
"context"
"fmt"
"mime/multipart"
"net/http"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
)
// LipsyncModel implements videogen.LipsyncProvider. The id selects which
// upstream llama-swap loads.
func (p *Provider) LipsyncModel(id string, opts ...videogen.LipsyncModelOption) (videogen.Lipsyncer, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = videogen.ApplyLipsyncModelOptions(opts)
return &lipsyncModel{p: p, id: id}, nil
}
type lipsyncModel struct {
p *Provider
id string
}
// Lipsync implements videogen.Lipsyncer.
func (m *lipsyncModel) Lipsync(ctx context.Context, req videogen.LipsyncRequest, opts ...videogen.LipsyncOption) (*videogen.Result, error) {
req = req.Apply(opts...)
if len(req.Image.Data) == 0 {
return nil, fmt.Errorf("%w: lipsync requires a portrait image", llm.ErrUnsupported)
}
if len(req.Audio) == 0 {
return nil, fmt.Errorf("%w: lipsync requires audio bytes", llm.ErrUnsupported)
}
if req.Preprocess != "" && req.Preprocess != "crop" && req.Preprocess != "full" {
return nil, fmt.Errorf("%w: lipsync preprocess must be \"crop\" or \"full\", got %q", llm.ErrUnsupported, req.Preprocess)
}
path, err := upstreamPath(m.id, "/v1/talking_head")
if err != nil {
return nil, err
}
// Two file parts — buildMultipart handles exactly one, so assemble by
// hand (mirrors videoModel.Generate).
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
fw, err := w.CreateFormFile("image", initImageFilename(req.Image.MIME))
if err != nil {
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
}
if _, err := fw.Write(req.Image.Data); err != nil {
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
}
fw, err = w.CreateFormFile("audio", transcriptionFilename(req.AudioFilename, req.AudioMIME))
if err != nil {
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
}
if _, err := fw.Write(req.Audio); err != nil {
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
}
still := ""
if req.Still {
still = "true"
}
enhance := ""
if req.Enhance {
enhance = "true"
}
if err := writeFormFields(w, "build lipsync form", []formField{
{"still", still, false},
{"enhance", enhance, false},
{"preprocess", req.Preprocess, false},
}); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
}
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes)
if err != nil {
return nil, err
}
return singleVideoResult(m.p.name, m.id, "lipsync", raw, respType)
}
@@ -0,0 +1,295 @@
package llamaswap
import (
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"testing"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
)
// webmFixture is a minimal EBML header so http.DetectContentType sniffs
// video/webm.
func webmFixture() []byte {
return append([]byte{0x1A, 0x45, 0xDF, 0xA3}, make([]byte, 20)...)
}
func TestLipsync(t *testing.T) {
png := pngFixture(t)
var gotPath, gotStill, gotEnhance, gotPreprocess string
var gotImage, gotAudio []byte
var gotImageName, gotAudioName string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotStill = r.FormValue("still")
gotEnhance = r.FormValue("enhance")
gotPreprocess = r.FormValue("preprocess")
if f, hdr, err := r.FormFile("image"); err == nil {
gotImage, _ = io.ReadAll(f)
gotImageName = hdr.Filename
f.Close()
} else {
t.Errorf("image part: %v", err)
}
if f, hdr, err := r.FormFile("audio"); err == nil {
gotAudio, _ = io.ReadAll(f)
gotAudioName = hdr.Filename
f.Close()
} else {
t.Errorf("audio part: %v", err)
}
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write(mp4Fixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ls, err := p.LipsyncModel("lipsync-sadtalker")
if err != nil {
t.Fatalf("LipsyncModel: %v", err)
}
res, err := ls.Lipsync(context.Background(),
videogen.LipsyncRequest{
Image: videogen.Image{MIME: "image/png", Data: png},
Audio: []byte("SPEECH"),
AudioMIME: "audio/wav",
},
videogen.WithLipsyncStill(), videogen.WithLipsyncEnhance(), videogen.WithLipsyncPreprocess("full"))
if err != nil {
t.Fatalf("Lipsync: %v", err)
}
if gotPath != "/upstream/lipsync-sadtalker/v1/talking_head" {
t.Errorf("path = %q", gotPath)
}
if gotStill != "true" || gotEnhance != "true" || gotPreprocess != "full" {
t.Errorf("still/enhance/preprocess = %q/%q/%q", gotStill, gotEnhance, gotPreprocess)
}
if string(gotImage) != string(png) || gotImageName != "frame.png" {
t.Errorf("image bytes/name = %d bytes/%q", len(gotImage), gotImageName)
}
if string(gotAudio) != "SPEECH" || gotAudioName != "audio.wav" {
t.Errorf("audio = %q name = %q", gotAudio, gotAudioName)
}
if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
}
}
func TestLipsyncOmitsUnsetFlags(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
for _, k := range []string{"still", "enhance", "preprocess"} {
if v, ok := r.MultipartForm.Value[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write(mp4Fixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ls, _ := p.LipsyncModel("lipsync-sadtalker")
if _, err := ls.Lipsync(context.Background(),
videogen.LipsyncRequest{Image: videogen.Image{Data: pngFixture(t)}, Audio: []byte("A")}); err != nil {
t.Fatalf("Lipsync: %v", err)
}
}
func TestLipsyncRejectsBadArgs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
ls, _ := p.LipsyncModel("lipsync-sadtalker")
if _, err := ls.Lipsync(context.Background(),
videogen.LipsyncRequest{Audio: []byte("A")}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("no image: err = %v, want ErrUnsupported", err)
}
if _, err := ls.Lipsync(context.Background(),
videogen.LipsyncRequest{Image: videogen.Image{Data: []byte{1}}}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("no audio: err = %v, want ErrUnsupported", err)
}
if _, err := ls.Lipsync(context.Background(),
videogen.LipsyncRequest{Image: videogen.Image{Data: []byte{1}}, Audio: []byte{1}, Preprocess: "zoom"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("preprocess zoom: err = %v, want ErrUnsupported", err)
}
}
func TestLipsyncRejectsNonVideoResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("<html>proxy error page</html>"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ls, _ := p.LipsyncModel("lipsync-sadtalker")
_, err := ls.Lipsync(context.Background(),
videogen.LipsyncRequest{Image: videogen.Image{Data: pngFixture(t)}, Audio: []byte("A")})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-video body", err)
}
}
func TestRemoveVideoBackground(t *testing.T) {
var gotPath, gotOutput, gotFilename string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotOutput = r.FormValue("output")
if _, hdr, err := r.FormFile("file"); err == nil {
gotFilename = hdr.Filename
} else {
t.Errorf("file part: %v", err)
}
w.Header().Set("Content-Type", "video/webm")
_, _ = w.Write(webmFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
vb, err := p.VideoBackgroundRemoverModel("mediautils")
if err != nil {
t.Fatalf("VideoBackgroundRemoverModel: %v", err)
}
res, err := vb.RemoveVideoBackground(context.Background(),
videogen.VideoBackgroundRemovalRequest{Video: mp4Fixture(), MIME: "video/mp4"},
videogen.WithVideoBackgroundOutput("alpha_webm"))
if err != nil {
t.Fatalf("RemoveVideoBackground: %v", err)
}
if gotPath != "/upstream/mediautils/v1/video/matte" {
t.Errorf("path = %q", gotPath)
}
if gotOutput != "alpha_webm" || gotFilename != "video.mp4" {
t.Errorf("output/filename = %q/%q", gotOutput, gotFilename)
}
if res.Video.MIME != "video/webm" || len(res.Video.Data) == 0 {
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
}
}
func TestRemoveVideoBackgroundOmitsDefaultOutput(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
if v, ok := r.MultipartForm.Value["output"]; ok {
t.Errorf("output field sent for default: %v; want omitted", v)
}
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write(mp4Fixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
vb, _ := p.VideoBackgroundRemoverModel("mediautils")
if _, err := vb.RemoveVideoBackground(context.Background(),
videogen.VideoBackgroundRemovalRequest{Video: mp4Fixture()}); err != nil {
t.Fatalf("RemoveVideoBackground: %v", err)
}
}
func TestRemoveVideoBackgroundRejectsBadArgs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
vb, _ := p.VideoBackgroundRemoverModel("mediautils")
if _, err := vb.RemoveVideoBackground(context.Background(),
videogen.VideoBackgroundRemovalRequest{}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("no video: err = %v, want ErrUnsupported", err)
}
if _, err := vb.RemoveVideoBackground(context.Background(),
videogen.VideoBackgroundRemovalRequest{Video: []byte{1}, Output: "gif"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("output gif: err = %v, want ErrUnsupported", err)
}
}
func TestUpscaleVideo(t *testing.T) {
var gotPath, gotScale string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotScale = r.FormValue("scale")
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write(mp4Fixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
vu, err := p.VideoUpscalerModel("mediautils")
if err != nil {
t.Fatalf("VideoUpscalerModel: %v", err)
}
res, err := vu.UpscaleVideo(context.Background(),
videogen.VideoUpscaleRequest{Video: mp4Fixture(), MIME: "video/mp4"},
videogen.WithVideoUpscaleScale(2))
if err != nil {
t.Fatalf("UpscaleVideo: %v", err)
}
if gotPath != "/upstream/mediautils/v1/video/upscale" {
t.Errorf("path = %q", gotPath)
}
if gotScale != "2" {
t.Errorf("scale = %q", gotScale)
}
if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
}
}
func TestUpscaleVideoRejectsBadArgs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
vu, _ := p.VideoUpscalerModel("mediautils")
if _, err := vu.UpscaleVideo(context.Background(), videogen.VideoUpscaleRequest{}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("no video: err = %v, want ErrUnsupported", err)
}
if _, err := vu.UpscaleVideo(context.Background(),
videogen.VideoUpscaleRequest{Video: []byte{1}, Scale: 3}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("scale 3: err = %v, want ErrUnsupported", err)
}
}
func TestUpscaleVideoRejectsNonVideoResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header()["Content-Type"] = nil // NO Content-Type at all
_, _ = w.Write([]byte("502 bad gateway"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
vu, _ := p.VideoUpscalerModel("mediautils")
_, err := vu.UpscaleVideo(context.Background(), videogen.VideoUpscaleRequest{Video: mp4Fixture()})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for headerless non-video body", err)
}
}
func TestVideoInputFilename(t *testing.T) {
cases := []struct {
filename, mime, want string
}{
{"clip.mp4", "", "clip.mp4"},
{"evil\r\nclip.mp4", "", "evilclip.mp4"},
{"", "video/mp4", "video.mp4"},
{"", "video/webm", "video.webm"},
{"", "video/quicktime", "video.mov"},
{"", "", "video"},
}
for _, tc := range cases {
if got := videoInputFilename(tc.filename, tc.mime); got != tc.want {
t.Errorf("videoInputFilename(%q, %q) = %q, want %q", tc.filename, tc.mime, got, tc.want)
}
}
}
+1 -8
View File
@@ -207,12 +207,5 @@ func (m *interpolatorModel) Interpolate(ctx context.Context, req videogen.Interp
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "interpolate response contained no video"}
}
mimeType := videoMIME(respType, raw)
if mimeType == "" {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "interpolate response is not a video"}
}
return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil
return singleVideoResult(m.p.name, m.id, "interpolate", raw, respType)
}
+19 -16
View File
@@ -100,22 +100,7 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ..
if err != nil {
return nil, err
}
if len(videoBytes) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video response contained no video"}
}
mimeType := videoMIME(contentType, videoBytes)
if mimeType == "" {
// A 2xx body that is neither declared nor sniffable as video is a
// misconfigured upstream (a JSON job envelope, an HTML error page
// behind a proxy) — fail loud rather than hand back garbage as a
// playable clip.
return nil, &llm.APIError{
Provider: m.p.name,
Model: m.id,
Message: fmt.Sprintf("video response is not a video (content-type %q)", contentType),
}
}
return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: mimeType}}, nil
return singleVideoResult(m.p.name, m.id, "video", videoBytes, contentType)
}
// videoMIME resolves the result MIME type: the response Content-Type when it
@@ -132,6 +117,24 @@ func videoMIME(contentType string, data []byte) string {
return ""
}
// singleVideoResult wraps one raw video body into a videogen.Result,
// requiring positive evidence of video-ness (declared video/* Content-Type
// or sniffed mp4/webm magic) so a 2xx body that is anything else — a JSON
// job envelope, an HTML error page behind a proxy — fails loud instead of
// coming back as "the clip". The video sibling of singleImageResult, shared
// by every surface whose response body IS the encoded clip.
func singleVideoResult(provider, model, verb string, raw []byte, contentType string) (*videogen.Result, error) {
if len(raw) == 0 {
return nil, &llm.APIError{Provider: provider, Model: model, Message: verb + " response contained no video"}
}
mimeType := videoMIME(contentType, raw)
if mimeType == "" {
return nil, &llm.APIError{Provider: provider, Model: model,
Message: fmt.Sprintf("%s response is not a video (Content-Type %q)", verb, contentType)}
}
return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil
}
// initImageFilename picks the multipart filename hint for the conditioning
// frame from its MIME subtype. The name is provider-chosen (never
// caller-supplied), so no sanitization is needed.
+217
View File
@@ -0,0 +1,217 @@
// videochain.go implements videogen.ChainerProvider against the videoutils
// chain orchestrator reached through llama-swap's /upstream passthrough
// (ADR-0025):
//
// POST /upstream/<id>/v1/video/chain JSON submit -> {job_id}
// GET /upstream/<id>/v1/jobs/{id} -> {status,segment,total,segments}
// GET /upstream/<id>/v1/jobs/{id}/result -> encoded clip
// GET /upstream/<id>/v1/jobs/{id}/segments/{n} -> encoded clip
//
// Unlike the ACE-Step music path this client does NOT hide the job queue
// behind a blocking call: a chain runs through multiple GPU swaps for many
// minutes, and the caller (mort's long-video tool) owns the poll loop so it
// can deliver PARTIAL results — completed segments survive a mid-chain
// failure and stay fetchable via ChainSegmentResult.
package llamaswap
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"math"
"net/http"
"strconv"
"strings"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
)
// ChainerModel implements videogen.ChainerProvider. The id selects which
// upstream llama-swap loads (the videoutils orchestrator, which in turn
// drives the generation model through llama-swap itself).
func (p *Provider) ChainerModel(id string, opts ...videogen.ChainerModelOption) (videogen.Chainer, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = videogen.ApplyChainerModelOptions(opts)
return &chainerModel{p: p, id: id}, nil
}
type chainerModel struct {
p *Provider
id string
}
// chainSubmitRequest is the videoutils POST /v1/video/chain shape. The init
// image rides as base64 in the JSON body (`init_image_b64`) — the submit is
// JSON, not multipart, per the pinned host contract.
type chainSubmitRequest struct {
Segments []chainSegmentWire `json:"segments"`
InitImageB64 string `json:"init_image_b64,omitempty"`
SmoothJoins bool `json:"smooth_joins,omitempty"`
Size string `json:"size,omitempty"`
}
type chainSegmentWire struct {
Prompt string `json:"prompt"`
Seconds float64 `json:"seconds,omitempty"`
}
// SubmitChain implements videogen.Chainer.
func (m *chainerModel) SubmitChain(ctx context.Context, req videogen.ChainRequest) (string, error) {
if len(req.Segments) == 0 {
return "", fmt.Errorf("%w: video chain requires at least one segment", llm.ErrUnsupported)
}
wire := chainSubmitRequest{
SmoothJoins: req.SmoothJoins,
Size: strings.TrimSpace(req.Size),
}
for i, seg := range req.Segments {
if strings.TrimSpace(seg.Prompt) == "" {
return "", fmt.Errorf("%w: video chain segment %d requires a prompt", llm.ErrUnsupported, i)
}
// NaN/±Inf would otherwise surface as an obscure json.Marshal error
// (NaN fails every comparison; +Inf passes the >= 0 check).
if seg.Seconds < 0 || math.IsNaN(seg.Seconds) || math.IsInf(seg.Seconds, 0) {
return "", fmt.Errorf("%w: video chain segment %d seconds must be a finite value >= 0, got %g", llm.ErrUnsupported, i, seg.Seconds)
}
wire.Segments = append(wire.Segments, chainSegmentWire{Prompt: seg.Prompt, Seconds: seg.Seconds})
}
if len(req.InitImage) > 0 {
wire.InitImageB64 = base64.StdEncoding.EncodeToString(req.InitImage)
}
path, err := upstreamPath(m.id, "/v1/video/chain")
if err != nil {
return "", err
}
// Tolerant envelope: {"job_id": ...} per the contract, with data-wrapped
// and bare-id fallbacks (musicgen release_task precedent).
var resp struct {
JobID string `json:"job_id"`
ID string `json:"id"`
Data struct {
JobID string `json:"job_id"`
ID string `json:"id"`
} `json:"data"`
}
if err := m.p.doJSON(ctx, http.MethodPost, path, m.id, &wire, &resp); err != nil {
return "", err
}
for _, id := range []string{resp.JobID, resp.Data.JobID, resp.ID, resp.Data.ID} {
if id != "" {
return id, nil
}
}
return "", &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video chain submit returned no job_id"}
}
// chainJobResponse is the GET /v1/jobs/{id} shape. `segments` entries are
// tolerated as bare strings or objects keyed by id/segment_id.
type chainJobResponse struct {
Status string `json:"status"`
Segment int `json:"segment"`
Total int `json:"total"`
Segments []json.RawMessage `json:"segments"`
}
// ChainStatus implements videogen.Chainer.
func (m *chainerModel) ChainStatus(ctx context.Context, jobID string) (*videogen.ChainJob, error) {
path, err := m.jobPath(jobID, "")
if err != nil {
return nil, err
}
var raw json.RawMessage
if err := m.p.doJSON(ctx, http.MethodGet, path, m.id, nil, &raw); err != nil {
return nil, err
}
var out chainJobResponse
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("llama-swap: decode chain job status: %w", err)
}
job := &videogen.ChainJob{
Status: out.Status,
Segment: out.Segment,
Total: out.Total,
Raw: raw,
}
// Entries that carry no usable id — JSON null (which unmarshals into a
// string as a no-op, leaving ""), an empty string, or an object with
// neither key — are SKIPPED, never appended as "": SegmentIDs promises
// fetchable artifacts, and the full payload stays in Raw for callers
// that want the unfiltered list.
for _, entry := range out.Segments {
var s string
if json.Unmarshal(entry, &s) == nil {
if s != "" {
job.SegmentIDs = append(job.SegmentIDs, s)
}
continue
}
var obj struct {
ID string `json:"id"`
SegmentID string `json:"segment_id"`
}
if json.Unmarshal(entry, &obj) == nil {
switch {
case obj.ID != "":
job.SegmentIDs = append(job.SegmentIDs, obj.ID)
case obj.SegmentID != "":
job.SegmentIDs = append(job.SegmentIDs, obj.SegmentID)
}
}
}
if job.Status == "" {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: "chain job status payload carried no status: " + truncateForError(raw)}
}
return job, nil
}
// ChainResult implements videogen.Chainer.
func (m *chainerModel) ChainResult(ctx context.Context, jobID string) (*videogen.Result, error) {
path, err := m.jobPath(jobID, "/result")
if err != nil {
return nil, err
}
raw, respType, err := m.p.doRaw(ctx, http.MethodGet, path, m.id, "", nil, maxVideoResponseBytes)
if err != nil {
return nil, err
}
return singleVideoResult(m.p.name, m.id, "video chain result", raw, respType)
}
// ChainSegmentResult implements videogen.Chainer.
func (m *chainerModel) ChainSegmentResult(ctx context.Context, jobID string, n int) (*videogen.Result, error) {
if n < 0 {
return nil, fmt.Errorf("%w: chain segment index must be >= 0, got %d", llm.ErrUnsupported, n)
}
path, err := m.jobPath(jobID, "/segments/"+strconv.Itoa(n))
if err != nil {
return nil, err
}
raw, respType, err := m.p.doRaw(ctx, http.MethodGet, path, m.id, "", nil, maxVideoResponseBytes)
if err != nil {
return nil, err
}
return singleVideoResult(m.p.name, m.id, "video chain segment", raw, respType)
}
// jobPath builds /upstream/<model>/v1/jobs/<jobID><suffix>, refusing job ids
// that carry path structure. The id is SERVER-SUPPLIED (echoed back from
// SubmitChain), so like upstreamPath's rest-component checks this rejects
// rather than escapes — a hostile/buggy upstream must not be able to steer
// the follow-up request at another proxy endpoint.
func (m *chainerModel) jobPath(jobID, suffix string) (string, error) {
if strings.TrimSpace(jobID) == "" {
return "", fmt.Errorf("llama-swap: chain job call requires a job id")
}
// '%' is rejected alongside the literal path characters: job ids never
// legitimately carry percent-escapes, and %2F/%2E%2E would decode back
// into path structure server-side.
if strings.ContainsAny(jobID, "/?#%") || strings.Contains(jobID, "..") {
return "", fmt.Errorf("llama-swap: invalid chain job id %q (contains path structure)", jobID)
}
return upstreamPath(m.id, "/v1/jobs/"+jobID+suffix)
}
+322
View File
@@ -0,0 +1,322 @@
package llamaswap
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"math"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
)
func TestSubmitChain(t *testing.T) {
var gotPath string
var gotBody map[string]any
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
_ = json.NewDecoder(r.Body).Decode(&gotBody)
_, _ = w.Write([]byte(`{"job_id":"job-123"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, err := p.ChainerModel("videoutils")
if err != nil {
t.Fatalf("ChainerModel: %v", err)
}
jobID, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
Segments: []videogen.ChainSegment{
{Prompt: "a cat walks in", Seconds: 5},
{Prompt: "the cat sits down", Seconds: 5},
},
InitImage: []byte("IMG"),
SmoothJoins: true,
Size: "1280x704",
})
if err != nil {
t.Fatalf("SubmitChain: %v", err)
}
if jobID != "job-123" {
t.Errorf("jobID = %q", jobID)
}
if gotPath != "/upstream/videoutils/v1/video/chain" {
t.Errorf("path = %q", gotPath)
}
segments, _ := gotBody["segments"].([]any)
if len(segments) != 2 {
t.Fatalf("segments = %v", gotBody["segments"])
}
first, _ := segments[0].(map[string]any)
if first["prompt"] != "a cat walks in" || first["seconds"] != 5.0 {
t.Errorf("segment[0] = %v", first)
}
if gotBody["init_image_b64"] != base64.StdEncoding.EncodeToString([]byte("IMG")) {
t.Errorf("init_image_b64 = %v", gotBody["init_image_b64"])
}
if gotBody["smooth_joins"] != true || gotBody["size"] != "1280x704" {
t.Errorf("smooth_joins/size = %v/%v", gotBody["smooth_joins"], gotBody["size"])
}
}
func TestSubmitChainOmitsUnsetFields(t *testing.T) {
var gotBody map[string]any
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewDecoder(r.Body).Decode(&gotBody)
_, _ = w.Write([]byte(`{"data":{"job_id":"job-9"}}`)) // data-wrapped envelope
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
jobID, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
Segments: []videogen.ChainSegment{{Prompt: "a dog"}},
})
if err != nil {
t.Fatalf("SubmitChain: %v", err)
}
if jobID != "job-9" {
t.Errorf("jobID = %q, want data-wrapped id", jobID)
}
for _, k := range []string{"init_image_b64", "smooth_joins", "size"} {
if v, ok := gotBody[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
seg, _ := gotBody["segments"].([]any)
if first, _ := seg[0].(map[string]any); first == nil {
t.Fatalf("segments = %v", gotBody["segments"])
} else if _, ok := first["seconds"]; ok {
t.Error("zero seconds sent; want omitted")
}
}
func TestSubmitChainRejectsBadArgs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
ch, _ := p.ChainerModel("videoutils")
if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("no segments: err = %v, want ErrUnsupported", err)
}
if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
Segments: []videogen.ChainSegment{{Prompt: " "}},
}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("empty prompt: err = %v, want ErrUnsupported", err)
}
if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
Segments: []videogen.ChainSegment{{Prompt: "x", Seconds: -1}},
}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("negative seconds: err = %v, want ErrUnsupported", err)
}
for name, bad := range map[string]float64{
"NaN": math.NaN(),
"+Inf": math.Inf(1),
"-Inf": math.Inf(-1),
} {
if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
Segments: []videogen.ChainSegment{{Prompt: "x", Seconds: bad}},
}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("%s seconds: err = %v, want ErrUnsupported", name, err)
}
}
}
func TestSubmitChainRejectsMissingJobID(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"ok":true}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
_, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
Segments: []videogen.ChainSegment{{Prompt: "x"}},
})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for missing job_id", err)
}
}
func TestChainStatus(t *testing.T) {
var gotPath string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
_, _ = w.Write([]byte(`{"status":"running","segment":2,"total":3,"segments":["seg-0"]}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
job, err := ch.ChainStatus(context.Background(), "job-123")
if err != nil {
t.Fatalf("ChainStatus: %v", err)
}
if gotPath != "/upstream/videoutils/v1/jobs/job-123" {
t.Errorf("path = %q", gotPath)
}
if job.Status != "running" || job.Segment != 2 || job.Total != 3 {
t.Errorf("job = %+v", job)
}
if !reflect.DeepEqual(job.SegmentIDs, []string{"seg-0"}) {
t.Errorf("SegmentIDs = %v", job.SegmentIDs)
}
if job.Raw == nil {
t.Error("Raw = nil, want raw payload")
}
}
func TestChainStatusToleratesObjectSegments(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"status":"done","segment":2,"total":2,"segments":[{"id":"seg-0"},{"segment_id":"seg-1"}]}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
job, err := ch.ChainStatus(context.Background(), "job-123")
if err != nil {
t.Fatalf("ChainStatus: %v", err)
}
if !reflect.DeepEqual(job.SegmentIDs, []string{"seg-0", "seg-1"}) {
t.Errorf("SegmentIDs = %v", job.SegmentIDs)
}
}
func TestChainStatusSkipsIDLessSegments(t *testing.T) {
// null, "", an id-less object, and a mistyped entry must all be
// skipped — never appended as "" (SegmentIDs promises fetchable
// artifacts; the unfiltered list stays in Raw).
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"status":"running","segment":3,"total":5,` +
`"segments":[null,"seg-0","",{},{"id":"seg-1"},{"segment_id":""},42]}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
job, err := ch.ChainStatus(context.Background(), "job-123")
if err != nil {
t.Fatalf("ChainStatus: %v", err)
}
if !reflect.DeepEqual(job.SegmentIDs, []string{"seg-0", "seg-1"}) {
t.Errorf("SegmentIDs = %v, want [seg-0 seg-1]", job.SegmentIDs)
}
}
func TestChainStatusRejectsStatuslessPayload(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"detail":"no such job"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
_, err := ch.ChainStatus(context.Background(), "job-123")
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for statusless payload", err)
}
}
func TestChainJobPathRejectsHostileIDs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
ch, _ := p.ChainerModel("videoutils")
for _, bad := range []string{"", "a/b", "a?b", "a#b", "..", "a..b", "a%2Fb", "%2e%2e", "a%b"} {
if _, err := ch.ChainStatus(context.Background(), bad); err == nil {
t.Errorf("ChainStatus(%q) succeeded; want error", bad)
}
if _, err := ch.ChainResult(context.Background(), bad); err == nil {
t.Errorf("ChainResult(%q) succeeded; want error", bad)
}
}
}
func TestChainResult(t *testing.T) {
var gotPath string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write(mp4Fixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
res, err := ch.ChainResult(context.Background(), "job-123")
if err != nil {
t.Fatalf("ChainResult: %v", err)
}
if gotPath != "/upstream/videoutils/v1/jobs/job-123/result" {
t.Errorf("path = %q", gotPath)
}
if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
}
}
func TestChainSegmentResult(t *testing.T) {
var gotPath string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write(mp4Fixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
res, err := ch.ChainSegmentResult(context.Background(), "job-123", 1)
if err != nil {
t.Fatalf("ChainSegmentResult: %v", err)
}
if gotPath != "/upstream/videoutils/v1/jobs/job-123/segments/1" {
t.Errorf("path = %q", gotPath)
}
if res.Video.MIME != "video/mp4" {
t.Fatalf("video = %q", res.Video.MIME)
}
if _, err := ch.ChainSegmentResult(context.Background(), "job-123", -1); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("negative segment: err = %v, want ErrUnsupported", err)
}
}
func TestChainResultRejectsNonVideoResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"status":"running"}`)) // a status page, not the clip
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
_, err := ch.ChainResult(context.Background(), "job-123")
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-video body", err)
}
}
func TestChainSurfacesAPIError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`{"error":{"message":"job not found"}}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ch, _ := p.ChainerModel("videoutils")
_, err := ch.ChainStatus(context.Background(), "job-void")
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %T %v, want *llm.APIError", err, err)
}
if apiErr.Status != http.StatusNotFound || apiErr.Message != "job not found" || apiErr.Model != "videoutils" {
t.Errorf("apiErr = %+v", apiErr)
}
}
+139
View File
@@ -0,0 +1,139 @@
// videoutil.go implements the videogen.VideoBackgroundRemovalProvider and
// videogen.VideoUpscaleProvider surfaces against the mediautils shim reached
// through llama-swap's /upstream passthrough (ADR-0025):
//
// matte POST /upstream/<id>/v1/video/matte (Robust Video Matting)
// upscale POST /upstream/<id>/v1/video/upscale (per-frame Real-ESRGAN)
//
// Both are one-file multipart in, encoded clip out — the video siblings of
// mediautil.go's still-image surfaces.
package llamaswap
import (
"context"
"fmt"
"mime"
"net/http"
"strconv"
"strings"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
)
// --- video background removal (matting) ---
// VideoBackgroundRemoverModel implements
// videogen.VideoBackgroundRemovalProvider against the mediautils shim's
// POST /v1/video/matte. The id selects which upstream llama-swap loads.
func (p *Provider) VideoBackgroundRemoverModel(id string, opts ...videogen.VideoBackgroundRemoverModelOption) (videogen.VideoBackgroundRemover, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = videogen.ApplyVideoBackgroundRemoverModelOptions(opts)
return &videoMatteModel{p: p, id: id}, nil
}
type videoMatteModel struct {
p *Provider
id string
}
// RemoveVideoBackground implements videogen.VideoBackgroundRemover.
func (m *videoMatteModel) RemoveVideoBackground(ctx context.Context, req videogen.VideoBackgroundRemovalRequest, opts ...videogen.VideoBackgroundRemovalOption) (*videogen.Result, error) {
req = req.Apply(opts...)
if len(req.Video) == 0 {
return nil, fmt.Errorf("%w: video background removal requires a video", llm.ErrUnsupported)
}
if req.Output != "" && req.Output != "greenscreen_mp4" && req.Output != "alpha_webm" {
return nil, fmt.Errorf("%w: video matte output must be \"greenscreen_mp4\" or \"alpha_webm\", got %q", llm.ErrUnsupported, req.Output)
}
path, err := upstreamPath(m.id, "/v1/video/matte")
if err != nil {
return nil, err
}
body, contentType, err := buildMultipart("build video-matte form",
filePart{field: "file", filename: videoInputFilename(req.Filename, req.MIME), data: req.Video},
[]formField{{"output", req.Output, false}})
if err != nil {
return nil, err
}
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes)
if err != nil {
return nil, err
}
return singleVideoResult(m.p.name, m.id, "video matte", raw, respType)
}
// --- video upscale ---
// VideoUpscalerModel implements videogen.VideoUpscaleProvider against the
// mediautils shim's POST /v1/video/upscale.
func (p *Provider) VideoUpscalerModel(id string, opts ...videogen.VideoUpscalerModelOption) (videogen.VideoUpscaler, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = videogen.ApplyVideoUpscalerModelOptions(opts)
return &videoUpscaleModel{p: p, id: id}, nil
}
type videoUpscaleModel struct {
p *Provider
id string
}
// UpscaleVideo implements videogen.VideoUpscaler.
func (m *videoUpscaleModel) UpscaleVideo(ctx context.Context, req videogen.VideoUpscaleRequest, opts ...videogen.VideoUpscaleOption) (*videogen.Result, error) {
req = req.Apply(opts...)
if len(req.Video) == 0 {
return nil, fmt.Errorf("%w: video upscale requires a video", llm.ErrUnsupported)
}
if req.Scale != 0 && req.Scale != 2 && req.Scale != 4 {
return nil, fmt.Errorf("%w: video upscale scale must be 2 or 4, got %d", llm.ErrUnsupported, req.Scale)
}
path, err := upstreamPath(m.id, "/v1/video/upscale")
if err != nil {
return nil, err
}
scale := ""
if req.Scale != 0 {
scale = strconv.Itoa(req.Scale)
}
body, contentType, err := buildMultipart("build video-upscale form",
filePart{field: "file", filename: videoInputFilename(req.Filename, req.MIME), data: req.Video},
[]formField{{"scale", scale, false}})
if err != nil {
return nil, err
}
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes)
if err != nil {
return nil, err
}
return singleVideoResult(m.p.name, m.id, "video upscale", raw, respType)
}
// videoInputFilename picks the multipart filename hint for a caller-supplied
// video: the caller's (sanitized — upload metadata is untrusted), else one
// derived from the MIME subtype ("video.mp4"), else "video". Mirrors
// transcriptionFilename.
func videoInputFilename(filename, mimeType string) string {
if name := sanitizeFilename(filename); name != "" {
return name
}
mt := strings.ToLower(strings.TrimSpace(mimeType))
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
mt = parsed
}
switch mt {
case "video/mp4":
return "video.mp4"
case "video/webm":
return "video.webm"
case "video/quicktime":
return "video.mov"
case "video/x-matroska":
return "video.mkv"
default:
return "video"
}
}