feat: wave-3 video surfaces — lipsync, video matte, video upscale, chain jobs (ADR-0025)

- videogen.LipSyncer/LipsyncProvider: SadTalker talking heads via POST /upstream/<id>/v1/talking_head (multipart image+audio parts, still/enhance/preprocess flags) -> mp4. - videogen.VideoBackgroundRemover/VideoBackgroundRemovalProvider: POST /upstream/<id>/v1/video/matte (output greenscreen_mp4|alpha_webm). - videogen.VideoUpscaler/VideoUpscaleProvider: POST /upstream/<id>/v1/video/upscale (scale 2|4). - videogen.Chainer/ChainerProvider: async long-video chain-job client — SubmitChain (JSON POST /v1/video/chain, init_image_b64), ChainStatus (GET /v1/jobs/{id}, tolerant segment-id decode), ChainResult, ChainSegmentResult (partial delivery after mid-chain failure); hostile job-id path rejection. - Shared singleVideoResult validation (positive video evidence) + a videoInputFilename hint helper; httptest contract tests per surface; ADR-0025 (index row deferred — MJ-A backfills the ADR index table and parallel edits would conflict). Co-Authored-By: Claude Fable 5 <[email protected]>
2026-07-16 17:07:27 -04:00
parent cd43009672
commit 5f175ecf82
10 changed files with 1453 additions and 0 deletions
@@ -0,0 +1,295 @@
+package llamaswap
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+	"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
+)
+
+// webmFixture is a minimal EBML header so http.DetectContentType sniffs
+// video/webm.
+func webmFixture() []byte {
+	return append([]byte{0x1A, 0x45, 0xDF, 0xA3}, make([]byte, 20)...)
+}
+
+func TestLipsync(t *testing.T) {
+	png := pngFixture(t)
+	var gotPath, gotStill, gotEnhance, gotPreprocess string
+	var gotImage, gotAudio []byte
+	var gotImageName, gotAudioName string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		if err := r.ParseMultipartForm(1 << 20); err != nil {
+			t.Fatalf("parse multipart: %v", err)
+		}
+		gotStill = r.FormValue("still")
+		gotEnhance = r.FormValue("enhance")
+		gotPreprocess = r.FormValue("preprocess")
+		if f, hdr, err := r.FormFile("image"); err == nil {
+			gotImage, _ = io.ReadAll(f)
+			gotImageName = hdr.Filename
+			f.Close()
+		} else {
+			t.Errorf("image part: %v", err)
+		}
+		if f, hdr, err := r.FormFile("audio"); err == nil {
+			gotAudio, _ = io.ReadAll(f)
+			gotAudioName = hdr.Filename
+			f.Close()
+		} else {
+			t.Errorf("audio part: %v", err)
+		}
+		w.Header().Set("Content-Type", "video/mp4")
+		_, _ = w.Write(mp4Fixture())
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	ls, err := p.LipsyncModel("lipsync-sadtalker")
+	if err != nil {
+		t.Fatalf("LipsyncModel: %v", err)
+	}
+	res, err := ls.Lipsync(context.Background(),
+		videogen.LipsyncRequest{
+			Image:     videogen.Image{MIME: "image/png", Data: png},
+			Audio:     []byte("SPEECH"),
+			AudioMIME: "audio/wav",
+		},
+		videogen.WithLipsyncStill(), videogen.WithLipsyncEnhance(), videogen.WithLipsyncPreprocess("full"))
+	if err != nil {
+		t.Fatalf("Lipsync: %v", err)
+	}
+	if gotPath != "/upstream/lipsync-sadtalker/v1/talking_head" {
+		t.Errorf("path = %q", gotPath)
+	}
+	if gotStill != "true" || gotEnhance != "true" || gotPreprocess != "full" {
+		t.Errorf("still/enhance/preprocess = %q/%q/%q", gotStill, gotEnhance, gotPreprocess)
+	}
+	if string(gotImage) != string(png) || gotImageName != "frame.png" {
+		t.Errorf("image bytes/name = %d bytes/%q", len(gotImage), gotImageName)
+	}
+	if string(gotAudio) != "SPEECH" || gotAudioName != "audio.wav" {
+		t.Errorf("audio = %q name = %q", gotAudio, gotAudioName)
+	}
+	if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
+		t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
+	}
+}
+
+func TestLipsyncOmitsUnsetFlags(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if err := r.ParseMultipartForm(1 << 20); err != nil {
+			t.Fatalf("parse multipart: %v", err)
+		}
+		for _, k := range []string{"still", "enhance", "preprocess"} {
+			if v, ok := r.MultipartForm.Value[k]; ok {
+				t.Errorf("unset request sent %q = %v, want omitted", k, v)
+			}
+		}
+		w.Header().Set("Content-Type", "video/mp4")
+		_, _ = w.Write(mp4Fixture())
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	ls, _ := p.LipsyncModel("lipsync-sadtalker")
+	if _, err := ls.Lipsync(context.Background(),
+		videogen.LipsyncRequest{Image: videogen.Image{Data: pngFixture(t)}, Audio: []byte("A")}); err != nil {
+		t.Fatalf("Lipsync: %v", err)
+	}
+}
+
+func TestLipsyncRejectsBadArgs(t *testing.T) {
+	p := New(WithBaseURL("http://unused"))
+	ls, _ := p.LipsyncModel("lipsync-sadtalker")
+	if _, err := ls.Lipsync(context.Background(),
+		videogen.LipsyncRequest{Audio: []byte("A")}); !errors.Is(err, llm.ErrUnsupported) {
+		t.Errorf("no image: err = %v, want ErrUnsupported", err)
+	}
+	if _, err := ls.Lipsync(context.Background(),
+		videogen.LipsyncRequest{Image: videogen.Image{Data: []byte{1}}}); !errors.Is(err, llm.ErrUnsupported) {
+		t.Errorf("no audio: err = %v, want ErrUnsupported", err)
+	}
+	if _, err := ls.Lipsync(context.Background(),
+		videogen.LipsyncRequest{Image: videogen.Image{Data: []byte{1}}, Audio: []byte{1}, Preprocess: "zoom"}); !errors.Is(err, llm.ErrUnsupported) {
+		t.Errorf("preprocess zoom: err = %v, want ErrUnsupported", err)
+	}
+}
+
+func TestLipsyncRejectsNonVideoResponse(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		_, _ = w.Write([]byte("<html>proxy error page</html>"))
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	ls, _ := p.LipsyncModel("lipsync-sadtalker")
+	_, err := ls.Lipsync(context.Background(),
+		videogen.LipsyncRequest{Image: videogen.Image{Data: pngFixture(t)}, Audio: []byte("A")})
+	var apiErr *llm.APIError
+	if !errors.As(err, &apiErr) {
+		t.Fatalf("err = %v, want APIError for non-video body", err)
+	}
+}
+
+func TestRemoveVideoBackground(t *testing.T) {
+	var gotPath, gotOutput, gotFilename string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		if err := r.ParseMultipartForm(1 << 20); err != nil {
+			t.Fatalf("parse multipart: %v", err)
+		}
+		gotOutput = r.FormValue("output")
+		if _, hdr, err := r.FormFile("file"); err == nil {
+			gotFilename = hdr.Filename
+		} else {
+			t.Errorf("file part: %v", err)
+		}
+		w.Header().Set("Content-Type", "video/webm")
+		_, _ = w.Write(webmFixture())
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	vb, err := p.VideoBackgroundRemoverModel("mediautils")
+	if err != nil {
+		t.Fatalf("VideoBackgroundRemoverModel: %v", err)
+	}
+	res, err := vb.RemoveVideoBackground(context.Background(),
+		videogen.VideoBackgroundRemovalRequest{Video: mp4Fixture(), MIME: "video/mp4"},
+		videogen.WithVideoBackgroundOutput("alpha_webm"))
+	if err != nil {
+		t.Fatalf("RemoveVideoBackground: %v", err)
+	}
+	if gotPath != "/upstream/mediautils/v1/video/matte" {
+		t.Errorf("path = %q", gotPath)
+	}
+	if gotOutput != "alpha_webm" || gotFilename != "video.mp4" {
+		t.Errorf("output/filename = %q/%q", gotOutput, gotFilename)
+	}
+	if res.Video.MIME != "video/webm" || len(res.Video.Data) == 0 {
+		t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
+	}
+}
+
+func TestRemoveVideoBackgroundOmitsDefaultOutput(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if err := r.ParseMultipartForm(1 << 20); err != nil {
+			t.Fatalf("parse multipart: %v", err)
+		}
+		if v, ok := r.MultipartForm.Value["output"]; ok {
+			t.Errorf("output field sent for default: %v; want omitted", v)
+		}
+		w.Header().Set("Content-Type", "video/mp4")
+		_, _ = w.Write(mp4Fixture())
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	vb, _ := p.VideoBackgroundRemoverModel("mediautils")
+	if _, err := vb.RemoveVideoBackground(context.Background(),
+		videogen.VideoBackgroundRemovalRequest{Video: mp4Fixture()}); err != nil {
+		t.Fatalf("RemoveVideoBackground: %v", err)
+	}
+}
+
+func TestRemoveVideoBackgroundRejectsBadArgs(t *testing.T) {
+	p := New(WithBaseURL("http://unused"))
+	vb, _ := p.VideoBackgroundRemoverModel("mediautils")
+	if _, err := vb.RemoveVideoBackground(context.Background(),
+		videogen.VideoBackgroundRemovalRequest{}); !errors.Is(err, llm.ErrUnsupported) {
+		t.Errorf("no video: err = %v, want ErrUnsupported", err)
+	}
+	if _, err := vb.RemoveVideoBackground(context.Background(),
+		videogen.VideoBackgroundRemovalRequest{Video: []byte{1}, Output: "gif"}); !errors.Is(err, llm.ErrUnsupported) {
+		t.Errorf("output gif: err = %v, want ErrUnsupported", err)
+	}
+}
+
+func TestUpscaleVideo(t *testing.T) {
+	var gotPath, gotScale string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		if err := r.ParseMultipartForm(1 << 20); err != nil {
+			t.Fatalf("parse multipart: %v", err)
+		}
+		gotScale = r.FormValue("scale")
+		w.Header().Set("Content-Type", "video/mp4")
+		_, _ = w.Write(mp4Fixture())
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	vu, err := p.VideoUpscalerModel("mediautils")
+	if err != nil {
+		t.Fatalf("VideoUpscalerModel: %v", err)
+	}
+	res, err := vu.UpscaleVideo(context.Background(),
+		videogen.VideoUpscaleRequest{Video: mp4Fixture(), MIME: "video/mp4"},
+		videogen.WithVideoUpscaleScale(2))
+	if err != nil {
+		t.Fatalf("UpscaleVideo: %v", err)
+	}
+	if gotPath != "/upstream/mediautils/v1/video/upscale" {
+		t.Errorf("path = %q", gotPath)
+	}
+	if gotScale != "2" {
+		t.Errorf("scale = %q", gotScale)
+	}
+	if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
+		t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
+	}
+}
+
+func TestUpscaleVideoRejectsBadArgs(t *testing.T) {
+	p := New(WithBaseURL("http://unused"))
+	vu, _ := p.VideoUpscalerModel("mediautils")
+	if _, err := vu.UpscaleVideo(context.Background(), videogen.VideoUpscaleRequest{}); !errors.Is(err, llm.ErrUnsupported) {
+		t.Errorf("no video: err = %v, want ErrUnsupported", err)
+	}
+	if _, err := vu.UpscaleVideo(context.Background(),
+		videogen.VideoUpscaleRequest{Video: []byte{1}, Scale: 3}); !errors.Is(err, llm.ErrUnsupported) {
+		t.Errorf("scale 3: err = %v, want ErrUnsupported", err)
+	}
+}
+
+func TestUpscaleVideoRejectsNonVideoResponse(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header()["Content-Type"] = nil // NO Content-Type at all
+		_, _ = w.Write([]byte("502 bad gateway"))
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	vu, _ := p.VideoUpscalerModel("mediautils")
+	_, err := vu.UpscaleVideo(context.Background(), videogen.VideoUpscaleRequest{Video: mp4Fixture()})
+	var apiErr *llm.APIError
+	if !errors.As(err, &apiErr) {
+		t.Fatalf("err = %v, want APIError for headerless non-video body", err)
+	}
+}
+
+func TestVideoInputFilename(t *testing.T) {
+	cases := []struct {
+		filename, mime, want string
+	}{
+		{"clip.mp4", "", "clip.mp4"},
+		{"evil\r\nclip.mp4", "", "evilclip.mp4"},
+		{"", "video/mp4", "video.mp4"},
+		{"", "video/webm", "video.webm"},
+		{"", "video/quicktime", "video.mov"},
+		{"", "", "video"},
+	}
+	for _, tc := range cases {
+		if got := videoInputFilename(tc.filename, tc.mime); got != tc.want {
+			t.Errorf("videoInputFilename(%q, %q) = %q, want %q", tc.filename, tc.mime, got, tc.want)
+		}
+	}
+}