From 5f175ecf82cafa7c1be6731dd9d643019b882fb2 Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Thu, 16 Jul 2026 17:07:27 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20wave-3=20video=20surfaces=20=E2=80=94?= =?UTF-8?q?=20lipsync,=20video=20matte,=20video=20upscale,=20chain=20jobs?= =?UTF-8?q?=20(ADR-0025)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - videogen.LipSyncer/LipsyncProvider: SadTalker talking heads via POST /upstream//v1/talking_head (multipart image+audio parts, still/enhance/preprocess flags) -> mp4. - videogen.VideoBackgroundRemover/VideoBackgroundRemovalProvider: POST /upstream//v1/video/matte (output greenscreen_mp4|alpha_webm). - videogen.VideoUpscaler/VideoUpscaleProvider: POST /upstream//v1/video/upscale (scale 2|4). - videogen.Chainer/ChainerProvider: async long-video chain-job client — SubmitChain (JSON POST /v1/video/chain, init_image_b64), ChainStatus (GET /v1/jobs/{id}, tolerant segment-id decode), ChainResult, ChainSegmentResult (partial delivery after mid-chain failure); hostile job-id path rejection. - Shared singleVideoResult validation (positive video evidence) + a videoInputFilename hint helper; httptest contract tests per surface; ADR-0025 (index row deferred — MJ-A backfills the ADR index table and parallel edits would conflict). Co-Authored-By: Claude Fable 5 --- docs/adr/0025-videogen-wave3-surfaces.md | 67 +++++ provider/llamaswap/lipsync.go | 113 +++++++ provider/llamaswap/lipsync_videoutil_test.go | 295 +++++++++++++++++++ provider/llamaswap/videochain.go | 204 +++++++++++++ provider/llamaswap/videochain_test.go | 289 ++++++++++++++++++ provider/llamaswap/videoutil.go | 139 +++++++++ videogen/chain.go | 99 +++++++ videogen/lipsync.go | 93 ++++++ videogen/matte.go | 80 +++++ videogen/vupscale.go | 74 +++++ 10 files changed, 1453 insertions(+) create mode 100644 docs/adr/0025-videogen-wave3-surfaces.md create mode 100644 provider/llamaswap/lipsync.go create mode 100644 provider/llamaswap/lipsync_videoutil_test.go create mode 100644 provider/llamaswap/videochain.go create mode 100644 provider/llamaswap/videochain_test.go create mode 100644 provider/llamaswap/videoutil.go create mode 100644 videogen/chain.go create mode 100644 videogen/lipsync.go create mode 100644 videogen/matte.go create mode 100644 videogen/vupscale.go diff --git a/docs/adr/0025-videogen-wave3-surfaces.md b/docs/adr/0025-videogen-wave3-surfaces.md new file mode 100644 index 0000000..8365adb --- /dev/null +++ b/docs/adr/0025-videogen-wave3-surfaces.md @@ -0,0 +1,67 @@ +# ADR-0025: Wave-3 video surfaces (lipsync, video matte, video upscale, chain jobs) + +Status: Accepted (2026-07-16) + +## Context + +The llama-swap host is gaining wave-3 video capabilities (spec: mort +docs/specs/2026-07-16-llamaswap-wave3.md): SadTalker talking heads, Robust +Video Matting and per-frame Real-ESRGAN on the mediautils shim, and a +videoutils orchestrator that generates LONG videos as a chain of i2v +segments (generate → extract last frame → continue → concat → optional RIFE +smoothing), exposed as an async job API following the ACE-Step precedent. + +## Decision + +1. **Three new optional `videogen` interfaces**, ADR-0016→0024 conventions: + - `videogen.LipSyncer` / `LipsyncProvider` — `LipsyncRequest{Image, + Audio, AudioMIME, AudioFilename, Still, Enhance, Preprocess}` → + `POST /upstream//v1/talking_head` (multipart `image` + `audio` + file parts + optional `still`/`enhance`/`preprocess` fields) → mp4. + Sync and minutes-slow (Hunyuan precedent); context deadline is the + budget. + - `videogen.VideoBackgroundRemover` / `VideoBackgroundRemovalProvider` — + `VideoBackgroundRemovalRequest{Video, MIME, Filename, Output}` → + `POST /upstream//v1/video/matte`. `Output` ∈ + `greenscreen_mp4` (universally playable) | `alpha_webm` (true + transparency); "" = backend default. + - `videogen.VideoUpscaler` / `VideoUpscaleProvider` — + `VideoUpscaleRequest{Video, MIME, Filename, Scale}` (2 or 4) → + `POST /upstream//v1/video/upscale` → mp4. +2. **The chain client is deliberately ASYNC** (`videogen.Chainer` / + `ChainerProvider`), unlike musicgen's blocking Generate (ADR-0021): a + chain holds the GPU through multiple model swaps for many minutes, and + the caller must be able to poll progress AND fetch completed segments + after a mid-chain failure — partial delivery is mandatory (never discard + multi-minute GPU output), which a blocking one-call contract cannot + express. + - `SubmitChain(ctx, ChainRequest{Segments[{Prompt,Seconds}], InitImage, + SmoothJoins, Size}) (jobID, error)` — JSON + `POST /upstream//v1/video/chain`, init image as base64 + `init_image_b64` (JSON submit, not multipart, per the pinned host + contract). + - `ChainStatus(ctx, jobID) (*ChainJob{Status, Segment, Total, + SegmentIDs, Raw})` — `GET /v1/jobs/{id}`; polling doubles as a + liveness signal for the shim's idle TTL. Segment entries are tolerated + as strings or `{id|segment_id}` objects. + - `ChainResult(ctx, jobID)` / `ChainSegmentResult(ctx, jobID, n)` — + `GET /v1/jobs/{id}/result` and `/v1/jobs/{id}/segments/{n}`. + - Job ids are echoed server input: job paths reject ids carrying path + structure (`/?#`, `..`), the upstreamPath smuggling rule. +3. **Binary success bodies are validated before wrapping** (ADR-0020 rule): + all four clip-returning calls go through a shared `singleVideoResult` + (positive evidence of video-ness — declared video/* Content-Type or + sniffed mp4/webm magic — so a JSON status page or proxy error can never + become "the clip"). + +## Consequences + +- videogen grows from two surfaces (Model, Interpolator) to six; the + chain client is the package's first async surface — the job-API shape + deferred in ADR-0019/0021 now exists where the workload actually + demands it. +- The wire shapes are pinned by the netherstorm videoutils/mediautils/ + SadTalker image builds; host smoke tests are the drift defence. +- mort's long-video tool owns the poll loop, timeout budget, and + partial-result envelope; majordomo only guarantees the artifacts stay + fetchable. diff --git a/provider/llamaswap/lipsync.go b/provider/llamaswap/lipsync.go new file mode 100644 index 0000000..544be0b --- /dev/null +++ b/provider/llamaswap/lipsync.go @@ -0,0 +1,113 @@ +// lipsync.go implements videogen.LipsyncProvider against a SadTalker shim +// reached through llama-swap's /upstream passthrough (ADR-0025): +// +// POST /upstream//v1/talking_head multipart image,audio[,still,enhance,preprocess] +// +// The response body IS the encoded clip (same contract as /v1/videos/sync), +// hence the video-sized response cap and the same MIME resolution rules. +// Generation is sync and slow (minutes) — bound calls with a context +// deadline (Hunyuan precedent). +package llamaswap + +import ( + "bytes" + "context" + "fmt" + "mime/multipart" + "net/http" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" +) + +// LipsyncModel implements videogen.LipsyncProvider. The id selects which +// upstream llama-swap loads. +func (p *Provider) LipsyncModel(id string, opts ...videogen.LipsyncModelOption) (videogen.LipSyncer, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = videogen.ApplyLipsyncModelOptions(opts) + return &lipsyncModel{p: p, id: id}, nil +} + +type lipsyncModel struct { + p *Provider + id string +} + +// Lipsync implements videogen.LipSyncer. +func (m *lipsyncModel) Lipsync(ctx context.Context, req videogen.LipsyncRequest, opts ...videogen.LipsyncOption) (*videogen.Result, error) { + req = req.Apply(opts...) + if len(req.Image.Data) == 0 { + return nil, fmt.Errorf("%w: lipsync requires a portrait image", llm.ErrUnsupported) + } + if len(req.Audio) == 0 { + return nil, fmt.Errorf("%w: lipsync requires audio bytes", llm.ErrUnsupported) + } + if req.Preprocess != "" && req.Preprocess != "crop" && req.Preprocess != "full" { + return nil, fmt.Errorf("%w: lipsync preprocess must be \"crop\" or \"full\", got %q", llm.ErrUnsupported, req.Preprocess) + } + path, err := upstreamPath(m.id, "/v1/talking_head") + if err != nil { + return nil, err + } + + // Two file parts — buildMultipart handles exactly one, so assemble by + // hand (mirrors videoModel.Generate). + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + fw, err := w.CreateFormFile("image", initImageFilename(req.Image.MIME)) + if err != nil { + return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) + } + if _, err := fw.Write(req.Image.Data); err != nil { + return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) + } + fw, err = w.CreateFormFile("audio", transcriptionFilename(req.AudioFilename, req.AudioMIME)) + if err != nil { + return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) + } + if _, err := fw.Write(req.Audio); err != nil { + return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) + } + still := "" + if req.Still { + still = "true" + } + enhance := "" + if req.Enhance { + enhance = "true" + } + if err := writeFormFields(w, "build lipsync form", []formField{ + {"still", still, false}, + {"enhance", enhance, false}, + {"preprocess", req.Preprocess, false}, + }); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) + } + + raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes) + if err != nil { + return nil, err + } + return singleVideoResult(m.p.name, m.id, "lipsync", raw, respType) +} + +// singleVideoResult wraps one raw video body into a videogen.Result, +// requiring positive evidence of video-ness (declared video/* Content-Type +// or sniffed mp4/webm magic) so a proxy error page can never become "the +// clip" — the video sibling of singleImageResult. +func singleVideoResult(provider, model, verb string, raw []byte, contentType string) (*videogen.Result, error) { + if len(raw) == 0 { + return nil, &llm.APIError{Provider: provider, Model: model, Message: verb + " response contained no video"} + } + mimeType := videoMIME(contentType, raw) + if mimeType == "" { + return nil, &llm.APIError{Provider: provider, Model: model, + Message: fmt.Sprintf("%s response is not a video (Content-Type %q)", verb, contentType)} + } + return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil +} diff --git a/provider/llamaswap/lipsync_videoutil_test.go b/provider/llamaswap/lipsync_videoutil_test.go new file mode 100644 index 0000000..3601406 --- /dev/null +++ b/provider/llamaswap/lipsync_videoutil_test.go @@ -0,0 +1,295 @@ +package llamaswap + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "testing" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" +) + +// webmFixture is a minimal EBML header so http.DetectContentType sniffs +// video/webm. +func webmFixture() []byte { + return append([]byte{0x1A, 0x45, 0xDF, 0xA3}, make([]byte, 20)...) +} + +func TestLipsync(t *testing.T) { + png := pngFixture(t) + var gotPath, gotStill, gotEnhance, gotPreprocess string + var gotImage, gotAudio []byte + var gotImageName, gotAudioName string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotStill = r.FormValue("still") + gotEnhance = r.FormValue("enhance") + gotPreprocess = r.FormValue("preprocess") + if f, hdr, err := r.FormFile("image"); err == nil { + gotImage, _ = io.ReadAll(f) + gotImageName = hdr.Filename + f.Close() + } else { + t.Errorf("image part: %v", err) + } + if f, hdr, err := r.FormFile("audio"); err == nil { + gotAudio, _ = io.ReadAll(f) + gotAudioName = hdr.Filename + f.Close() + } else { + t.Errorf("audio part: %v", err) + } + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(mp4Fixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ls, err := p.LipsyncModel("lipsync-sadtalker") + if err != nil { + t.Fatalf("LipsyncModel: %v", err) + } + res, err := ls.Lipsync(context.Background(), + videogen.LipsyncRequest{ + Image: videogen.Image{MIME: "image/png", Data: png}, + Audio: []byte("SPEECH"), + AudioMIME: "audio/wav", + }, + videogen.WithLipsyncStill(), videogen.WithLipsyncEnhance(), videogen.WithLipsyncPreprocess("full")) + if err != nil { + t.Fatalf("Lipsync: %v", err) + } + if gotPath != "/upstream/lipsync-sadtalker/v1/talking_head" { + t.Errorf("path = %q", gotPath) + } + if gotStill != "true" || gotEnhance != "true" || gotPreprocess != "full" { + t.Errorf("still/enhance/preprocess = %q/%q/%q", gotStill, gotEnhance, gotPreprocess) + } + if string(gotImage) != string(png) || gotImageName != "frame.png" { + t.Errorf("image bytes/name = %d bytes/%q", len(gotImage), gotImageName) + } + if string(gotAudio) != "SPEECH" || gotAudioName != "audio.wav" { + t.Errorf("audio = %q name = %q", gotAudio, gotAudioName) + } + if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 { + t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data)) + } +} + +func TestLipsyncOmitsUnsetFlags(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + for _, k := range []string{"still", "enhance", "preprocess"} { + if v, ok := r.MultipartForm.Value[k]; ok { + t.Errorf("unset request sent %q = %v, want omitted", k, v) + } + } + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(mp4Fixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ls, _ := p.LipsyncModel("lipsync-sadtalker") + if _, err := ls.Lipsync(context.Background(), + videogen.LipsyncRequest{Image: videogen.Image{Data: pngFixture(t)}, Audio: []byte("A")}); err != nil { + t.Fatalf("Lipsync: %v", err) + } +} + +func TestLipsyncRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + ls, _ := p.LipsyncModel("lipsync-sadtalker") + if _, err := ls.Lipsync(context.Background(), + videogen.LipsyncRequest{Audio: []byte("A")}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no image: err = %v, want ErrUnsupported", err) + } + if _, err := ls.Lipsync(context.Background(), + videogen.LipsyncRequest{Image: videogen.Image{Data: []byte{1}}}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no audio: err = %v, want ErrUnsupported", err) + } + if _, err := ls.Lipsync(context.Background(), + videogen.LipsyncRequest{Image: videogen.Image{Data: []byte{1}}, Audio: []byte{1}, Preprocess: "zoom"}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("preprocess zoom: err = %v, want ErrUnsupported", err) + } +} + +func TestLipsyncRejectsNonVideoResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("proxy error page")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ls, _ := p.LipsyncModel("lipsync-sadtalker") + _, err := ls.Lipsync(context.Background(), + videogen.LipsyncRequest{Image: videogen.Image{Data: pngFixture(t)}, Audio: []byte("A")}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for non-video body", err) + } +} + +func TestRemoveVideoBackground(t *testing.T) { + var gotPath, gotOutput, gotFilename string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotOutput = r.FormValue("output") + if _, hdr, err := r.FormFile("file"); err == nil { + gotFilename = hdr.Filename + } else { + t.Errorf("file part: %v", err) + } + w.Header().Set("Content-Type", "video/webm") + _, _ = w.Write(webmFixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + vb, err := p.VideoBackgroundRemoverModel("mediautils") + if err != nil { + t.Fatalf("VideoBackgroundRemoverModel: %v", err) + } + res, err := vb.RemoveVideoBackground(context.Background(), + videogen.VideoBackgroundRemovalRequest{Video: mp4Fixture(), MIME: "video/mp4"}, + videogen.WithVideoBackgroundOutput("alpha_webm")) + if err != nil { + t.Fatalf("RemoveVideoBackground: %v", err) + } + if gotPath != "/upstream/mediautils/v1/video/matte" { + t.Errorf("path = %q", gotPath) + } + if gotOutput != "alpha_webm" || gotFilename != "video.mp4" { + t.Errorf("output/filename = %q/%q", gotOutput, gotFilename) + } + if res.Video.MIME != "video/webm" || len(res.Video.Data) == 0 { + t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data)) + } +} + +func TestRemoveVideoBackgroundOmitsDefaultOutput(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + if v, ok := r.MultipartForm.Value["output"]; ok { + t.Errorf("output field sent for default: %v; want omitted", v) + } + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(mp4Fixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + vb, _ := p.VideoBackgroundRemoverModel("mediautils") + if _, err := vb.RemoveVideoBackground(context.Background(), + videogen.VideoBackgroundRemovalRequest{Video: mp4Fixture()}); err != nil { + t.Fatalf("RemoveVideoBackground: %v", err) + } +} + +func TestRemoveVideoBackgroundRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + vb, _ := p.VideoBackgroundRemoverModel("mediautils") + if _, err := vb.RemoveVideoBackground(context.Background(), + videogen.VideoBackgroundRemovalRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no video: err = %v, want ErrUnsupported", err) + } + if _, err := vb.RemoveVideoBackground(context.Background(), + videogen.VideoBackgroundRemovalRequest{Video: []byte{1}, Output: "gif"}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("output gif: err = %v, want ErrUnsupported", err) + } +} + +func TestUpscaleVideo(t *testing.T) { + var gotPath, gotScale string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotScale = r.FormValue("scale") + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(mp4Fixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + vu, err := p.VideoUpscalerModel("mediautils") + if err != nil { + t.Fatalf("VideoUpscalerModel: %v", err) + } + res, err := vu.UpscaleVideo(context.Background(), + videogen.VideoUpscaleRequest{Video: mp4Fixture(), MIME: "video/mp4"}, + videogen.WithVideoUpscaleScale(2)) + if err != nil { + t.Fatalf("UpscaleVideo: %v", err) + } + if gotPath != "/upstream/mediautils/v1/video/upscale" { + t.Errorf("path = %q", gotPath) + } + if gotScale != "2" { + t.Errorf("scale = %q", gotScale) + } + if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 { + t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data)) + } +} + +func TestUpscaleVideoRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + vu, _ := p.VideoUpscalerModel("mediautils") + if _, err := vu.UpscaleVideo(context.Background(), videogen.VideoUpscaleRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no video: err = %v, want ErrUnsupported", err) + } + if _, err := vu.UpscaleVideo(context.Background(), + videogen.VideoUpscaleRequest{Video: []byte{1}, Scale: 3}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("scale 3: err = %v, want ErrUnsupported", err) + } +} + +func TestUpscaleVideoRejectsNonVideoResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header()["Content-Type"] = nil // NO Content-Type at all + _, _ = w.Write([]byte("502 bad gateway")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + vu, _ := p.VideoUpscalerModel("mediautils") + _, err := vu.UpscaleVideo(context.Background(), videogen.VideoUpscaleRequest{Video: mp4Fixture()}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for headerless non-video body", err) + } +} + +func TestVideoInputFilename(t *testing.T) { + cases := []struct { + filename, mime, want string + }{ + {"clip.mp4", "", "clip.mp4"}, + {"evil\r\nclip.mp4", "", "evilclip.mp4"}, + {"", "video/mp4", "video.mp4"}, + {"", "video/webm", "video.webm"}, + {"", "video/quicktime", "video.mov"}, + {"", "", "video"}, + } + for _, tc := range cases { + if got := videoInputFilename(tc.filename, tc.mime); got != tc.want { + t.Errorf("videoInputFilename(%q, %q) = %q, want %q", tc.filename, tc.mime, got, tc.want) + } + } +} diff --git a/provider/llamaswap/videochain.go b/provider/llamaswap/videochain.go new file mode 100644 index 0000000..1e70e60 --- /dev/null +++ b/provider/llamaswap/videochain.go @@ -0,0 +1,204 @@ +// videochain.go implements videogen.ChainerProvider against the videoutils +// chain orchestrator reached through llama-swap's /upstream passthrough +// (ADR-0025): +// +// POST /upstream//v1/video/chain JSON submit -> {job_id} +// GET /upstream//v1/jobs/{id} -> {status,segment,total,segments} +// GET /upstream//v1/jobs/{id}/result -> encoded clip +// GET /upstream//v1/jobs/{id}/segments/{n} -> encoded clip +// +// Unlike the ACE-Step music path this client does NOT hide the job queue +// behind a blocking call: a chain runs through multiple GPU swaps for many +// minutes, and the caller (mort's long-video tool) owns the poll loop so it +// can deliver PARTIAL results — completed segments survive a mid-chain +// failure and stay fetchable via ChainSegmentResult. +package llamaswap + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" +) + +// ChainerModel implements videogen.ChainerProvider. The id selects which +// upstream llama-swap loads (the videoutils orchestrator, which in turn +// drives the generation model through llama-swap itself). +func (p *Provider) ChainerModel(id string, opts ...videogen.ChainerModelOption) (videogen.Chainer, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = videogen.ApplyChainerModelOptions(opts) + return &chainerModel{p: p, id: id}, nil +} + +type chainerModel struct { + p *Provider + id string +} + +// chainSubmitRequest is the videoutils POST /v1/video/chain shape. The init +// image rides as base64 in the JSON body (`init_image_b64`) — the submit is +// JSON, not multipart, per the pinned host contract. +type chainSubmitRequest struct { + Segments []chainSegmentWire `json:"segments"` + InitImageB64 string `json:"init_image_b64,omitempty"` + SmoothJoins bool `json:"smooth_joins,omitempty"` + Size string `json:"size,omitempty"` +} + +type chainSegmentWire struct { + Prompt string `json:"prompt"` + Seconds float64 `json:"seconds,omitempty"` +} + +// SubmitChain implements videogen.Chainer. +func (m *chainerModel) SubmitChain(ctx context.Context, req videogen.ChainRequest) (string, error) { + if len(req.Segments) == 0 { + return "", fmt.Errorf("%w: video chain requires at least one segment", llm.ErrUnsupported) + } + wire := chainSubmitRequest{ + SmoothJoins: req.SmoothJoins, + Size: strings.TrimSpace(req.Size), + } + for i, seg := range req.Segments { + if strings.TrimSpace(seg.Prompt) == "" { + return "", fmt.Errorf("%w: video chain segment %d requires a prompt", llm.ErrUnsupported, i) + } + if seg.Seconds < 0 { + return "", fmt.Errorf("%w: video chain segment %d seconds must be >= 0, got %g", llm.ErrUnsupported, i, seg.Seconds) + } + wire.Segments = append(wire.Segments, chainSegmentWire{Prompt: seg.Prompt, Seconds: seg.Seconds}) + } + if len(req.InitImage) > 0 { + wire.InitImageB64 = base64.StdEncoding.EncodeToString(req.InitImage) + } + path, err := upstreamPath(m.id, "/v1/video/chain") + if err != nil { + return "", err + } + // Tolerant envelope: {"job_id": ...} per the contract, with data-wrapped + // and bare-id fallbacks (musicgen release_task precedent). + var resp struct { + JobID string `json:"job_id"` + ID string `json:"id"` + Data struct { + JobID string `json:"job_id"` + ID string `json:"id"` + } `json:"data"` + } + if err := m.p.doJSON(ctx, http.MethodPost, path, m.id, &wire, &resp); err != nil { + return "", err + } + for _, id := range []string{resp.JobID, resp.Data.JobID, resp.ID, resp.Data.ID} { + if id != "" { + return id, nil + } + } + return "", &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video chain submit returned no job_id"} +} + +// chainJobResponse is the GET /v1/jobs/{id} shape. `segments` entries are +// tolerated as bare strings or objects keyed by id/segment_id. +type chainJobResponse struct { + Status string `json:"status"` + Segment int `json:"segment"` + Total int `json:"total"` + Segments []json.RawMessage `json:"segments"` +} + +// ChainStatus implements videogen.Chainer. +func (m *chainerModel) ChainStatus(ctx context.Context, jobID string) (*videogen.ChainJob, error) { + path, err := m.jobPath(jobID, "") + if err != nil { + return nil, err + } + var raw json.RawMessage + if err := m.p.doJSON(ctx, http.MethodGet, path, m.id, nil, &raw); err != nil { + return nil, err + } + var out chainJobResponse + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("llama-swap: decode chain job status: %w", err) + } + job := &videogen.ChainJob{ + Status: out.Status, + Segment: out.Segment, + Total: out.Total, + Raw: raw, + } + for _, entry := range out.Segments { + var s string + if json.Unmarshal(entry, &s) == nil { + job.SegmentIDs = append(job.SegmentIDs, s) + continue + } + var obj struct { + ID string `json:"id"` + SegmentID string `json:"segment_id"` + } + if json.Unmarshal(entry, &obj) == nil { + switch { + case obj.ID != "": + job.SegmentIDs = append(job.SegmentIDs, obj.ID) + case obj.SegmentID != "": + job.SegmentIDs = append(job.SegmentIDs, obj.SegmentID) + } + } + } + if job.Status == "" { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, + Message: "chain job status payload carried no status: " + truncateForError(raw)} + } + return job, nil +} + +// ChainResult implements videogen.Chainer. +func (m *chainerModel) ChainResult(ctx context.Context, jobID string) (*videogen.Result, error) { + path, err := m.jobPath(jobID, "/result") + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodGet, path, m.id, "", nil, maxVideoResponseBytes) + if err != nil { + return nil, err + } + return singleVideoResult(m.p.name, m.id, "video chain result", raw, respType) +} + +// ChainSegmentResult implements videogen.Chainer. +func (m *chainerModel) ChainSegmentResult(ctx context.Context, jobID string, n int) (*videogen.Result, error) { + if n < 0 { + return nil, fmt.Errorf("%w: chain segment index must be >= 0, got %d", llm.ErrUnsupported, n) + } + path, err := m.jobPath(jobID, "/segments/"+strconv.Itoa(n)) + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodGet, path, m.id, "", nil, maxVideoResponseBytes) + if err != nil { + return nil, err + } + return singleVideoResult(m.p.name, m.id, "video chain segment", raw, respType) +} + +// jobPath builds /upstream//v1/jobs/, refusing job ids +// that carry path structure. The id is SERVER-SUPPLIED (echoed back from +// SubmitChain), so like upstreamPath's rest-component checks this rejects +// rather than escapes — a hostile/buggy upstream must not be able to steer +// the follow-up request at another proxy endpoint. +func (m *chainerModel) jobPath(jobID, suffix string) (string, error) { + if strings.TrimSpace(jobID) == "" { + return "", fmt.Errorf("llama-swap: chain job call requires a job id") + } + if strings.ContainsAny(jobID, "/?#") || strings.Contains(jobID, "..") { + return "", fmt.Errorf("llama-swap: invalid chain job id %q (contains path structure)", jobID) + } + return upstreamPath(m.id, "/v1/jobs/"+jobID+suffix) +} diff --git a/provider/llamaswap/videochain_test.go b/provider/llamaswap/videochain_test.go new file mode 100644 index 0000000..d157bb6 --- /dev/null +++ b/provider/llamaswap/videochain_test.go @@ -0,0 +1,289 @@ +package llamaswap + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" +) + +func TestSubmitChain(t *testing.T) { + var gotPath string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"job_id":"job-123"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, err := p.ChainerModel("videoutils") + if err != nil { + t.Fatalf("ChainerModel: %v", err) + } + jobID, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{ + Segments: []videogen.ChainSegment{ + {Prompt: "a cat walks in", Seconds: 5}, + {Prompt: "the cat sits down", Seconds: 5}, + }, + InitImage: []byte("IMG"), + SmoothJoins: true, + Size: "1280x704", + }) + if err != nil { + t.Fatalf("SubmitChain: %v", err) + } + if jobID != "job-123" { + t.Errorf("jobID = %q", jobID) + } + if gotPath != "/upstream/videoutils/v1/video/chain" { + t.Errorf("path = %q", gotPath) + } + segments, _ := gotBody["segments"].([]any) + if len(segments) != 2 { + t.Fatalf("segments = %v", gotBody["segments"]) + } + first, _ := segments[0].(map[string]any) + if first["prompt"] != "a cat walks in" || first["seconds"] != 5.0 { + t.Errorf("segment[0] = %v", first) + } + if gotBody["init_image_b64"] != base64.StdEncoding.EncodeToString([]byte("IMG")) { + t.Errorf("init_image_b64 = %v", gotBody["init_image_b64"]) + } + if gotBody["smooth_joins"] != true || gotBody["size"] != "1280x704" { + t.Errorf("smooth_joins/size = %v/%v", gotBody["smooth_joins"], gotBody["size"]) + } +} + +func TestSubmitChainOmitsUnsetFields(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"data":{"job_id":"job-9"}}`)) // data-wrapped envelope + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + jobID, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{ + Segments: []videogen.ChainSegment{{Prompt: "a dog"}}, + }) + if err != nil { + t.Fatalf("SubmitChain: %v", err) + } + if jobID != "job-9" { + t.Errorf("jobID = %q, want data-wrapped id", jobID) + } + for _, k := range []string{"init_image_b64", "smooth_joins", "size"} { + if v, ok := gotBody[k]; ok { + t.Errorf("unset request sent %q = %v, want omitted", k, v) + } + } + seg, _ := gotBody["segments"].([]any) + if first, _ := seg[0].(map[string]any); first == nil { + t.Fatalf("segments = %v", gotBody["segments"]) + } else if _, ok := first["seconds"]; ok { + t.Error("zero seconds sent; want omitted") + } +} + +func TestSubmitChainRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + ch, _ := p.ChainerModel("videoutils") + if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no segments: err = %v, want ErrUnsupported", err) + } + if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{ + Segments: []videogen.ChainSegment{{Prompt: " "}}, + }); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("empty prompt: err = %v, want ErrUnsupported", err) + } + if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{ + Segments: []videogen.ChainSegment{{Prompt: "x", Seconds: -1}}, + }); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("negative seconds: err = %v, want ErrUnsupported", err) + } +} + +func TestSubmitChainRejectsMissingJobID(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{ + Segments: []videogen.ChainSegment{{Prompt: "x"}}, + }) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for missing job_id", err) + } +} + +func TestChainStatus(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _, _ = w.Write([]byte(`{"status":"running","segment":2,"total":3,"segments":["seg-0"]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + job, err := ch.ChainStatus(context.Background(), "job-123") + if err != nil { + t.Fatalf("ChainStatus: %v", err) + } + if gotPath != "/upstream/videoutils/v1/jobs/job-123" { + t.Errorf("path = %q", gotPath) + } + if job.Status != "running" || job.Segment != 2 || job.Total != 3 { + t.Errorf("job = %+v", job) + } + if !reflect.DeepEqual(job.SegmentIDs, []string{"seg-0"}) { + t.Errorf("SegmentIDs = %v", job.SegmentIDs) + } + if job.Raw == nil { + t.Error("Raw = nil, want raw payload") + } +} + +func TestChainStatusToleratesObjectSegments(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"status":"done","segment":2,"total":2,"segments":[{"id":"seg-0"},{"segment_id":"seg-1"}]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + job, err := ch.ChainStatus(context.Background(), "job-123") + if err != nil { + t.Fatalf("ChainStatus: %v", err) + } + if !reflect.DeepEqual(job.SegmentIDs, []string{"seg-0", "seg-1"}) { + t.Errorf("SegmentIDs = %v", job.SegmentIDs) + } +} + +func TestChainStatusRejectsStatuslessPayload(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"detail":"no such job"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + _, err := ch.ChainStatus(context.Background(), "job-123") + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for statusless payload", err) + } +} + +func TestChainJobPathRejectsHostileIDs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + ch, _ := p.ChainerModel("videoutils") + for _, bad := range []string{"", "a/b", "a?b", "a#b", "..", "a..b"} { + if _, err := ch.ChainStatus(context.Background(), bad); err == nil { + t.Errorf("ChainStatus(%q) succeeded; want error", bad) + } + if _, err := ch.ChainResult(context.Background(), bad); err == nil { + t.Errorf("ChainResult(%q) succeeded; want error", bad) + } + } +} + +func TestChainResult(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(mp4Fixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + res, err := ch.ChainResult(context.Background(), "job-123") + if err != nil { + t.Fatalf("ChainResult: %v", err) + } + if gotPath != "/upstream/videoutils/v1/jobs/job-123/result" { + t.Errorf("path = %q", gotPath) + } + if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 { + t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data)) + } +} + +func TestChainSegmentResult(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(mp4Fixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + res, err := ch.ChainSegmentResult(context.Background(), "job-123", 1) + if err != nil { + t.Fatalf("ChainSegmentResult: %v", err) + } + if gotPath != "/upstream/videoutils/v1/jobs/job-123/segments/1" { + t.Errorf("path = %q", gotPath) + } + if res.Video.MIME != "video/mp4" { + t.Fatalf("video = %q", res.Video.MIME) + } + + if _, err := ch.ChainSegmentResult(context.Background(), "job-123", -1); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("negative segment: err = %v, want ErrUnsupported", err) + } +} + +func TestChainResultRejectsNonVideoResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"running"}`)) // a status page, not the clip + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + _, err := ch.ChainResult(context.Background(), "job-123") + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for non-video body", err) + } +} + +func TestChainSurfacesAPIError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"message":"job not found"}}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ch, _ := p.ChainerModel("videoutils") + _, err := ch.ChainStatus(context.Background(), "job-void") + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %T %v, want *llm.APIError", err, err) + } + if apiErr.Status != http.StatusNotFound || apiErr.Message != "job not found" || apiErr.Model != "videoutils" { + t.Errorf("apiErr = %+v", apiErr) + } +} diff --git a/provider/llamaswap/videoutil.go b/provider/llamaswap/videoutil.go new file mode 100644 index 0000000..e29a524 --- /dev/null +++ b/provider/llamaswap/videoutil.go @@ -0,0 +1,139 @@ +// videoutil.go implements the videogen.VideoBackgroundRemovalProvider and +// videogen.VideoUpscaleProvider surfaces against the mediautils shim reached +// through llama-swap's /upstream passthrough (ADR-0025): +// +// matte POST /upstream//v1/video/matte (Robust Video Matting) +// upscale POST /upstream//v1/video/upscale (per-frame Real-ESRGAN) +// +// Both are one-file multipart in, encoded clip out — the video siblings of +// mediautil.go's still-image surfaces. +package llamaswap + +import ( + "context" + "fmt" + "mime" + "net/http" + "strconv" + "strings" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" +) + +// --- video background removal (matting) --- + +// VideoBackgroundRemoverModel implements +// videogen.VideoBackgroundRemovalProvider against the mediautils shim's +// POST /v1/video/matte. The id selects which upstream llama-swap loads. +func (p *Provider) VideoBackgroundRemoverModel(id string, opts ...videogen.VideoBackgroundRemoverModelOption) (videogen.VideoBackgroundRemover, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = videogen.ApplyVideoBackgroundRemoverModelOptions(opts) + return &videoMatteModel{p: p, id: id}, nil +} + +type videoMatteModel struct { + p *Provider + id string +} + +// RemoveVideoBackground implements videogen.VideoBackgroundRemover. +func (m *videoMatteModel) RemoveVideoBackground(ctx context.Context, req videogen.VideoBackgroundRemovalRequest, opts ...videogen.VideoBackgroundRemovalOption) (*videogen.Result, error) { + req = req.Apply(opts...) + if len(req.Video) == 0 { + return nil, fmt.Errorf("%w: video background removal requires a video", llm.ErrUnsupported) + } + if req.Output != "" && req.Output != "greenscreen_mp4" && req.Output != "alpha_webm" { + return nil, fmt.Errorf("%w: video matte output must be \"greenscreen_mp4\" or \"alpha_webm\", got %q", llm.ErrUnsupported, req.Output) + } + path, err := upstreamPath(m.id, "/v1/video/matte") + if err != nil { + return nil, err + } + body, contentType, err := buildMultipart("build video-matte form", + filePart{field: "file", filename: videoInputFilename(req.Filename, req.MIME), data: req.Video}, + []formField{{"output", req.Output, false}}) + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes) + if err != nil { + return nil, err + } + return singleVideoResult(m.p.name, m.id, "video matte", raw, respType) +} + +// --- video upscale --- + +// VideoUpscalerModel implements videogen.VideoUpscaleProvider against the +// mediautils shim's POST /v1/video/upscale. +func (p *Provider) VideoUpscalerModel(id string, opts ...videogen.VideoUpscalerModelOption) (videogen.VideoUpscaler, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = videogen.ApplyVideoUpscalerModelOptions(opts) + return &videoUpscaleModel{p: p, id: id}, nil +} + +type videoUpscaleModel struct { + p *Provider + id string +} + +// UpscaleVideo implements videogen.VideoUpscaler. +func (m *videoUpscaleModel) UpscaleVideo(ctx context.Context, req videogen.VideoUpscaleRequest, opts ...videogen.VideoUpscaleOption) (*videogen.Result, error) { + req = req.Apply(opts...) + if len(req.Video) == 0 { + return nil, fmt.Errorf("%w: video upscale requires a video", llm.ErrUnsupported) + } + if req.Scale != 0 && req.Scale != 2 && req.Scale != 4 { + return nil, fmt.Errorf("%w: video upscale scale must be 2 or 4, got %d", llm.ErrUnsupported, req.Scale) + } + path, err := upstreamPath(m.id, "/v1/video/upscale") + if err != nil { + return nil, err + } + scale := "" + if req.Scale != 0 { + scale = strconv.Itoa(req.Scale) + } + body, contentType, err := buildMultipart("build video-upscale form", + filePart{field: "file", filename: videoInputFilename(req.Filename, req.MIME), data: req.Video}, + []formField{{"scale", scale, false}}) + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes) + if err != nil { + return nil, err + } + return singleVideoResult(m.p.name, m.id, "video upscale", raw, respType) +} + +// videoInputFilename picks the multipart filename hint for a caller-supplied +// video: the caller's (sanitized — upload metadata is untrusted), else one +// derived from the MIME subtype ("video.mp4"), else "video". Mirrors +// transcriptionFilename. +func videoInputFilename(filename, mimeType string) string { + if name := sanitizeFilename(filename); name != "" { + return name + } + mt := strings.ToLower(strings.TrimSpace(mimeType)) + if parsed, _, err := mime.ParseMediaType(mt); err == nil { + mt = parsed + } + switch mt { + case "video/mp4": + return "video.mp4" + case "video/webm": + return "video.webm" + case "video/quicktime": + return "video.mov" + case "video/x-matroska": + return "video.mkv" + default: + return "video" + } +} diff --git a/videogen/chain.go b/videogen/chain.go new file mode 100644 index 0000000..5903383 --- /dev/null +++ b/videogen/chain.go @@ -0,0 +1,99 @@ +package videogen + +import "context" + +// ChainSegment is one prompt in a multi-segment ("long video") chain. +type ChainSegment struct { + // Prompt describes this segment. Required. + Prompt string + + // Seconds is the segment's requested length; 0 = backend default. + Seconds float64 +} + +// ChainRequest asks a chain orchestrator to generate a long video as a +// sequence of segments, each continuing from the previous segment's last +// frame. Zero values mean "backend default" (ADR-0025). +type ChainRequest struct { + // Segments are the per-segment prompts in order. At least one required. + Segments []ChainSegment + + // InitImage optionally conditions the FIRST segment on a starting frame + // (image-to-video); nil = pure text-to-video. + InitImage []byte + + // SmoothJoins asks the orchestrator to interpolate across segment + // boundaries (RIFE-style) so cuts don't pop. + SmoothJoins bool + + // Size is the requested resolution, e.g. "1280x704"; "" = backend + // default. + Size string +} + +// ChainJob is a chain job's progress snapshot. +type ChainJob struct { + // Status is the backend's job state, passed through verbatim + // (e.g. "queued", "running", "done", "failed"). + Status string + + // Segment is the segment currently being generated (1-based); Total is + // the segment count. + Segment int + Total int + + // SegmentIDs are the COMPLETED per-segment artifacts, fetchable via + // ChainSegmentResult — a mid-chain failure still leaves these + // retrievable, so multi-minute GPU output is never discarded. + SegmentIDs []string + + // Raw is the provider-native job payload. May be nil. + Raw any +} + +// Chainer drives a multi-segment video-chain job. Unlike Model.Generate it +// is deliberately ASYNC — a chain runs through multiple GPU loads for many +// minutes, so callers submit, poll, and fetch instead of holding one +// blocking call open. +type Chainer interface { + // SubmitChain starts a chain job and returns its job id. + SubmitChain(ctx context.Context, req ChainRequest) (string, error) + + // ChainStatus reports the job's progress. Polling also signals liveness + // to backends that unload idle orchestrators. + ChainStatus(ctx context.Context, jobID string) (*ChainJob, error) + + // ChainResult fetches the finished, concatenated clip. + ChainResult(ctx context.Context, jobID string) (*Result, error) + + // ChainSegmentResult fetches one completed segment's clip (n indexes the + // job's segment list) — the partial-delivery path when a chain dies + // mid-run. + ChainSegmentResult(ctx context.Context, jobID string, n int) (*Result, error) +} + +// ChainerModelOption configures a Chainer at construction time. Reserved for +// future per-model settings. +type ChainerModelOption func(*ChainerModelConfig) + +// ChainerModelConfig carries per-model construction settings. +type ChainerModelConfig struct{} + +// ApplyChainerModelOptions folds options into a config. +func ApplyChainerModelOptions(opts []ChainerModelOption) ChainerModelConfig { + var cfg ChainerModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// ChainerProvider mints Chainers bound to one backend. +type ChainerProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // ChainerModel returns a Chainer bound to the given id (passed through + // to the backend verbatim; no catalog validation). + ChainerModel(id string, opts ...ChainerModelOption) (Chainer, error) +} diff --git a/videogen/lipsync.go b/videogen/lipsync.go new file mode 100644 index 0000000..2548b9d --- /dev/null +++ b/videogen/lipsync.go @@ -0,0 +1,93 @@ +package videogen + +import "context" + +// LipsyncRequest asks a talking-head backend (SadTalker style) to animate a +// still portrait so it speaks the given audio. Zero values mean "backend +// default" (ADR-0025). +type LipsyncRequest struct { + // Image is the portrait to animate. Required. + Image Image + + // Audio is the encoded speech the head lip-syncs to. Required. Carried + // as bytes (never a URL), mirroring audio.TranscriptionRequest. + Audio []byte + + // AudioMIME is the audio MIME type (e.g. "audio/wav"); "" = let the + // backend sniff it. + AudioMIME string + + // AudioFilename is the multipart filename hint some backends key their + // format detection on; "" derives one from AudioMIME or falls back to + // "audio". + AudioFilename string + + // Still reduces head motion to blinks and lip movement (less uncanny on + // formal portraits); false = backend default motion. + Still bool + + // Enhance runs the backend's face enhancer over the output frames. + Enhance bool + + // Preprocess selects how the backend frames the face: "crop" (animate + // the face crop) or "full" (paste the animated face back into the whole + // image); "" = backend default. + Preprocess string +} + +// LipsyncOption mutates a LipsyncRequest before it is sent. +type LipsyncOption func(*LipsyncRequest) + +// WithLipsyncStill reduces head motion to blinks and lip movement. +func WithLipsyncStill() LipsyncOption { return func(r *LipsyncRequest) { r.Still = true } } + +// WithLipsyncEnhance runs the backend's face enhancer over the output. +func WithLipsyncEnhance() LipsyncOption { return func(r *LipsyncRequest) { r.Enhance = true } } + +// WithLipsyncPreprocess selects the face framing ("crop" or "full"). +func WithLipsyncPreprocess(p string) LipsyncOption { + return func(r *LipsyncRequest) { r.Preprocess = p } +} + +// Apply returns a copy of the request with all options applied. +func (r LipsyncRequest) Apply(opts ...LipsyncOption) LipsyncRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// LipSyncer animates still portraits into talking-head clips. Its own small +// interface rather than a method on Model: lip-syncers are not text-to-video +// generators — they bind to a different backend id entirely. +type LipSyncer interface { + // Lipsync returns the talking-head clip. Generation is slow (minutes); + // bound the call with a context deadline. + Lipsync(ctx context.Context, req LipsyncRequest, opts ...LipsyncOption) (*Result, error) +} + +// LipsyncModelOption configures a LipSyncer at construction time. Reserved +// for future per-model settings. +type LipsyncModelOption func(*LipsyncModelConfig) + +// LipsyncModelConfig carries per-model construction settings. +type LipsyncModelConfig struct{} + +// ApplyLipsyncModelOptions folds options into a config. +func ApplyLipsyncModelOptions(opts []LipsyncModelOption) LipsyncModelConfig { + var cfg LipsyncModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// LipsyncProvider mints LipSyncers bound to one backend. +type LipsyncProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // LipsyncModel returns a LipSyncer bound to the given id (passed through + // to the backend verbatim; no catalog validation). + LipsyncModel(id string, opts ...LipsyncModelOption) (LipSyncer, error) +} diff --git a/videogen/matte.go b/videogen/matte.go new file mode 100644 index 0000000..4943056 --- /dev/null +++ b/videogen/matte.go @@ -0,0 +1,80 @@ +package videogen + +import "context" + +// VideoBackgroundRemovalRequest asks a video-matting backend (Robust Video +// Matting style) to separate the foreground subject from the background of a +// clip. Zero values mean "backend default" (ADR-0025). +type VideoBackgroundRemovalRequest struct { + // Video is the encoded clip to matte. Required. Carried as bytes (never + // a URL). + Video []byte + + // MIME is the video MIME type (e.g. "video/mp4"); "" = let the backend + // sniff it. + MIME string + + // Filename is the multipart filename hint some backends key their format + // detection on; "" derives one from MIME ("video.mp4") or falls back to + // "video". + Filename string + + // Output selects the delivery container: "greenscreen_mp4" (subject over + // solid green, universally playable) or "alpha_webm" (true transparency, + // VP9 alpha channel); "" = backend default. + Output string +} + +// VideoBackgroundRemovalOption mutates a VideoBackgroundRemovalRequest +// before it is sent. +type VideoBackgroundRemovalOption func(*VideoBackgroundRemovalRequest) + +// WithVideoBackgroundOutput selects the delivery container +// ("greenscreen_mp4" or "alpha_webm"). +func WithVideoBackgroundOutput(o string) VideoBackgroundRemovalOption { + return func(r *VideoBackgroundRemovalRequest) { r.Output = o } +} + +// Apply returns a copy of the request with all options applied. +func (r VideoBackgroundRemovalRequest) Apply(opts ...VideoBackgroundRemovalOption) VideoBackgroundRemovalRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// VideoBackgroundRemover mattes the subject out of video clips — the moving- +// picture sibling of imagegen.BackgroundRemover. +type VideoBackgroundRemover interface { + // RemoveVideoBackground returns the matted clip. Matting is slow + // (per-frame inference); bound the call with a context deadline. + RemoveVideoBackground(ctx context.Context, req VideoBackgroundRemovalRequest, opts ...VideoBackgroundRemovalOption) (*Result, error) +} + +// VideoBackgroundRemoverModelOption configures a VideoBackgroundRemover at +// construction time. Reserved for future per-model settings. +type VideoBackgroundRemoverModelOption func(*VideoBackgroundRemoverModelConfig) + +// VideoBackgroundRemoverModelConfig carries per-model construction settings. +type VideoBackgroundRemoverModelConfig struct{} + +// ApplyVideoBackgroundRemoverModelOptions folds options into a config. +func ApplyVideoBackgroundRemoverModelOptions(opts []VideoBackgroundRemoverModelOption) VideoBackgroundRemoverModelConfig { + var cfg VideoBackgroundRemoverModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// VideoBackgroundRemovalProvider mints VideoBackgroundRemovers bound to one +// backend. +type VideoBackgroundRemovalProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // VideoBackgroundRemoverModel returns a VideoBackgroundRemover bound to + // the given id (passed through to the backend verbatim; no catalog + // validation). + VideoBackgroundRemoverModel(id string, opts ...VideoBackgroundRemoverModelOption) (VideoBackgroundRemover, error) +} diff --git a/videogen/vupscale.go b/videogen/vupscale.go new file mode 100644 index 0000000..ae8976a --- /dev/null +++ b/videogen/vupscale.go @@ -0,0 +1,74 @@ +package videogen + +import "context" + +// VideoUpscaleRequest asks a super-resolution backend (per-frame Real-ESRGAN +// style) to enlarge a clip. Zero values mean "backend default" (ADR-0025). +type VideoUpscaleRequest struct { + // Video is the encoded clip to upscale. Required. Carried as bytes + // (never a URL). + Video []byte + + // MIME is the video MIME type (e.g. "video/mp4"); "" = let the backend + // sniff it. + MIME string + + // Filename is the multipart filename hint some backends key their format + // detection on; "" derives one from MIME ("video.mp4") or falls back to + // "video". + Filename string + + // Scale is the enlargement factor (2 or 4 on the reference backend); + // 0 = backend default. + Scale int +} + +// VideoUpscaleOption mutates a VideoUpscaleRequest before it is sent. +type VideoUpscaleOption func(*VideoUpscaleRequest) + +// WithVideoUpscaleScale sets the enlargement factor. +func WithVideoUpscaleScale(s int) VideoUpscaleOption { + return func(r *VideoUpscaleRequest) { r.Scale = s } +} + +// Apply returns a copy of the request with all options applied. +func (r VideoUpscaleRequest) Apply(opts ...VideoUpscaleOption) VideoUpscaleRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// VideoUpscaler enlarges video clips frame by frame — the moving-picture +// sibling of imagegen.Upscaler. +type VideoUpscaler interface { + // UpscaleVideo returns the enlarged clip. Upscaling is slow (per-frame + // inference); bound the call with a context deadline. + UpscaleVideo(ctx context.Context, req VideoUpscaleRequest, opts ...VideoUpscaleOption) (*Result, error) +} + +// VideoUpscalerModelOption configures a VideoUpscaler at construction time. +// Reserved for future per-model settings. +type VideoUpscalerModelOption func(*VideoUpscalerModelConfig) + +// VideoUpscalerModelConfig carries per-model construction settings. +type VideoUpscalerModelConfig struct{} + +// ApplyVideoUpscalerModelOptions folds options into a config. +func ApplyVideoUpscalerModelOptions(opts []VideoUpscalerModelOption) VideoUpscalerModelConfig { + var cfg VideoUpscalerModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// VideoUpscaleProvider mints VideoUpscalers bound to one backend. +type VideoUpscaleProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // VideoUpscalerModel returns a VideoUpscaler bound to the given id + // (passed through to the backend verbatim; no catalog validation). + VideoUpscalerModel(id string, opts ...VideoUpscalerModelOption) (VideoUpscaler, error) +}