diff --git a/README.md b/README.md index 85ff031..0ecaf6b 100644 --- a/README.md +++ b/README.md @@ -272,17 +272,30 @@ tr, err := tm.Transcribe(ctx, audio.TranscriptionRequest{ voices, err := ls.ListVoices(ctx, "kokoro") // []string of voice ids ``` -## Video: text-to-video + image-to-video +## Video: text-to-video, image-to-video, first-last-frame Video generation lives in the `videogen` package (ADR-0019), mirroring imagegen/audio: one small `Model` contract, zero values mean backend -defaults, bytes in/out. Text-to-video and image-to-video are one surface — -a nil `InitImage` is a pure text prompt; setting it conditions generation -on that frame (hybrid checkpoints like Wan 2.2 TI2V serve both). First -backend: llama-swap (blocking `/v1/videos/sync`, vLLM-Omni style — the +defaults, bytes in/out. All modes are one surface, selected by which +keyframes are set rather than by a mode flag: + +| `InitImage` | `LastImage` | mode | +|---|---|---| +| nil | nil | text-to-video | +| set | nil | image-to-video (hybrid checkpoints like Wan 2.2 TI2V serve both) | +| set | set | first-last-frame — both ends pinned | +| nil | set | pin the destination, model invents the approach | + +First backend: llama-swap (blocking `/v1/videos/sync`, vLLM-Omni style — the response body is the encoded clip, so `Result` carries a single `Video`). Generation runs for minutes; bound the call with a context deadline. +**`LastImage` support is per-model and cannot be detected.** A backend that +does not understand a trailing keyframe ignores the part and returns an +ordinary clip — indistinguishable from success. There is no capability bit, +because the contract has no way to learn one, so a caller depending on the +pin must establish support out of band. + ```go vm, _ := ls.VideoModel("videogen-wan22-5b") res, err := vm.Generate(ctx, videogen.Request{Prompt: "a cat surfing"}, diff --git a/provider/llamaswap/faceswap.go b/provider/llamaswap/faceswap.go index 8434ce8..294a765 100644 --- a/provider/llamaswap/faceswap.go +++ b/provider/llamaswap/faceswap.go @@ -207,10 +207,12 @@ func parseSwapReport(header string) []imagegen.SwappedFace { // imageFilename picks a multipart filename for an image part. The shim reads // bytes, not names, but a plausible extension keeps server-side sniffing and // request logs honest. base distinguishes the parts of a multi-file form -// ("target"/"source") so a log line says which one was malformed. +// ("target"/"source", "frame"/"frame_last") so a log line says which one was +// malformed — and, for the video keyframes, so a backend that stages uploads +// by filename cannot have the second overwrite the first. // -// initImageFilename (video.go) is this function with base fixed to "frame" -// and delegates here — two copies of one extension table is how they drift. +// Every caller routes through here: two copies of one extension table is how +// they drift. func imageFilename(mimeType, base string) string { if base == "" { base = "image" diff --git a/provider/llamaswap/lipsync.go b/provider/llamaswap/lipsync.go index 279aa3b..2d594c8 100644 --- a/provider/llamaswap/lipsync.go +++ b/provider/llamaswap/lipsync.go @@ -56,7 +56,7 @@ func (m *lipsyncModel) Lipsync(ctx context.Context, req videogen.LipsyncRequest, // hand (mirrors videoModel.Generate). var buf bytes.Buffer w := multipart.NewWriter(&buf) - fw, err := w.CreateFormFile("image", initImageFilename(req.Image.MIME)) + fw, err := w.CreateFormFile("image", imageFilename(req.Image.MIME, "frame")) if err != nil { return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err) } diff --git a/provider/llamaswap/video.go b/provider/llamaswap/video.go index 23f21ef..a694977 100644 --- a/provider/llamaswap/video.go +++ b/provider/llamaswap/video.go @@ -38,10 +38,13 @@ type videoModel struct { // bound the call with a context deadline. // // Parameter names follow vLLM-Omni's videos API (num_frames, fps, -// num_inference_steps, guidance_scale); the conditioning frame is sent as an -// `input_reference` file part, following OpenAI's videos API. Upstreams +// num_inference_steps, guidance_scale); the leading conditioning frame is sent +// as an `input_reference` file part, following OpenAI's videos API, and a +// trailing keyframe (Request.LastImage) as `input_reference_last`. Upstreams // ignore fields they don't understand, and optional fields stay off the wire -// entirely so the model's own defaults apply. +// entirely so the model's own defaults apply — which is also why a backend +// without first-last-frame support returns an ordinary clip here rather than +// an error. func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) { req = req.Apply(opts...) if strings.TrimSpace(req.Prompt) == "" { @@ -56,6 +59,9 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts .. if req.InitImage != nil && len(req.InitImage.Data) == 0 { return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported) } + if req.LastImage != nil && len(req.LastImage.Data) == 0 { + return nil, fmt.Errorf("%w: video last image has no bytes", llm.ErrUnsupported) + } width, height, err := parseSize(req.Size) if err != nil { return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err) @@ -83,12 +89,19 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts .. return nil, err } if req.InitImage != nil { - fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME)) - if err != nil { - return nil, fmt.Errorf("llama-swap: build video form: %w", err) + if err := writeImagePart(w, "input_reference", "frame", req.InitImage); err != nil { + return nil, err } - if _, err := fw.Write(req.InitImage.Data); err != nil { - return nil, fmt.Errorf("llama-swap: build video form: %w", err) + } + // The trailing keyframe rides a SEPARATE part rather than a second + // `input_reference`: multipart permits repeated names, but the receiving + // end would then have to rely on part ORDER to tell first from last, and + // an ordering contract that is invisible in the field name is one nobody + // can see they have broken. A backend that does not know the name ignores + // the part, which is the same degradation as any other unknown field. + if req.LastImage != nil { + if err := writeImagePart(w, "input_reference_last", "frame_last", req.LastImage); err != nil { + return nil, err } } if err := w.Close(); err != nil { @@ -134,11 +147,27 @@ func singleVideoResult(provider, model, verb string, raw []byte, contentType str return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil } -// initImageFilename picks the multipart filename hint for the conditioning -// frame from its MIME subtype. The name is provider-chosen (never -// caller-supplied), so no sanitization is needed. -func initImageFilename(mimeType string) string { - return imageFilename(mimeType, "frame") +// writeImagePart attaches one conditioning frame under the given field name, +// with a filename derived from nameStem. Shared by the first- and last-frame +// parts so the two cannot drift in how they encode, which is the usual way a +// second copy of a block goes wrong. +// +// The two frames MUST carry DISTINCT filenames, not merely distinct field +// names. Backends commonly stage an uploaded frame under a name derived from +// the filename — our own ComfyUI shim posts to /upload/image with +// overwrite=true — so two parts sharing "frame.png" would have the second +// clobber the first, and BOTH keyframe inputs would then resolve to the same +// stored image. The clip would render clean, pinned at both ends to the same +// frame, with nothing anywhere reporting a problem. +func writeImagePart(w *multipart.Writer, field, nameStem string, img *videogen.Image) error { + fw, err := w.CreateFormFile(field, imageFilename(img.MIME, nameStem)) + if err != nil { + return fmt.Errorf("llama-swap: build video form: %w", err) + } + if _, err := fw.Write(img.Data); err != nil { + return fmt.Errorf("llama-swap: build video form: %w", err) + } + return nil } // formatInt renders an optional int pointer for a form field; nil = "" (omit). diff --git a/provider/llamaswap/video_test.go b/provider/llamaswap/video_test.go index 9958b00..81a6ae5 100644 --- a/provider/llamaswap/video_test.go +++ b/provider/llamaswap/video_test.go @@ -223,3 +223,134 @@ func TestVideoGenerateNonVideoBodyErrors(t *testing.T) { t.Errorf("message = %q, want mention of non-video body", apiErr.Message) } } + +// Both keyframes reach the wire, under DISTINCT field names. +// +// The distinct-name property is the actual contract with the backend shim: the +// two frames could have shared one repeated `input_reference` name, and then +// which is first and which is last would depend on multipart part ORDER — an +// ordering contract invisible in the payload, that nothing would notice +// breaking. Asserting the names is what pins it. +func TestVideoGenerateSendsBothKeyframes(t *testing.T) { + var gotFirst, gotLast []byte + var firstName, lastName string + var sawLastPart bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(32 << 20); err != nil { + t.Errorf("parse form: %v", err) + return + } + if f, hdr, err := r.FormFile("input_reference"); err == nil { + gotFirst, _ = io.ReadAll(f) + firstName = hdr.Filename + f.Close() + } + if f, hdr, err := r.FormFile("input_reference_last"); err == nil { + sawLastPart = true + gotLast, _ = io.ReadAll(f) + lastName = hdr.Filename + f.Close() + } + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write([]byte("fake-mp4-bytes")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + vm, err := p.VideoModel("videogen-minimax-h3") + if err != nil { + t.Fatalf("VideoModel: %v", err) + } + + first, _ := base64.StdEncoding.DecodeString(onePixelPNG) + last := append(append([]byte{}, first...), 0x00) // distinguishable from first + + if _, err := vm.Generate(context.Background(), videogen.Request{ + Prompt: "a cat surfing", + InitImage: &videogen.Image{MIME: "image/png", Data: first}, + LastImage: &videogen.Image{MIME: "image/png", Data: last}, + }); err != nil { + t.Fatalf("Generate: %v", err) + } + + if !sawLastPart { + t.Fatal("input_reference_last was not sent — a pinned end frame would be silently dropped") + } + if string(gotFirst) != string(first) { + t.Errorf("input_reference = %d bytes, want %d", len(gotFirst), len(first)) + } + if string(gotLast) != string(last) { + t.Errorf("input_reference_last = %d bytes, want %d", len(gotLast), len(last)) + } + // The two must not be the same bytes, or a swap/aliasing bug reads as a pass. + if string(gotFirst) == string(gotLast) { + t.Error("both parts carry identical bytes — the frames are being aliased") + } + // DISTINCT FILENAMES, not just distinct field names. Backends stage an + // uploaded frame under a name derived from the filename (our ComfyUI shim + // posts to /upload/image with overwrite=true), so two parts sharing + // "frame.png" would have the second clobber the first and BOTH keyframes + // would resolve to the same stored image — a clip pinned at both ends to + // the same frame, rendering cleanly with nothing reporting a fault. + if firstName == "" || lastName == "" { + t.Fatalf("filenames = %q / %q, want both set", firstName, lastName) + } + if firstName == lastName { + t.Errorf("both parts use filename %q — the second upload would clobber the first", firstName) + } +} + +// LastImage alone (no InitImage) is a legitimate request: pin the destination +// and let the model invent the approach. It must not require a first frame. +func TestVideoGenerateLastImageAloneIsAllowed(t *testing.T) { + var sawFirst, sawLast bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(32 << 20); err != nil { + t.Errorf("parse form: %v", err) + return + } + if f, _, err := r.FormFile("input_reference"); err == nil { + sawFirst = true + f.Close() + } + if f, _, err := r.FormFile("input_reference_last"); err == nil { + sawLast = true + f.Close() + } + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write([]byte("fake-mp4-bytes")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + vm, _ := p.VideoModel("videogen-minimax-h3") + frame, _ := base64.StdEncoding.DecodeString(onePixelPNG) + + if _, err := vm.Generate(context.Background(), + videogen.Request{Prompt: "arrive here"}, + videogen.WithLastImage(videogen.Image{MIME: "image/png", Data: frame}), + ); err != nil { + t.Fatalf("Generate: %v", err) + } + if sawFirst { + t.Error("input_reference sent, want omitted") + } + if !sawLast { + t.Error("input_reference_last omitted, want sent") + } +} + +// An empty LastImage is rejected before the request is built, matching +// InitImage's existing contract — a zero-byte frame reaching the backend is a +// confusing upstream error instead of a clear local one. +func TestVideoGenerateRejectsEmptyLastImage(t *testing.T) { + p := New(WithBaseURL("http://unused")) + vm, _ := p.VideoModel("videogen-minimax-h3") + _, err := vm.Generate(context.Background(), videogen.Request{ + Prompt: "x", + LastImage: &videogen.Image{MIME: "image/png"}, + }) + if !errors.Is(err, llm.ErrUnsupported) { + t.Fatalf("err = %v, want llm.ErrUnsupported", err) + } +} diff --git a/videogen/videogen.go b/videogen/videogen.go index c20ff32..5dec93c 100644 --- a/videogen/videogen.go +++ b/videogen/videogen.go @@ -12,6 +12,8 @@ // InitImage is a pure text prompt, a non-nil InitImage conditions generation // on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same // checkpoint, so unlike imagegen there is no separate Editor-style interface. +// LastImage extends the same surface to the other end of the clip, so one +// Request covers t2v, i2v, and first-last-frame-to-video without a mode flag. // // The first implementation is provider/llamaswap, which targets the blocking // OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is @@ -51,6 +53,19 @@ type Request struct { // nil = pure text-to-video. InitImage *Image + // LastImage conditions generation on an ENDING frame. With InitImage it + // pins both ends (first-last-frame-to-video); alone it pins only the + // destination and lets the backend invent the approach. + // + // Support is per-model and NOT advertised anywhere in this contract: a + // backend that does not understand a trailing keyframe ignores it and + // returns an ordinary clip, which is indistinguishable from success. + // There is no capability bit to consult, because the contract has no way + // to learn one. A caller that needs to know whether the pin actually took + // effect must establish that out of band — by configuration it controls, + // not by inspecting the result. + LastImage *Image + // Size is the requested resolution, e.g. "1280x704"; "" = backend default. Size string @@ -92,6 +107,10 @@ type Option func(*Request) // WithInitImage conditions generation on a starting frame (image-to-video). func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } } +// WithLastImage conditions generation on an ending frame. Combined with +// WithInitImage this pins both ends of the clip. +func WithLastImage(img Image) Option { return func(r *Request) { r.LastImage = &img } } + // WithSize sets the requested resolution (e.g. "1280x704"). func WithSize(size string) Option { return func(r *Request) { r.Size = size } }