feat(videogen): LastImage — pin the trailing keyframe (first-last-frame-to-video) #26

Merged
steve merged 4 commits from feat/videogen-last-frame into main 2026-08-08 07:10:27 +00:00
3 changed files with 163 additions and 5 deletions
Showing only changes of commit 44fcfbb273 - Show all commits
+29 -5
View File
@@ -56,6 +56,9 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ..
if req.InitImage != nil && len(req.InitImage.Data) == 0 {
return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported)
}
if req.LastImage != nil && len(req.LastImage.Data) == 0 {
return nil, fmt.Errorf("%w: video last image has no bytes", llm.ErrUnsupported)
}
width, height, err := parseSize(req.Size)
if err != nil {
return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
@@ -83,12 +86,19 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ..
return nil, err
}
if req.InitImage != nil {
fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME))
if err != nil {
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
if err := writeImagePart(w, "input_reference", req.InitImage); err != nil {
return nil, err
}
if _, err := fw.Write(req.InitImage.Data); err != nil {
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
}
// The trailing keyframe rides a SEPARATE part rather than a second
// `input_reference`: multipart permits repeated names, but the receiving
// end would then have to rely on part ORDER to tell first from last, and
// an ordering contract that is invisible in the field name is one nobody
Review

Same first/last-frame rationale duplicated across ~6 comment sites (video.go x3, videogen.go, README, tests x2) — drift risk; keep one canonical copy

maintainability · flagged by 1 model

  • provider/llamaswap/video.go:96 (and siblings) — the "distinct name vs. part-order" rationale is duplicated across ~6 sites. The same argument appears in the Generate doc comment (video.go:42-47), the inline comment before the LastImage block (video.go:96-101), the writeImagePart doc (video.go:157-168), videogen.go:56-67, README.md, and the two test doc comments (video_test.go). One authoritative explanation is warranted, but the near-verbatim copies are a drift hazard. Suggest keepin…

🪰 Gadfly · advisory

⚪ **Same first/last-frame rationale duplicated across ~6 comment sites (video.go x3, videogen.go, README, tests x2) — drift risk; keep one canonical copy** _maintainability · flagged by 1 model_ - **`provider/llamaswap/video.go:96` (and siblings) — the "distinct name vs. part-order" rationale is duplicated across ~6 sites.** The same argument appears in the `Generate` doc comment (video.go:42-47), the inline comment before the LastImage block (video.go:96-101), the `writeImagePart` doc (video.go:157-168), videogen.go:56-67, README.md, and the two test doc comments (video_test.go). One authoritative explanation is warranted, but the near-verbatim copies are a drift hazard. Suggest keepin… <sub>🪰 Gadfly · advisory</sub>
// can see they have broken. A backend that does not know the name ignores
// the part, which is the same degradation as any other unknown field.
if req.LastImage != nil {
if err := writeImagePart(w, "input_reference_last", req.LastImage); err != nil {
return nil, err
}
}
if err := w.Close(); err != nil {
@@ -141,6 +151,20 @@ func initImageFilename(mimeType string) string {
return imageFilename(mimeType, "frame")
}
// writeImagePart attaches one conditioning frame under the given field name.
// Shared by the first- and last-frame parts so the two cannot drift in how
// they encode, which is the usual way a second copy of a block goes wrong.
func writeImagePart(w *multipart.Writer, field string, img *videogen.Image) error {
fw, err := w.CreateFormFile(field, initImageFilename(img.MIME))
if err != nil {
return fmt.Errorf("llama-swap: build video form: %w", err)
}
if _, err := fw.Write(img.Data); err != nil {
return fmt.Errorf("llama-swap: build video form: %w", err)
}
return nil
}
// formatInt renders an optional int pointer for a form field; nil = "" (omit).
func formatInt(v *int) string {
if v == nil {
+116
View File
@@ -223,3 +223,119 @@ func TestVideoGenerateNonVideoBodyErrors(t *testing.T) {
t.Errorf("message = %q, want mention of non-video body", apiErr.Message)
}
}
// Both keyframes reach the wire, under DISTINCT field names.
//
// The distinct-name property is the actual contract with the backend shim: the
// two frames could have shared one repeated `input_reference` name, and then
// which is first and which is last would depend on multipart part ORDER — an
// ordering contract invisible in the payload, that nothing would notice
// breaking. Asserting the names is what pins it.
func TestVideoGenerateSendsBothKeyframes(t *testing.T) {
var gotFirst, gotLast []byte
var sawLastPart bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(32 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if f, _, err := r.FormFile("input_reference"); err == nil {
gotFirst, _ = io.ReadAll(f)
f.Close()
}
if f, _, err := r.FormFile("input_reference_last"); err == nil {
sawLastPart = true
gotLast, _ = io.ReadAll(f)
f.Close()
}
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write([]byte("fake-mp4-bytes"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
vm, err := p.VideoModel("videogen-minimax-h3")
if err != nil {
t.Fatalf("VideoModel: %v", err)
}
first, _ := base64.StdEncoding.DecodeString(onePixelPNG)
last := append(append([]byte{}, first...), 0x00) // distinguishable from first
if _, err := vm.Generate(context.Background(), videogen.Request{
Prompt: "a cat surfing",
InitImage: &videogen.Image{MIME: "image/png", Data: first},
LastImage: &videogen.Image{MIME: "image/png", Data: last},
}); err != nil {
t.Fatalf("Generate: %v", err)
}
if !sawLastPart {
t.Fatal("input_reference_last was not sent — a pinned end frame would be silently dropped")
}
if string(gotFirst) != string(first) {
t.Errorf("input_reference = %d bytes, want %d", len(gotFirst), len(first))
}
if string(gotLast) != string(last) {
t.Errorf("input_reference_last = %d bytes, want %d", len(gotLast), len(last))
}
// The two must not be the same bytes, or a swap/aliasing bug reads as a pass.
if string(gotFirst) == string(gotLast) {
t.Error("both parts carry identical bytes — the frames are being aliased")
}
}
// LastImage alone (no InitImage) is a legitimate request: pin the destination
// and let the model invent the approach. It must not require a first frame.
func TestVideoGenerateLastImageAloneIsAllowed(t *testing.T) {
var sawFirst, sawLast bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(32 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if f, _, err := r.FormFile("input_reference"); err == nil {
sawFirst = true
f.Close()
}
if f, _, err := r.FormFile("input_reference_last"); err == nil {
sawLast = true
f.Close()
}
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write([]byte("fake-mp4-bytes"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
vm, _ := p.VideoModel("videogen-minimax-h3")
frame, _ := base64.StdEncoding.DecodeString(onePixelPNG)
if _, err := vm.Generate(context.Background(),
videogen.Request{Prompt: "arrive here"},
videogen.WithLastImage(videogen.Image{MIME: "image/png", Data: frame}),
); err != nil {
t.Fatalf("Generate: %v", err)
}
if sawFirst {
t.Error("input_reference sent, want omitted")
}
if !sawLast {
t.Error("input_reference_last omitted, want sent")
}
}
// An empty LastImage is rejected before the request is built, matching
// InitImage's existing contract — a zero-byte frame reaching the backend is a
// confusing upstream error instead of a clear local one.
func TestVideoGenerateRejectsEmptyLastImage(t *testing.T) {
p := New(WithBaseURL("http://unused"))
vm, _ := p.VideoModel("videogen-minimax-h3")
_, err := vm.Generate(context.Background(), videogen.Request{
Prompt: "x",
LastImage: &videogen.Image{MIME: "image/png"},
})
if !errors.Is(err, llm.ErrUnsupported) {
t.Fatalf("err = %v, want llm.ErrUnsupported", err)
}
}
+18
View File
@@ -12,6 +12,8 @@
// InitImage is a pure text prompt, a non-nil InitImage conditions generation
// on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same
// checkpoint, so unlike imagegen there is no separate Editor-style interface.
// LastImage extends the same surface to the other end of the clip, so one
// Request covers t2v, i2v, and first-last-frame-to-video without a mode flag.
//
// The first implementation is provider/llamaswap, which targets the blocking
// OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is
@@ -51,6 +53,18 @@ type Request struct {
// nil = pure text-to-video.
InitImage *Image
// LastImage conditions generation on an ENDING frame. With InitImage it
// pins both ends (first-last-frame-to-video); alone it pins only the
// destination and lets the backend invent the approach.
//
// Support is per-model and NOT advertised anywhere in this contract: a
// backend that does not understand a trailing keyframe ignores it and
// returns an ordinary clip, which is indistinguishable from success. A
// caller that needs to know whether the pin took effect must establish
// that out of band — see the note on LastImage support in
// provider/llamaswap.
LastImage *Image
// Size is the requested resolution, e.g. "1280x704"; "" = backend default.
Size string
@@ -92,6 +106,10 @@ type Option func(*Request)
// WithInitImage conditions generation on a starting frame (image-to-video).
func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } }
// WithLastImage conditions generation on an ending frame. Combined with
// WithInitImage this pins both ends of the clip.
func WithLastImage(img Image) Option { return func(r *Request) { r.LastImage = &img } }
// WithSize sets the requested resolution (e.g. "1280x704").
func WithSize(size string) Option { return func(r *Request) { r.Size = size } }