feat(videogen): LastImage — pin the trailing keyframe (first-last-frame-to-video)
Gadfly review (reusable) / review (pull_request) Successful in 3m41s
Adversarial Review (Gadfly) / review (pull_request) Successful in 3m41s
CI / Tidy (pull_request) Successful in 9m26s
CI / Build & Test (pull_request) Successful in 9m48s

videogen.Request gains LastImage alongside InitImage, so one Request covers
t2v, i2v and FL2V without a mode flag. With InitImage it pins both ends of the
clip; alone it pins the destination and lets the backend invent the approach.

The llamaswap provider sends it as a SEPARATE `input_reference_last` part
rather than a second `input_reference`. Multipart permits repeated names, but
then which frame is first and which is last depends on part ORDER — an
ordering contract invisible in the payload, that nothing notices breaking. A
backend that does not know the new name ignores the part, the same degradation
as any other unknown field.

Both parts go through one writeImagePart helper so their encoding cannot
drift, and an empty LastImage is rejected up front exactly as InitImage
already is.

Support is per-model and deliberately NOT advertised in this contract: a
backend that ignores a trailing keyframe returns an ordinary clip, which is
indistinguishable from success. The doc comment says so, because a caller that
needs to know whether the pin took effect has to establish that out of band —
and the mort side gates on a convar for exactly this reason.

Motivated by mort's #1567 (long-form video): with both ends pinned, drift
becomes structurally bounded inside each shot instead of compounding across an
autoregressive chain.

Tests break-checked: sending the last frame under the shared name fails both
the distinct-name assertion and the last-alone case.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01PLjgrxvHjm1sJgUu9zBPH9
This commit is contained in:
2026-08-08 02:47:39 -04:00
co-authored by Claude Opus 5
parent 203895696c
commit 44fcfbb273
3 changed files with 163 additions and 5 deletions
+29 -5
View File
@@ -56,6 +56,9 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ..
if req.InitImage != nil && len(req.InitImage.Data) == 0 { if req.InitImage != nil && len(req.InitImage.Data) == 0 {
return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported) return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported)
} }
if req.LastImage != nil && len(req.LastImage.Data) == 0 {
return nil, fmt.Errorf("%w: video last image has no bytes", llm.ErrUnsupported)
}
width, height, err := parseSize(req.Size) width, height, err := parseSize(req.Size)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err) return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
@@ -83,12 +86,19 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ..
return nil, err return nil, err
} }
if req.InitImage != nil { if req.InitImage != nil {
fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME)) if err := writeImagePart(w, "input_reference", req.InitImage); err != nil {
if err != nil { return nil, err
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
} }
if _, err := fw.Write(req.InitImage.Data); err != nil { }
return nil, fmt.Errorf("llama-swap: build video form: %w", err) // The trailing keyframe rides a SEPARATE part rather than a second
// `input_reference`: multipart permits repeated names, but the receiving
// end would then have to rely on part ORDER to tell first from last, and
// an ordering contract that is invisible in the field name is one nobody
// can see they have broken. A backend that does not know the name ignores
// the part, which is the same degradation as any other unknown field.
if req.LastImage != nil {
if err := writeImagePart(w, "input_reference_last", req.LastImage); err != nil {
return nil, err
} }
} }
if err := w.Close(); err != nil { if err := w.Close(); err != nil {
@@ -141,6 +151,20 @@ func initImageFilename(mimeType string) string {
return imageFilename(mimeType, "frame") return imageFilename(mimeType, "frame")
} }
// writeImagePart attaches one conditioning frame under the given field name.
// Shared by the first- and last-frame parts so the two cannot drift in how
// they encode, which is the usual way a second copy of a block goes wrong.
func writeImagePart(w *multipart.Writer, field string, img *videogen.Image) error {
fw, err := w.CreateFormFile(field, initImageFilename(img.MIME))
if err != nil {
return fmt.Errorf("llama-swap: build video form: %w", err)
}
if _, err := fw.Write(img.Data); err != nil {
return fmt.Errorf("llama-swap: build video form: %w", err)
}
return nil
}
// formatInt renders an optional int pointer for a form field; nil = "" (omit). // formatInt renders an optional int pointer for a form field; nil = "" (omit).
func formatInt(v *int) string { func formatInt(v *int) string {
if v == nil { if v == nil {
+116
View File
@@ -223,3 +223,119 @@ func TestVideoGenerateNonVideoBodyErrors(t *testing.T) {
t.Errorf("message = %q, want mention of non-video body", apiErr.Message) t.Errorf("message = %q, want mention of non-video body", apiErr.Message)
} }
} }
// Both keyframes reach the wire, under DISTINCT field names.
//
// The distinct-name property is the actual contract with the backend shim: the
// two frames could have shared one repeated `input_reference` name, and then
// which is first and which is last would depend on multipart part ORDER — an
// ordering contract invisible in the payload, that nothing would notice
// breaking. Asserting the names is what pins it.
func TestVideoGenerateSendsBothKeyframes(t *testing.T) {
var gotFirst, gotLast []byte
var sawLastPart bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(32 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if f, _, err := r.FormFile("input_reference"); err == nil {
gotFirst, _ = io.ReadAll(f)
f.Close()
}
if f, _, err := r.FormFile("input_reference_last"); err == nil {
sawLastPart = true
gotLast, _ = io.ReadAll(f)
f.Close()
}
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write([]byte("fake-mp4-bytes"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
vm, err := p.VideoModel("videogen-minimax-h3")
if err != nil {
t.Fatalf("VideoModel: %v", err)
}
first, _ := base64.StdEncoding.DecodeString(onePixelPNG)
last := append(append([]byte{}, first...), 0x00) // distinguishable from first
if _, err := vm.Generate(context.Background(), videogen.Request{
Prompt: "a cat surfing",
InitImage: &videogen.Image{MIME: "image/png", Data: first},
LastImage: &videogen.Image{MIME: "image/png", Data: last},
}); err != nil {
t.Fatalf("Generate: %v", err)
}
if !sawLastPart {
t.Fatal("input_reference_last was not sent — a pinned end frame would be silently dropped")
}
if string(gotFirst) != string(first) {
t.Errorf("input_reference = %d bytes, want %d", len(gotFirst), len(first))
}
if string(gotLast) != string(last) {
t.Errorf("input_reference_last = %d bytes, want %d", len(gotLast), len(last))
}
// The two must not be the same bytes, or a swap/aliasing bug reads as a pass.
if string(gotFirst) == string(gotLast) {
t.Error("both parts carry identical bytes — the frames are being aliased")
}
}
// LastImage alone (no InitImage) is a legitimate request: pin the destination
// and let the model invent the approach. It must not require a first frame.
func TestVideoGenerateLastImageAloneIsAllowed(t *testing.T) {
var sawFirst, sawLast bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(32 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if f, _, err := r.FormFile("input_reference"); err == nil {
sawFirst = true
f.Close()
}
if f, _, err := r.FormFile("input_reference_last"); err == nil {
sawLast = true
f.Close()
}
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write([]byte("fake-mp4-bytes"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
vm, _ := p.VideoModel("videogen-minimax-h3")
frame, _ := base64.StdEncoding.DecodeString(onePixelPNG)
if _, err := vm.Generate(context.Background(),
videogen.Request{Prompt: "arrive here"},
videogen.WithLastImage(videogen.Image{MIME: "image/png", Data: frame}),
); err != nil {
t.Fatalf("Generate: %v", err)
}
if sawFirst {
t.Error("input_reference sent, want omitted")
}
if !sawLast {
t.Error("input_reference_last omitted, want sent")
}
}
// An empty LastImage is rejected before the request is built, matching
// InitImage's existing contract — a zero-byte frame reaching the backend is a
// confusing upstream error instead of a clear local one.
func TestVideoGenerateRejectsEmptyLastImage(t *testing.T) {
p := New(WithBaseURL("http://unused"))
vm, _ := p.VideoModel("videogen-minimax-h3")
_, err := vm.Generate(context.Background(), videogen.Request{
Prompt: "x",
LastImage: &videogen.Image{MIME: "image/png"},
})
if !errors.Is(err, llm.ErrUnsupported) {
t.Fatalf("err = %v, want llm.ErrUnsupported", err)
}
}
+18
View File
@@ -12,6 +12,8 @@
// InitImage is a pure text prompt, a non-nil InitImage conditions generation // InitImage is a pure text prompt, a non-nil InitImage conditions generation
// on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same // on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same
// checkpoint, so unlike imagegen there is no separate Editor-style interface. // checkpoint, so unlike imagegen there is no separate Editor-style interface.
// LastImage extends the same surface to the other end of the clip, so one
// Request covers t2v, i2v, and first-last-frame-to-video without a mode flag.
// //
// The first implementation is provider/llamaswap, which targets the blocking // The first implementation is provider/llamaswap, which targets the blocking
// OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is // OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is
@@ -51,6 +53,18 @@ type Request struct {
// nil = pure text-to-video. // nil = pure text-to-video.
InitImage *Image InitImage *Image
// LastImage conditions generation on an ENDING frame. With InitImage it
// pins both ends (first-last-frame-to-video); alone it pins only the
// destination and lets the backend invent the approach.
//
// Support is per-model and NOT advertised anywhere in this contract: a
// backend that does not understand a trailing keyframe ignores it and
// returns an ordinary clip, which is indistinguishable from success. A
// caller that needs to know whether the pin took effect must establish
// that out of band — see the note on LastImage support in
// provider/llamaswap.
LastImage *Image
// Size is the requested resolution, e.g. "1280x704"; "" = backend default. // Size is the requested resolution, e.g. "1280x704"; "" = backend default.
Size string Size string
@@ -92,6 +106,10 @@ type Option func(*Request)
// WithInitImage conditions generation on a starting frame (image-to-video). // WithInitImage conditions generation on a starting frame (image-to-video).
func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } } func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } }
// WithLastImage conditions generation on an ending frame. Combined with
// WithInitImage this pins both ends of the clip.
func WithLastImage(img Image) Option { return func(r *Request) { r.LastImage = &img } }
// WithSize sets the requested resolution (e.g. "1280x704"). // WithSize sets the requested resolution (e.g. "1280x704").
func WithSize(size string) Option { return func(r *Request) { r.Size = size } } func WithSize(size string) Option { return func(r *Request) { r.Size = size } }