From 588e0924653d0411bd689eb65a98358c023256a0 Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Sat, 8 Aug 2026 02:59:12 -0400 Subject: [PATCH] =?UTF-8?q?docs(videogen):=20gadfly=20=E2=80=94=20README?= =?UTF-8?q?=20FL2V=20section,=20and=20stop=20pointing=20at=20a=20note=20th?= =?UTF-8?q?at=20does=20not=20exist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README documented only t2v/i2v. Now a table of the four keyframe combinations, plus the undetectable-support caveat, which is the one thing a caller cannot work out for itself. - The LastImage doc comment said "see the note on LastImage support in provider/llamaswap" — there was no such note. A pointer to something that does not exist is worse than no pointer; the comment is now self-contained. - Generate's doc described only input_reference; it now names input_reference_last and explains why an unsupporting backend returns a clip rather than an error. The 2/4 finding (writeImagePart reusing the "frame" base for both parts) was already fixed in dbc9689 — from the receiving end, where the consequence is concrete rather than stylistic: ComfyUI stages uploads by FILENAME with overwrite=true, so a shared name means the second clobbers the first and both keyframes resolve to one image. Not taken: initImageFilename's name is no longer misleading (writeImagePart stopped calling it), and it is still used by lipsync.go so it is not dead. The empty-LastImage test stays standalone — it mirrors the existing standalone empty-InitImage coverage rather than a table this file does not have. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PLjgrxvHjm1sJgUu9zBPH9 --- README.md | 23 ++++++++++++++++++----- provider/llamaswap/video.go | 9 ++++++--- videogen/videogen.go | 9 +++++---- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 85ff031..0ecaf6b 100644 --- a/README.md +++ b/README.md @@ -272,17 +272,30 @@ tr, err := tm.Transcribe(ctx, audio.TranscriptionRequest{ voices, err := ls.ListVoices(ctx, "kokoro") // []string of voice ids ``` -## Video: text-to-video + image-to-video +## Video: text-to-video, image-to-video, first-last-frame Video generation lives in the `videogen` package (ADR-0019), mirroring imagegen/audio: one small `Model` contract, zero values mean backend -defaults, bytes in/out. Text-to-video and image-to-video are one surface — -a nil `InitImage` is a pure text prompt; setting it conditions generation -on that frame (hybrid checkpoints like Wan 2.2 TI2V serve both). First -backend: llama-swap (blocking `/v1/videos/sync`, vLLM-Omni style — the +defaults, bytes in/out. All modes are one surface, selected by which +keyframes are set rather than by a mode flag: + +| `InitImage` | `LastImage` | mode | +|---|---|---| +| nil | nil | text-to-video | +| set | nil | image-to-video (hybrid checkpoints like Wan 2.2 TI2V serve both) | +| set | set | first-last-frame — both ends pinned | +| nil | set | pin the destination, model invents the approach | + +First backend: llama-swap (blocking `/v1/videos/sync`, vLLM-Omni style — the response body is the encoded clip, so `Result` carries a single `Video`). Generation runs for minutes; bound the call with a context deadline. +**`LastImage` support is per-model and cannot be detected.** A backend that +does not understand a trailing keyframe ignores the part and returns an +ordinary clip — indistinguishable from success. There is no capability bit, +because the contract has no way to learn one, so a caller depending on the +pin must establish support out of band. + ```go vm, _ := ls.VideoModel("videogen-wan22-5b") res, err := vm.Generate(ctx, videogen.Request{Prompt: "a cat surfing"}, diff --git a/provider/llamaswap/video.go b/provider/llamaswap/video.go index d9e3308..f74a35b 100644 --- a/provider/llamaswap/video.go +++ b/provider/llamaswap/video.go @@ -38,10 +38,13 @@ type videoModel struct { // bound the call with a context deadline. // // Parameter names follow vLLM-Omni's videos API (num_frames, fps, -// num_inference_steps, guidance_scale); the conditioning frame is sent as an -// `input_reference` file part, following OpenAI's videos API. Upstreams +// num_inference_steps, guidance_scale); the leading conditioning frame is sent +// as an `input_reference` file part, following OpenAI's videos API, and a +// trailing keyframe (Request.LastImage) as `input_reference_last`. Upstreams // ignore fields they don't understand, and optional fields stay off the wire -// entirely so the model's own defaults apply. +// entirely so the model's own defaults apply — which is also why a backend +// without first-last-frame support returns an ordinary clip here rather than +// an error. func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) { req = req.Apply(opts...) if strings.TrimSpace(req.Prompt) == "" { diff --git a/videogen/videogen.go b/videogen/videogen.go index f5f16f2..5dec93c 100644 --- a/videogen/videogen.go +++ b/videogen/videogen.go @@ -59,10 +59,11 @@ type Request struct { // // Support is per-model and NOT advertised anywhere in this contract: a // backend that does not understand a trailing keyframe ignores it and - // returns an ordinary clip, which is indistinguishable from success. A - // caller that needs to know whether the pin took effect must establish - // that out of band — see the note on LastImage support in - // provider/llamaswap. + // returns an ordinary clip, which is indistinguishable from success. + // There is no capability bit to consult, because the contract has no way + // to learn one. A caller that needs to know whether the pin actually took + // effect must establish that out of band — by configuration it controls, + // not by inspecting the result. LastImage *Image // Size is the requested resolution, e.g. "1280x704"; "" = backend default.