feat(videogen): canonical video-generation surface + llama-swap client #13
@@ -34,6 +34,11 @@ majordomo Registry, Parse, env-DSN loading, chain executor, re-exports
|
||||
Tool/Toolbox, Capabilities, Stream, Model, Provider, errors
|
||||
imagegen/ canonical text-to-image contract: Request/Result/Model/
|
||||
Provider (separate from llm; Image = llm.ImagePart) (ADR-0016)
|
||||
audio/ canonical speech contracts: SpeechModel (TTS) +
|
||||
TranscriptionModel (STT), same conventions (ADR-0017)
|
||||
videogen/ canonical video-generation contract: one Model for
|
||||
text-to-video + image-to-video (InitImage), single-clip
|
||||
Result (Image = llm.ImagePart) (ADR-0019)
|
||||
health/ clock-injected health tracker (bench/backoff)
|
||||
media/ image normalization to target capabilities (sniff real
|
||||
format, downscale, transcode, byte ladder; ErrUnsupported
|
||||
@@ -44,7 +49,8 @@ majordomo Registry, Parse, env-DSN loading, chain executor, re-exports
|
||||
provider/ollama/ one native /api/chat client serving the ollama,
|
||||
ollama-cloud, and foreman built-ins via presets
|
||||
provider/llamaswap/ llama-swap proxy: chat delegates to provider/openai,
|
||||
plus management methods + imagegen image client (ADR-0015)
|
||||
plus management methods + imagegen/audio/videogen
|
||||
clients (ADR-0015..0019)
|
||||
provider/google/ Gemini on google.golang.org/genai (the one approved
|
||||
dependency; lazy client, raw-JSON-schema tools,
|
||||
ThinkingLevel reasoning, iter.Pull2 streaming)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# ADR-0019: videogen — canonical video-generation surface
|
||||
|
||||
**Status:** Accepted — 2026-07-12
|
||||
|
||||
## Context
|
||||
|
||||
mort is adding local video generation (Wan 2.2 / LTX-class models on a 24GB
|
||||
GPU) behind the same llama-swap instance that serves imagegen and audio. Like
|
||||
those modalities, video generation shares none of the chat machinery, so it
|
||||
needs its own small contract package (the ADR-0016/0017 pattern). Three shape
|
||||
questions: one interface for text-to-video and image-to-video or two, batch or
|
||||
single result, and which wire endpoint the llama-swap provider targets.
|
||||
|
||||
## Decision
|
||||
|
||||
- **New `videogen/` package** with the established conventions: `Request` /
|
||||
`Result` / `Option` + `Apply`, `Model` / `ModelOption` / `Provider`, zero
|
||||
values mean backend default, `Image = llm.ImagePart`.
|
||||
- **Text-to-video and image-to-video are one surface.** `Request.InitImage
|
||||
*Image` (nil = pure text-to-video) instead of an imagegen-style separate
|
||||
`Editor` interface: hybrid checkpoints (Wan 2.2 TI2V) serve both modes from
|
||||
the same model and endpoint, so a second interface would duplicate the
|
||||
request shape for no dispatch benefit.
|
||||
- **`Result` carries a single `Video`, not a batch.** The blocking sync
|
||||
endpoint answers with the encoded clip as the response body — one request,
|
||||
one clip. Batching multi-minute generations behind one HTTP request is the
|
||||
wrong shape; if batch ever matters it arrives with an async job surface,
|
||||
not by widening this one.
|
||||
- **provider/llamaswap targets `POST {base}/v1/videos/sync`**
|
||||
(multipart/form-data, llama-swap routes by the `model` form field; the
|
||||
response body is the video). This is vLLM-Omni's blocking videos endpoint,
|
||||
and the steve/llama-swap fork dispatches it as a model route. Parameter
|
||||
names follow vLLM-Omni (`num_frames`, `fps`, `num_inference_steps`,
|
||||
`guidance_scale`); the conditioning frame is an `input_reference` file part
|
||||
per OpenAI's videos API. Optional fields stay off the wire so per-model
|
||||
launch-flag defaults apply — the imagegen convention.
|
||||
- **No polling in v1.** The async `POST /v1/videos` + `GET /v1/videos/{id}`
|
||||
job flow is deliberately not wrapped: callers (mort's skill tools) already
|
||||
run synchronous-with-generous-timeout and bound the call with a context
|
||||
deadline. An async `Job` surface is a compatible later addition.
|
||||
|
||||
## Consequences
|
||||
|
||||
- A `videogen.Video` is its own type (`Data []byte`, `MIME string`); there is
|
||||
no `llm.VideoPart`, and no chat-side video-input support is implied.
|
||||
- Duration is expressed as `NumFrames` + `FPS` (the diffusion-native knobs),
|
||||
not seconds; callers wanting seconds convert at their edge.
|
||||
- Any upstream exposing the same `/v1/videos/sync` shape (e.g. a ComfyUI
|
||||
shim) works unchanged; the contract does not name an engine.
|
||||
@@ -0,0 +1,177 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
// VideoModel implements videogen.Provider, binding a video-generation model
|
||||
// served by llama-swap (routed to a vLLM-Omni-style upstream, or any shim
|
||||
// exposing the same /v1/videos/sync shape). The id is passed through verbatim
|
||||
// and selects which upstream llama-swap loads.
|
||||
func (p *Provider) VideoModel(id string, opts ...videogen.ModelOption) (videogen.Model, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = videogen.ApplyModelOptions(opts)
|
||||
return &videoModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type videoModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Generate implements videogen.Model via POST {base}/v1/videos/sync
|
||||
// (multipart/form-data — llama-swap routes by the `model` form field). The
|
||||
// blocking sync endpoint answers with the encoded video itself, so the
|
||||
// response body is the result; there is no job id to poll. Generation runs
|
||||
// for minutes — the provider client carries no timeout by design, and callers
|
||||
// bound the call with a context deadline.
|
||||
//
|
||||
// Parameter names follow vLLM-Omni's videos API (num_frames, fps,
|
||||
// num_inference_steps, guidance_scale); the conditioning frame is sent as an
|
||||
// `input_reference` file part, following OpenAI's videos API. Upstreams
|
||||
// ignore fields they don't understand, and optional fields stay off the wire
|
||||
// entirely so the model's own defaults apply.
|
||||
func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Prompt) == "" {
|
||||
|
|
||||
return nil, fmt.Errorf("%w: video generation requires a prompt", llm.ErrUnsupported)
|
||||
}
|
||||
if req.NumFrames < 0 {
|
||||
return nil, fmt.Errorf("%w: video frame count must be >= 0, got %d", llm.ErrUnsupported, req.NumFrames)
|
||||
}
|
||||
if req.FPS < 0 {
|
||||
return nil, fmt.Errorf("%w: video fps must be >= 0, got %d", llm.ErrUnsupported, req.FPS)
|
||||
}
|
||||
if req.InitImage != nil && len(req.InitImage.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported)
|
||||
}
|
||||
width, height, err := parseSize(req.Size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
|
||||
|
gitea-actions
commented
🟠 Multipart body buffering doubles memory for init image data error-handling, maintainability, performance, security · flagged by 3 models
🪰 Gadfly · advisory 🟠 **Multipart body buffering doubles memory for init image data**
_error-handling, maintainability, performance, security · flagged by 3 models_
- **`provider/llamaswap/video.go:65-103`** — The multipart request body is fully buffered in a `bytes.Buffer` before sending. When `InitImage` is present, the image bytes exist in both `req.InitImage.Data` and the form buffer simultaneously, temporarily doubling request-side memory. For high-resolution conditioning frames this is a needless copy. **Fix:** since `doRaw` already accepts an `io.Reader`, pipe the multipart output directly via `io.Pipe()` instead of buffering the entire form.
<sub>🪰 Gadfly · advisory</sub>
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
// Required fields always go on the wire (an empty model id should fail
|
||||
// loudly upstream, not silently vanish); optional ones only when set.
|
||||
fields := []struct {
|
||||
key, value string
|
||||
required bool
|
||||
}{
|
||||
{"model", m.id, true},
|
||||
{"prompt", req.Prompt, true},
|
||||
{"negative_prompt", req.NegativePrompt, false},
|
||||
{"width", formatInt(width), false},
|
||||
{"height", formatInt(height), false},
|
||||
{"num_frames", formatNonZero(req.NumFrames), false},
|
||||
{"fps", formatNonZero(req.FPS), false},
|
||||
{"num_inference_steps", formatInt(req.Steps), false},
|
||||
{"guidance_scale", formatFloat(req.GuidanceScale), false},
|
||||
{"seed", formatInt64(req.Seed), false},
|
||||
|
gitea-actions
commented
🔴 Unbounded InitImage.Data with no size validation allows memory exhaustion via arbitrarily large conditioning frames security · flagged by 1 model
🪰 Gadfly · advisory 🔴 **Unbounded InitImage.Data with no size validation allows memory exhaustion via arbitrarily large conditioning frames**
_security · flagged by 1 model_
- **`provider/llamaswap/video.go:82-88`** — Unbounded `InitImage.Data` with no size validation. The conditioning frame (`req.InitImage.Data`) is written directly into the multipart form without any maximum-size check. An attacker can supply an arbitrarily large image, causing the client to exhaust memory building the request body. This is an asymmetric DoS vector: the request is unbounded while `doRaw` caps the response at 64 MiB. **Fix:** Add a size limit (e.g., 50–100 MiB) and reject oversized…
<sub>🪰 Gadfly · advisory</sub>
|
||||
}
|
||||
for _, f := range fields {
|
||||
if !f.required && f.value == "" {
|
||||
continue
|
||||
}
|
||||
if err := w.WriteField(f.key, f.value); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||
}
|
||||
}
|
||||
if req.InitImage != nil {
|
||||
fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||
}
|
||||
if _, err := fw.Write(req.InitImage.Data); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||
}
|
||||
}
|
||||
|
gitea-actions
commented
🟡 64 MiB response cap (sized for JSON bodies) hard-fails realistic binary video clips instead of returning them error-handling · flagged by 1 model 🪰 Gadfly · advisory 🟡 **64 MiB response cap (sized for JSON bodies) hard-fails realistic binary video clips instead of returning them**
_error-handling · flagged by 1 model_
<sub>🪰 Gadfly · advisory</sub>
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||
}
|
||||
|
||||
videoBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/videos/sync", m.id, w.FormDataContentType(), &buf)
|
||||
|
gitea-actions
commented
🔴 doRaw's 64 MB maxResponseBytes cap will reject legitimate large video responses correctness, error-handling, performance · flagged by 3 models
🪰 Gadfly · advisory 🔴 **doRaw's 64 MB maxResponseBytes cap will reject legitimate large video responses**
_correctness, error-handling, performance · flagged by 3 models_
- **`provider/llamaswap/video.go:105` — video responses capped at 64 MB via shared `doRaw`.** `Generate` reads the result through `m.p.doRaw(...)`, which wraps the body in `io.LimitReader(resp.Body, maxResponseBytes+1)` and hard-errors when `len(data) > maxResponseBytes` (`audio.go:289-295`; `llamaswap.go:46` defines `maxResponseBytes = 64 << 20`). The whole design of this PR is that the sync endpoint's response body **is** the encoded video. A legitimate generated clip — a multi-second high-bit…
<sub>🪰 Gadfly · advisory</sub>
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(videoBytes) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video response contained no video"}
|
||||
}
|
||||
return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: videoMIME(contentType, videoBytes)}}, nil
|
||||
}
|
||||
|
||||
// videoMIME resolves the result MIME type: the response Content-Type when it
|
||||
// is a concrete video type, else content sniffing, else video/mp4 (the
|
||||
// endpoint's default container).
|
||||
func videoMIME(contentType string, data []byte) string {
|
||||
if mt, _, err := mime.ParseMediaType(contentType); err == nil && strings.HasPrefix(mt, "video/") {
|
||||
return mt
|
||||
}
|
||||
if mt := http.DetectContentType(data); strings.HasPrefix(mt, "video/") {
|
||||
|
gitea-actions
commented
🟡 videoMIME content-sniffing branch is effectively dead (DetectContentType has no video signatures) correctness · flagged by 1 model
🪰 Gadfly · advisory 🟡 **videoMIME content-sniffing branch is effectively dead (DetectContentType has no video signatures)**
_correctness · flagged by 1 model_
- **`provider/llamaswap/video.go:122` — `videoMIME` sniffing branch is effectively dead.** `http.DetectContentType` does not recognize video container formats (its sniff table has no MP4 `ftyp`/WebM `EBML` signatures, returning `application/octet-stream` for such bytes), so the `strings.HasPrefix(mt, "video/")` sniff check at line 122 can essentially never succeed; the function always lands on the Content-Type branch or the `video/mp4` fallback. Not a logic bug (the fallback is safe), but the sn…
<sub>🪰 Gadfly · advisory</sub>
|
||||
return mt
|
||||
}
|
||||
return "video/mp4"
|
||||
}
|
||||
|
||||
// initImageFilename picks the multipart filename hint for the conditioning
|
||||
// frame from its MIME subtype. The name is provider-chosen (never
|
||||
// caller-supplied), so no sanitization is needed.
|
||||
func initImageFilename(mimeType string) string {
|
||||
mt := strings.ToLower(strings.TrimSpace(mimeType))
|
||||
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
||||
mt = parsed
|
||||
}
|
||||
switch mt {
|
||||
case "image/jpeg", "image/jpg":
|
||||
return "frame.jpg"
|
||||
case "image/webp":
|
||||
return "frame.webp"
|
||||
default: // stable-diffusion outputs and unknowns: PNG is the safe hint
|
||||
|
gitea-actions
commented
⚪ stale 'stable-diffusion' comment copy-pasted from image.go into initImageFilename maintainability · flagged by 1 model
🪰 Gadfly · advisory ⚪ **stale 'stable-diffusion' comment copy-pasted from image.go into initImageFilename**
_maintainability · flagged by 1 model_
- **`provider/llamaswap/video.go:141` — stale "stable-diffusion" comment in `initImageFilename`**
<sub>🪰 Gadfly · advisory</sub>
|
||||
return "frame.png"
|
||||
}
|
||||
}
|
||||
|
||||
// formatInt renders an optional int pointer for a form field; nil = "" (omit).
|
||||
func formatInt(v *int) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return strconv.Itoa(*v)
|
||||
}
|
||||
|
||||
// formatInt64 renders an optional int64 pointer for a form field; nil = "" (omit).
|
||||
func formatInt64(v *int64) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return strconv.FormatInt(*v, 10)
|
||||
}
|
||||
|
||||
// formatFloat renders an optional float pointer for a form field; nil = "" (omit).
|
||||
func formatFloat(v *float64) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return strconv.FormatFloat(*v, 'g', -1, 64)
|
||||
}
|
||||
|
||||
// formatNonZero renders a non-negative int for a form field; 0 = "" (omit,
|
||||
// backend default).
|
||||
func formatNonZero(v int) string {
|
||||
if v == 0 {
|
||||
return ""
|
||||
}
|
||||
return strconv.Itoa(v)
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
func TestVideoGenerate(t *testing.T) {
|
||||
var gotPath, gotContentType string
|
||||
var gotForm map[string]string
|
||||
var gotFrame []byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
gotContentType = r.Header.Get("Content-Type")
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
|
gitea-actions
commented
🟡 Nil dereference in test when ParseMultipartForm fails error-handling · flagged by 1 model
🪰 Gadfly · advisory 🟡 **Nil dereference in test when ParseMultipartForm fails**
_error-handling · flagged by 1 model_
- **`provider/llamaswap/video_test.go:24-28`** — `TestVideoGenerate`: if `r.ParseMultipartForm` fails, the test logs the error via `t.Errorf` but continues executing. `r.MultipartForm` is then nil, and line 28 (`r.MultipartForm.Value`) panics with a nil-dereference, obscuring the real parse failure. Same pattern in `TestVideoGenerateOmitsUnsetOverrides` at line 96-97. (Verified by reading `video_test.go:24-28` and `video_test.go:96-97`.)
<sub>🪰 Gadfly · advisory</sub>
|
||||
t.Errorf("parse form: %v", err)
|
||||
}
|
||||
gotForm = map[string]string{}
|
||||
for k, v := range r.MultipartForm.Value {
|
||||
gotForm[k] = v[0]
|
||||
}
|
||||
if f, _, err := r.FormFile("input_reference"); err == nil {
|
||||
gotFrame, _ = io.ReadAll(f)
|
||||
f.Close()
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("fake-mp4-bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vm, err := p.VideoModel("videogen-wan")
|
||||
if err != nil {
|
||||
t.Fatalf("VideoModel: %v", err)
|
||||
}
|
||||
|
||||
frame, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
res, err := vm.Generate(context.Background(),
|
||||
videogen.Request{Prompt: "a cat surfing", InitImage: &videogen.Image{MIME: "image/png", Data: frame}},
|
||||
videogen.WithSize("1280x704"),
|
||||
videogen.WithNumFrames(81),
|
||||
videogen.WithFPS(16),
|
||||
videogen.WithSteps(4),
|
||||
videogen.WithGuidanceScale(1.0),
|
||||
videogen.WithNegativePrompt("blurry"),
|
||||
videogen.WithSeed(42),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if string(res.Video.Data) != "fake-mp4-bytes" || res.Video.MIME != "video/mp4" {
|
||||
t.Fatalf("video = %d bytes, MIME %q", len(res.Video.Data), res.Video.MIME)
|
||||
}
|
||||
|
||||
if gotPath != "/v1/videos/sync" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if !strings.HasPrefix(gotContentType, "multipart/form-data") {
|
||||
t.Errorf("content-type = %q", gotContentType)
|
||||
}
|
||||
want := map[string]string{
|
||||
"model": "videogen-wan",
|
||||
"prompt": "a cat surfing",
|
||||
"negative_prompt": "blurry",
|
||||
"width": "1280",
|
||||
"height": "704",
|
||||
"num_frames": "81",
|
||||
"fps": "16",
|
||||
"num_inference_steps": "4",
|
||||
"guidance_scale": "1",
|
||||
"seed": "42",
|
||||
}
|
||||
for k, v := range want {
|
||||
if gotForm[k] != v {
|
||||
t.Errorf("form[%q] = %q, want %q", k, gotForm[k], v)
|
||||
}
|
||||
}
|
||||
if string(gotFrame) != string(frame) {
|
||||
t.Errorf("input_reference = %d bytes, want %d", len(gotFrame), len(frame))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoGenerateOmitsUnsetOverrides(t *testing.T) {
|
||||
var gotForm map[string][]string
|
||||
var hadFrame bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseMultipartForm(32 << 20)
|
||||
gotForm = r.MultipartForm.Value
|
||||
_, _, err := r.FormFile("input_reference")
|
||||
hadFrame = err == nil
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("v"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vm, _ := p.VideoModel("videogen-wan")
|
||||
|
gitea-actions
commented
🟡 Ignored VideoModel error leads to nil-dereference panic in multiple tests error-handling · flagged by 1 model
🪰 Gadfly · advisory 🟡 **Ignored VideoModel error leads to nil-dereference panic in multiple tests**
_error-handling · flagged by 1 model_
- **`provider/llamaswap/video_test.go:106,122,151,166`** — Four tests ignore the error return from `p.VideoModel(...)`: `vm, _ := p.VideoModel(...)`. If `VideoModel` returns an error (e.g. `requireBaseURL` fails), `vm` is nil and the subsequent `vm.Generate(...)` panics with a nil-dereference instead of a clear test failure. (Verified by reading `video_test.go:106-107`, `122`, `151`, `166`.)
<sub>🪰 Gadfly · advisory</sub>
|
||||
if _, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"}); err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
for _, k := range []string{"negative_prompt", "width", "height", "num_frames", "fps", "num_inference_steps", "guidance_scale", "seed"} {
|
||||
if _, ok := gotForm[k]; ok {
|
||||
t.Errorf("form field %q sent, want omitted", k)
|
||||
}
|
||||
}
|
||||
if hadFrame {
|
||||
t.Error("input_reference sent, want omitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoGenerateValidation(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused.invalid"))
|
||||
vm, _ := p.VideoModel("videogen-wan")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
req videogen.Request
|
||||
}{
|
||||
{"empty prompt", videogen.Request{}},
|
||||
{"negative frames", videogen.Request{Prompt: "x", NumFrames: -1}},
|
||||
{"negative fps", videogen.Request{Prompt: "x", FPS: -1}},
|
||||
{"empty init image", videogen.Request{Prompt: "x", InitImage: &videogen.Image{}}},
|
||||
{"bad size", videogen.Request{Prompt: "x", Size: "banana"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := vm.Generate(context.Background(), tc.req)
|
||||
if !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Fatalf("err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoGenerateUpstreamError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, `{"error":{"message":"boom"}}`, http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vm, _ := p.VideoModel("videogen-wan")
|
||||
_, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want *llm.APIError", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoGenerateEmptyResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vm, _ := p.VideoModel("videogen-wan")
|
||||
_, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want *llm.APIError for empty body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoModelRequiresBaseURL(t *testing.T) {
|
||||
p := New()
|
||||
if _, err := p.VideoModel("videogen-wan"); err == nil {
|
||||
t.Fatal("VideoModel with no base URL should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoMIME(t *testing.T) {
|
||||
cases := []struct {
|
||||
contentType string
|
||||
data []byte
|
||||
want string
|
||||
}{
|
||||
{"video/webm", []byte("x"), "video/webm"},
|
||||
{"video/mp4; charset=binary", []byte("x"), "video/mp4"},
|
||||
{"application/octet-stream", []byte("x"), "video/mp4"},
|
||||
{"", []byte("x"), "video/mp4"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := videoMIME(tc.contentType, tc.data); got != tc.want {
|
||||
t.Errorf("videoMIME(%q) = %q, want %q", tc.contentType, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
// Package videogen is majordomo's canonical video-generation surface. Like
|
||||
// imagegen and audio, it is a deliberately separate contract from the llm
|
||||
// package: video generation shares none of the chat message/tool/stream
|
||||
// machinery, so it gets its own small Provider/Model interface rather than
|
||||
// overloading llm.Model (ADR-0019).
|
||||
//
|
||||
// Zero values mean "backend default" throughout, mirroring imagegen: an empty
|
||||
// Size leaves the backend's default resolution, zero NumFrames/FPS the
|
||||
// backend's default clip length and rate.
|
||||
//
|
||||
// Text-to-video and image-to-video are one surface: a Request with a nil
|
||||
// InitImage is a pure text prompt, a non-nil InitImage conditions generation
|
||||
// on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same
|
||||
// checkpoint, so unlike imagegen there is no separate Editor-style interface.
|
||||
//
|
||||
// The first implementation is provider/llamaswap, which targets the blocking
|
||||
// OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is
|
||||
// the encoded video itself, so one request yields exactly one clip — Result
|
||||
// carries a single Video, not a batch.
|
||||
package videogen
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// Image is a conditioning input frame (bytes + MIME). Aliased to
|
||||
// llm.ImagePart so chat-sourced images feed image-to-video without
|
||||
// conversion, mirroring imagegen.Image.
|
||||
type Image = llm.ImagePart
|
||||
|
||||
// Video is one generated video: raw encoded bytes plus a MIME type
|
||||
// (e.g. "video/mp4").
|
||||
type Video struct {
|
||||
// Data is the encoded video container.
|
||||
Data []byte
|
||||
|
||||
// MIME is the video MIME type, e.g. "video/mp4".
|
||||
MIME string
|
||||
}
|
||||
|
||||
// Request is a video generation request. Zero values mean "backend default" —
|
||||
// for llama-swap-served models that is the per-model default baked into the
|
||||
// upstream launch flags. A caller overrides only what it explicitly sets.
|
||||
type Request struct {
|
||||
// Prompt is the text description of the video to generate.
|
||||
Prompt string
|
||||
|
||||
// InitImage conditions generation on a starting frame (image-to-video);
|
||||
// nil = pure text-to-video.
|
||||
InitImage *Image
|
||||
|
||||
// Size is the requested resolution, e.g. "1280x704"; "" = backend default.
|
||||
Size string
|
||||
|
||||
// NumFrames is the clip length in frames; 0 = backend default.
|
||||
NumFrames int
|
||||
|
||||
// FPS is the frame rate of the generated clip; 0 = backend default.
|
||||
FPS int
|
||||
|
||||
// Steps is the number of diffusion steps; nil = backend default.
|
||||
Steps *int
|
||||
|
||||
// GuidanceScale is the guidance strength; nil = backend default.
|
||||
// Architecture-sensitive (distilled models want low or none), so prefer
|
||||
// leaving it nil unless the caller knows the target model.
|
||||
GuidanceScale *float64
|
||||
|
||||
// NegativePrompt steers generation away from concepts; "" = none.
|
||||
NegativePrompt string
|
||||
|
||||
// Seed fixes the RNG seed for reproducible output; nil = random.
|
||||
Seed *int64
|
||||
}
|
||||
|
||||
// Result is the canonical video-generation result.
|
||||
type Result struct {
|
||||
// Video is the generated clip.
|
||||
Video Video
|
||||
|
||||
// Raw is the provider-native response object, an escape hatch for
|
||||
// provider-specific fields. May be nil; never required for normal use.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// Option mutates a Request before it is sent. Options passed to Generate are
|
||||
// applied to a copy of the request, so a Request value can be reused.
|
||||
type Option func(*Request)
|
||||
|
||||
// WithInitImage conditions generation on a starting frame (image-to-video).
|
||||
func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } }
|
||||
|
||||
// WithSize sets the requested resolution (e.g. "1280x704").
|
||||
func WithSize(size string) Option { return func(r *Request) { r.Size = size } }
|
||||
|
||||
// WithNumFrames sets the clip length in frames.
|
||||
func WithNumFrames(n int) Option { return func(r *Request) { r.NumFrames = n } }
|
||||
|
||||
// WithFPS sets the frame rate of the generated clip.
|
||||
func WithFPS(fps int) Option { return func(r *Request) { r.FPS = fps } }
|
||||
|
||||
// WithSteps overrides the number of diffusion steps.
|
||||
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
|
||||
|
||||
// WithGuidanceScale overrides the guidance strength.
|
||||
func WithGuidanceScale(s float64) Option { return func(r *Request) { r.GuidanceScale = &s } }
|
||||
|
||||
// WithNegativePrompt sets a negative prompt.
|
||||
func WithNegativePrompt(s string) Option { return func(r *Request) { r.NegativePrompt = s } }
|
||||
|
||||
// WithSeed fixes the RNG seed for reproducible output.
|
||||
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
|
||||
|
||||
// Apply returns a copy of the request with all options applied. Providers call
|
||||
// this once at the top of Generate.
|
||||
func (r Request) Apply(opts ...Option) Request {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Model generates a video clip from a text prompt and optional conditioning
|
||||
// frame. It is intentionally narrower than llm.Model — no Stream, no
|
||||
// Capabilities, no tool calls.
|
||||
type Model interface {
|
||||
// Generate produces one clip for the request. Generation is slow
|
||||
// (minutes on consumer hardware) and the call blocks until the clip is
|
||||
// ready; callers bound it with a context deadline.
|
||||
Generate(ctx context.Context, req Request, opts ...Option) (*Result, error)
|
||||
}
|
||||
|
||||
// ModelOption configures a Model at construction time (Provider.VideoModel).
|
||||
// Reserved for future per-model settings; present now so the interface is
|
||||
// forward-compatible.
|
||||
type ModelOption func(*ModelConfig)
|
||||
|
||||
// ModelConfig carries per-model construction settings.
|
||||
type ModelConfig struct{}
|
||||
|
||||
// ApplyModelOptions folds options into a config.
|
||||
func ApplyModelOptions(opts []ModelOption) ModelConfig {
|
||||
var cfg ModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// Provider mints video Models bound to one backend. It mirrors llm.Provider
|
||||
// but for video generation.
|
||||
type Provider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// VideoModel returns a Model bound to the given id (passed through to the
|
||||
// backend verbatim; no catalog validation).
|
||||
VideoModel(id string, opts ...ModelOption) (Model, error)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package videogen
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestApplyDoesNotMutateOriginal(t *testing.T) {
|
||||
orig := Request{Prompt: "a cat"}
|
||||
got := orig.Apply(
|
||||
WithSize("1280x704"),
|
||||
WithNumFrames(81),
|
||||
WithFPS(16),
|
||||
WithSteps(4),
|
||||
WithGuidanceScale(1.0),
|
||||
WithNegativePrompt("blurry"),
|
||||
WithSeed(42),
|
||||
WithInitImage(Image{MIME: "image/png", Data: []byte{1}}),
|
||||
)
|
||||
|
||||
if orig.Size != "" || orig.NumFrames != 0 || orig.FPS != 0 || orig.Steps != nil ||
|
||||
orig.GuidanceScale != nil || orig.NegativePrompt != "" || orig.Seed != nil || orig.InitImage != nil {
|
||||
t.Fatalf("original mutated: %+v", orig)
|
||||
}
|
||||
|
||||
if got.Size != "1280x704" || got.NumFrames != 81 || got.FPS != 16 {
|
||||
t.Errorf("size/frames/fps = %q/%d/%d", got.Size, got.NumFrames, got.FPS)
|
||||
}
|
||||
if got.Steps == nil || *got.Steps != 4 {
|
||||
t.Errorf("steps = %v", got.Steps)
|
||||
}
|
||||
if got.GuidanceScale == nil || *got.GuidanceScale != 1.0 {
|
||||
t.Errorf("guidance = %v", got.GuidanceScale)
|
||||
}
|
||||
if got.NegativePrompt != "blurry" {
|
||||
t.Errorf("negative = %q", got.NegativePrompt)
|
||||
}
|
||||
if got.Seed == nil || *got.Seed != 42 {
|
||||
t.Errorf("seed = %v", got.Seed)
|
||||
}
|
||||
if got.InitImage == nil || got.InitImage.MIME != "image/png" {
|
||||
t.Errorf("init image = %+v", got.InitImage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyModelOptions(t *testing.T) {
|
||||
called := false
|
||||
ApplyModelOptions([]ModelOption{func(*ModelConfig) { called = true }})
|
||||
if !called {
|
||||
t.Fatal("model option not applied")
|
||||
}
|
||||
}
|
||||
🔴 No material issues found in error handling.
error-handling · flagged by 2 models
provider/llamaswap/video.go:48-63* Checks every error from multipart form operations (WriteField,CreateFormFile,Write,Close) — verified atvideo.go:88-102* Handles thedoRawresponse error and empty-body edge case — verified atvideo.go:105-111*…🪰 Gadfly · advisory