feat(videogen): canonical video-generation surface + llama-swap client

New videogen/ contract package (ADR-0019): Request/Result/Model/Provider with the imagegen conventions. Text-to-video and image-to-video are one surface (Request.InitImage, nil = t2v) since hybrid checkpoints like Wan 2.2 TI2V serve both from one model; Result carries a single clip. provider/llamaswap gains VideoModel(id) targeting the blocking POST {base}/v1/videos/sync (multipart, model-routed by the fork's new video routes): vLLM-Omni parameter names, OpenAI-style input_reference file part, optional fields stay off the wire so per-model launch-flag defaults apply. CLAUDE.md package map picks up audio/ (missed in #12) and videogen/. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AXQxVhXBw8PwFAtsVrXSmj
2026-07-12 09:31:08 -04:00
parent 5ed18abb94
commit 4629c6af0f
6 changed files with 640 additions and 1 deletions
@@ -0,0 +1,177 @@
+package llamaswap
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+	"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
+)
+
+// VideoModel implements videogen.Provider, binding a video-generation model
+// served by llama-swap (routed to a vLLM-Omni-style upstream, or any shim
+// exposing the same /v1/videos/sync shape). The id is passed through verbatim
+// and selects which upstream llama-swap loads.
+func (p *Provider) VideoModel(id string, opts ...videogen.ModelOption) (videogen.Model, error) {
+	if err := p.requireBaseURL(); err != nil {
+		return nil, err
+	}
+	_ = videogen.ApplyModelOptions(opts)
+	return &videoModel{p: p, id: id}, nil
+}
+
+type videoModel struct {
+	p  *Provider
+	id string
+}
+
+// Generate implements videogen.Model via POST {base}/v1/videos/sync
+// (multipart/form-data — llama-swap routes by the `model` form field). The
+// blocking sync endpoint answers with the encoded video itself, so the
+// response body is the result; there is no job id to poll. Generation runs
+// for minutes — the provider client carries no timeout by design, and callers
+// bound the call with a context deadline.
+//
+// Parameter names follow vLLM-Omni's videos API (num_frames, fps,
+// num_inference_steps, guidance_scale); the conditioning frame is sent as an
+// `input_reference` file part, following OpenAI's videos API. Upstreams
+// ignore fields they don't understand, and optional fields stay off the wire
+// entirely so the model's own defaults apply.
+func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) {
+	req = req.Apply(opts...)
+	if strings.TrimSpace(req.Prompt) == "" {
+		return nil, fmt.Errorf("%w: video generation requires a prompt", llm.ErrUnsupported)
+	}
+	if req.NumFrames < 0 {
+		return nil, fmt.Errorf("%w: video frame count must be >= 0, got %d", llm.ErrUnsupported, req.NumFrames)
+	}
+	if req.FPS < 0 {
+		return nil, fmt.Errorf("%w: video fps must be >= 0, got %d", llm.ErrUnsupported, req.FPS)
+	}
+	if req.InitImage != nil && len(req.InitImage.Data) == 0 {
+		return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported)
+	}
+	width, height, err := parseSize(req.Size)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
+	}
+
+	var buf bytes.Buffer
+	w := multipart.NewWriter(&buf)
+	// Required fields always go on the wire (an empty model id should fail
+	// loudly upstream, not silently vanish); optional ones only when set.
+	fields := []struct {
+		key, value string
+		required   bool
+	}{
+		{"model", m.id, true},
+		{"prompt", req.Prompt, true},
+		{"negative_prompt", req.NegativePrompt, false},
+		{"width", formatInt(width), false},
+		{"height", formatInt(height), false},
+		{"num_frames", formatNonZero(req.NumFrames), false},
+		{"fps", formatNonZero(req.FPS), false},
+		{"num_inference_steps", formatInt(req.Steps), false},
+		{"guidance_scale", formatFloat(req.GuidanceScale), false},
+		{"seed", formatInt64(req.Seed), false},
+	}
+	for _, f := range fields {
+		if !f.required && f.value == "" {
+			continue
+		}
+		if err := w.WriteField(f.key, f.value); err != nil {
+			return nil, fmt.Errorf("llama-swap: build video form: %w", err)
+		}
+	}
+	if req.InitImage != nil {
+		fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME))
+		if err != nil {
+			return nil, fmt.Errorf("llama-swap: build video form: %w", err)
+		}
+		if _, err := fw.Write(req.InitImage.Data); err != nil {
+			return nil, fmt.Errorf("llama-swap: build video form: %w", err)
+		}
+	}
+	if err := w.Close(); err != nil {
+		return nil, fmt.Errorf("llama-swap: build video form: %w", err)
+	}
+
+	videoBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/videos/sync", m.id, w.FormDataContentType(), &buf)
+	if err != nil {
+		return nil, err
+	}
+	if len(videoBytes) == 0 {
+		return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video response contained no video"}
+	}
+	return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: videoMIME(contentType, videoBytes)}}, nil
+}
+
+// videoMIME resolves the result MIME type: the response Content-Type when it
+// is a concrete video type, else content sniffing, else video/mp4 (the
+// endpoint's default container).
+func videoMIME(contentType string, data []byte) string {
+	if mt, _, err := mime.ParseMediaType(contentType); err == nil && strings.HasPrefix(mt, "video/") {
+		return mt
+	}
+	if mt := http.DetectContentType(data); strings.HasPrefix(mt, "video/") {
+		return mt
+	}
+	return "video/mp4"
+}
+
+// initImageFilename picks the multipart filename hint for the conditioning
+// frame from its MIME subtype. The name is provider-chosen (never
+// caller-supplied), so no sanitization is needed.
+func initImageFilename(mimeType string) string {
+	mt := strings.ToLower(strings.TrimSpace(mimeType))
+	if parsed, _, err := mime.ParseMediaType(mt); err == nil {
+		mt = parsed
+	}
+	switch mt {
+	case "image/jpeg", "image/jpg":
+		return "frame.jpg"
+	case "image/webp":
+		return "frame.webp"
+	default: // stable-diffusion outputs and unknowns: PNG is the safe hint
+		return "frame.png"
+	}
+}
+
+// formatInt renders an optional int pointer for a form field; nil = "" (omit).
+func formatInt(v *int) string {
+	if v == nil {
+		return ""
+	}
+	return strconv.Itoa(*v)
+}
+
+// formatInt64 renders an optional int64 pointer for a form field; nil = "" (omit).
+func formatInt64(v *int64) string {
+	if v == nil {
+		return ""
+	}
+	return strconv.FormatInt(*v, 10)
+}
+
+// formatFloat renders an optional float pointer for a form field; nil = "" (omit).
+func formatFloat(v *float64) string {
+	if v == nil {
+		return ""
+	}
+	return strconv.FormatFloat(*v, 'g', -1, 64)
+}
+
+// formatNonZero renders a non-negative int for a form field; 0 = "" (omit,
+// backend default).
+func formatNonZero(v int) string {
+	if v == 0 {
+		return ""
+	}
+	return strconv.Itoa(v)
+}
@@ -0,0 +1,197 @@
+package llamaswap
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+	"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
+)
+
+func TestVideoGenerate(t *testing.T) {
+	var gotPath, gotContentType string
+	var gotForm map[string]string
+	var gotFrame []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		gotContentType = r.Header.Get("Content-Type")
+		if err := r.ParseMultipartForm(32 << 20); err != nil {
+			t.Errorf("parse form: %v", err)
+		}
+		gotForm = map[string]string{}
+		for k, v := range r.MultipartForm.Value {
+			gotForm[k] = v[0]
+		}
+		if f, _, err := r.FormFile("input_reference"); err == nil {
+			gotFrame, _ = io.ReadAll(f)
+			f.Close()
+		}
+		w.Header().Set("Content-Type", "video/mp4")
+		_, _ = w.Write([]byte("fake-mp4-bytes"))
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	vm, err := p.VideoModel("videogen-wan")
+	if err != nil {
+		t.Fatalf("VideoModel: %v", err)
+	}
+
+	frame, _ := base64.StdEncoding.DecodeString(onePixelPNG)
+	res, err := vm.Generate(context.Background(),
+		videogen.Request{Prompt: "a cat surfing", InitImage: &videogen.Image{MIME: "image/png", Data: frame}},
+		videogen.WithSize("1280x704"),
+		videogen.WithNumFrames(81),
+		videogen.WithFPS(16),
+		videogen.WithSteps(4),
+		videogen.WithGuidanceScale(1.0),
+		videogen.WithNegativePrompt("blurry"),
+		videogen.WithSeed(42),
+	)
+	if err != nil {
+		t.Fatalf("Generate: %v", err)
+	}
+	if string(res.Video.Data) != "fake-mp4-bytes" || res.Video.MIME != "video/mp4" {
+		t.Fatalf("video = %d bytes, MIME %q", len(res.Video.Data), res.Video.MIME)
+	}
+
+	if gotPath != "/v1/videos/sync" {
+		t.Errorf("path = %q", gotPath)
+	}
+	if !strings.HasPrefix(gotContentType, "multipart/form-data") {
+		t.Errorf("content-type = %q", gotContentType)
+	}
+	want := map[string]string{
+		"model":               "videogen-wan",
+		"prompt":              "a cat surfing",
+		"negative_prompt":     "blurry",
+		"width":               "1280",
+		"height":              "704",
+		"num_frames":          "81",
+		"fps":                 "16",
+		"num_inference_steps": "4",
+		"guidance_scale":      "1",
+		"seed":                "42",
+	}
+	for k, v := range want {
+		if gotForm[k] != v {
+			t.Errorf("form[%q] = %q, want %q", k, gotForm[k], v)
+		}
+	}
+	if string(gotFrame) != string(frame) {
+		t.Errorf("input_reference = %d bytes, want %d", len(gotFrame), len(frame))
+	}
+}
+
+func TestVideoGenerateOmitsUnsetOverrides(t *testing.T) {
+	var gotForm map[string][]string
+	var hadFrame bool
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseMultipartForm(32 << 20)
+		gotForm = r.MultipartForm.Value
+		_, _, err := r.FormFile("input_reference")
+		hadFrame = err == nil
+		w.Header().Set("Content-Type", "video/mp4")
+		_, _ = w.Write([]byte("v"))
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	vm, _ := p.VideoModel("videogen-wan")
+	if _, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"}); err != nil {
+		t.Fatalf("Generate: %v", err)
+	}
+	for _, k := range []string{"negative_prompt", "width", "height", "num_frames", "fps", "num_inference_steps", "guidance_scale", "seed"} {
+		if _, ok := gotForm[k]; ok {
+			t.Errorf("form field %q sent, want omitted", k)
+		}
+	}
+	if hadFrame {
+		t.Error("input_reference sent, want omitted")
+	}
+}
+
+func TestVideoGenerateValidation(t *testing.T) {
+	p := New(WithBaseURL("http://unused.invalid"))
+	vm, _ := p.VideoModel("videogen-wan")
+
+	cases := []struct {
+		name string
+		req  videogen.Request
+	}{
+		{"empty prompt", videogen.Request{}},
+		{"negative frames", videogen.Request{Prompt: "x", NumFrames: -1}},
+		{"negative fps", videogen.Request{Prompt: "x", FPS: -1}},
+		{"empty init image", videogen.Request{Prompt: "x", InitImage: &videogen.Image{}}},
+		{"bad size", videogen.Request{Prompt: "x", Size: "banana"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := vm.Generate(context.Background(), tc.req)
+			if !errors.Is(err, llm.ErrUnsupported) {
+				t.Fatalf("err = %v, want ErrUnsupported", err)
+			}
+		})
+	}
+}
+
+func TestVideoGenerateUpstreamError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"error":{"message":"boom"}}`, http.StatusInternalServerError)
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	vm, _ := p.VideoModel("videogen-wan")
+	_, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"})
+	var apiErr *llm.APIError
+	if !errors.As(err, &apiErr) {
+		t.Fatalf("err = %v, want *llm.APIError", err)
+	}
+}
+
+func TestVideoGenerateEmptyResponse(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "video/mp4")
+	}))
+	defer srv.Close()
+
+	p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
+	vm, _ := p.VideoModel("videogen-wan")
+	_, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"})
+	var apiErr *llm.APIError
+	if !errors.As(err, &apiErr) {
+		t.Fatalf("err = %v, want *llm.APIError for empty body", err)
+	}
+}
+
+func TestVideoModelRequiresBaseURL(t *testing.T) {
+	p := New()
+	if _, err := p.VideoModel("videogen-wan"); err == nil {
+		t.Fatal("VideoModel with no base URL should error")
+	}
+}
+
+func TestVideoMIME(t *testing.T) {
+	cases := []struct {
+		contentType string
+		data        []byte
+		want        string
+	}{
+		{"video/webm", []byte("x"), "video/webm"},
+		{"video/mp4; charset=binary", []byte("x"), "video/mp4"},
+		{"application/octet-stream", []byte("x"), "video/mp4"},
+		{"", []byte("x"), "video/mp4"},
+	}
+	for _, tc := range cases {
+		if got := videoMIME(tc.contentType, tc.data); got != tc.want {
+			t.Errorf("videoMIME(%q) = %q, want %q", tc.contentType, got, tc.want)
+		}
+	}
+}