feat(videogen): canonical video-generation surface + llama-swap client
New videogen/ contract package (ADR-0019): Request/Result/Model/Provider
with the imagegen conventions. Text-to-video and image-to-video are one
surface (Request.InitImage, nil = t2v) since hybrid checkpoints like
Wan 2.2 TI2V serve both from one model; Result carries a single clip.
provider/llamaswap gains VideoModel(id) targeting the blocking
POST {base}/v1/videos/sync (multipart, model-routed by the fork's new
video routes): vLLM-Omni parameter names, OpenAI-style input_reference
file part, optional fields stay off the wire so per-model launch-flag
defaults apply. CLAUDE.md package map picks up audio/ (missed in #12)
and videogen/.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AXQxVhXBw8PwFAtsVrXSmj
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
// Package videogen is majordomo's canonical video-generation surface. Like
|
||||
// imagegen and audio, it is a deliberately separate contract from the llm
|
||||
// package: video generation shares none of the chat message/tool/stream
|
||||
// machinery, so it gets its own small Provider/Model interface rather than
|
||||
// overloading llm.Model (ADR-0019).
|
||||
//
|
||||
// Zero values mean "backend default" throughout, mirroring imagegen: an empty
|
||||
// Size leaves the backend's default resolution, zero NumFrames/FPS the
|
||||
// backend's default clip length and rate.
|
||||
//
|
||||
// Text-to-video and image-to-video are one surface: a Request with a nil
|
||||
// InitImage is a pure text prompt, a non-nil InitImage conditions generation
|
||||
// on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same
|
||||
// checkpoint, so unlike imagegen there is no separate Editor-style interface.
|
||||
//
|
||||
// The first implementation is provider/llamaswap, which targets the blocking
|
||||
// OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is
|
||||
// the encoded video itself, so one request yields exactly one clip — Result
|
||||
// carries a single Video, not a batch.
|
||||
package videogen
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// Image is a conditioning input frame (bytes + MIME). Aliased to
|
||||
// llm.ImagePart so chat-sourced images feed image-to-video without
|
||||
// conversion, mirroring imagegen.Image.
|
||||
type Image = llm.ImagePart
|
||||
|
||||
// Video is one generated video: raw encoded bytes plus a MIME type
|
||||
// (e.g. "video/mp4").
|
||||
type Video struct {
|
||||
// Data is the encoded video container.
|
||||
Data []byte
|
||||
|
||||
// MIME is the video MIME type, e.g. "video/mp4".
|
||||
MIME string
|
||||
}
|
||||
|
||||
// Request is a video generation request. Zero values mean "backend default" —
|
||||
// for llama-swap-served models that is the per-model default baked into the
|
||||
// upstream launch flags. A caller overrides only what it explicitly sets.
|
||||
type Request struct {
|
||||
// Prompt is the text description of the video to generate.
|
||||
Prompt string
|
||||
|
||||
// InitImage conditions generation on a starting frame (image-to-video);
|
||||
// nil = pure text-to-video.
|
||||
InitImage *Image
|
||||
|
||||
// Size is the requested resolution, e.g. "1280x704"; "" = backend default.
|
||||
Size string
|
||||
|
||||
// NumFrames is the clip length in frames; 0 = backend default.
|
||||
NumFrames int
|
||||
|
||||
// FPS is the frame rate of the generated clip; 0 = backend default.
|
||||
FPS int
|
||||
|
||||
// Steps is the number of diffusion steps; nil = backend default.
|
||||
Steps *int
|
||||
|
||||
// GuidanceScale is the guidance strength; nil = backend default.
|
||||
// Architecture-sensitive (distilled models want low or none), so prefer
|
||||
// leaving it nil unless the caller knows the target model.
|
||||
GuidanceScale *float64
|
||||
|
||||
// NegativePrompt steers generation away from concepts; "" = none.
|
||||
NegativePrompt string
|
||||
|
||||
// Seed fixes the RNG seed for reproducible output; nil = random.
|
||||
Seed *int64
|
||||
}
|
||||
|
||||
// Result is the canonical video-generation result.
|
||||
type Result struct {
|
||||
// Video is the generated clip.
|
||||
Video Video
|
||||
|
||||
// Raw is the provider-native response object, an escape hatch for
|
||||
// provider-specific fields. May be nil; never required for normal use.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// Option mutates a Request before it is sent. Options passed to Generate are
|
||||
// applied to a copy of the request, so a Request value can be reused.
|
||||
type Option func(*Request)
|
||||
|
||||
// WithInitImage conditions generation on a starting frame (image-to-video).
|
||||
func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } }
|
||||
|
||||
// WithSize sets the requested resolution (e.g. "1280x704").
|
||||
func WithSize(size string) Option { return func(r *Request) { r.Size = size } }
|
||||
|
||||
// WithNumFrames sets the clip length in frames.
|
||||
func WithNumFrames(n int) Option { return func(r *Request) { r.NumFrames = n } }
|
||||
|
||||
// WithFPS sets the frame rate of the generated clip.
|
||||
func WithFPS(fps int) Option { return func(r *Request) { r.FPS = fps } }
|
||||
|
||||
// WithSteps overrides the number of diffusion steps.
|
||||
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
|
||||
|
||||
// WithGuidanceScale overrides the guidance strength.
|
||||
func WithGuidanceScale(s float64) Option { return func(r *Request) { r.GuidanceScale = &s } }
|
||||
|
||||
// WithNegativePrompt sets a negative prompt.
|
||||
func WithNegativePrompt(s string) Option { return func(r *Request) { r.NegativePrompt = s } }
|
||||
|
||||
// WithSeed fixes the RNG seed for reproducible output.
|
||||
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
|
||||
|
||||
// Apply returns a copy of the request with all options applied. Providers call
|
||||
// this once at the top of Generate.
|
||||
func (r Request) Apply(opts ...Option) Request {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Model generates a video clip from a text prompt and optional conditioning
|
||||
// frame. It is intentionally narrower than llm.Model — no Stream, no
|
||||
// Capabilities, no tool calls.
|
||||
type Model interface {
|
||||
// Generate produces one clip for the request. Generation is slow
|
||||
// (minutes on consumer hardware) and the call blocks until the clip is
|
||||
// ready; callers bound it with a context deadline.
|
||||
Generate(ctx context.Context, req Request, opts ...Option) (*Result, error)
|
||||
}
|
||||
|
||||
// ModelOption configures a Model at construction time (Provider.VideoModel).
|
||||
// Reserved for future per-model settings; present now so the interface is
|
||||
// forward-compatible.
|
||||
type ModelOption func(*ModelConfig)
|
||||
|
||||
// ModelConfig carries per-model construction settings.
|
||||
type ModelConfig struct{}
|
||||
|
||||
// ApplyModelOptions folds options into a config.
|
||||
func ApplyModelOptions(opts []ModelOption) ModelConfig {
|
||||
var cfg ModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// Provider mints video Models bound to one backend. It mirrors llm.Provider
|
||||
// but for video generation.
|
||||
type Provider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// VideoModel returns a Model bound to the given id (passed through to the
|
||||
// backend verbatim; no catalog validation).
|
||||
VideoModel(id string, opts ...ModelOption) (Model, error)
|
||||
}
|
||||
Reference in New Issue
Block a user