majordomo/videogen/videogen.go

// Package videogen is majordomo's canonical video-generation surface. Like
// imagegen and audio, it is a deliberately separate contract from the llm
// package: video generation shares none of the chat message/tool/stream
// machinery, so it gets its own small Provider/Model interface rather than
// overloading llm.Model (ADR-0019).
//
// Zero values mean "backend default" throughout, mirroring imagegen: an empty
// Size leaves the backend's default resolution, zero NumFrames/FPS the
// backend's default clip length and rate.
//
// Text-to-video and image-to-video are one surface: a Request with a nil
// InitImage is a pure text prompt, a non-nil InitImage conditions generation
// on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same
// checkpoint, so unlike imagegen there is no separate Editor-style interface.
// LastImage extends the same surface to the other end of the clip, so one
// Request covers t2v, i2v, and first-last-frame-to-video without a mode flag.
//
// The first implementation is provider/llamaswap, which targets the blocking
// OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is
// the encoded video itself, so one request yields exactly one clip — Result
// carries a single Video, not a batch.
package videogen

import (
	"context"

	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)

// Image is a conditioning input frame (bytes + MIME). Aliased to
// llm.ImagePart so chat-sourced images feed image-to-video without
// conversion, mirroring imagegen.Image.
type Image = llm.ImagePart

// Video is one generated video: raw encoded bytes plus a MIME type
// (e.g. "video/mp4").
type Video struct {
	// Data is the encoded video container.
	Data []byte

	// MIME is the video MIME type, e.g. "video/mp4".
	MIME string
}

// Request is a video generation request. Zero values mean "backend default" —
// for llama-swap-served models that is the per-model default baked into the
// upstream launch flags. A caller overrides only what it explicitly sets.
type Request struct {
	// Prompt is the text description of the video to generate.
	Prompt string

	// InitImage conditions generation on a starting frame (image-to-video);
	// nil = pure text-to-video.
	InitImage *Image

	// LastImage conditions generation on an ENDING frame. With InitImage it
	// pins both ends (first-last-frame-to-video); alone it pins only the
	// destination and lets the backend invent the approach.
	//
	// Support is per-model and NOT advertised anywhere in this contract: a
	// backend that does not understand a trailing keyframe ignores it and
	// returns an ordinary clip, which is indistinguishable from success. A
	// caller that needs to know whether the pin took effect must establish
	// that out of band — see the note on LastImage support in
	// provider/llamaswap.
	LastImage *Image

	// Size is the requested resolution, e.g. "1280x704"; "" = backend default.
	Size string

	// NumFrames is the clip length in frames; 0 = backend default.
	NumFrames int

	// FPS is the frame rate of the generated clip; 0 = backend default.
	FPS int

	// Steps is the number of diffusion steps; nil = backend default.
	Steps *int

	// GuidanceScale is the guidance strength; nil = backend default.
	// Architecture-sensitive (distilled models want low or none), so prefer
	// leaving it nil unless the caller knows the target model.
	GuidanceScale *float64

	// NegativePrompt steers generation away from concepts; "" = none.
	NegativePrompt string

	// Seed fixes the RNG seed for reproducible output; nil = random.
	Seed *int64
}

// Result is the canonical video-generation result.
type Result struct {
	// Video is the generated clip.
	Video Video

	// Raw is the provider-native response object, an escape hatch for
	// provider-specific fields. May be nil; never required for normal use.
	Raw any
}

// Option mutates a Request before it is sent. Options passed to Generate are
// applied to a copy of the request, so a Request value can be reused.
type Option func(*Request)

// WithInitImage conditions generation on a starting frame (image-to-video).
func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } }

// WithLastImage conditions generation on an ending frame. Combined with
// WithInitImage this pins both ends of the clip.
func WithLastImage(img Image) Option { return func(r *Request) { r.LastImage = &img } }

// WithSize sets the requested resolution (e.g. "1280x704").
func WithSize(size string) Option { return func(r *Request) { r.Size = size } }

// WithNumFrames sets the clip length in frames.
func WithNumFrames(n int) Option { return func(r *Request) { r.NumFrames = n } }

// WithFPS sets the frame rate of the generated clip.
func WithFPS(fps int) Option { return func(r *Request) { r.FPS = fps } }

// WithSteps overrides the number of diffusion steps.
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }

// WithGuidanceScale overrides the guidance strength.
func WithGuidanceScale(s float64) Option { return func(r *Request) { r.GuidanceScale = &s } }

// WithNegativePrompt sets a negative prompt.
func WithNegativePrompt(s string) Option { return func(r *Request) { r.NegativePrompt = s } }

// WithSeed fixes the RNG seed for reproducible output.
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }

// Apply returns a copy of the request with all options applied. Providers call
// this once at the top of Generate.
func (r Request) Apply(opts ...Option) Request {
	for _, opt := range opts {
		opt(&r)
	}
	return r
}

// Model generates a video clip from a text prompt and optional conditioning
// frame. It is intentionally narrower than llm.Model — no Stream, no
// Capabilities, no tool calls.
type Model interface {
	// Generate produces one clip for the request. Generation is slow
	// (minutes on consumer hardware) and the call blocks until the clip is
	// ready; callers bound it with a context deadline.
	Generate(ctx context.Context, req Request, opts ...Option) (*Result, error)
}

// ModelOption configures a Model at construction time (Provider.VideoModel).
// Reserved for future per-model settings; present now so the interface is
// forward-compatible.
type ModelOption func(*ModelConfig)

// ModelConfig carries per-model construction settings.
type ModelConfig struct{}

// ApplyModelOptions folds options into a config.
func ApplyModelOptions(opts []ModelOption) ModelConfig {
	var cfg ModelConfig
	for _, opt := range opts {
		opt(&cfg)
	}
	return cfg
}

// Provider mints video Models bound to one backend. It mirrors llm.Provider
// but for video generation.
type Provider interface {
	// Name is the registry identifier for the provider.
	Name() string

	// VideoModel returns a Model bound to the given id (passed through to the
	// backend verbatim; no catalog validation).
	VideoModel(id string, opts ...ModelOption) (Model, error)
}