majordomo/provider/llamaswap/video.go

package llamaswap

import (
	"bytes"
	"context"
	"fmt"
	"mime/multipart"
	"net/http"
	"strconv"
	"strings"

	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
	"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
)

// VideoModel implements videogen.Provider, binding a video-generation model
// served by llama-swap (routed to a vLLM-Omni-style upstream, or any shim
// exposing the same /v1/videos/sync shape). The id is passed through verbatim
// and selects which upstream llama-swap loads.
func (p *Provider) VideoModel(id string, opts ...videogen.ModelOption) (videogen.Model, error) {
	if err := p.requireBaseURL(); err != nil {
		return nil, err
	}
	_ = videogen.ApplyModelOptions(opts)
	return &videoModel{p: p, id: id}, nil
}

type videoModel struct {
	p  *Provider
	id string
}

// Generate implements videogen.Model via POST {base}/v1/videos/sync
// (multipart/form-data — llama-swap routes by the `model` form field). The
// blocking sync endpoint answers with the encoded video itself, so the
// response body is the result; there is no job id to poll. Generation runs
// for minutes — the provider client carries no timeout by design, and callers
// bound the call with a context deadline.
//
// Parameter names follow vLLM-Omni's videos API (num_frames, fps,
// num_inference_steps, guidance_scale); the leading conditioning frame is sent
// as an `input_reference` file part, following OpenAI's videos API, and a
// trailing keyframe (Request.LastImage) as `input_reference_last`. Upstreams
// ignore fields they don't understand, and optional fields stay off the wire
// entirely so the model's own defaults apply — which is also why a backend
// without first-last-frame support returns an ordinary clip here rather than
// an error.
func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) {
	req = req.Apply(opts...)
	if strings.TrimSpace(req.Prompt) == "" {
		return nil, fmt.Errorf("%w: video generation requires a prompt", llm.ErrUnsupported)
	}
	if req.NumFrames < 0 {
		return nil, fmt.Errorf("%w: video frame count must be >= 0, got %d", llm.ErrUnsupported, req.NumFrames)
	}
	if req.FPS < 0 {
		return nil, fmt.Errorf("%w: video fps must be >= 0, got %d", llm.ErrUnsupported, req.FPS)
	}
	if req.InitImage != nil && len(req.InitImage.Data) == 0 {
		return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported)
	}
	if req.LastImage != nil && len(req.LastImage.Data) == 0 {
		return nil, fmt.Errorf("%w: video last image has no bytes", llm.ErrUnsupported)
	}
	width, height, err := parseSize(req.Size)
	if err != nil {
		return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
	}

	var buf bytes.Buffer
	w := multipart.NewWriter(&buf)
	if err := writeFormFields(w, "build video form", []formField{
		{"model", m.id, true},
		{"prompt", req.Prompt, true},
		{"negative_prompt", req.NegativePrompt, false},
		// Resolution rides the wire twice: width/height (vLLM-Omni's
		// names) AND the equivalent OpenAI-style size string, since
		// upstreams silently ignore fields they don't understand and the
		// values can never disagree.
		{"width", formatInt(width), false},
		{"height", formatInt(height), false},
		{"size", strings.TrimSpace(req.Size), false},
		{"num_frames", formatNonZero(req.NumFrames), false},
		{"fps", formatNonZero(req.FPS), false},
		{"num_inference_steps", formatInt(req.Steps), false},
		{"guidance_scale", formatFloat(req.GuidanceScale), false},
		{"seed", formatInt64(req.Seed), false},
	}); err != nil {
		return nil, err
	}
	if req.InitImage != nil {
		if err := writeImagePart(w, "input_reference", "frame", req.InitImage); err != nil {
			return nil, err
		}
	}
	// The trailing keyframe rides a SEPARATE part rather than a second
	// `input_reference`: multipart permits repeated names, but the receiving
	// end would then have to rely on part ORDER to tell first from last, and
	// an ordering contract that is invisible in the field name is one nobody
	// can see they have broken. A backend that does not know the name ignores
	// the part, which is the same degradation as any other unknown field.
	if req.LastImage != nil {
		if err := writeImagePart(w, "input_reference_last", "frame_last", req.LastImage); err != nil {
			return nil, err
		}
	}
	if err := w.Close(); err != nil {
		return nil, fmt.Errorf("llama-swap: build video form: %w", err)
	}

	videoBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/videos/sync", m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes)
	if err != nil {
		return nil, err
	}
	return singleVideoResult(m.p.name, m.id, "video", videoBytes, contentType)
}

// videoMIME resolves the result MIME type: the response Content-Type when it
// is a concrete video type, else content sniffing (mp4/webm magic bytes),
// else "" — the caller treats undetectable as an upstream error, unlike the
// audio path where the request's format param implies the container.
func videoMIME(contentType string, data []byte) string {
	if mt := mimeFromContentType(contentType, "video/"); mt != "" {
		return mt
	}
	if mt := http.DetectContentType(data); strings.HasPrefix(mt, "video/") {
		return mt
	}
	return ""
}

// singleVideoResult wraps one raw video body into a videogen.Result,
// requiring positive evidence of video-ness (declared video/* Content-Type
// or sniffed mp4/webm magic) so a 2xx body that is anything else — a JSON
// job envelope, an HTML error page behind a proxy — fails loud instead of
// coming back as "the clip". The video sibling of singleImageResult, shared
// by every surface whose response body IS the encoded clip.
func singleVideoResult(provider, model, verb string, raw []byte, contentType string) (*videogen.Result, error) {
	if len(raw) == 0 {
		return nil, &llm.APIError{Provider: provider, Model: model, Message: verb + " response contained no video"}
	}
	mimeType := videoMIME(contentType, raw)
	if mimeType == "" {
		return nil, &llm.APIError{Provider: provider, Model: model,
			Message: fmt.Sprintf("%s response is not a video (Content-Type %q)", verb, contentType)}
	}
	return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil
}

// initImageFilename picks the multipart filename hint for the conditioning
// frame from its MIME subtype. The name is provider-chosen (never
// caller-supplied), so no sanitization is needed.
func initImageFilename(mimeType string) string {
	return imageFilename(mimeType, "frame")
}

// writeImagePart attaches one conditioning frame under the given field name,
// with a filename derived from nameStem. Shared by the first- and last-frame
// parts so the two cannot drift in how they encode, which is the usual way a
// second copy of a block goes wrong.
//
// The two frames MUST carry DISTINCT filenames, not merely distinct field
// names. Backends commonly stage an uploaded frame under a name derived from
// the filename — our own ComfyUI shim posts to /upload/image with
// overwrite=true — so two parts sharing "frame.png" would have the second
// clobber the first, and BOTH keyframe inputs would then resolve to the same
// stored image. The clip would render clean, pinned at both ends to the same
// frame, with nothing anywhere reporting a problem.
func writeImagePart(w *multipart.Writer, field, nameStem string, img *videogen.Image) error {
	fw, err := w.CreateFormFile(field, imageFilename(img.MIME, nameStem))
	if err != nil {
		return fmt.Errorf("llama-swap: build video form: %w", err)
	}
	if _, err := fw.Write(img.Data); err != nil {
		return fmt.Errorf("llama-swap: build video form: %w", err)
	}
	return nil
}

// formatInt renders an optional int pointer for a form field; nil = "" (omit).
func formatInt(v *int) string {
	if v == nil {
		return ""
	}
	return strconv.Itoa(*v)
}

// formatInt64 renders an optional int64 pointer for a form field; nil = "" (omit).
func formatInt64(v *int64) string {
	if v == nil {
		return ""
	}
	return strconv.FormatInt(*v, 10)
}

// formatFloat renders an optional float pointer for a form field; nil = "" (omit).
func formatFloat(v *float64) string {
	if v == nil {
		return ""
	}
	return strconv.FormatFloat(*v, 'g', -1, 64)
}

// formatNonZero renders a non-negative int for a form field; 0 = "" (omit,
// backend default).
func formatNonZero(v int) string {
	if v == 0 {
		return ""
	}
	return strconv.Itoa(v)
}