majordomo/provider/llamaswap/video.go

package llamaswap

import (
	"bytes"
	"context"
	"fmt"
	"mime"
	"mime/multipart"
	"net/http"
	"strconv"
	"strings"

	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
	"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
)

// VideoModel implements videogen.Provider, binding a video-generation model
// served by llama-swap (routed to a vLLM-Omni-style upstream, or any shim
// exposing the same /v1/videos/sync shape). The id is passed through verbatim
// and selects which upstream llama-swap loads.
func (p *Provider) VideoModel(id string, opts ...videogen.ModelOption) (videogen.Model, error) {
	if err := p.requireBaseURL(); err != nil {
		return nil, err
	}
	_ = videogen.ApplyModelOptions(opts)
	return &videoModel{p: p, id: id}, nil
}

type videoModel struct {
	p  *Provider
	id string
}

// Generate implements videogen.Model via POST {base}/v1/videos/sync
// (multipart/form-data — llama-swap routes by the `model` form field). The
// blocking sync endpoint answers with the encoded video itself, so the
// response body is the result; there is no job id to poll. Generation runs
// for minutes — the provider client carries no timeout by design, and callers
// bound the call with a context deadline.
//
// Parameter names follow vLLM-Omni's videos API (num_frames, fps,
// num_inference_steps, guidance_scale); the conditioning frame is sent as an
// `input_reference` file part, following OpenAI's videos API. Upstreams
// ignore fields they don't understand, and optional fields stay off the wire
// entirely so the model's own defaults apply.
func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) {
	req = req.Apply(opts...)
	if strings.TrimSpace(req.Prompt) == "" {
		return nil, fmt.Errorf("%w: video generation requires a prompt", llm.ErrUnsupported)
	}
	if req.NumFrames < 0 {
		return nil, fmt.Errorf("%w: video frame count must be >= 0, got %d", llm.ErrUnsupported, req.NumFrames)
	}
	if req.FPS < 0 {
		return nil, fmt.Errorf("%w: video fps must be >= 0, got %d", llm.ErrUnsupported, req.FPS)
	}
	if req.InitImage != nil && len(req.InitImage.Data) == 0 {
		return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported)
	}
	width, height, err := parseSize(req.Size)
	if err != nil {
		return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
	}

	var buf bytes.Buffer
	w := multipart.NewWriter(&buf)
	// Required fields always go on the wire (an empty model id should fail
	// loudly upstream, not silently vanish); optional ones only when set.
	fields := []struct {
		key, value string
		required   bool
	}{
		{"model", m.id, true},
		{"prompt", req.Prompt, true},
		{"negative_prompt", req.NegativePrompt, false},
		{"width", formatInt(width), false},
		{"height", formatInt(height), false},
		{"num_frames", formatNonZero(req.NumFrames), false},
		{"fps", formatNonZero(req.FPS), false},
		{"num_inference_steps", formatInt(req.Steps), false},
		{"guidance_scale", formatFloat(req.GuidanceScale), false},
		{"seed", formatInt64(req.Seed), false},
	}
	for _, f := range fields {
		if !f.required && f.value == "" {
			continue
		}
		if err := w.WriteField(f.key, f.value); err != nil {
			return nil, fmt.Errorf("llama-swap: build video form: %w", err)
		}
	}
	if req.InitImage != nil {
		fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME))
		if err != nil {
			return nil, fmt.Errorf("llama-swap: build video form: %w", err)
		}
		if _, err := fw.Write(req.InitImage.Data); err != nil {
			return nil, fmt.Errorf("llama-swap: build video form: %w", err)
		}
	}
	if err := w.Close(); err != nil {
		return nil, fmt.Errorf("llama-swap: build video form: %w", err)
	}

	videoBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/videos/sync", m.id, w.FormDataContentType(), &buf)
	if err != nil {
		return nil, err
	}
	if len(videoBytes) == 0 {
		return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video response contained no video"}
	}
	return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: videoMIME(contentType, videoBytes)}}, nil
}

// videoMIME resolves the result MIME type: the response Content-Type when it
// is a concrete video type, else content sniffing, else video/mp4 (the
// endpoint's default container).
func videoMIME(contentType string, data []byte) string {
	if mt, _, err := mime.ParseMediaType(contentType); err == nil && strings.HasPrefix(mt, "video/") {
		return mt
	}
	if mt := http.DetectContentType(data); strings.HasPrefix(mt, "video/") {
		return mt
	}
	return "video/mp4"
}

// initImageFilename picks the multipart filename hint for the conditioning
// frame from its MIME subtype. The name is provider-chosen (never
// caller-supplied), so no sanitization is needed.
func initImageFilename(mimeType string) string {
	mt := strings.ToLower(strings.TrimSpace(mimeType))
	if parsed, _, err := mime.ParseMediaType(mt); err == nil {
		mt = parsed
	}
	switch mt {
	case "image/jpeg", "image/jpg":
		return "frame.jpg"
	case "image/webp":
		return "frame.webp"
	default: // stable-diffusion outputs and unknowns: PNG is the safe hint
		return "frame.png"
	}
}

// formatInt renders an optional int pointer for a form field; nil = "" (omit).
func formatInt(v *int) string {
	if v == nil {
		return ""
	}
	return strconv.Itoa(*v)
}

// formatInt64 renders an optional int64 pointer for a form field; nil = "" (omit).
func formatInt64(v *int64) string {
	if v == nil {
		return ""
	}
	return strconv.FormatInt(*v, 10)
}

// formatFloat renders an optional float pointer for a form field; nil = "" (omit).
func formatFloat(v *float64) string {
	if v == nil {
		return ""
	}
	return strconv.FormatFloat(*v, 'g', -1, 64)
}

// formatNonZero renders a non-negative int for a form field; 0 = "" (omit,
// backend default).
func formatNonZero(v int) string {
	if v == 0 {
		return ""
	}
	return strconv.Itoa(v)
}