package llamaswap import ( "bytes" "context" "fmt" "mime" "mime/multipart" "net/http" "strconv" "strings" "gitea.stevedudenhoeffer.com/steve/majordomo/llm" "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" ) // VideoModel implements videogen.Provider, binding a video-generation model // served by llama-swap (routed to a vLLM-Omni-style upstream, or any shim // exposing the same /v1/videos/sync shape). The id is passed through verbatim // and selects which upstream llama-swap loads. func (p *Provider) VideoModel(id string, opts ...videogen.ModelOption) (videogen.Model, error) { if err := p.requireBaseURL(); err != nil { return nil, err } _ = videogen.ApplyModelOptions(opts) return &videoModel{p: p, id: id}, nil } type videoModel struct { p *Provider id string } // Generate implements videogen.Model via POST {base}/v1/videos/sync // (multipart/form-data — llama-swap routes by the `model` form field). The // blocking sync endpoint answers with the encoded video itself, so the // response body is the result; there is no job id to poll. Generation runs // for minutes — the provider client carries no timeout by design, and callers // bound the call with a context deadline. // // Parameter names follow vLLM-Omni's videos API (num_frames, fps, // num_inference_steps, guidance_scale); the conditioning frame is sent as an // `input_reference` file part, following OpenAI's videos API. Upstreams // ignore fields they don't understand, and optional fields stay off the wire // entirely so the model's own defaults apply. func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) { req = req.Apply(opts...) if strings.TrimSpace(req.Prompt) == "" { return nil, fmt.Errorf("%w: video generation requires a prompt", llm.ErrUnsupported) } if req.NumFrames < 0 { return nil, fmt.Errorf("%w: video frame count must be >= 0, got %d", llm.ErrUnsupported, req.NumFrames) } if req.FPS < 0 { return nil, fmt.Errorf("%w: video fps must be >= 0, got %d", llm.ErrUnsupported, req.FPS) } if req.InitImage != nil && len(req.InitImage.Data) == 0 { return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported) } width, height, err := parseSize(req.Size) if err != nil { return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err) } var buf bytes.Buffer w := multipart.NewWriter(&buf) if err := writeFormFields(w, "build video form", []formField{ {"model", m.id, true}, {"prompt", req.Prompt, true}, {"negative_prompt", req.NegativePrompt, false}, // Resolution rides the wire twice: width/height (vLLM-Omni's // names) AND the equivalent OpenAI-style size string, since // upstreams silently ignore fields they don't understand and the // values can never disagree. {"width", formatInt(width), false}, {"height", formatInt(height), false}, {"size", strings.TrimSpace(req.Size), false}, {"num_frames", formatNonZero(req.NumFrames), false}, {"fps", formatNonZero(req.FPS), false}, {"num_inference_steps", formatInt(req.Steps), false}, {"guidance_scale", formatFloat(req.GuidanceScale), false}, {"seed", formatInt64(req.Seed), false}, }); err != nil { return nil, err } if req.InitImage != nil { fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME)) if err != nil { return nil, fmt.Errorf("llama-swap: build video form: %w", err) } if _, err := fw.Write(req.InitImage.Data); err != nil { return nil, fmt.Errorf("llama-swap: build video form: %w", err) } } if err := w.Close(); err != nil { return nil, fmt.Errorf("llama-swap: build video form: %w", err) } videoBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/videos/sync", m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes) if err != nil { return nil, err } if len(videoBytes) == 0 { return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video response contained no video"} } mimeType := videoMIME(contentType, videoBytes) if mimeType == "" { // A 2xx body that is neither declared nor sniffable as video is a // misconfigured upstream (a JSON job envelope, an HTML error page // behind a proxy) — fail loud rather than hand back garbage as a // playable clip. return nil, &llm.APIError{ Provider: m.p.name, Model: m.id, Message: fmt.Sprintf("video response is not a video (content-type %q)", contentType), } } return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: mimeType}}, nil } // videoMIME resolves the result MIME type: the response Content-Type when it // is a concrete video type, else content sniffing (mp4/webm magic bytes), // else "" — the caller treats undetectable as an upstream error, unlike the // audio path where the request's format param implies the container. func videoMIME(contentType string, data []byte) string { if mt := mimeFromContentType(contentType, "video/"); mt != "" { return mt } if mt := http.DetectContentType(data); strings.HasPrefix(mt, "video/") { return mt } return "" } // initImageFilename picks the multipart filename hint for the conditioning // frame from its MIME subtype. The name is provider-chosen (never // caller-supplied), so no sanitization is needed. func initImageFilename(mimeType string) string { mt := strings.ToLower(strings.TrimSpace(mimeType)) if parsed, _, err := mime.ParseMediaType(mt); err == nil { mt = parsed } switch mt { case "image/jpeg", "image/jpg": return "frame.jpg" case "image/webp": return "frame.webp" default: // unknown MIME — PNG is the safe hint return "frame.png" } } // formatInt renders an optional int pointer for a form field; nil = "" (omit). func formatInt(v *int) string { if v == nil { return "" } return strconv.Itoa(*v) } // formatInt64 renders an optional int64 pointer for a form field; nil = "" (omit). func formatInt64(v *int64) string { if v == nil { return "" } return strconv.FormatInt(*v, 10) } // formatFloat renders an optional float pointer for a form field; nil = "" (omit). func formatFloat(v *float64) string { if v == nil { return "" } return strconv.FormatFloat(*v, 'g', -1, 64) } // formatNonZero renders a non-negative int for a form field; 0 = "" (omit, // backend default). func formatNonZero(v int) string { if v == 0 { return "" } return strconv.Itoa(v) }