package llamaswap import ( "bytes" "context" "fmt" "mime/multipart" "net/http" "strconv" "strings" "gitea.stevedudenhoeffer.com/steve/majordomo/llm" "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" ) // VideoModel implements videogen.Provider, binding a video-generation model // served by llama-swap (routed to a vLLM-Omni-style upstream, or any shim // exposing the same /v1/videos/sync shape). The id is passed through verbatim // and selects which upstream llama-swap loads. func (p *Provider) VideoModel(id string, opts ...videogen.ModelOption) (videogen.Model, error) { if err := p.requireBaseURL(); err != nil { return nil, err } _ = videogen.ApplyModelOptions(opts) return &videoModel{p: p, id: id}, nil } type videoModel struct { p *Provider id string } // Generate implements videogen.Model via POST {base}/v1/videos/sync // (multipart/form-data — llama-swap routes by the `model` form field). The // blocking sync endpoint answers with the encoded video itself, so the // response body is the result; there is no job id to poll. Generation runs // for minutes — the provider client carries no timeout by design, and callers // bound the call with a context deadline. // // Parameter names follow vLLM-Omni's videos API (num_frames, fps, // num_inference_steps, guidance_scale); the leading conditioning frame is sent // as an `input_reference` file part, following OpenAI's videos API, and a // trailing keyframe (Request.LastImage) as `input_reference_last`. Upstreams // ignore fields they don't understand, and optional fields stay off the wire // entirely so the model's own defaults apply — which is also why a backend // without first-last-frame support returns an ordinary clip here rather than // an error. func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) { req = req.Apply(opts...) if strings.TrimSpace(req.Prompt) == "" { return nil, fmt.Errorf("%w: video generation requires a prompt", llm.ErrUnsupported) } if req.NumFrames < 0 { return nil, fmt.Errorf("%w: video frame count must be >= 0, got %d", llm.ErrUnsupported, req.NumFrames) } if req.FPS < 0 { return nil, fmt.Errorf("%w: video fps must be >= 0, got %d", llm.ErrUnsupported, req.FPS) } if req.InitImage != nil && len(req.InitImage.Data) == 0 { return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported) } if req.LastImage != nil && len(req.LastImage.Data) == 0 { return nil, fmt.Errorf("%w: video last image has no bytes", llm.ErrUnsupported) } width, height, err := parseSize(req.Size) if err != nil { return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err) } var buf bytes.Buffer w := multipart.NewWriter(&buf) if err := writeFormFields(w, "build video form", []formField{ {"model", m.id, true}, {"prompt", req.Prompt, true}, {"negative_prompt", req.NegativePrompt, false}, // Resolution rides the wire twice: width/height (vLLM-Omni's // names) AND the equivalent OpenAI-style size string, since // upstreams silently ignore fields they don't understand and the // values can never disagree. {"width", formatInt(width), false}, {"height", formatInt(height), false}, {"size", strings.TrimSpace(req.Size), false}, {"num_frames", formatNonZero(req.NumFrames), false}, {"fps", formatNonZero(req.FPS), false}, {"num_inference_steps", formatInt(req.Steps), false}, {"guidance_scale", formatFloat(req.GuidanceScale), false}, {"seed", formatInt64(req.Seed), false}, }); err != nil { return nil, err } if req.InitImage != nil { if err := writeImagePart(w, "input_reference", "frame", req.InitImage); err != nil { return nil, err } } // The trailing keyframe rides a SEPARATE part rather than a second // `input_reference`: multipart permits repeated names, but the receiving // end would then have to rely on part ORDER to tell first from last, and // an ordering contract that is invisible in the field name is one nobody // can see they have broken. A backend that does not know the name ignores // the part, which is the same degradation as any other unknown field. if req.LastImage != nil { if err := writeImagePart(w, "input_reference_last", "frame_last", req.LastImage); err != nil { return nil, err } } if err := w.Close(); err != nil { return nil, fmt.Errorf("llama-swap: build video form: %w", err) } videoBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/videos/sync", m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes) if err != nil { return nil, err } return singleVideoResult(m.p.name, m.id, "video", videoBytes, contentType) } // videoMIME resolves the result MIME type: the response Content-Type when it // is a concrete video type, else content sniffing (mp4/webm magic bytes), // else "" — the caller treats undetectable as an upstream error, unlike the // audio path where the request's format param implies the container. func videoMIME(contentType string, data []byte) string { if mt := mimeFromContentType(contentType, "video/"); mt != "" { return mt } if mt := http.DetectContentType(data); strings.HasPrefix(mt, "video/") { return mt } return "" } // singleVideoResult wraps one raw video body into a videogen.Result, // requiring positive evidence of video-ness (declared video/* Content-Type // or sniffed mp4/webm magic) so a 2xx body that is anything else — a JSON // job envelope, an HTML error page behind a proxy — fails loud instead of // coming back as "the clip". The video sibling of singleImageResult, shared // by every surface whose response body IS the encoded clip. func singleVideoResult(provider, model, verb string, raw []byte, contentType string) (*videogen.Result, error) { if len(raw) == 0 { return nil, &llm.APIError{Provider: provider, Model: model, Message: verb + " response contained no video"} } mimeType := videoMIME(contentType, raw) if mimeType == "" { return nil, &llm.APIError{Provider: provider, Model: model, Message: fmt.Sprintf("%s response is not a video (Content-Type %q)", verb, contentType)} } return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil } // initImageFilename picks the multipart filename hint for the conditioning // frame from its MIME subtype. The name is provider-chosen (never // caller-supplied), so no sanitization is needed. func initImageFilename(mimeType string) string { return imageFilename(mimeType, "frame") } // writeImagePart attaches one conditioning frame under the given field name, // with a filename derived from nameStem. Shared by the first- and last-frame // parts so the two cannot drift in how they encode, which is the usual way a // second copy of a block goes wrong. // // The two frames MUST carry DISTINCT filenames, not merely distinct field // names. Backends commonly stage an uploaded frame under a name derived from // the filename — our own ComfyUI shim posts to /upload/image with // overwrite=true — so two parts sharing "frame.png" would have the second // clobber the first, and BOTH keyframe inputs would then resolve to the same // stored image. The clip would render clean, pinned at both ends to the same // frame, with nothing anywhere reporting a problem. func writeImagePart(w *multipart.Writer, field, nameStem string, img *videogen.Image) error { fw, err := w.CreateFormFile(field, imageFilename(img.MIME, nameStem)) if err != nil { return fmt.Errorf("llama-swap: build video form: %w", err) } if _, err := fw.Write(img.Data); err != nil { return fmt.Errorf("llama-swap: build video form: %w", err) } return nil } // formatInt renders an optional int pointer for a form field; nil = "" (omit). func formatInt(v *int) string { if v == nil { return "" } return strconv.Itoa(*v) } // formatInt64 renders an optional int64 pointer for a form field; nil = "" (omit). func formatInt64(v *int64) string { if v == nil { return "" } return strconv.FormatInt(*v, 10) } // formatFloat renders an optional float pointer for a form field; nil = "" (omit). func formatFloat(v *float64) string { if v == nil { return "" } return strconv.FormatFloat(*v, 'g', -1, 64) } // formatNonZero renders a non-negative int for a form field; 0 = "" (omit, // backend default). func formatNonZero(v int) string { if v == 0 { return "" } return strconv.Itoa(v) }