feat: wave-3 video surfaces — lipsync, video matte, video upscale, chain jobs (ADR-0025)
- videogen.LipSyncer/LipsyncProvider: SadTalker talking heads via
POST /upstream/<id>/v1/talking_head (multipart image+audio parts,
still/enhance/preprocess flags) -> mp4.
- videogen.VideoBackgroundRemover/VideoBackgroundRemovalProvider:
POST /upstream/<id>/v1/video/matte (output greenscreen_mp4|alpha_webm).
- videogen.VideoUpscaler/VideoUpscaleProvider:
POST /upstream/<id>/v1/video/upscale (scale 2|4).
- videogen.Chainer/ChainerProvider: async long-video chain-job client —
SubmitChain (JSON POST /v1/video/chain, init_image_b64), ChainStatus
(GET /v1/jobs/{id}, tolerant segment-id decode), ChainResult,
ChainSegmentResult (partial delivery after mid-chain failure); hostile
job-id path rejection.
- Shared singleVideoResult validation (positive video evidence) + a
videoInputFilename hint helper; httptest contract tests per surface;
ADR-0025 (index row deferred — MJ-A backfills the ADR index table and
parallel edits would conflict).
Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
// videoutil.go implements the videogen.VideoBackgroundRemovalProvider and
|
||||
// videogen.VideoUpscaleProvider surfaces against the mediautils shim reached
|
||||
// through llama-swap's /upstream passthrough (ADR-0025):
|
||||
//
|
||||
// matte POST /upstream/<id>/v1/video/matte (Robust Video Matting)
|
||||
// upscale POST /upstream/<id>/v1/video/upscale (per-frame Real-ESRGAN)
|
||||
//
|
||||
// Both are one-file multipart in, encoded clip out — the video siblings of
|
||||
// mediautil.go's still-image surfaces.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
// --- video background removal (matting) ---
|
||||
|
||||
// VideoBackgroundRemoverModel implements
|
||||
// videogen.VideoBackgroundRemovalProvider against the mediautils shim's
|
||||
// POST /v1/video/matte. The id selects which upstream llama-swap loads.
|
||||
func (p *Provider) VideoBackgroundRemoverModel(id string, opts ...videogen.VideoBackgroundRemoverModelOption) (videogen.VideoBackgroundRemover, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = videogen.ApplyVideoBackgroundRemoverModelOptions(opts)
|
||||
return &videoMatteModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type videoMatteModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// RemoveVideoBackground implements videogen.VideoBackgroundRemover.
|
||||
func (m *videoMatteModel) RemoveVideoBackground(ctx context.Context, req videogen.VideoBackgroundRemovalRequest, opts ...videogen.VideoBackgroundRemovalOption) (*videogen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Video) == 0 {
|
||||
return nil, fmt.Errorf("%w: video background removal requires a video", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Output != "" && req.Output != "greenscreen_mp4" && req.Output != "alpha_webm" {
|
||||
return nil, fmt.Errorf("%w: video matte output must be \"greenscreen_mp4\" or \"alpha_webm\", got %q", llm.ErrUnsupported, req.Output)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/video/matte")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, contentType, err := buildMultipart("build video-matte form",
|
||||
filePart{field: "file", filename: videoInputFilename(req.Filename, req.MIME), data: req.Video},
|
||||
[]formField{{"output", req.Output, false}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleVideoResult(m.p.name, m.id, "video matte", raw, respType)
|
||||
}
|
||||
|
||||
// --- video upscale ---
|
||||
|
||||
// VideoUpscalerModel implements videogen.VideoUpscaleProvider against the
|
||||
// mediautils shim's POST /v1/video/upscale.
|
||||
func (p *Provider) VideoUpscalerModel(id string, opts ...videogen.VideoUpscalerModelOption) (videogen.VideoUpscaler, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = videogen.ApplyVideoUpscalerModelOptions(opts)
|
||||
return &videoUpscaleModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type videoUpscaleModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// UpscaleVideo implements videogen.VideoUpscaler.
|
||||
func (m *videoUpscaleModel) UpscaleVideo(ctx context.Context, req videogen.VideoUpscaleRequest, opts ...videogen.VideoUpscaleOption) (*videogen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Video) == 0 {
|
||||
return nil, fmt.Errorf("%w: video upscale requires a video", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Scale != 0 && req.Scale != 2 && req.Scale != 4 {
|
||||
return nil, fmt.Errorf("%w: video upscale scale must be 2 or 4, got %d", llm.ErrUnsupported, req.Scale)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/video/upscale")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scale := ""
|
||||
if req.Scale != 0 {
|
||||
scale = strconv.Itoa(req.Scale)
|
||||
}
|
||||
body, contentType, err := buildMultipart("build video-upscale form",
|
||||
filePart{field: "file", filename: videoInputFilename(req.Filename, req.MIME), data: req.Video},
|
||||
[]formField{{"scale", scale, false}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleVideoResult(m.p.name, m.id, "video upscale", raw, respType)
|
||||
}
|
||||
|
||||
// videoInputFilename picks the multipart filename hint for a caller-supplied
|
||||
// video: the caller's (sanitized — upload metadata is untrusted), else one
|
||||
// derived from the MIME subtype ("video.mp4"), else "video". Mirrors
|
||||
// transcriptionFilename.
|
||||
func videoInputFilename(filename, mimeType string) string {
|
||||
if name := sanitizeFilename(filename); name != "" {
|
||||
return name
|
||||
}
|
||||
mt := strings.ToLower(strings.TrimSpace(mimeType))
|
||||
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
||||
mt = parsed
|
||||
}
|
||||
switch mt {
|
||||
case "video/mp4":
|
||||
return "video.mp4"
|
||||
case "video/webm":
|
||||
return "video.webm"
|
||||
case "video/quicktime":
|
||||
return "video.mov"
|
||||
case "video/x-matroska":
|
||||
return "video.mkv"
|
||||
default:
|
||||
return "video"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user