diff --git a/audio/diarize.go b/audio/diarize.go new file mode 100644 index 0000000..66bd444 --- /dev/null +++ b/audio/diarize.go @@ -0,0 +1,113 @@ +package audio + +import "context" + +// DiarizationRequest is a speaker-labelled transcription request ("who said +// what"). Audio is carried as bytes (never a URL), mirroring +// TranscriptionRequest. Zero values mean "backend default" (ADR-0020). +type DiarizationRequest struct { + // Audio is the encoded audio (or video container) to transcribe. + Audio []byte + + // MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend + // sniff it. + MIME string + + // Filename is the multipart filename hint some backends key their format + // detection on; "" derives one from MIME or falls back to "audio". + Filename string + + // Language is a BCP-47/ISO-639 hint (e.g. "en"); "" = auto-detect. + Language string + + // MinSpeakers / MaxSpeakers bound the speaker count when the caller knows + // it; 0 = let the backend estimate. + MinSpeakers int + MaxSpeakers int +} + +// DiarizationSegment is one speaker turn. +type DiarizationSegment struct { + // Start and End are offsets into the audio, in seconds. + Start float64 + End float64 + + // Speaker is the backend's per-file speaker label (e.g. "SPEAKER_00"). + // Labels are relative to this one file — they are NOT stable identities + // across files. + Speaker string + + // Text is the transcript of this turn. + Text string +} + +// DiarizationResult is the canonical diarization result. +type DiarizationResult struct { + // Text is the full transcript, unlabelled. + Text string + + // Language is the detected (or requested) language code; may be "". + Language string + + // Segments are the speaker turns in time order. + Segments []DiarizationSegment + + // Raw is the provider-native response object. May be nil. + Raw any +} + +// DiarizationOption mutates a DiarizationRequest before it is sent. +type DiarizationOption func(*DiarizationRequest) + +// WithDiarizationLanguage sets the language hint (e.g. "en"). +func WithDiarizationLanguage(l string) DiarizationOption { + return func(r *DiarizationRequest) { r.Language = l } +} + +// WithSpeakerBounds bounds the expected speaker count (0 leaves a bound +// unset). +func WithSpeakerBounds(minSpeakers, maxSpeakers int) DiarizationOption { + return func(r *DiarizationRequest) { + r.MinSpeakers, r.MaxSpeakers = minSpeakers, maxSpeakers + } +} + +// Apply returns a copy of the request with all options applied. +func (r DiarizationRequest) Apply(opts ...DiarizationOption) DiarizationRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// DiarizationModel transcribes audio with speaker labels. +type DiarizationModel interface { + // Diarize converts the request's audio into speaker-labelled segments. + Diarize(ctx context.Context, req DiarizationRequest, opts ...DiarizationOption) (*DiarizationResult, error) +} + +// DiarizationModelOption configures a DiarizationModel at construction time. +// Reserved for future per-model settings. +type DiarizationModelOption func(*DiarizationModelConfig) + +// DiarizationModelConfig carries per-model construction settings. +type DiarizationModelConfig struct{} + +// ApplyDiarizationModelOptions folds options into a config. +func ApplyDiarizationModelOptions(opts []DiarizationModelOption) DiarizationModelConfig { + var cfg DiarizationModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// DiarizationProvider mints diarization models bound to one backend. +type DiarizationProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // DiarizationModel returns a DiarizationModel bound to the given id + // (passed through to the backend verbatim; no catalog validation). + DiarizationModel(id string, opts ...DiarizationModelOption) (DiarizationModel, error) +} diff --git a/docs/adr/0020-upstream-passthrough-media-surfaces.md b/docs/adr/0020-upstream-passthrough-media-surfaces.md new file mode 100644 index 0000000..5e654a8 --- /dev/null +++ b/docs/adr/0020-upstream-passthrough-media-surfaces.md @@ -0,0 +1,62 @@ +# ADR-0020: Upstream-passthrough media surfaces (mask, upscale, background removal, interpolation, diarization, meshgen) + +Status: Accepted (2026-07-12) + +## Context + +The llama-swap host is growing capabilities whose native HTTP APIs are not +OpenAI-shaped and carry no routable `model` field: rembg (`/api/remove`), +a Real-ESRGAN+RIFE shim (`/v1/upscale`, `/v1/interpolate`), a WhisperX +diarization sidecar (`/asr?diarize=true`), and Hunyuan3D (`/generate`). +llama-swap's fork already exposes a generic passthrough — +`/upstream//` — that pins the model, runs the normal +load/swap queue, and proxies the rest of the path verbatim. + +Separately, sd-server's `/sdapi/v1/img2img` accepts an inpainting `mask` +field that `imagegen.EditRequest` could not express. + +## Decision + +1. **Route odd-shaped upstreams through `/upstream//`** via a + shared `upstreamPath` helper (model-id validation identical to `Unload`: + reject `/?#`, never escape). No per-endpoint llama-swap routes; the + passthrough is the contract. +2. **Grow the existing leaf contracts instead of inventing parallel ones**, + keeping the ADR-0016→0019 conventions (functional options, zero value = + backend default, bytes-only I/O, `Raw` escape hatch, provider mints + model): + - `imagegen.EditRequest.Mask` (white = repaint, black = keep; backends + without mask support must reject, not ignore). + - `imagegen.Upscaler` / `UpscaleProvider` — super-resolution is not a + diffusion model, so it binds its own backend id. + - `imagegen.BackgroundRemover` / `BackgroundRemovalProvider` — request + `Net` selects the remover's internal network (rembg `model=` param); + `OnlyMask` returns the segmentation mask (the natural mask source for + inpainting). + - `videogen.Interpolator` / `InterpolationProvider` — fps boost or + slow-mo (`SlowMo` plays synthesized frames at the source rate; audio + stripped by the backend in that mode). + - `audio.DiarizationModel` / `DiarizationProvider` — speaker-labelled + segments; per-file `SPEAKER_NN` labels are explicitly NOT stable + identities. +3. **New `meshgen` leaf package** for image→3D (`Mesh{Data, Format, MIME}`, + formats glb/stl/obj). Image-to-3D only: text-to-3D is the caller-owned + composition imagegen → meshgen. +4. **Binary success bodies are validated before wrapping**: one-image + results reject non-image payloads, interpolation reuses the videogen + MIME rules and 512MB cap, meshes reject JSON-shaped "success" bodies + (a queue-full detail page must never become "the mesh"), and + diarization requires `output=json` because the vtt/srt shapes drop + speaker labels. + +## Consequences + +- provider/llamaswap gains five surfaces with no new wire machinery beyond + `upstreamPath` + a shared one-file-multipart builder. +- The passthrough couples majordomo to the fork's `/upstream` route + (upstream llama-swap has it too) and to each upstream's native API shape; + those shapes are pinned by the netherstorm image builds, not by version + negotiation — smoke tests on the host are the drift defence. +- `upstream.ignorePaths` (llama-swap config) can 409 asset-looking paths on + cold models; none of the paths used here match the default pattern, but + new surfaces must check. diff --git a/imagegen/background.go b/imagegen/background.go new file mode 100644 index 0000000..93c52a4 --- /dev/null +++ b/imagegen/background.go @@ -0,0 +1,76 @@ +package imagegen + +import "context" + +// BackgroundRemovalRequest asks a matting/segmentation backend to cut the +// subject out of an image. Zero values mean "backend default" (ADR-0020). +type BackgroundRemovalRequest struct { + // Image is the image to process. Required. + Image Image + + // Net selects the remover's internal network where the backend offers + // several (rembg: "u2net", "birefnet-general", "isnet-general-use", ...); + // "" = backend default. This is NOT the provider model id — that is fixed + // when the BackgroundRemover is minted. + Net string + + // OnlyMask returns the black/white segmentation mask instead of the + // cutout — white where the subject is. Useful as an inpainting mask + // (EditRequest.Mask semantics after inversion: the mask marks the + // FOREGROUND, an inpaint mask marks the region to REPAINT). + OnlyMask bool +} + +// BackgroundRemovalOption mutates a BackgroundRemovalRequest before it is sent. +type BackgroundRemovalOption func(*BackgroundRemovalRequest) + +// WithBackgroundNet selects the remover's internal network. +func WithBackgroundNet(n string) BackgroundRemovalOption { + return func(r *BackgroundRemovalRequest) { r.Net = n } +} + +// WithOnlyMask requests the segmentation mask instead of the cutout. +func WithOnlyMask() BackgroundRemovalOption { + return func(r *BackgroundRemovalRequest) { r.OnlyMask = true } +} + +// Apply returns a copy of the request with all options applied. +func (r BackgroundRemovalRequest) Apply(opts ...BackgroundRemovalOption) BackgroundRemovalRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// BackgroundRemover cuts subjects out of images (RGBA PNG with transparent +// background, or the mask alone with OnlyMask). +type BackgroundRemover interface { + // RemoveBackground returns the cutout (or mask) as a one-image Result. + RemoveBackground(ctx context.Context, req BackgroundRemovalRequest, opts ...BackgroundRemovalOption) (*Result, error) +} + +// BackgroundRemovalModelOption configures a BackgroundRemover at construction +// time. Reserved for future per-model settings. +type BackgroundRemovalModelOption func(*BackgroundRemovalModelConfig) + +// BackgroundRemovalModelConfig carries per-model construction settings. +type BackgroundRemovalModelConfig struct{} + +// ApplyBackgroundRemovalModelOptions folds options into a config. +func ApplyBackgroundRemovalModelOptions(opts []BackgroundRemovalModelOption) BackgroundRemovalModelConfig { + var cfg BackgroundRemovalModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// BackgroundRemovalProvider mints BackgroundRemovers bound to one backend. +type BackgroundRemovalProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // BackgroundRemovalModel returns a BackgroundRemover bound to the given + // id (passed through to the backend verbatim; no catalog validation). + BackgroundRemovalModel(id string, opts ...BackgroundRemovalModelOption) (BackgroundRemover, error) +} diff --git a/imagegen/edit.go b/imagegen/edit.go index cecdbaa..6f16b3d 100644 --- a/imagegen/edit.go +++ b/imagegen/edit.go @@ -12,6 +12,12 @@ type EditRequest struct { // Init is the initial image the edit starts from. Required. Init Image + // Mask restricts the edit to a region (inpainting): a single-channel or + // RGB image the same size as Init where WHITE pixels are repainted and + // BLACK pixels are kept. Empty = whole-image edit. Backends without mask + // support must reject a masked request rather than silently ignoring it. + Mask Image + // Strength is the denoising strength in [0,1] — how far the result may // depart from Init (0 = return the input, 1 = ignore it); nil = backend // default. @@ -45,6 +51,9 @@ type EditRequest struct { // are applied to a copy of the request, so an EditRequest value can be reused. type EditOption func(*EditRequest) +// WithEditMask restricts the edit to a region (white = repaint, black = keep). +func WithEditMask(m Image) EditOption { return func(r *EditRequest) { r.Mask = m } } + // WithEditStrength sets the denoising strength in [0,1]. func WithEditStrength(s float64) EditOption { return func(r *EditRequest) { r.Strength = &s } } diff --git a/imagegen/upscale.go b/imagegen/upscale.go new file mode 100644 index 0000000..e5be7f7 --- /dev/null +++ b/imagegen/upscale.go @@ -0,0 +1,62 @@ +package imagegen + +import "context" + +// UpscaleRequest asks a super-resolution backend to enlarge an image. Zero +// values mean "backend default", mirroring EditRequest (ADR-0020). +type UpscaleRequest struct { + // Image is the image to upscale. Required. + Image Image + + // Scale is the enlargement factor (2 or 4 on the reference backend); + // 0 = backend default. + Scale int +} + +// UpscaleOption mutates an UpscaleRequest before it is sent. +type UpscaleOption func(*UpscaleRequest) + +// WithUpscaleScale sets the enlargement factor. +func WithUpscaleScale(s int) UpscaleOption { return func(r *UpscaleRequest) { r.Scale = s } } + +// Apply returns a copy of the request with all options applied. +func (r UpscaleRequest) Apply(opts ...UpscaleOption) UpscaleRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// Upscaler enlarges images with a super-resolution model. It is its own +// small interface (not a method on Model) because upscalers are not +// diffusion models — they bind to a different backend id entirely. +type Upscaler interface { + // Upscale returns the enlarged image (a one-image Result). + Upscale(ctx context.Context, req UpscaleRequest, opts ...UpscaleOption) (*Result, error) +} + +// UpscaleModelOption configures an Upscaler at construction time. Reserved +// for future per-model settings (mirrors ModelOption). +type UpscaleModelOption func(*UpscaleModelConfig) + +// UpscaleModelConfig carries per-model construction settings. +type UpscaleModelConfig struct{} + +// ApplyUpscaleModelOptions folds options into a config. +func ApplyUpscaleModelOptions(opts []UpscaleModelOption) UpscaleModelConfig { + var cfg UpscaleModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// UpscaleProvider mints Upscalers bound to one backend. +type UpscaleProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // UpscaleModel returns an Upscaler bound to the given id (passed through + // to the backend verbatim; no catalog validation). + UpscaleModel(id string, opts ...UpscaleModelOption) (Upscaler, error) +} diff --git a/meshgen/meshgen.go b/meshgen/meshgen.go new file mode 100644 index 0000000..e8fa976 --- /dev/null +++ b/meshgen/meshgen.go @@ -0,0 +1,152 @@ +// Package meshgen is majordomo's canonical image-to-3D surface. Like +// imagegen/audio/videogen, it is a deliberately separate leaf contract from +// the llm package: mesh generation shares none of the chat machinery, so it +// gets its own small Provider/Model interfaces (ADR-0020, following the +// ADR-0016→0019 lineage: functional options, zero values = backend default, +// bytes-only I/O, Raw escape hatch). +// +// The first implementation is provider/llamaswap, which targets a +// Hunyuan3D-2.1-style api_server reached through the /upstream passthrough. +// The surface is image-to-3D only: text-to-3D is the composition +// imagegen.Generate → meshgen.Generate, owned by the caller. +package meshgen + +import ( + "context" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" +) + +// Image is the conditioning input (bytes + MIME), aliased to llm.ImagePart so +// chat- or imagegen-sourced images feed mesh generation without conversion. +type Image = llm.ImagePart + +// Mesh is one generated 3D asset: raw encoded bytes plus its format. +type Mesh struct { + // Data is the encoded mesh (binary GLB/STL/OBJ container). + Data []byte + + // Format is the lowercase container name ("glb", "stl", "obj"). + Format string + + // MIME is the corresponding MIME type, e.g. "model/gltf-binary", + // "model/stl". + MIME string +} + +// Request is an image-to-3D generation request. Zero values mean "backend +// default". +type Request struct { + // Image is the source image the mesh is reconstructed from. Required. + Image Image + + // Format is the requested output container ("glb", "stl", "obj"); + // "" = backend default (glb). + Format string + + // Texture asks the backend to also synthesize surface textures. Much + // slower and much more VRAM-hungry than shape-only on the reference + // backend; leave false for print-pipeline geometry. + Texture bool + + // RemoveBackground lets the backend cut the subject out first; nil = + // backend default (true on Hunyuan3D). Set explicitly to false when the + // input is already a clean cutout. + RemoveBackground *bool + + // OctreeResolution is the shape-decoder grid resolution; 0 = backend + // default. + OctreeResolution int + + // Steps is the number of diffusion steps; nil = backend default. + Steps *int + + // GuidanceScale is the guidance strength; nil = backend default. + GuidanceScale *float64 + + // Seed fixes the RNG seed for reproducible output; nil = backend default. + Seed *int64 + + // FaceCount caps the output mesh's face count; 0 = backend default. + FaceCount int +} + +// Result is the canonical mesh-generation result. +type Result struct { + // Mesh is the generated asset. + Mesh Mesh + + // Raw is the provider-native response object. May be nil. + Raw any +} + +// Option mutates a Request before it is sent. Options passed to Generate are +// applied to a copy of the request, so a Request value can be reused. +type Option func(*Request) + +// WithFormat sets the output container ("glb", "stl", "obj"). +func WithFormat(f string) Option { return func(r *Request) { r.Format = f } } + +// WithTexture enables texture synthesis. +func WithTexture() Option { return func(r *Request) { r.Texture = true } } + +// WithRemoveBackground sets the backend's background-removal preprocessing. +func WithRemoveBackground(v bool) Option { + return func(r *Request) { r.RemoveBackground = &v } +} + +// WithOctreeResolution sets the shape-decoder grid resolution. +func WithOctreeResolution(n int) Option { return func(r *Request) { r.OctreeResolution = n } } + +// WithSteps overrides the number of diffusion steps. +func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } } + +// WithGuidanceScale overrides the guidance strength. +func WithGuidanceScale(s float64) Option { return func(r *Request) { r.GuidanceScale = &s } } + +// WithSeed fixes the RNG seed. +func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } } + +// WithFaceCount caps the output mesh's face count. +func WithFaceCount(n int) Option { return func(r *Request) { r.FaceCount = n } } + +// Apply returns a copy of the request with all options applied. Providers +// call this once at the top of Generate. +func (r Request) Apply(opts ...Option) Request { + for _, opt := range opts { + opt(&r) + } + return r +} + +// Model generates 3D meshes from images. +type Model interface { + // Generate reconstructs a mesh from the request's image. + Generate(ctx context.Context, req Request, opts ...Option) (*Result, error) +} + +// ModelOption configures a Model at construction time. Reserved for future +// per-model settings. +type ModelOption func(*ModelConfig) + +// ModelConfig carries per-model construction settings. +type ModelConfig struct{} + +// ApplyModelOptions folds options into a config. +func ApplyModelOptions(opts []ModelOption) ModelConfig { + var cfg ModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// Provider mints mesh models bound to one backend. +type Provider interface { + // Name is the registry identifier for the provider. + Name() string + + // MeshModel returns a Model bound to the given id (passed through to the + // backend verbatim; no catalog validation). + MeshModel(id string, opts ...ModelOption) (Model, error) +} diff --git a/provider/llamaswap/diarize.go b/provider/llamaswap/diarize.go new file mode 100644 index 0000000..aae29fc --- /dev/null +++ b/provider/llamaswap/diarize.go @@ -0,0 +1,122 @@ +// diarize.go implements audio.DiarizationProvider against a +// whisper-asr-webservice upstream (WhisperX engine) reached through +// llama-swap's /upstream passthrough (ADR-0020): +// +// POST /upstream//asr?output=json&diarize=true[&language&min_speakers&max_speakers] +// +// output=json is load-bearing: the vtt/srt output formats DROP the speaker +// labels; only the json shape carries per-segment `speaker` fields. +package llamaswap + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" +) + +// DiarizationModel implements audio.DiarizationProvider. The id selects +// which upstream llama-swap loads (the pyannote-equipped WhisperX sidecar, +// not the plain whisper.cpp model). +func (p *Provider) DiarizationModel(id string, opts ...audio.DiarizationModelOption) (audio.DiarizationModel, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = audio.ApplyDiarizationModelOptions(opts) + return &diarizationModel{p: p, id: id}, nil +} + +type diarizationModel struct { + p *Provider + id string +} + +// asrResponse is whisper-asr-webservice's output=json shape (the subset this +// package relies on; per-word entries are ignored — segment granularity is +// the contract). +type asrResponse struct { + Text string `json:"text"` + Language string `json:"language"` + Segments []struct { + Start float64 `json:"start"` + End float64 `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker"` + } `json:"segments"` +} + +// Diarize implements audio.DiarizationModel. +func (m *diarizationModel) Diarize(ctx context.Context, req audio.DiarizationRequest, opts ...audio.DiarizationOption) (*audio.DiarizationResult, error) { + req = req.Apply(opts...) + if len(req.Audio) == 0 { + return nil, fmt.Errorf("%w: diarization requires audio bytes", llm.ErrUnsupported) + } + if req.MinSpeakers < 0 || req.MaxSpeakers < 0 || + (req.MaxSpeakers > 0 && req.MinSpeakers > req.MaxSpeakers) { + return nil, fmt.Errorf("%w: invalid speaker bounds [%d,%d]", llm.ErrUnsupported, req.MinSpeakers, req.MaxSpeakers) + } + + q := url.Values{} + q.Set("output", "json") + q.Set("diarize", "true") + if req.Language != "" { + q.Set("language", req.Language) + } + if req.MinSpeakers > 0 { + q.Set("min_speakers", strconv.Itoa(req.MinSpeakers)) + } + if req.MaxSpeakers > 0 { + q.Set("max_speakers", strconv.Itoa(req.MaxSpeakers)) + } + path, err := upstreamPath(m.id, "/asr?"+q.Encode()) + if err != nil { + return nil, err + } + + body, contentType, err := buildMultipart("build diarization form", + filePart{field: "audio_file", filename: diarizationFilename(req), data: req.Audio}, + nil) + if err != nil { + return nil, err + } + raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes) + if err != nil { + return nil, err + } + + var out asrResponse + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("llama-swap: decode diarization response: %w", err) + } + res := &audio.DiarizationResult{ + Text: out.Text, + Language: out.Language, + Raw: json.RawMessage(raw), + } + for _, s := range out.Segments { + res.Segments = append(res.Segments, audio.DiarizationSegment{ + Start: s.Start, + End: s.End, + Speaker: s.Speaker, + Text: s.Text, + }) + } + if len(res.Segments) == 0 && res.Text == "" { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "diarization response contained no transcript"} + } + return res, nil +} + +// diarizationFilename mirrors transcriptionFilename for the diarization +// request shape (same untrusted-metadata sanitization rules). +func diarizationFilename(req audio.DiarizationRequest) string { + return transcriptionFilename(audio.TranscriptionRequest{ + Filename: req.Filename, + MIME: req.MIME, + }) +} diff --git a/provider/llamaswap/edit_test.go b/provider/llamaswap/edit_test.go index a8b765d..5c5b3e0 100644 --- a/provider/llamaswap/edit_test.go +++ b/provider/llamaswap/edit_test.go @@ -108,3 +108,47 @@ func TestImageEditValidation(t *testing.T) { t.Errorf("out-of-range strength: err = %v, want ErrUnsupported", err) } } + +func TestImageEditWithMask(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + im, _ := p.ImageModel("sd") + ed := im.(imagegen.Editor) + + mask := imagegen.Image{MIME: "image/png", Data: []byte{0xDE, 0xAD}} + if _, err := ed.Edit(context.Background(), + imagegen.EditRequest{Prompt: "replace the sky", Init: editInit(t)}, + imagegen.WithEditMask(mask), + ); err != nil { + t.Fatalf("Edit: %v", err) + } + if got := gotBody["mask"]; got != base64.StdEncoding.EncodeToString(mask.Data) { + t.Errorf("mask = %v, want the b64 mask", got) + } +} + +func TestImageEditWithoutMaskOmitsField(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + im, _ := p.ImageModel("sd") + ed := im.(imagegen.Editor) + if _, err := ed.Edit(context.Background(), + imagegen.EditRequest{Prompt: "p", Init: editInit(t)}); err != nil { + t.Fatalf("Edit: %v", err) + } + if _, ok := gotBody["mask"]; ok { + t.Error("mask field sent for unmasked edit; want omitted") + } +} diff --git a/provider/llamaswap/image.go b/provider/llamaswap/image.go index e10af4c..a6248f4 100644 --- a/provider/llamaswap/image.go +++ b/provider/llamaswap/image.go @@ -128,6 +128,11 @@ type img2imgRequest struct { txt2imgRequest InitImages []string `json:"init_images"` DenoisingStrength *float64 `json:"denoising_strength,omitempty"` + // Mask enables inpainting: base64 image, white = repaint, black = keep + // (sd-server also accepts a data URL; plain base64 keeps symmetry with + // init_images). sd-server has no mask_blur/inpaint_full_res — callers + // wanting soft edges pre-feather the mask. + Mask string `json:"mask,omitempty"` } // Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img. @@ -148,6 +153,9 @@ func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts .. InitImages: []string{base64.StdEncoding.EncodeToString(req.Init.Data)}, DenoisingStrength: req.Strength, } + if len(req.Mask.Data) > 0 { + wire.Mask = base64.StdEncoding.EncodeToString(req.Mask.Data) + } var resp txt2imgResponse if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/img2img", m.id, &wire, &resp); err != nil { diff --git a/provider/llamaswap/mediautil.go b/provider/llamaswap/mediautil.go new file mode 100644 index 0000000..3387b4c --- /dev/null +++ b/provider/llamaswap/mediautil.go @@ -0,0 +1,204 @@ +// mediautil.go implements the imagegen.UpscaleProvider, +// imagegen.BackgroundRemovalProvider, and videogen.InterpolationProvider +// surfaces against upstreams reached through llama-swap's /upstream +// passthrough (ADR-0020): +// +// upscale POST /upstream//v1/upscale (mediautils shim) +// background POST /upstream//api/remove (rembg server) +// interpolate POST /upstream//v1/interpolate (mediautils shim) +// +// All three are one-file multipart in, raw bytes out. +package llamaswap + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + + "gitea.stevedudenhoeffer.com/steve/majordomo/imagegen" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" +) + +// --- upscale --- + +// UpscaleModel implements imagegen.UpscaleProvider against the mediautils +// shim's POST /v1/upscale. The id selects which upstream llama-swap loads. +func (p *Provider) UpscaleModel(id string, opts ...imagegen.UpscaleModelOption) (imagegen.Upscaler, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = imagegen.ApplyUpscaleModelOptions(opts) + return &upscaleModel{p: p, id: id}, nil +} + +type upscaleModel struct { + p *Provider + id string +} + +// Upscale implements imagegen.Upscaler. +func (m *upscaleModel) Upscale(ctx context.Context, req imagegen.UpscaleRequest, opts ...imagegen.UpscaleOption) (*imagegen.Result, error) { + req = req.Apply(opts...) + if len(req.Image.Data) == 0 { + return nil, fmt.Errorf("%w: upscale requires an image", llm.ErrUnsupported) + } + if req.Scale != 0 && req.Scale != 2 && req.Scale != 4 { + return nil, fmt.Errorf("%w: upscale scale must be 2 or 4, got %d", llm.ErrUnsupported, req.Scale) + } + path, err := upstreamPath(m.id, "/v1/upscale") + if err != nil { + return nil, err + } + scale := "" + if req.Scale != 0 { + scale = strconv.Itoa(req.Scale) + } + body, contentType, err := buildMultipart("build upscale form", + filePart{field: "file", filename: "image.png", data: req.Image.Data}, + []formField{{"scale", scale, false}}) + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes) + if err != nil { + return nil, err + } + return singleImageResult(m.p.name, m.id, "upscale", raw, respType) +} + +// --- background removal --- + +// BackgroundRemovalModel implements imagegen.BackgroundRemovalProvider +// against a rembg server's POST /api/remove. +func (p *Provider) BackgroundRemovalModel(id string, opts ...imagegen.BackgroundRemovalModelOption) (imagegen.BackgroundRemover, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = imagegen.ApplyBackgroundRemovalModelOptions(opts) + return &backgroundModel{p: p, id: id}, nil +} + +type backgroundModel struct { + p *Provider + id string +} + +// RemoveBackground implements imagegen.BackgroundRemover. rembg's `model` +// form field is the request's Net (its internal network); the llama-swap +// model id only picks the upstream. `om=true` returns the black/white +// foreground mask instead of the RGBA cutout. +func (m *backgroundModel) RemoveBackground(ctx context.Context, req imagegen.BackgroundRemovalRequest, opts ...imagegen.BackgroundRemovalOption) (*imagegen.Result, error) { + req = req.Apply(opts...) + if len(req.Image.Data) == 0 { + return nil, fmt.Errorf("%w: background removal requires an image", llm.ErrUnsupported) + } + path, err := upstreamPath(m.id, "/api/remove") + if err != nil { + return nil, err + } + om := "" + if req.OnlyMask { + om = "true" + } + body, contentType, err := buildMultipart("build background-removal form", + filePart{field: "file", filename: "image.png", data: req.Image.Data}, + []formField{ + {"model", req.Net, false}, + {"om", om, false}, + }) + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes) + if err != nil { + return nil, err + } + return singleImageResult(m.p.name, m.id, "background removal", raw, respType) +} + +// singleImageResult wraps one raw image body into an imagegen.Result, +// rejecting empty or non-image payloads (a proxy error page must not become +// "the image"). +func singleImageResult(provider, model, verb string, raw []byte, contentType string) (*imagegen.Result, error) { + if len(raw) == 0 { + return nil, &llm.APIError{Provider: provider, Model: model, Message: verb + " response contained no image"} + } + mimeType := sniffImageMIME(raw) + if ct := strings.TrimSpace(contentType); ct != "" && !strings.HasPrefix(ct, "image/") && !strings.HasPrefix(http.DetectContentType(raw), "image/") { + return nil, &llm.APIError{Provider: provider, Model: model, Message: fmt.Sprintf("%s response is not an image (Content-Type %q)", verb, ct)} + } + return &imagegen.Result{Images: []llm.ImagePart{{MIME: mimeType, Data: raw}}}, nil +} + +// --- frame interpolation --- + +// InterpolatorModel implements videogen.InterpolationProvider against the +// mediautils shim's POST /v1/interpolate. +func (p *Provider) InterpolatorModel(id string, opts ...videogen.InterpolatorModelOption) (videogen.Interpolator, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = videogen.ApplyInterpolatorModelOptions(opts) + return &interpolatorModel{p: p, id: id}, nil +} + +type interpolatorModel struct { + p *Provider + id string +} + +// Interpolate implements videogen.Interpolator. The response body IS the +// encoded clip (same contract as /v1/videos/sync), hence the video-sized +// response cap and the same MIME resolution rules. +func (m *interpolatorModel) Interpolate(ctx context.Context, req videogen.InterpolateRequest, opts ...videogen.InterpolateOption) (*videogen.Result, error) { + req = req.Apply(opts...) + if len(req.Video.Data) == 0 { + return nil, fmt.Errorf("%w: interpolation requires a video", llm.ErrUnsupported) + } + if req.Multi != 0 && req.Multi != 2 && req.Multi != 4 { + return nil, fmt.Errorf("%w: interpolation multi must be 2 or 4, got %d", llm.ErrUnsupported, req.Multi) + } + if req.TargetFPS < 0 { + return nil, fmt.Errorf("%w: target fps must be >= 0, got %d", llm.ErrUnsupported, req.TargetFPS) + } + path, err := upstreamPath(m.id, "/v1/interpolate") + if err != nil { + return nil, err + } + multi := "" + if req.Multi != 0 { + multi = strconv.Itoa(req.Multi) + } + mode := "" + targetFPS := "" + if req.SlowMo { + mode = "slowmo" + } else if req.TargetFPS > 0 { + targetFPS = strconv.Itoa(req.TargetFPS) + } + body, contentType, err := buildMultipart("build interpolate form", + filePart{field: "file", filename: "video.mp4", data: req.Video.Data}, + []formField{ + {"multi", multi, false}, + {"mode", mode, false}, + {"target_fps", targetFPS, false}, + }) + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "interpolate response contained no video"} + } + mimeType := videoMIME(respType, raw) + if mimeType == "" { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "interpolate response is not a video"} + } + return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil +} diff --git a/provider/llamaswap/mediautil_test.go b/provider/llamaswap/mediautil_test.go new file mode 100644 index 0000000..6737c12 --- /dev/null +++ b/provider/llamaswap/mediautil_test.go @@ -0,0 +1,253 @@ +package llamaswap + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "gitea.stevedudenhoeffer.com/steve/majordomo/imagegen" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/videogen" +) + +func pngFixture(t *testing.T) []byte { + t.Helper() + raw, err := base64.StdEncoding.DecodeString(onePixelPNG) + if err != nil { + t.Fatalf("decode fixture: %v", err) + } + return raw +} + +func TestUpscale(t *testing.T) { + png := pngFixture(t) + var gotPath, gotScale, gotFilename string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Errorf("parse multipart: %v", err) + } + gotScale = r.FormValue("scale") + if f, hdr, err := r.FormFile("file"); err == nil { + gotFilename = hdr.Filename + f.Close() + } else { + t.Errorf("file part: %v", err) + } + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(png) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + up, err := p.UpscaleModel("mediautils") + if err != nil { + t.Fatalf("UpscaleModel: %v", err) + } + res, err := up.Upscale(context.Background(), + imagegen.UpscaleRequest{Image: imagegen.Image{MIME: "image/png", Data: png}}, + imagegen.WithUpscaleScale(2)) + if err != nil { + t.Fatalf("Upscale: %v", err) + } + if gotPath != "/upstream/mediautils/v1/upscale" { + t.Errorf("path = %q", gotPath) + } + if gotScale != "2" || gotFilename != "image.png" { + t.Errorf("scale/filename = %q/%q", gotScale, gotFilename) + } + if len(res.Images) != 1 || res.Images[0].MIME != "image/png" { + t.Fatalf("images = %+v", res.Images) + } +} + +func TestUpscaleRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + up, _ := p.UpscaleModel("mediautils") + if _, err := up.Upscale(context.Background(), imagegen.UpscaleRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no image: err = %v, want ErrUnsupported", err) + } + if _, err := up.Upscale(context.Background(), + imagegen.UpscaleRequest{Image: imagegen.Image{Data: []byte{1}}, Scale: 3}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("scale 3: err = %v, want ErrUnsupported", err) + } +} + +func TestUpscaleRejectsNonImageResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("proxy error page")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + up, _ := p.UpscaleModel("mediautils") + _, err := up.Upscale(context.Background(), + imagegen.UpscaleRequest{Image: imagegen.Image{Data: pngFixture(t)}}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for non-image body", err) + } +} + +func TestRemoveBackground(t *testing.T) { + png := pngFixture(t) + var gotPath, gotNet, gotOM string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Errorf("parse multipart: %v", err) + } + gotNet = r.FormValue("model") + gotOM = r.FormValue("om") + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(png) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + br, err := p.BackgroundRemovalModel("rembg") + if err != nil { + t.Fatalf("BackgroundRemovalModel: %v", err) + } + res, err := br.RemoveBackground(context.Background(), + imagegen.BackgroundRemovalRequest{Image: imagegen.Image{Data: png}}, + imagegen.WithBackgroundNet("birefnet-general"), imagegen.WithOnlyMask()) + if err != nil { + t.Fatalf("RemoveBackground: %v", err) + } + if gotPath != "/upstream/rembg/api/remove" { + t.Errorf("path = %q", gotPath) + } + if gotNet != "birefnet-general" || gotOM != "true" { + t.Errorf("net/om = %q/%q", gotNet, gotOM) + } + if len(res.Images) != 1 { + t.Fatalf("images = %+v", res.Images) + } +} + +func TestRemoveBackgroundOmitsUnsetFields(t *testing.T) { + png := pngFixture(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Errorf("parse multipart: %v", err) + } + if _, ok := r.MultipartForm.Value["model"]; ok { + t.Error("model field sent for default net; want omitted") + } + if _, ok := r.MultipartForm.Value["om"]; ok { + t.Error("om field sent for cutout mode; want omitted") + } + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(png) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + br, _ := p.BackgroundRemovalModel("rembg") + if _, err := br.RemoveBackground(context.Background(), + imagegen.BackgroundRemovalRequest{Image: imagegen.Image{Data: png}}); err != nil { + t.Fatalf("RemoveBackground: %v", err) + } +} + +// mp4Fixture is a minimal ftyp box so http.DetectContentType sniffs video/mp4. +func mp4Fixture() []byte { + return []byte{0, 0, 0, 24, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm', + 0, 0, 2, 0, 'i', 's', 'o', 'm', 'i', 's', 'o', '2'} +} + +func TestInterpolate(t *testing.T) { + var gotPath, gotMulti, gotMode, gotTargetFPS string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Errorf("parse multipart: %v", err) + } + gotMulti = r.FormValue("multi") + gotMode = r.FormValue("mode") + gotTargetFPS = r.FormValue("target_fps") + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(mp4Fixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ip, err := p.InterpolatorModel("mediautils") + if err != nil { + t.Fatalf("InterpolatorModel: %v", err) + } + res, err := ip.Interpolate(context.Background(), + videogen.InterpolateRequest{Video: videogen.Video{Data: mp4Fixture(), MIME: "video/mp4"}}, + videogen.WithInterpolateMulti(2), videogen.WithTargetFPS(60)) + if err != nil { + t.Fatalf("Interpolate: %v", err) + } + if gotPath != "/upstream/mediautils/v1/interpolate" { + t.Errorf("path = %q", gotPath) + } + if gotMulti != "2" || gotMode != "" || gotTargetFPS != "60" { + t.Errorf("multi/mode/target_fps = %q/%q/%q", gotMulti, gotMode, gotTargetFPS) + } + if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 { + t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data)) + } +} + +func TestInterpolateSlowMoSetsModeAndDropsTargetFPS(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Errorf("parse multipart: %v", err) + } + if got := r.FormValue("mode"); got != "slowmo" { + t.Errorf("mode = %q, want slowmo", got) + } + if _, ok := r.MultipartForm.Value["target_fps"]; ok { + t.Error("target_fps sent in slowmo mode; want omitted") + } + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write(mp4Fixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ip, _ := p.InterpolatorModel("mediautils") + _, err := ip.Interpolate(context.Background(), + videogen.InterpolateRequest{Video: videogen.Video{Data: mp4Fixture()}, TargetFPS: 60}, + videogen.WithSlowMo()) + if err != nil { + t.Fatalf("Interpolate: %v", err) + } +} + +func TestInterpolateRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + ip, _ := p.InterpolatorModel("mediautils") + if _, err := ip.Interpolate(context.Background(), videogen.InterpolateRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no video: err = %v, want ErrUnsupported", err) + } + if _, err := ip.Interpolate(context.Background(), + videogen.InterpolateRequest{Video: videogen.Video{Data: []byte{1}}, Multi: 3}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("multi 3: err = %v, want ErrUnsupported", err) + } +} + +func TestUpstreamPathRejectsSeparators(t *testing.T) { + for _, bad := range []string{"", "a/b", "a?b", "a#b"} { + if _, err := upstreamPath(bad, "/x"); err == nil { + t.Errorf("upstreamPath(%q) succeeded; want error", bad) + } + } + got, err := upstreamPath("rembg", "api/remove") + if err != nil || got != "/upstream/rembg/api/remove" { + t.Errorf("upstreamPath = %q, %v", got, err) + } + if !strings.HasPrefix(got, "/upstream/") { + t.Errorf("path prefix wrong: %q", got) + } +} diff --git a/provider/llamaswap/mesh.go b/provider/llamaswap/mesh.go new file mode 100644 index 0000000..796a008 --- /dev/null +++ b/provider/llamaswap/mesh.go @@ -0,0 +1,128 @@ +// mesh.go implements meshgen.Provider against a Hunyuan3D-2.1-style +// api_server reached through llama-swap's /upstream passthrough (ADR-0020): +// +// POST /upstream//generate JSON {image: , type: "glb"|"stl"|...} +// +// The response body IS the encoded mesh (binary, Content-Type +// application/octet-stream), so one request yields exactly one asset. +package llamaswap + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "strings" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/meshgen" +) + +// maxMeshResponseBytes caps the /generate body — a high-resolution textured +// mesh legitimately passes the 64MB JSON cap, but stays far under video +// scale. +const maxMeshResponseBytes = 256 << 20 + +// MeshModel implements meshgen.Provider. The id selects which upstream +// llama-swap loads. +func (p *Provider) MeshModel(id string, opts ...meshgen.ModelOption) (meshgen.Model, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = meshgen.ApplyModelOptions(opts) + return &meshModel{p: p, id: id}, nil +} + +type meshModel struct { + p *Provider + id string +} + +// hunyuanGenerateRequest is the Hunyuan3D api_server /generate shape. +// Optional fields are pointers/omitempty so unset values fall back to the +// server's defaults (mirrors the sd-server wire structs). +type hunyuanGenerateRequest struct { + Image string `json:"image"` + Type string `json:"type,omitempty"` + Texture bool `json:"texture"` + RemoveBackground *bool `json:"remove_background,omitempty"` + OctreeResolution int `json:"octree_resolution,omitempty"` + NumInferenceSteps *int `json:"num_inference_steps,omitempty"` + GuidanceScale *float64 `json:"guidance_scale,omitempty"` + Seed *int64 `json:"seed,omitempty"` + FaceCount int `json:"face_count,omitempty"` +} + +// meshFormats maps the supported output containers to MIME types. +var meshFormats = map[string]string{ + "glb": "model/gltf-binary", + "stl": "model/stl", + "obj": "model/obj", +} + +// Generate implements meshgen.Model. +func (m *meshModel) Generate(ctx context.Context, req meshgen.Request, opts ...meshgen.Option) (*meshgen.Result, error) { + req = req.Apply(opts...) + if len(req.Image.Data) == 0 { + return nil, fmt.Errorf("%w: mesh generation requires an image", llm.ErrUnsupported) + } + format := strings.ToLower(strings.TrimSpace(req.Format)) + if format == "" { + format = "glb" + } + mimeType, ok := meshFormats[format] + if !ok { + return nil, fmt.Errorf("%w: unsupported mesh format %q (want glb, stl, or obj)", llm.ErrUnsupported, req.Format) + } + if req.OctreeResolution < 0 || req.FaceCount < 0 { + return nil, fmt.Errorf("%w: octree resolution and face count must be >= 0", llm.ErrUnsupported) + } + + path, err := upstreamPath(m.id, "/generate") + if err != nil { + return nil, err + } + wire := hunyuanGenerateRequest{ + Image: base64.StdEncoding.EncodeToString(req.Image.Data), + Type: format, + Texture: req.Texture, + RemoveBackground: req.RemoveBackground, + OctreeResolution: req.OctreeResolution, + NumInferenceSteps: req.Steps, + GuidanceScale: req.GuidanceScale, + Seed: req.Seed, + FaceCount: req.FaceCount, + } + + // doRaw + manual JSON encode (not doJSON): the SUCCESS body is binary + // mesh bytes, not JSON. + encoded, err := json.Marshal(wire) + if err != nil { + return nil, fmt.Errorf("llama-swap: encode mesh request: %w", err) + } + raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, "application/json", bytes.NewReader(encoded), maxMeshResponseBytes) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "mesh response contained no data"} + } + // A JSON body on the success path is the server reporting a soft error + // (or an API drift) — never hand it back as "the mesh". + if first := strings.TrimLeft(string(raw[:min(len(raw), 64)]), " \t\r\n"); strings.HasPrefix(first, "{") || strings.HasPrefix(first, "[") { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, + Message: "mesh response is JSON, not mesh bytes: " + truncateForError(raw)} + } + return &meshgen.Result{Mesh: meshgen.Mesh{Data: raw, Format: format, MIME: mimeType}}, nil +} + +// truncateForError bounds a payload quoted into an error message. +func truncateForError(b []byte) string { + const cap = 500 + if len(b) > cap { + return string(b[:cap]) + "..." + } + return string(b) +} diff --git a/provider/llamaswap/mesh_diarize_test.go b/provider/llamaswap/mesh_diarize_test.go new file mode 100644 index 0000000..0164c71 --- /dev/null +++ b/provider/llamaswap/mesh_diarize_test.go @@ -0,0 +1,170 @@ +package llamaswap + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/meshgen" +) + +func TestMeshGenerate(t *testing.T) { + png := pngFixture(t) + stl := []byte("solid mort\nendsolid mort\n") + var gotPath string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(&gotBody) + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(stl) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + mm, err := p.MeshModel("image3d-hunyuan21") + if err != nil { + t.Fatalf("MeshModel: %v", err) + } + res, err := mm.Generate(context.Background(), + meshgen.Request{Image: meshgen.Image{MIME: "image/png", Data: png}}, + meshgen.WithFormat("stl"), meshgen.WithSeed(7)) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if gotPath != "/upstream/image3d-hunyuan21/generate" { + t.Errorf("path = %q", gotPath) + } + if gotBody["type"] != "stl" || gotBody["image"] != base64.StdEncoding.EncodeToString(png) { + t.Errorf("type/image = %v/(b64 mismatch)", gotBody["type"]) + } + if gotBody["seed"] != float64(7) { + t.Errorf("seed = %v", gotBody["seed"]) + } + if _, ok := gotBody["octree_resolution"]; ok { + t.Error("octree_resolution sent when unset; want omitted") + } + if res.Mesh.Format != "stl" || res.Mesh.MIME != "model/stl" || string(res.Mesh.Data) != string(stl) { + t.Fatalf("mesh = %+v", res.Mesh) + } +} + +func TestMeshGenerateRejectsJSONBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write([]byte(` {"detail": "queue full"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + mm, _ := p.MeshModel("image3d-hunyuan21") + _, err := mm.Generate(context.Background(), + meshgen.Request{Image: meshgen.Image{Data: pngFixture(t)}}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for JSON success body", err) + } +} + +func TestMeshGenerateRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + mm, _ := p.MeshModel("image3d-hunyuan21") + if _, err := mm.Generate(context.Background(), meshgen.Request{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no image: err = %v, want ErrUnsupported", err) + } + if _, err := mm.Generate(context.Background(), + meshgen.Request{Image: meshgen.Image{Data: []byte{1}}, Format: "fbx"}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("format fbx: err = %v, want ErrUnsupported", err) + } +} + +func TestDiarize(t *testing.T) { + var gotPath string + var gotQuery map[string][]string + var gotField string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotQuery = r.URL.Query() + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Errorf("parse multipart: %v", err) + } + if f, _, err := r.FormFile("audio_file"); err == nil { + gotField = "audio_file" + f.Close() + } + _, _ = w.Write([]byte(`{ + "text": "hello there. general kenobi.", + "language": "en", + "segments": [ + {"start": 0.0, "end": 1.2, "text": "hello there.", "speaker": "SPEAKER_00"}, + {"start": 1.4, "end": 3.0, "text": "general kenobi.", "speaker": "SPEAKER_01"} + ] + }`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + dm, err := p.DiarizationModel("whisperx-diarize") + if err != nil { + t.Fatalf("DiarizationModel: %v", err) + } + res, err := dm.Diarize(context.Background(), + audio.DiarizationRequest{Audio: []byte("RIFFfake"), MIME: "audio/wav"}, + audio.WithSpeakerBounds(2, 4), audio.WithDiarizationLanguage("en")) + if err != nil { + t.Fatalf("Diarize: %v", err) + } + if gotPath != "/upstream/whisperx-diarize/asr" { + t.Errorf("path = %q", gotPath) + } + for k, want := range map[string]string{ + "output": "json", "diarize": "true", + "min_speakers": "2", "max_speakers": "4", "language": "en", + } { + if got := gotQuery[k]; len(got) != 1 || got[0] != want { + t.Errorf("query %s = %v, want %q", k, got, want) + } + } + if gotField != "audio_file" { + t.Error("audio_file part missing") + } + if len(res.Segments) != 2 || res.Segments[1].Speaker != "SPEAKER_01" { + t.Fatalf("segments = %+v", res.Segments) + } + if res.Language != "en" || res.Text == "" { + t.Errorf("language/text = %q/%q", res.Language, res.Text) + } +} + +func TestDiarizeRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + dm, _ := p.DiarizationModel("whisperx-diarize") + if _, err := dm.Diarize(context.Background(), audio.DiarizationRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no audio: err = %v, want ErrUnsupported", err) + } + if _, err := dm.Diarize(context.Background(), + audio.DiarizationRequest{Audio: []byte{1}, MinSpeakers: 5, MaxSpeakers: 2}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("bounds 5>2: err = %v, want ErrUnsupported", err) + } +} + +func TestDiarizeEmptyTranscriptIsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"text": "", "segments": []}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + dm, _ := p.DiarizationModel("whisperx-diarize") + _, err := dm.Diarize(context.Background(), audio.DiarizationRequest{Audio: []byte{1}}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for empty transcript", err) + } +} diff --git a/provider/llamaswap/upstream.go b/provider/llamaswap/upstream.go new file mode 100644 index 0000000..85cd2f6 --- /dev/null +++ b/provider/llamaswap/upstream.go @@ -0,0 +1,61 @@ +package llamaswap + +import ( + "bytes" + "fmt" + "mime/multipart" + "strings" +) + +// upstreamPath builds a path through llama-swap's generic /upstream// +// passthrough, which pins the model (triggering the normal load/swap queue) +// and forwards the remaining path to the upstream verbatim. This is how the +// provider reaches upstreams whose native APIs carry no routable `model` +// field (rembg, mediautils, WhisperX, ACE-Step, ...) without needing a +// llama-swap route per endpoint (ADR-0020). +// +// Why reject rather than escape: same rationale as Unload — model ids +// legitimately contain ":" but never path-structure characters, and escaping +// would mask a config error instead of surfacing it. +func upstreamPath(model, rest string) (string, error) { + if strings.TrimSpace(model) == "" { + return "", fmt.Errorf("llama-swap: upstream call requires a model id") + } + if strings.ContainsAny(model, "/?#") { + return "", fmt.Errorf("llama-swap: invalid model id %q for upstream call (contains a path separator)", model) + } + if !strings.HasPrefix(rest, "/") { + rest = "/" + rest + } + return "/upstream/" + model + rest, nil +} + +// filePart is the single file entry of a media multipart form. +type filePart struct { + field string // form field name ("file", "audio_file", ...) + filename string // already sanitized + data []byte +} + +// buildMultipart assembles a one-file multipart body: the file part first, +// then the given fields (optional fields skipped when empty, matching +// writeFormFields). wrap labels errors. Returns the body and its content +// type. +func buildMultipart(wrap string, file filePart, fields []formField) (*bytes.Buffer, string, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + fw, err := w.CreateFormFile(file.field, file.filename) + if err != nil { + return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err) + } + if _, err := fw.Write(file.data); err != nil { + return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err) + } + if err := writeFormFields(w, wrap, fields); err != nil { + return nil, "", err + } + if err := w.Close(); err != nil { + return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err) + } + return &buf, w.FormDataContentType(), nil +} diff --git a/videogen/interpolate.go b/videogen/interpolate.go new file mode 100644 index 0000000..d870124 --- /dev/null +++ b/videogen/interpolate.go @@ -0,0 +1,82 @@ +package videogen + +import "context" + +// InterpolateRequest asks a frame-interpolation backend (RIFE-style) to +// synthesize intermediate frames in an existing clip. Zero values mean +// "backend default" (ADR-0020). +type InterpolateRequest struct { + // Video is the clip to interpolate. Required. + Video Video + + // Multi is the frame multiplier (2 or 4 on the reference backend); + // 0 = backend default (2). + Multi int + + // SlowMo plays the synthesized frames at the SOURCE frame rate instead of + // multiplying it: Multi-times slow motion. The backend strips audio in + // this mode (time-stretched audio is noise). + SlowMo bool + + // TargetFPS resamples the smoothed clip to a specific frame rate + // (e.g. 60); 0 = Multi times the source rate. Ignored in SlowMo mode. + TargetFPS int +} + +// InterpolateOption mutates an InterpolateRequest before it is sent. +type InterpolateOption func(*InterpolateRequest) + +// WithInterpolateMulti sets the frame multiplier. +func WithInterpolateMulti(m int) InterpolateOption { + return func(r *InterpolateRequest) { r.Multi = m } +} + +// WithSlowMo switches to slow-motion output. +func WithSlowMo() InterpolateOption { return func(r *InterpolateRequest) { r.SlowMo = true } } + +// WithTargetFPS resamples the smoothed clip to a specific frame rate. +func WithTargetFPS(fps int) InterpolateOption { + return func(r *InterpolateRequest) { r.TargetFPS = fps } +} + +// Apply returns a copy of the request with all options applied. +func (r InterpolateRequest) Apply(opts ...InterpolateOption) InterpolateRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// Interpolator synthesizes intermediate frames (fps boost or slow-mo). Its +// own small interface rather than a method on Model: interpolators are not +// generators — they bind to a different backend id entirely. +type Interpolator interface { + // Interpolate returns the smoothed (or slowed) clip. + Interpolate(ctx context.Context, req InterpolateRequest, opts ...InterpolateOption) (*Result, error) +} + +// InterpolatorModelOption configures an Interpolator at construction time. +// Reserved for future per-model settings. +type InterpolatorModelOption func(*InterpolatorModelConfig) + +// InterpolatorModelConfig carries per-model construction settings. +type InterpolatorModelConfig struct{} + +// ApplyInterpolatorModelOptions folds options into a config. +func ApplyInterpolatorModelOptions(opts []InterpolatorModelOption) InterpolatorModelConfig { + var cfg InterpolatorModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// InterpolationProvider mints Interpolators bound to one backend. +type InterpolationProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // InterpolatorModel returns an Interpolator bound to the given id (passed + // through to the backend verbatim; no catalog validation). + InterpolatorModel(id string, opts ...InterpolatorModelOption) (Interpolator, error) +}