feat: media expansion surfaces — edit mask, upscale, background removal, interpolation, diarization, meshgen (ADR-0020)
- imagegen.EditRequest.Mask -> sd-server img2img inpainting (white=repaint) - imagegen.Upscaler + BackgroundRemover, videogen.Interpolator, audio.DiarizationModel: new optional provider-minted surfaces - NEW meshgen leaf package (image->3D, glb/stl/obj) - provider/llamaswap: all five via the /upstream/<model>/<path> passthrough (upstreamPath helper, shared one-file multipart builder); binary success bodies validated (non-image, non-video, JSON-mesh rejection); diarization pins output=json (vtt/srt drop speaker labels) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,122 @@
|
||||
// diarize.go implements audio.DiarizationProvider against a
|
||||
// whisper-asr-webservice upstream (WhisperX engine) reached through
|
||||
// llama-swap's /upstream passthrough (ADR-0020):
|
||||
//
|
||||
// POST /upstream/<id>/asr?output=json&diarize=true[&language&min_speakers&max_speakers]
|
||||
//
|
||||
// output=json is load-bearing: the vtt/srt output formats DROP the speaker
|
||||
// labels; only the json shape carries per-segment `speaker` fields.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// DiarizationModel implements audio.DiarizationProvider. The id selects
|
||||
// which upstream llama-swap loads (the pyannote-equipped WhisperX sidecar,
|
||||
// not the plain whisper.cpp model).
|
||||
func (p *Provider) DiarizationModel(id string, opts ...audio.DiarizationModelOption) (audio.DiarizationModel, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = audio.ApplyDiarizationModelOptions(opts)
|
||||
return &diarizationModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type diarizationModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// asrResponse is whisper-asr-webservice's output=json shape (the subset this
|
||||
// package relies on; per-word entries are ignored — segment granularity is
|
||||
// the contract).
|
||||
type asrResponse struct {
|
||||
Text string `json:"text"`
|
||||
Language string `json:"language"`
|
||||
Segments []struct {
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
Speaker string `json:"speaker"`
|
||||
} `json:"segments"`
|
||||
}
|
||||
|
||||
// Diarize implements audio.DiarizationModel.
|
||||
func (m *diarizationModel) Diarize(ctx context.Context, req audio.DiarizationRequest, opts ...audio.DiarizationOption) (*audio.DiarizationResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Audio) == 0 {
|
||||
return nil, fmt.Errorf("%w: diarization requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
if req.MinSpeakers < 0 || req.MaxSpeakers < 0 ||
|
||||
(req.MaxSpeakers > 0 && req.MinSpeakers > req.MaxSpeakers) {
|
||||
return nil, fmt.Errorf("%w: invalid speaker bounds [%d,%d]", llm.ErrUnsupported, req.MinSpeakers, req.MaxSpeakers)
|
||||
}
|
||||
|
||||
q := url.Values{}
|
||||
q.Set("output", "json")
|
||||
q.Set("diarize", "true")
|
||||
if req.Language != "" {
|
||||
q.Set("language", req.Language)
|
||||
}
|
||||
if req.MinSpeakers > 0 {
|
||||
q.Set("min_speakers", strconv.Itoa(req.MinSpeakers))
|
||||
}
|
||||
if req.MaxSpeakers > 0 {
|
||||
q.Set("max_speakers", strconv.Itoa(req.MaxSpeakers))
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/asr?"+q.Encode())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
body, contentType, err := buildMultipart("build diarization form",
|
||||
filePart{field: "audio_file", filename: diarizationFilename(req), data: req.Audio},
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out asrResponse
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: decode diarization response: %w", err)
|
||||
}
|
||||
res := &audio.DiarizationResult{
|
||||
Text: out.Text,
|
||||
Language: out.Language,
|
||||
Raw: json.RawMessage(raw),
|
||||
}
|
||||
for _, s := range out.Segments {
|
||||
res.Segments = append(res.Segments, audio.DiarizationSegment{
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Speaker: s.Speaker,
|
||||
Text: s.Text,
|
||||
})
|
||||
}
|
||||
if len(res.Segments) == 0 && res.Text == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "diarization response contained no transcript"}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// diarizationFilename mirrors transcriptionFilename for the diarization
|
||||
// request shape (same untrusted-metadata sanitization rules).
|
||||
func diarizationFilename(req audio.DiarizationRequest) string {
|
||||
return transcriptionFilename(audio.TranscriptionRequest{
|
||||
Filename: req.Filename,
|
||||
MIME: req.MIME,
|
||||
})
|
||||
}
|
||||
@@ -108,3 +108,47 @@ func TestImageEditValidation(t *testing.T) {
|
||||
t.Errorf("out-of-range strength: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageEditWithMask(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
im, _ := p.ImageModel("sd")
|
||||
ed := im.(imagegen.Editor)
|
||||
|
||||
mask := imagegen.Image{MIME: "image/png", Data: []byte{0xDE, 0xAD}}
|
||||
if _, err := ed.Edit(context.Background(),
|
||||
imagegen.EditRequest{Prompt: "replace the sky", Init: editInit(t)},
|
||||
imagegen.WithEditMask(mask),
|
||||
); err != nil {
|
||||
t.Fatalf("Edit: %v", err)
|
||||
}
|
||||
if got := gotBody["mask"]; got != base64.StdEncoding.EncodeToString(mask.Data) {
|
||||
t.Errorf("mask = %v, want the b64 mask", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageEditWithoutMaskOmitsField(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
im, _ := p.ImageModel("sd")
|
||||
ed := im.(imagegen.Editor)
|
||||
if _, err := ed.Edit(context.Background(),
|
||||
imagegen.EditRequest{Prompt: "p", Init: editInit(t)}); err != nil {
|
||||
t.Fatalf("Edit: %v", err)
|
||||
}
|
||||
if _, ok := gotBody["mask"]; ok {
|
||||
t.Error("mask field sent for unmasked edit; want omitted")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -128,6 +128,11 @@ type img2imgRequest struct {
|
||||
txt2imgRequest
|
||||
InitImages []string `json:"init_images"`
|
||||
DenoisingStrength *float64 `json:"denoising_strength,omitempty"`
|
||||
// Mask enables inpainting: base64 image, white = repaint, black = keep
|
||||
// (sd-server also accepts a data URL; plain base64 keeps symmetry with
|
||||
// init_images). sd-server has no mask_blur/inpaint_full_res — callers
|
||||
// wanting soft edges pre-feather the mask.
|
||||
Mask string `json:"mask,omitempty"`
|
||||
}
|
||||
|
||||
// Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img.
|
||||
@@ -148,6 +153,9 @@ func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ..
|
||||
InitImages: []string{base64.StdEncoding.EncodeToString(req.Init.Data)},
|
||||
DenoisingStrength: req.Strength,
|
||||
}
|
||||
if len(req.Mask.Data) > 0 {
|
||||
wire.Mask = base64.StdEncoding.EncodeToString(req.Mask.Data)
|
||||
}
|
||||
|
||||
var resp txt2imgResponse
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/img2img", m.id, &wire, &resp); err != nil {
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
// mediautil.go implements the imagegen.UpscaleProvider,
|
||||
// imagegen.BackgroundRemovalProvider, and videogen.InterpolationProvider
|
||||
// surfaces against upstreams reached through llama-swap's /upstream
|
||||
// passthrough (ADR-0020):
|
||||
//
|
||||
// upscale POST /upstream/<id>/v1/upscale (mediautils shim)
|
||||
// background POST /upstream/<id>/api/remove (rembg server)
|
||||
// interpolate POST /upstream/<id>/v1/interpolate (mediautils shim)
|
||||
//
|
||||
// All three are one-file multipart in, raw bytes out.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
// --- upscale ---
|
||||
|
||||
// UpscaleModel implements imagegen.UpscaleProvider against the mediautils
|
||||
// shim's POST /v1/upscale. The id selects which upstream llama-swap loads.
|
||||
func (p *Provider) UpscaleModel(id string, opts ...imagegen.UpscaleModelOption) (imagegen.Upscaler, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = imagegen.ApplyUpscaleModelOptions(opts)
|
||||
return &upscaleModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type upscaleModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Upscale implements imagegen.Upscaler.
|
||||
func (m *upscaleModel) Upscale(ctx context.Context, req imagegen.UpscaleRequest, opts ...imagegen.UpscaleOption) (*imagegen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Image.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: upscale requires an image", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Scale != 0 && req.Scale != 2 && req.Scale != 4 {
|
||||
return nil, fmt.Errorf("%w: upscale scale must be 2 or 4, got %d", llm.ErrUnsupported, req.Scale)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/upscale")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scale := ""
|
||||
if req.Scale != 0 {
|
||||
scale = strconv.Itoa(req.Scale)
|
||||
}
|
||||
body, contentType, err := buildMultipart("build upscale form",
|
||||
filePart{field: "file", filename: "image.png", data: req.Image.Data},
|
||||
[]formField{{"scale", scale, false}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleImageResult(m.p.name, m.id, "upscale", raw, respType)
|
||||
}
|
||||
|
||||
// --- background removal ---
|
||||
|
||||
// BackgroundRemovalModel implements imagegen.BackgroundRemovalProvider
|
||||
// against a rembg server's POST /api/remove.
|
||||
func (p *Provider) BackgroundRemovalModel(id string, opts ...imagegen.BackgroundRemovalModelOption) (imagegen.BackgroundRemover, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = imagegen.ApplyBackgroundRemovalModelOptions(opts)
|
||||
return &backgroundModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type backgroundModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// RemoveBackground implements imagegen.BackgroundRemover. rembg's `model`
|
||||
// form field is the request's Net (its internal network); the llama-swap
|
||||
// model id only picks the upstream. `om=true` returns the black/white
|
||||
// foreground mask instead of the RGBA cutout.
|
||||
func (m *backgroundModel) RemoveBackground(ctx context.Context, req imagegen.BackgroundRemovalRequest, opts ...imagegen.BackgroundRemovalOption) (*imagegen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Image.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: background removal requires an image", llm.ErrUnsupported)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/api/remove")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
om := ""
|
||||
if req.OnlyMask {
|
||||
om = "true"
|
||||
}
|
||||
body, contentType, err := buildMultipart("build background-removal form",
|
||||
filePart{field: "file", filename: "image.png", data: req.Image.Data},
|
||||
[]formField{
|
||||
{"model", req.Net, false},
|
||||
{"om", om, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleImageResult(m.p.name, m.id, "background removal", raw, respType)
|
||||
}
|
||||
|
||||
// singleImageResult wraps one raw image body into an imagegen.Result,
|
||||
// rejecting empty or non-image payloads (a proxy error page must not become
|
||||
// "the image").
|
||||
func singleImageResult(provider, model, verb string, raw []byte, contentType string) (*imagegen.Result, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: provider, Model: model, Message: verb + " response contained no image"}
|
||||
}
|
||||
mimeType := sniffImageMIME(raw)
|
||||
if ct := strings.TrimSpace(contentType); ct != "" && !strings.HasPrefix(ct, "image/") && !strings.HasPrefix(http.DetectContentType(raw), "image/") {
|
||||
return nil, &llm.APIError{Provider: provider, Model: model, Message: fmt.Sprintf("%s response is not an image (Content-Type %q)", verb, ct)}
|
||||
}
|
||||
return &imagegen.Result{Images: []llm.ImagePart{{MIME: mimeType, Data: raw}}}, nil
|
||||
}
|
||||
|
||||
// --- frame interpolation ---
|
||||
|
||||
// InterpolatorModel implements videogen.InterpolationProvider against the
|
||||
// mediautils shim's POST /v1/interpolate.
|
||||
func (p *Provider) InterpolatorModel(id string, opts ...videogen.InterpolatorModelOption) (videogen.Interpolator, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = videogen.ApplyInterpolatorModelOptions(opts)
|
||||
return &interpolatorModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type interpolatorModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Interpolate implements videogen.Interpolator. The response body IS the
|
||||
// encoded clip (same contract as /v1/videos/sync), hence the video-sized
|
||||
// response cap and the same MIME resolution rules.
|
||||
func (m *interpolatorModel) Interpolate(ctx context.Context, req videogen.InterpolateRequest, opts ...videogen.InterpolateOption) (*videogen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Video.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: interpolation requires a video", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Multi != 0 && req.Multi != 2 && req.Multi != 4 {
|
||||
return nil, fmt.Errorf("%w: interpolation multi must be 2 or 4, got %d", llm.ErrUnsupported, req.Multi)
|
||||
}
|
||||
if req.TargetFPS < 0 {
|
||||
return nil, fmt.Errorf("%w: target fps must be >= 0, got %d", llm.ErrUnsupported, req.TargetFPS)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/interpolate")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
multi := ""
|
||||
if req.Multi != 0 {
|
||||
multi = strconv.Itoa(req.Multi)
|
||||
}
|
||||
mode := ""
|
||||
targetFPS := ""
|
||||
if req.SlowMo {
|
||||
mode = "slowmo"
|
||||
} else if req.TargetFPS > 0 {
|
||||
targetFPS = strconv.Itoa(req.TargetFPS)
|
||||
}
|
||||
body, contentType, err := buildMultipart("build interpolate form",
|
||||
filePart{field: "file", filename: "video.mp4", data: req.Video.Data},
|
||||
[]formField{
|
||||
{"multi", multi, false},
|
||||
{"mode", mode, false},
|
||||
{"target_fps", targetFPS, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "interpolate response contained no video"}
|
||||
}
|
||||
mimeType := videoMIME(respType, raw)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "interpolate response is not a video"}
|
||||
}
|
||||
return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
func pngFixture(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
raw, err := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
if err != nil {
|
||||
t.Fatalf("decode fixture: %v", err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func TestUpscale(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
var gotPath, gotScale, gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
gotScale = r.FormValue("scale")
|
||||
if f, hdr, err := r.FormFile("file"); err == nil {
|
||||
gotFilename = hdr.Filename
|
||||
f.Close()
|
||||
} else {
|
||||
t.Errorf("file part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(png)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
up, err := p.UpscaleModel("mediautils")
|
||||
if err != nil {
|
||||
t.Fatalf("UpscaleModel: %v", err)
|
||||
}
|
||||
res, err := up.Upscale(context.Background(),
|
||||
imagegen.UpscaleRequest{Image: imagegen.Image{MIME: "image/png", Data: png}},
|
||||
imagegen.WithUpscaleScale(2))
|
||||
if err != nil {
|
||||
t.Fatalf("Upscale: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/mediautils/v1/upscale" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotScale != "2" || gotFilename != "image.png" {
|
||||
t.Errorf("scale/filename = %q/%q", gotScale, gotFilename)
|
||||
}
|
||||
if len(res.Images) != 1 || res.Images[0].MIME != "image/png" {
|
||||
t.Fatalf("images = %+v", res.Images)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpscaleRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
up, _ := p.UpscaleModel("mediautils")
|
||||
if _, err := up.Upscale(context.Background(), imagegen.UpscaleRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no image: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := up.Upscale(context.Background(),
|
||||
imagegen.UpscaleRequest{Image: imagegen.Image{Data: []byte{1}}, Scale: 3}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("scale 3: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpscaleRejectsNonImageResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte("<html>proxy error page</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
up, _ := p.UpscaleModel("mediautils")
|
||||
_, err := up.Upscale(context.Background(),
|
||||
imagegen.UpscaleRequest{Image: imagegen.Image{Data: pngFixture(t)}})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-image body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveBackground(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
var gotPath, gotNet, gotOM string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
gotNet = r.FormValue("model")
|
||||
gotOM = r.FormValue("om")
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(png)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
br, err := p.BackgroundRemovalModel("rembg")
|
||||
if err != nil {
|
||||
t.Fatalf("BackgroundRemovalModel: %v", err)
|
||||
}
|
||||
res, err := br.RemoveBackground(context.Background(),
|
||||
imagegen.BackgroundRemovalRequest{Image: imagegen.Image{Data: png}},
|
||||
imagegen.WithBackgroundNet("birefnet-general"), imagegen.WithOnlyMask())
|
||||
if err != nil {
|
||||
t.Fatalf("RemoveBackground: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/rembg/api/remove" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotNet != "birefnet-general" || gotOM != "true" {
|
||||
t.Errorf("net/om = %q/%q", gotNet, gotOM)
|
||||
}
|
||||
if len(res.Images) != 1 {
|
||||
t.Fatalf("images = %+v", res.Images)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveBackgroundOmitsUnsetFields(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
if _, ok := r.MultipartForm.Value["model"]; ok {
|
||||
t.Error("model field sent for default net; want omitted")
|
||||
}
|
||||
if _, ok := r.MultipartForm.Value["om"]; ok {
|
||||
t.Error("om field sent for cutout mode; want omitted")
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(png)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
br, _ := p.BackgroundRemovalModel("rembg")
|
||||
if _, err := br.RemoveBackground(context.Background(),
|
||||
imagegen.BackgroundRemovalRequest{Image: imagegen.Image{Data: png}}); err != nil {
|
||||
t.Fatalf("RemoveBackground: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// mp4Fixture is a minimal ftyp box so http.DetectContentType sniffs video/mp4.
|
||||
func mp4Fixture() []byte {
|
||||
return []byte{0, 0, 0, 24, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm',
|
||||
0, 0, 2, 0, 'i', 's', 'o', 'm', 'i', 's', 'o', '2'}
|
||||
}
|
||||
|
||||
func TestInterpolate(t *testing.T) {
|
||||
var gotPath, gotMulti, gotMode, gotTargetFPS string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
gotMulti = r.FormValue("multi")
|
||||
gotMode = r.FormValue("mode")
|
||||
gotTargetFPS = r.FormValue("target_fps")
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write(mp4Fixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ip, err := p.InterpolatorModel("mediautils")
|
||||
if err != nil {
|
||||
t.Fatalf("InterpolatorModel: %v", err)
|
||||
}
|
||||
res, err := ip.Interpolate(context.Background(),
|
||||
videogen.InterpolateRequest{Video: videogen.Video{Data: mp4Fixture(), MIME: "video/mp4"}},
|
||||
videogen.WithInterpolateMulti(2), videogen.WithTargetFPS(60))
|
||||
if err != nil {
|
||||
t.Fatalf("Interpolate: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/mediautils/v1/interpolate" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotMulti != "2" || gotMode != "" || gotTargetFPS != "60" {
|
||||
t.Errorf("multi/mode/target_fps = %q/%q/%q", gotMulti, gotMode, gotTargetFPS)
|
||||
}
|
||||
if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
|
||||
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterpolateSlowMoSetsModeAndDropsTargetFPS(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
if got := r.FormValue("mode"); got != "slowmo" {
|
||||
t.Errorf("mode = %q, want slowmo", got)
|
||||
}
|
||||
if _, ok := r.MultipartForm.Value["target_fps"]; ok {
|
||||
t.Error("target_fps sent in slowmo mode; want omitted")
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write(mp4Fixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ip, _ := p.InterpolatorModel("mediautils")
|
||||
_, err := ip.Interpolate(context.Background(),
|
||||
videogen.InterpolateRequest{Video: videogen.Video{Data: mp4Fixture()}, TargetFPS: 60},
|
||||
videogen.WithSlowMo())
|
||||
if err != nil {
|
||||
t.Fatalf("Interpolate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterpolateRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
ip, _ := p.InterpolatorModel("mediautils")
|
||||
if _, err := ip.Interpolate(context.Background(), videogen.InterpolateRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no video: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ip.Interpolate(context.Background(),
|
||||
videogen.InterpolateRequest{Video: videogen.Video{Data: []byte{1}}, Multi: 3}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("multi 3: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamPathRejectsSeparators(t *testing.T) {
|
||||
for _, bad := range []string{"", "a/b", "a?b", "a#b"} {
|
||||
if _, err := upstreamPath(bad, "/x"); err == nil {
|
||||
t.Errorf("upstreamPath(%q) succeeded; want error", bad)
|
||||
}
|
||||
}
|
||||
got, err := upstreamPath("rembg", "api/remove")
|
||||
if err != nil || got != "/upstream/rembg/api/remove" {
|
||||
t.Errorf("upstreamPath = %q, %v", got, err)
|
||||
}
|
||||
if !strings.HasPrefix(got, "/upstream/") {
|
||||
t.Errorf("path prefix wrong: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// mesh.go implements meshgen.Provider against a Hunyuan3D-2.1-style
|
||||
// api_server reached through llama-swap's /upstream passthrough (ADR-0020):
|
||||
//
|
||||
// POST /upstream/<id>/generate JSON {image: <b64>, type: "glb"|"stl"|...}
|
||||
//
|
||||
// The response body IS the encoded mesh (binary, Content-Type
|
||||
// application/octet-stream), so one request yields exactly one asset.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/meshgen"
|
||||
)
|
||||
|
||||
// maxMeshResponseBytes caps the /generate body — a high-resolution textured
|
||||
// mesh legitimately passes the 64MB JSON cap, but stays far under video
|
||||
// scale.
|
||||
const maxMeshResponseBytes = 256 << 20
|
||||
|
||||
// MeshModel implements meshgen.Provider. The id selects which upstream
|
||||
// llama-swap loads.
|
||||
func (p *Provider) MeshModel(id string, opts ...meshgen.ModelOption) (meshgen.Model, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = meshgen.ApplyModelOptions(opts)
|
||||
return &meshModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type meshModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// hunyuanGenerateRequest is the Hunyuan3D api_server /generate shape.
|
||||
// Optional fields are pointers/omitempty so unset values fall back to the
|
||||
// server's defaults (mirrors the sd-server wire structs).
|
||||
type hunyuanGenerateRequest struct {
|
||||
Image string `json:"image"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Texture bool `json:"texture"`
|
||||
RemoveBackground *bool `json:"remove_background,omitempty"`
|
||||
OctreeResolution int `json:"octree_resolution,omitempty"`
|
||||
NumInferenceSteps *int `json:"num_inference_steps,omitempty"`
|
||||
GuidanceScale *float64 `json:"guidance_scale,omitempty"`
|
||||
Seed *int64 `json:"seed,omitempty"`
|
||||
FaceCount int `json:"face_count,omitempty"`
|
||||
}
|
||||
|
||||
// meshFormats maps the supported output containers to MIME types.
|
||||
var meshFormats = map[string]string{
|
||||
"glb": "model/gltf-binary",
|
||||
"stl": "model/stl",
|
||||
"obj": "model/obj",
|
||||
}
|
||||
|
||||
// Generate implements meshgen.Model.
|
||||
func (m *meshModel) Generate(ctx context.Context, req meshgen.Request, opts ...meshgen.Option) (*meshgen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Image.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: mesh generation requires an image", llm.ErrUnsupported)
|
||||
}
|
||||
format := strings.ToLower(strings.TrimSpace(req.Format))
|
||||
if format == "" {
|
||||
format = "glb"
|
||||
}
|
||||
mimeType, ok := meshFormats[format]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: unsupported mesh format %q (want glb, stl, or obj)", llm.ErrUnsupported, req.Format)
|
||||
}
|
||||
if req.OctreeResolution < 0 || req.FaceCount < 0 {
|
||||
return nil, fmt.Errorf("%w: octree resolution and face count must be >= 0", llm.ErrUnsupported)
|
||||
}
|
||||
|
||||
path, err := upstreamPath(m.id, "/generate")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wire := hunyuanGenerateRequest{
|
||||
Image: base64.StdEncoding.EncodeToString(req.Image.Data),
|
||||
Type: format,
|
||||
Texture: req.Texture,
|
||||
RemoveBackground: req.RemoveBackground,
|
||||
OctreeResolution: req.OctreeResolution,
|
||||
NumInferenceSteps: req.Steps,
|
||||
GuidanceScale: req.GuidanceScale,
|
||||
Seed: req.Seed,
|
||||
FaceCount: req.FaceCount,
|
||||
}
|
||||
|
||||
// doRaw + manual JSON encode (not doJSON): the SUCCESS body is binary
|
||||
// mesh bytes, not JSON.
|
||||
encoded, err := json.Marshal(wire)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: encode mesh request: %w", err)
|
||||
}
|
||||
raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, "application/json", bytes.NewReader(encoded), maxMeshResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "mesh response contained no data"}
|
||||
}
|
||||
// A JSON body on the success path is the server reporting a soft error
|
||||
// (or an API drift) — never hand it back as "the mesh".
|
||||
if first := strings.TrimLeft(string(raw[:min(len(raw), 64)]), " \t\r\n"); strings.HasPrefix(first, "{") || strings.HasPrefix(first, "[") {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: "mesh response is JSON, not mesh bytes: " + truncateForError(raw)}
|
||||
}
|
||||
return &meshgen.Result{Mesh: meshgen.Mesh{Data: raw, Format: format, MIME: mimeType}}, nil
|
||||
}
|
||||
|
||||
// truncateForError bounds a payload quoted into an error message.
|
||||
func truncateForError(b []byte) string {
|
||||
const cap = 500
|
||||
if len(b) > cap {
|
||||
return string(b[:cap]) + "..."
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/meshgen"
|
||||
)
|
||||
|
||||
func TestMeshGenerate(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
stl := []byte("solid mort\nendsolid mort\n")
|
||||
var gotPath string
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write(stl)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
mm, err := p.MeshModel("image3d-hunyuan21")
|
||||
if err != nil {
|
||||
t.Fatalf("MeshModel: %v", err)
|
||||
}
|
||||
res, err := mm.Generate(context.Background(),
|
||||
meshgen.Request{Image: meshgen.Image{MIME: "image/png", Data: png}},
|
||||
meshgen.WithFormat("stl"), meshgen.WithSeed(7))
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/image3d-hunyuan21/generate" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotBody["type"] != "stl" || gotBody["image"] != base64.StdEncoding.EncodeToString(png) {
|
||||
t.Errorf("type/image = %v/(b64 mismatch)", gotBody["type"])
|
||||
}
|
||||
if gotBody["seed"] != float64(7) {
|
||||
t.Errorf("seed = %v", gotBody["seed"])
|
||||
}
|
||||
if _, ok := gotBody["octree_resolution"]; ok {
|
||||
t.Error("octree_resolution sent when unset; want omitted")
|
||||
}
|
||||
if res.Mesh.Format != "stl" || res.Mesh.MIME != "model/stl" || string(res.Mesh.Data) != string(stl) {
|
||||
t.Fatalf("mesh = %+v", res.Mesh)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeshGenerateRejectsJSONBody(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write([]byte(` {"detail": "queue full"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
mm, _ := p.MeshModel("image3d-hunyuan21")
|
||||
_, err := mm.Generate(context.Background(),
|
||||
meshgen.Request{Image: meshgen.Image{Data: pngFixture(t)}})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for JSON success body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeshGenerateRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
mm, _ := p.MeshModel("image3d-hunyuan21")
|
||||
if _, err := mm.Generate(context.Background(), meshgen.Request{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no image: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := mm.Generate(context.Background(),
|
||||
meshgen.Request{Image: meshgen.Image{Data: []byte{1}}, Format: "fbx"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("format fbx: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiarize(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotQuery map[string][]string
|
||||
var gotField string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
gotQuery = r.URL.Query()
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
if f, _, err := r.FormFile("audio_file"); err == nil {
|
||||
gotField = "audio_file"
|
||||
f.Close()
|
||||
}
|
||||
_, _ = w.Write([]byte(`{
|
||||
"text": "hello there. general kenobi.",
|
||||
"language": "en",
|
||||
"segments": [
|
||||
{"start": 0.0, "end": 1.2, "text": "hello there.", "speaker": "SPEAKER_00"},
|
||||
{"start": 1.4, "end": 3.0, "text": "general kenobi.", "speaker": "SPEAKER_01"}
|
||||
]
|
||||
}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
dm, err := p.DiarizationModel("whisperx-diarize")
|
||||
if err != nil {
|
||||
t.Fatalf("DiarizationModel: %v", err)
|
||||
}
|
||||
res, err := dm.Diarize(context.Background(),
|
||||
audio.DiarizationRequest{Audio: []byte("RIFFfake"), MIME: "audio/wav"},
|
||||
audio.WithSpeakerBounds(2, 4), audio.WithDiarizationLanguage("en"))
|
||||
if err != nil {
|
||||
t.Fatalf("Diarize: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/whisperx-diarize/asr" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
for k, want := range map[string]string{
|
||||
"output": "json", "diarize": "true",
|
||||
"min_speakers": "2", "max_speakers": "4", "language": "en",
|
||||
} {
|
||||
if got := gotQuery[k]; len(got) != 1 || got[0] != want {
|
||||
t.Errorf("query %s = %v, want %q", k, got, want)
|
||||
}
|
||||
}
|
||||
if gotField != "audio_file" {
|
||||
t.Error("audio_file part missing")
|
||||
}
|
||||
if len(res.Segments) != 2 || res.Segments[1].Speaker != "SPEAKER_01" {
|
||||
t.Fatalf("segments = %+v", res.Segments)
|
||||
}
|
||||
if res.Language != "en" || res.Text == "" {
|
||||
t.Errorf("language/text = %q/%q", res.Language, res.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiarizeRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
dm, _ := p.DiarizationModel("whisperx-diarize")
|
||||
if _, err := dm.Diarize(context.Background(), audio.DiarizationRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no audio: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := dm.Diarize(context.Background(),
|
||||
audio.DiarizationRequest{Audio: []byte{1}, MinSpeakers: 5, MaxSpeakers: 2}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("bounds 5>2: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiarizeEmptyTranscriptIsError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"text": "", "segments": []}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
dm, _ := p.DiarizationModel("whisperx-diarize")
|
||||
_, err := dm.Diarize(context.Background(), audio.DiarizationRequest{Audio: []byte{1}})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for empty transcript", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// upstreamPath builds a path through llama-swap's generic /upstream/<model>/
|
||||
// passthrough, which pins the model (triggering the normal load/swap queue)
|
||||
// and forwards the remaining path to the upstream verbatim. This is how the
|
||||
// provider reaches upstreams whose native APIs carry no routable `model`
|
||||
// field (rembg, mediautils, WhisperX, ACE-Step, ...) without needing a
|
||||
// llama-swap route per endpoint (ADR-0020).
|
||||
//
|
||||
// Why reject rather than escape: same rationale as Unload — model ids
|
||||
// legitimately contain ":" but never path-structure characters, and escaping
|
||||
// would mask a config error instead of surfacing it.
|
||||
func upstreamPath(model, rest string) (string, error) {
|
||||
if strings.TrimSpace(model) == "" {
|
||||
return "", fmt.Errorf("llama-swap: upstream call requires a model id")
|
||||
}
|
||||
if strings.ContainsAny(model, "/?#") {
|
||||
return "", fmt.Errorf("llama-swap: invalid model id %q for upstream call (contains a path separator)", model)
|
||||
}
|
||||
if !strings.HasPrefix(rest, "/") {
|
||||
rest = "/" + rest
|
||||
}
|
||||
return "/upstream/" + model + rest, nil
|
||||
}
|
||||
|
||||
// filePart is the single file entry of a media multipart form.
|
||||
type filePart struct {
|
||||
field string // form field name ("file", "audio_file", ...)
|
||||
filename string // already sanitized
|
||||
data []byte
|
||||
}
|
||||
|
||||
// buildMultipart assembles a one-file multipart body: the file part first,
|
||||
// then the given fields (optional fields skipped when empty, matching
|
||||
// writeFormFields). wrap labels errors. Returns the body and its content
|
||||
// type.
|
||||
func buildMultipart(wrap string, file filePart, fields []formField) (*bytes.Buffer, string, error) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
fw, err := w.CreateFormFile(file.field, file.filename)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err)
|
||||
}
|
||||
if _, err := fw.Write(file.data); err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err)
|
||||
}
|
||||
if err := writeFormFields(w, wrap, fields); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err)
|
||||
}
|
||||
return &buf, w.FormDataContentType(), nil
|
||||
}
|
||||
Reference in New Issue
Block a user