feat: media expansion surfaces — edit mask, upscale, background removal, interpolation, diarization, meshgen (ADR-0020)
- imagegen.EditRequest.Mask -> sd-server img2img inpainting (white=repaint) - imagegen.Upscaler + BackgroundRemover, videogen.Interpolator, audio.DiarizationModel: new optional provider-minted surfaces - NEW meshgen leaf package (image->3D, glb/stl/obj) - provider/llamaswap: all five via the /upstream/<model>/<path> passthrough (upstreamPath helper, shared one-file multipart builder); binary success bodies validated (non-image, non-video, JSON-mesh rejection); diarization pins output=json (vtt/srt drop speaker labels) Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
// mesh.go implements meshgen.Provider against a Hunyuan3D-2.1-style
|
||||
// api_server reached through llama-swap's /upstream passthrough (ADR-0020):
|
||||
//
|
||||
// POST /upstream/<id>/generate JSON {image: <b64>, type: "glb"|"stl"|...}
|
||||
//
|
||||
// The response body IS the encoded mesh (binary, Content-Type
|
||||
// application/octet-stream), so one request yields exactly one asset.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/meshgen"
|
||||
)
|
||||
|
||||
// maxMeshResponseBytes caps the /generate body — a high-resolution textured
|
||||
// mesh legitimately passes the 64MB JSON cap, but stays far under video
|
||||
// scale.
|
||||
const maxMeshResponseBytes = 256 << 20
|
||||
|
||||
// MeshModel implements meshgen.Provider. The id selects which upstream
|
||||
// llama-swap loads.
|
||||
func (p *Provider) MeshModel(id string, opts ...meshgen.ModelOption) (meshgen.Model, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = meshgen.ApplyModelOptions(opts)
|
||||
return &meshModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type meshModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// hunyuanGenerateRequest is the Hunyuan3D api_server /generate shape.
|
||||
// Optional fields are pointers/omitempty so unset values fall back to the
|
||||
// server's defaults (mirrors the sd-server wire structs).
|
||||
type hunyuanGenerateRequest struct {
|
||||
Image string `json:"image"`
|
||||
Type string `json:"type,omitempty"`
|
||||
Texture bool `json:"texture"`
|
||||
RemoveBackground *bool `json:"remove_background,omitempty"`
|
||||
OctreeResolution int `json:"octree_resolution,omitempty"`
|
||||
NumInferenceSteps *int `json:"num_inference_steps,omitempty"`
|
||||
GuidanceScale *float64 `json:"guidance_scale,omitempty"`
|
||||
Seed *int64 `json:"seed,omitempty"`
|
||||
FaceCount int `json:"face_count,omitempty"`
|
||||
}
|
||||
|
||||
// meshFormats maps the supported output containers to MIME types.
|
||||
var meshFormats = map[string]string{
|
||||
"glb": "model/gltf-binary",
|
||||
"stl": "model/stl",
|
||||
"obj": "model/obj",
|
||||
}
|
||||
|
||||
// Generate implements meshgen.Model.
|
||||
func (m *meshModel) Generate(ctx context.Context, req meshgen.Request, opts ...meshgen.Option) (*meshgen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Image.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: mesh generation requires an image", llm.ErrUnsupported)
|
||||
}
|
||||
format := strings.ToLower(strings.TrimSpace(req.Format))
|
||||
if format == "" {
|
||||
format = "glb"
|
||||
}
|
||||
mimeType, ok := meshFormats[format]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: unsupported mesh format %q (want glb, stl, or obj)", llm.ErrUnsupported, req.Format)
|
||||
}
|
||||
if req.OctreeResolution < 0 || req.FaceCount < 0 {
|
||||
return nil, fmt.Errorf("%w: octree resolution and face count must be >= 0", llm.ErrUnsupported)
|
||||
}
|
||||
|
||||
path, err := upstreamPath(m.id, "/generate")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wire := hunyuanGenerateRequest{
|
||||
Image: base64.StdEncoding.EncodeToString(req.Image.Data),
|
||||
Type: format,
|
||||
Texture: req.Texture,
|
||||
RemoveBackground: req.RemoveBackground,
|
||||
OctreeResolution: req.OctreeResolution,
|
||||
NumInferenceSteps: req.Steps,
|
||||
GuidanceScale: req.GuidanceScale,
|
||||
Seed: req.Seed,
|
||||
FaceCount: req.FaceCount,
|
||||
}
|
||||
|
||||
// doRaw + manual JSON encode (not doJSON): the SUCCESS body is binary
|
||||
// mesh bytes, not JSON.
|
||||
encoded, err := json.Marshal(wire)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: encode mesh request: %w", err)
|
||||
}
|
||||
raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, "application/json", bytes.NewReader(encoded), maxMeshResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "mesh response contained no data"}
|
||||
}
|
||||
// A JSON body on the success path is the server reporting a soft error
|
||||
// (or an API drift) — never hand it back as "the mesh".
|
||||
if first := strings.TrimLeft(string(raw[:min(len(raw), 64)]), " \t\r\n"); strings.HasPrefix(first, "{") || strings.HasPrefix(first, "[") {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: "mesh response is JSON, not mesh bytes: " + truncateForError(raw)}
|
||||
}
|
||||
return &meshgen.Result{Mesh: meshgen.Mesh{Data: raw, Format: format, MIME: mimeType}}, nil
|
||||
}
|
||||
|
||||
// truncateForError bounds a payload quoted into an error message.
|
||||
func truncateForError(b []byte) string {
|
||||
const cap = 500
|
||||
if len(b) > cap {
|
||||
return string(b[:cap]) + "..."
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
Reference in New Issue
Block a user