- audio.StemSeparator/StemSeparationProvider: Demucs zip transport via POST /upstream/<id>/v1/stems (Mode two -> two_stems=vocals; model + format fields); bounded zip unpack, entry name -> stem, ext -> MIME. - SFXModel reuses musicgen against the sync /upstream/<id>/v1/sfx route (JSON prompt/seconds/steps/cfg_scale/seed -> WAV); musicgen.Request gains CFGScale. - audio.SpeechEnhancer/SpeechEnhancementProvider: POST /upstream/<id>/v1/enhance -> WAV (result reuses SpeechResult). - SpeechRequest.ReferenceAudio/ReferenceMIME (+WithReferenceAudio): llamaswap switches to the chatterbox clone route POST /upstream/<id>/v1/audio/speech/upload (input + voice_file), wav MIME fallback. - TranscriptionRequest.Translate (+WithTranslate): translate=true form field, language=auto forced when no explicit hint (whisper.cpp default en would skip translation). - httptest contract tests (zip unpack, clone-route switch, translate + auto-language injection); ADR-0024 (index row deferred — MJ-A backfills the ADR index table and parallel edits would conflict). Co-Authored-By: Claude Fable 5 <[email protected]>
129 lines
4.3 KiB
Go
129 lines
4.3 KiB
Go
// Package musicgen is majordomo's canonical music-generation surface (full
|
|
// songs from a text prompt, optionally with lyrics). Like imagegen/audio/
|
|
// videogen/meshgen, it is a deliberately separate leaf contract from the llm
|
|
// package (ADR-0021, following the ADR-0016→0020 lineage: functional
|
|
// options, zero values = backend default, bytes-only I/O, Raw escape hatch).
|
|
//
|
|
// The first implementation is provider/llamaswap, which targets an
|
|
// ACE-Step-1.5-style async job API reached through the /upstream
|
|
// passthrough; Generate blocks (polling internally) so callers get the
|
|
// imagegen-style one-call contract — bound it with a context deadline.
|
|
package musicgen
|
|
|
|
import "context"
|
|
|
|
// Audio is one generated piece: raw encoded bytes plus a MIME type.
|
|
type Audio struct {
|
|
// Data is the encoded audio container.
|
|
Data []byte
|
|
|
|
// MIME is the audio MIME type, e.g. "audio/mpeg".
|
|
MIME string
|
|
}
|
|
|
|
// Request is a music-generation request. Zero values mean "backend default".
|
|
type Request struct {
|
|
// Prompt describes the music (genre, mood, instrumentation, tempo...).
|
|
Prompt string
|
|
|
|
// Lyrics are optional song lyrics for models that sing; "" =
|
|
// instrumental or model-written lyrics, per backend behavior.
|
|
Lyrics string
|
|
|
|
// DurationSeconds is the requested clip length; 0 = backend default.
|
|
DurationSeconds int
|
|
|
|
// Format is the audio container ("mp3", "wav", "flac", "opus", "aac");
|
|
// "" = backend default (mp3 on ACE-Step).
|
|
Format string
|
|
|
|
// Steps is the number of inference steps; nil = backend default.
|
|
Steps *int
|
|
|
|
// CFGScale is the classifier-free-guidance scale; nil = backend default.
|
|
// Architecture-sensitive, so prefer leaving it nil unless the caller
|
|
// knows the target model. Backends without the knob ignore it.
|
|
CFGScale *float64
|
|
|
|
// Seed fixes the RNG seed for reproducible output; nil = backend
|
|
// default (random).
|
|
Seed *int64
|
|
}
|
|
|
|
// Result is the canonical music-generation result.
|
|
type Result struct {
|
|
// Audio is the generated piece.
|
|
Audio Audio
|
|
|
|
// Raw is the provider-native response object (e.g. the job result with
|
|
// bpm/keyscale metadata). May be nil.
|
|
Raw any
|
|
}
|
|
|
|
// Option mutates a Request before it is sent. Options passed to Generate are
|
|
// applied to a copy of the request, so a Request value can be reused.
|
|
type Option func(*Request)
|
|
|
|
// WithLyrics sets song lyrics.
|
|
func WithLyrics(l string) Option { return func(r *Request) { r.Lyrics = l } }
|
|
|
|
// WithDuration sets the requested clip length in seconds.
|
|
func WithDuration(seconds int) Option {
|
|
return func(r *Request) { r.DurationSeconds = seconds }
|
|
}
|
|
|
|
// WithFormat sets the audio container format.
|
|
func WithFormat(f string) Option { return func(r *Request) { r.Format = f } }
|
|
|
|
// WithSteps overrides the number of inference steps.
|
|
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
|
|
|
|
// WithCFGScale overrides the classifier-free-guidance scale.
|
|
func WithCFGScale(s float64) Option { return func(r *Request) { r.CFGScale = &s } }
|
|
|
|
// WithSeed fixes the RNG seed.
|
|
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
|
|
|
|
// Apply returns a copy of the request with all options applied. Providers
|
|
// call this once at the top of Generate.
|
|
func (r Request) Apply(opts ...Option) Request {
|
|
for _, opt := range opts {
|
|
opt(&r)
|
|
}
|
|
return r
|
|
}
|
|
|
|
// Model generates music from text.
|
|
type Model interface {
|
|
// Generate renders the request as one audio clip. It blocks until the
|
|
// backend finishes (or ctx expires) even when the backend is an async
|
|
// job queue — polling is the provider's business, not the caller's.
|
|
Generate(ctx context.Context, req Request, opts ...Option) (*Result, error)
|
|
}
|
|
|
|
// ModelOption configures a Model at construction time. Reserved for future
|
|
// per-model settings.
|
|
type ModelOption func(*ModelConfig)
|
|
|
|
// ModelConfig carries per-model construction settings.
|
|
type ModelConfig struct{}
|
|
|
|
// ApplyModelOptions folds options into a config.
|
|
func ApplyModelOptions(opts []ModelOption) ModelConfig {
|
|
var cfg ModelConfig
|
|
for _, opt := range opts {
|
|
opt(&cfg)
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
// Provider mints music models bound to one backend.
|
|
type Provider interface {
|
|
// Name is the registry identifier for the provider.
|
|
Name() string
|
|
|
|
// MusicModel returns a Model bound to the given id (passed through to
|
|
// the backend verbatim; no catalog validation).
|
|
MusicModel(id string, opts ...ModelOption) (Model, error)
|
|
}
|