feat: wave-3 audio surfaces — stems, sfx, speech enhance, voice clone, translate (ADR-0024)
- audio.StemSeparator/StemSeparationProvider: Demucs zip transport via POST /upstream/<id>/v1/stems (Mode two -> two_stems=vocals; model + format fields); bounded zip unpack, entry name -> stem, ext -> MIME. - SFXModel reuses musicgen against the sync /upstream/<id>/v1/sfx route (JSON prompt/seconds/steps/cfg_scale/seed -> WAV); musicgen.Request gains CFGScale. - audio.SpeechEnhancer/SpeechEnhancementProvider: POST /upstream/<id>/v1/enhance -> WAV (result reuses SpeechResult). - SpeechRequest.ReferenceAudio/ReferenceMIME (+WithReferenceAudio): llamaswap switches to the chatterbox clone route POST /upstream/<id>/v1/audio/speech/upload (input + voice_file), wav MIME fallback. - TranscriptionRequest.Translate (+WithTranslate): translate=true form field, language=auto forced when no explicit hint (whisper.cpp default en would skip translation). - httptest contract tests (zip unpack, clone-route switch, translate + auto-language injection); ADR-0024 (index row deferred — MJ-A backfills the ADR index table and parallel edits would conflict). Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
// enhance.go implements audio.SpeechEnhancementProvider against a
|
||||
// DeepFilterNet shim (audioutils) reached through llama-swap's /upstream
|
||||
// passthrough (ADR-0024):
|
||||
//
|
||||
// POST /upstream/<id>/v1/enhance multipart file -> WAV
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// SpeechEnhancerModel implements audio.SpeechEnhancementProvider. The id
|
||||
// selects which upstream llama-swap loads.
|
||||
func (p *Provider) SpeechEnhancerModel(id string, opts ...audio.SpeechEnhancerModelOption) (audio.SpeechEnhancer, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = audio.ApplySpeechEnhancerModelOptions(opts)
|
||||
return &speechEnhancerModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type speechEnhancerModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Enhance implements audio.SpeechEnhancer. The endpoint always answers WAV.
|
||||
func (m *speechEnhancerModel) Enhance(ctx context.Context, req audio.EnhancementRequest, opts ...audio.EnhancementOption) (*audio.SpeechResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Audio) == 0 {
|
||||
return nil, fmt.Errorf("%w: speech enhancement requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/enhance")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, contentType, err := buildMultipart("build enhance form",
|
||||
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "enhance response contained no audio"}
|
||||
}
|
||||
mimeType := audioResultMIME(respType, raw)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("enhance response is not audio (Content-Type %q): %s", respType, truncateForError(raw))}
|
||||
}
|
||||
return &audio.SpeechResult{Audio: raw, MIME: mimeType}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user