feat: wave-3 audio surfaces — stems, sfx, speech enhance, voice clone, translate (ADR-0024)
- audio.StemSeparator/StemSeparationProvider: Demucs zip transport via POST /upstream/<id>/v1/stems (Mode two -> two_stems=vocals; model + format fields); bounded zip unpack, entry name -> stem, ext -> MIME. - SFXModel reuses musicgen against the sync /upstream/<id>/v1/sfx route (JSON prompt/seconds/steps/cfg_scale/seed -> WAV); musicgen.Request gains CFGScale. - audio.SpeechEnhancer/SpeechEnhancementProvider: POST /upstream/<id>/v1/enhance -> WAV (result reuses SpeechResult). - SpeechRequest.ReferenceAudio/ReferenceMIME (+WithReferenceAudio): llamaswap switches to the chatterbox clone route POST /upstream/<id>/v1/audio/speech/upload (input + voice_file), wav MIME fallback. - TranscriptionRequest.Translate (+WithTranslate): translate=true form field, language=auto forced when no explicit hint (whisper.cpp default en would skip translation). - httptest contract tests (zip unpack, clone-route switch, translate + auto-language injection); ADR-0024 (index row deferred — MJ-A backfills the ADR index table and parallel edits would conflict). Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
@@ -42,7 +43,11 @@ type speechRequest struct {
|
||||
Speed float64 `json:"speed,omitempty"`
|
||||
}
|
||||
|
||||
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech.
|
||||
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech, or —
|
||||
// when the request carries reference audio for voice cloning — via multipart
|
||||
// POST {base}/upstream/<id>/v1/audio/speech/upload (the chatterbox clone
|
||||
// route; verified live 2026-07-16: stateless per-request cloning, fields
|
||||
// `input` + `voice_file`).
|
||||
func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Input) == "" {
|
||||
@@ -51,6 +56,9 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
|
||||
if req.Speed < 0 {
|
||||
return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed)
|
||||
}
|
||||
if len(req.ReferenceAudio) > 0 {
|
||||
return m.speakWithReference(ctx, req)
|
||||
}
|
||||
wire := speechRequest{
|
||||
Model: m.id,
|
||||
Input: req.Input,
|
||||
@@ -72,6 +80,49 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
|
||||
return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil
|
||||
}
|
||||
|
||||
// speakWithReference performs zero-shot voice cloning via the upstream
|
||||
// passthrough clone route. The reference sample rides as the `voice_file`
|
||||
// part and the text as an `input` field; voice/format/speed still go on the
|
||||
// wire when set (upstreams ignore fields they don't understand). The route's
|
||||
// default container is WAV, so MIME resolution falls back to audio/wav (not
|
||||
// the JSON route's mp3) when neither the response header nor the requested
|
||||
// format decides.
|
||||
func (m *speechModel) speakWithReference(ctx context.Context, req audio.SpeechRequest) (*audio.SpeechResult, error) {
|
||||
path, err := upstreamPath(m.id, "/v1/audio/speech/upload")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
speed := ""
|
||||
if req.Speed != 0 {
|
||||
speed = strconv.FormatFloat(req.Speed, 'g', -1, 64)
|
||||
}
|
||||
body, formType, err := buildMultipart("build speech clone form",
|
||||
filePart{field: "voice_file", filename: transcriptionFilename("", req.ReferenceMIME), data: req.ReferenceAudio},
|
||||
[]formField{
|
||||
{"input", req.Input, true},
|
||||
{"voice", req.Voice, false},
|
||||
{"response_format", req.Format, false},
|
||||
{"speed", speed, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, formType, body, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech clone response contained no audio"}
|
||||
}
|
||||
mimeType := "audio/wav"
|
||||
if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
|
||||
mimeType = mt
|
||||
} else if req.Format != "" {
|
||||
mimeType = speechMIME("", req.Format)
|
||||
}
|
||||
return &audio.SpeechResult{Audio: audioBytes, MIME: mimeType}, nil
|
||||
}
|
||||
|
||||
// speechMIME resolves the result MIME type: the response Content-Type when it
|
||||
// is a concrete audio type, else a mapping from the requested format, else
|
||||
// audio/mpeg (the OpenAI endpoint's default container is mp3).
|
||||
@@ -118,13 +169,26 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.Transcrip
|
||||
return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
|
||||
// Translation is a per-request bool form field on whisper.cpp's server
|
||||
// (server.cpp:534, verified 2026-07-16). Its language DEFAULT is "en",
|
||||
// which would make translate a no-op — so an explicit language hint
|
||||
// wins, but an unset one is forced to auto-detection.
|
||||
translate := ""
|
||||
language := req.Language
|
||||
if req.Translate {
|
||||
translate = "true"
|
||||
if language == "" {
|
||||
language = "auto"
|
||||
}
|
||||
}
|
||||
buf, formType, err := buildMultipart("build transcription form",
|
||||
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
|
||||
[]formField{
|
||||
{"model", m.id, true},
|
||||
{"response_format", "json", true},
|
||||
{"language", req.Language, false},
|
||||
{"language", language, false},
|
||||
{"prompt", req.Prompt, false},
|
||||
{"translate", translate, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
Reference in New Issue
Block a user