feat: audio surfaces (TTS + transcription), imagegen.Editor, llamaswap health probe
- New leaf package `audio` (ADR-0017): SpeechModel/SpeechProvider and TranscriptionModel/TranscriptionProvider with imagegen conventions (zero value = backend default, functional options + Apply, bytes in/out, never URLs). Root re-exports added. - imagegen.Editor (ADR-0018): optional image-to-image interface — EditRequest carries the generation knobs plus Init image and denoising Strength; separate interface so existing Models keep compiling. - provider/llamaswap implements all of it: POST /v1/audio/speech (JSON, raw-audio response, MIME from Content-Type with format fallback), POST /v1/audio/transcriptions (multipart, response_format=json), ListVoices (GET /v1/audio/voices?model=, tolerant of string-list and object-list shapes), POST /sdapi/v1/img2img (txt2img wire + init_images/denoising_strength, shared image decode), and Health(ctx) (GET /health) — a cheap liveness probe for often-offline hosts. - Hermetic httptest coverage for every new wire shape and validation path; README sections + support-matrix footnote updated in the same commit (also corrects the stale /v1/images/generations claim — the image path has been SDAPI since the seed fix). First consumer: mort's llamaswap media tool cluster (status / image / TTS / STT agent tools against the netherstorm host). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AXQxVhXBw8PwFAtsVrXSmj
This commit is contained in:
+186
@@ -0,0 +1,186 @@
|
||||
// Package audio is majordomo's canonical speech surface: text-to-speech
|
||||
// (SpeechModel) and audio transcription (TranscriptionModel). Like imagegen,
|
||||
// it is a deliberately separate contract from the llm package — synthesis and
|
||||
// transcription share none of the chat message/tool/stream machinery, so they
|
||||
// get their own small Provider/Model interfaces rather than overloading
|
||||
// llm.Model (ADR-0017).
|
||||
//
|
||||
// Zero values mean "backend default" throughout, mirroring imagegen: an empty
|
||||
// Voice uses the model's default voice, an empty Format the backend's default
|
||||
// container, a zero Speed the natural rate.
|
||||
//
|
||||
// The first implementation is provider/llamaswap, which targets the OpenAI
|
||||
// /v1/audio/speech and /v1/audio/transcriptions endpoints routed to
|
||||
// kokoro/whisper.cpp-style upstreams.
|
||||
package audio
|
||||
|
||||
import "context"
|
||||
|
||||
// SpeechRequest is a text-to-speech request.
|
||||
type SpeechRequest struct {
|
||||
// Input is the text to speak.
|
||||
Input string
|
||||
|
||||
// Voice selects the voice; "" = the model's default voice.
|
||||
Voice string
|
||||
|
||||
// Format is the audio container ("mp3", "wav", "opus", ...);
|
||||
// "" = backend default.
|
||||
Format string
|
||||
|
||||
// Speed is the playback-rate multiplier; 0 = backend default (1.0).
|
||||
Speed float64
|
||||
}
|
||||
|
||||
// SpeechResult is the canonical synthesis result: raw audio bytes plus the
|
||||
// MIME type reported (or implied) by the backend.
|
||||
type SpeechResult struct {
|
||||
// Audio is the encoded audio.
|
||||
Audio []byte
|
||||
|
||||
// MIME is the audio MIME type, e.g. "audio/mpeg".
|
||||
MIME string
|
||||
|
||||
// Raw is the provider-native response object, an escape hatch for
|
||||
// provider-specific fields. May be nil; never required for normal use.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// SpeechOption mutates a SpeechRequest before it is sent. Options passed to
|
||||
// Speak are applied to a copy, so a request value can be reused.
|
||||
type SpeechOption func(*SpeechRequest)
|
||||
|
||||
// WithVoice selects the voice.
|
||||
func WithVoice(v string) SpeechOption { return func(r *SpeechRequest) { r.Voice = v } }
|
||||
|
||||
// WithFormat sets the audio container format (e.g. "mp3", "wav").
|
||||
func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Format = f } }
|
||||
|
||||
// WithSpeed sets the playback-rate multiplier.
|
||||
func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } }
|
||||
|
||||
// Apply returns a copy of the request with all options applied. Providers
|
||||
// call this once at the top of Speak.
|
||||
func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// SpeechModel synthesizes speech from text.
|
||||
type SpeechModel interface {
|
||||
// Speak renders the request's input text as audio.
|
||||
Speak(ctx context.Context, req SpeechRequest, opts ...SpeechOption) (*SpeechResult, error)
|
||||
}
|
||||
|
||||
// SpeechModelOption configures a SpeechModel at construction time. Reserved
|
||||
// for future per-model settings; present so the interface is
|
||||
// forward-compatible (mirrors imagegen.ModelOption).
|
||||
type SpeechModelOption func(*SpeechModelConfig)
|
||||
|
||||
// SpeechModelConfig carries per-model construction settings.
|
||||
type SpeechModelConfig struct{}
|
||||
|
||||
// ApplySpeechModelOptions folds options into a config.
|
||||
func ApplySpeechModelOptions(opts []SpeechModelOption) SpeechModelConfig {
|
||||
var cfg SpeechModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SpeechProvider mints speech models bound to one backend.
|
||||
type SpeechProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// SpeechModel returns a SpeechModel bound to the given id (passed through
|
||||
// to the backend verbatim; no catalog validation).
|
||||
SpeechModel(id string, opts ...SpeechModelOption) (SpeechModel, error)
|
||||
}
|
||||
|
||||
// TranscriptionRequest is a speech-to-text request. Audio is carried as bytes
|
||||
// (never a URL), mirroring llm.ImagePart's bytes-only contract.
|
||||
type TranscriptionRequest struct {
|
||||
// Audio is the encoded audio to transcribe.
|
||||
Audio []byte
|
||||
|
||||
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
|
||||
// sniff it.
|
||||
MIME string
|
||||
|
||||
// Filename is the multipart filename hint some backends key their format
|
||||
// detection on; "" derives one from MIME ("audio.mp3") or falls back to
|
||||
// "audio".
|
||||
Filename string
|
||||
|
||||
// Language is a BCP-47/ISO-639 hint (e.g. "en"); "" = auto-detect.
|
||||
Language string
|
||||
|
||||
// Prompt is optional context or vocabulary to bias decoding; "" = none.
|
||||
Prompt string
|
||||
}
|
||||
|
||||
// TranscriptionResult is the canonical transcription result.
|
||||
type TranscriptionResult struct {
|
||||
// Text is the transcript.
|
||||
Text string
|
||||
|
||||
// Raw is the provider-native response object. May be nil.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// TranscriptionOption mutates a TranscriptionRequest before it is sent.
|
||||
type TranscriptionOption func(*TranscriptionRequest)
|
||||
|
||||
// WithLanguage sets the language hint (e.g. "en").
|
||||
func WithLanguage(l string) TranscriptionOption {
|
||||
return func(r *TranscriptionRequest) { r.Language = l }
|
||||
}
|
||||
|
||||
// WithPrompt sets the decoding context/vocabulary hint.
|
||||
func WithPrompt(p string) TranscriptionOption {
|
||||
return func(r *TranscriptionRequest) { r.Prompt = p }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// TranscriptionModel transcribes audio to text.
|
||||
type TranscriptionModel interface {
|
||||
// Transcribe converts the request's audio into text.
|
||||
Transcribe(ctx context.Context, req TranscriptionRequest, opts ...TranscriptionOption) (*TranscriptionResult, error)
|
||||
}
|
||||
|
||||
// TranscriptionModelOption configures a TranscriptionModel at construction
|
||||
// time. Reserved for future per-model settings.
|
||||
type TranscriptionModelOption func(*TranscriptionModelConfig)
|
||||
|
||||
// TranscriptionModelConfig carries per-model construction settings.
|
||||
type TranscriptionModelConfig struct{}
|
||||
|
||||
// ApplyTranscriptionModelOptions folds options into a config.
|
||||
func ApplyTranscriptionModelOptions(opts []TranscriptionModelOption) TranscriptionModelConfig {
|
||||
var cfg TranscriptionModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// TranscriptionProvider mints transcription models bound to one backend.
|
||||
type TranscriptionProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// TranscriptionModel returns a TranscriptionModel bound to the given id
|
||||
// (passed through to the backend verbatim; no catalog validation).
|
||||
TranscriptionModel(id string, opts ...TranscriptionModelOption) (TranscriptionModel, error)
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package audio
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSpeechRequestApply(t *testing.T) {
|
||||
base := SpeechRequest{Input: "hello"}
|
||||
got := base.Apply(WithVoice("af_heart"), WithFormat("wav"), WithSpeed(1.5))
|
||||
|
||||
if got.Input != "hello" {
|
||||
t.Errorf("Input = %q, want %q", got.Input, "hello")
|
||||
}
|
||||
if got.Voice != "af_heart" || got.Format != "wav" || got.Speed != 1.5 {
|
||||
t.Errorf("got = %+v", got)
|
||||
}
|
||||
|
||||
// Apply must not mutate the receiver (options apply to a copy).
|
||||
if base.Voice != "" || base.Format != "" || base.Speed != 0 {
|
||||
t.Errorf("base mutated: %+v", base)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscriptionRequestApply(t *testing.T) {
|
||||
base := TranscriptionRequest{Audio: []byte{1, 2, 3}}
|
||||
got := base.Apply(WithLanguage("en"), WithPrompt("names"))
|
||||
|
||||
if got.Language != "en" || got.Prompt != "names" {
|
||||
t.Errorf("got = %+v", got)
|
||||
}
|
||||
if base.Language != "" || base.Prompt != "" {
|
||||
t.Errorf("base mutated: %+v", base)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyModelOptions(t *testing.T) {
|
||||
// No options yet; just verify they return usable zero configs.
|
||||
_ = ApplySpeechModelOptions(nil)
|
||||
_ = ApplyTranscriptionModelOptions(nil)
|
||||
}
|
||||
Reference in New Issue
Block a user