majordomo/audio/audio.go

// Package audio is majordomo's canonical speech surface: text-to-speech
// (SpeechModel) and audio transcription (TranscriptionModel). Like imagegen,
// it is a deliberately separate contract from the llm package — synthesis and
// transcription share none of the chat message/tool/stream machinery, so they
// get their own small Provider/Model interfaces rather than overloading
// llm.Model (ADR-0017).
//
// Zero values mean "backend default" throughout, mirroring imagegen: an empty
// Voice uses the model's default voice, an empty Format the backend's default
// container, a zero Speed the natural rate.
//
// The first implementation is provider/llamaswap, which targets the OpenAI
// /v1/audio/speech and /v1/audio/transcriptions endpoints routed to
// kokoro/whisper.cpp-style upstreams.
package audio

import "context"

// SpeechRequest is a text-to-speech request.
type SpeechRequest struct {
	// Input is the text to speak.
	Input string

	// Voice selects the voice; "" = the model's default voice.
	Voice string

	// Format is the audio container ("mp3", "wav", "opus", ...);
	// "" = backend default.
	Format string

	// Speed is the playback-rate multiplier; 0 = backend default (1.0).
	Speed float64

	// ReferenceAudio is a short voice sample for zero-shot voice cloning
	// (chatterbox style): when set, the model speaks Input in the sampled
	// voice instead of a named Voice. nil = normal synthesis. Backends
	// without cloning support must reject a reference-carrying request
	// rather than silently ignoring it.
	ReferenceAudio []byte

	// ReferenceMIME is the reference audio's MIME type (e.g. "audio/wav");
	// "" = let the backend sniff it.
	ReferenceMIME string
}

// SpeechResult is the canonical synthesis result: raw audio bytes plus the
// MIME type reported (or implied) by the backend.
type SpeechResult struct {
	// Audio is the encoded audio.
	Audio []byte

	// MIME is the audio MIME type, e.g. "audio/mpeg".
	MIME string

	// Raw is the provider-native response object, an escape hatch for
	// provider-specific fields. May be nil; never required for normal use.
	Raw any
}

// SpeechOption mutates a SpeechRequest before it is sent. Options passed to
// Speak are applied to a copy, so a request value can be reused.
type SpeechOption func(*SpeechRequest)

// WithVoice selects the voice.
func WithVoice(v string) SpeechOption { return func(r *SpeechRequest) { r.Voice = v } }

// WithFormat sets the audio container format (e.g. "mp3", "wav").
func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Format = f } }

// WithSpeed sets the playback-rate multiplier.
func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } }

// WithReferenceAudio provides a voice sample for zero-shot voice cloning.
func WithReferenceAudio(data []byte, mime string) SpeechOption {
	return func(r *SpeechRequest) { r.ReferenceAudio, r.ReferenceMIME = data, mime }
}

// Apply returns a copy of the request with all options applied. Providers
// call this once at the top of Speak.
func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest {
	for _, opt := range opts {
		opt(&r)
	}
	return r
}

// SpeechModel synthesizes speech from text.
type SpeechModel interface {
	// Speak renders the request's input text as audio.
	Speak(ctx context.Context, req SpeechRequest, opts ...SpeechOption) (*SpeechResult, error)
}

// SpeechModelOption configures a SpeechModel at construction time. Reserved
// for future per-model settings; present so the interface is
// forward-compatible (mirrors imagegen.ModelOption).
type SpeechModelOption func(*SpeechModelConfig)

// SpeechModelConfig carries per-model construction settings.
type SpeechModelConfig struct{}

// ApplySpeechModelOptions folds options into a config.
func ApplySpeechModelOptions(opts []SpeechModelOption) SpeechModelConfig {
	var cfg SpeechModelConfig
	for _, opt := range opts {
		opt(&cfg)
	}
	return cfg
}

// SpeechProvider mints speech models bound to one backend.
type SpeechProvider interface {
	// Name is the registry identifier for the provider.
	Name() string

	// SpeechModel returns a SpeechModel bound to the given id (passed through
	// to the backend verbatim; no catalog validation).
	SpeechModel(id string, opts ...SpeechModelOption) (SpeechModel, error)
}

// TranscriptionRequest is a speech-to-text request. Audio is carried as bytes
// (never a URL), mirroring llm.ImagePart's bytes-only contract.
type TranscriptionRequest struct {
	// Audio is the encoded audio to transcribe.
	Audio []byte

	// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
	// sniff it.
	MIME string

	// Filename is the multipart filename hint some backends key their format
	// detection on; "" derives one from MIME ("audio.mp3") or falls back to
	// "audio".
	Filename string

	// Language is a BCP-47/ISO-639 hint (e.g. "en"); "" = auto-detect.
	Language string

	// Prompt is optional context or vocabulary to bias decoding; "" = none.
	Prompt string

	// Translate requests an English translation of the speech instead of a
	// same-language transcript (whisper's translate task). When set and no
	// Language is given, providers must force source-language auto-detection
	// — a backend whose default language is "en" would otherwise skip
	// translation entirely.
	Translate bool
}

// TranscriptionResult is the canonical transcription result.
type TranscriptionResult struct {
	// Text is the transcript.
	Text string

	// Raw is the provider-native response object. May be nil.
	Raw any
}

// TranscriptionOption mutates a TranscriptionRequest before it is sent.
type TranscriptionOption func(*TranscriptionRequest)

// WithLanguage sets the language hint (e.g. "en").
func WithLanguage(l string) TranscriptionOption {
	return func(r *TranscriptionRequest) { r.Language = l }
}

// WithPrompt sets the decoding context/vocabulary hint.
func WithPrompt(p string) TranscriptionOption {
	return func(r *TranscriptionRequest) { r.Prompt = p }
}

// WithTranslate requests an English translation instead of a transcript.
func WithTranslate() TranscriptionOption {
	return func(r *TranscriptionRequest) { r.Translate = true }
}

// Apply returns a copy of the request with all options applied.
func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest {
	for _, opt := range opts {
		opt(&r)
	}
	return r
}

// TranscriptionModel transcribes audio to text.
type TranscriptionModel interface {
	// Transcribe converts the request's audio into text.
	Transcribe(ctx context.Context, req TranscriptionRequest, opts ...TranscriptionOption) (*TranscriptionResult, error)
}

// TranscriptionModelOption configures a TranscriptionModel at construction
// time. Reserved for future per-model settings.
type TranscriptionModelOption func(*TranscriptionModelConfig)

// TranscriptionModelConfig carries per-model construction settings.
type TranscriptionModelConfig struct{}

// ApplyTranscriptionModelOptions folds options into a config.
func ApplyTranscriptionModelOptions(opts []TranscriptionModelOption) TranscriptionModelConfig {
	var cfg TranscriptionModelConfig
	for _, opt := range opts {
		opt(&cfg)
	}
	return cfg
}

// TranscriptionProvider mints transcription models bound to one backend.
type TranscriptionProvider interface {
	// Name is the registry identifier for the provider.
	Name() string

	// TranscriptionModel returns a TranscriptionModel bound to the given id
	// (passed through to the backend verbatim; no catalog validation).
	TranscriptionModel(id string, opts ...TranscriptionModelOption) (TranscriptionModel, error)
}