majordomo/provider/llamaswap/audio.go

package llamaswap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)

// SpeechModel implements audio.SpeechProvider, binding a text-to-speech model
// served by llama-swap (routed to a kokoro/chatterbox-style OpenAI-compatible
// upstream). The id is passed through verbatim and selects which upstream
// llama-swap loads.
func (p *Provider) SpeechModel(id string, opts ...audio.SpeechModelOption) (audio.SpeechModel, error) {
	if err := p.requireBaseURL(); err != nil {
		return nil, err
	}
	_ = audio.ApplySpeechModelOptions(opts)
	return &speechModel{p: p, id: id}, nil
}

type speechModel struct {
	p  *Provider
	id string
}

// speechRequest is the OpenAI /v1/audio/speech request shape. llama-swap
// routes by the `model` field in the body.
type speechRequest struct {
	Model          string  `json:"model"`
	Input          string  `json:"input"`
	Voice          string  `json:"voice,omitempty"`
	ResponseFormat string  `json:"response_format,omitempty"`
	Speed          float64 `json:"speed,omitempty"`
}

// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech, or —
// when the request carries reference audio for voice cloning — via multipart
// POST {base}/upstream/<id>/v1/audio/speech/upload (the chatterbox clone
// route; verified live 2026-07-16: stateless per-request cloning, fields
// `input` + `voice_file`).
func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) {
	req = req.Apply(opts...)
	if strings.TrimSpace(req.Input) == "" {
		return nil, fmt.Errorf("%w: speech synthesis requires input text", llm.ErrUnsupported)
	}
	if req.Speed < 0 {
		return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed)
	}
	if len(req.ReferenceAudio) > 0 {
		return m.speakWithReference(ctx, req)
	}
	wire := speechRequest{
		Model:          m.id,
		Input:          req.Input,
		Voice:          req.Voice,
		ResponseFormat: req.Format,
		Speed:          req.Speed,
	}
	body, err := json.Marshal(wire)
	if err != nil {
		return nil, fmt.Errorf("llama-swap: encode speech request: %w", err)
	}
	audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/audio/speech", m.id, "application/json", bytes.NewReader(body), maxResponseBytes)
	if err != nil {
		return nil, err
	}
	if len(audioBytes) == 0 {
		return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech response contained no audio"}
	}
	return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil
}

// speakWithReference performs zero-shot voice cloning via the upstream
// passthrough clone route. The reference sample rides as the `voice_file`
// part and the text as an `input` field; voice/format/speed still go on the
// wire when set (upstreams ignore fields they don't understand). The result
// MIME comes from the response header or content sniffing (audioResultMIME,
// same validation as the sfx/enhance surfaces) — a JSON soft error or an
// HTML proxy page must never be wrapped up as audio bytes.
func (m *speechModel) speakWithReference(ctx context.Context, req audio.SpeechRequest) (*audio.SpeechResult, error) {
	upPath, err := upstreamPath(m.id, "/v1/audio/speech/upload")
	if err != nil {
		return nil, err
	}
	speed := ""
	if req.Speed != 0 {
		speed = strconv.FormatFloat(req.Speed, 'g', -1, 64)
	}
	body, formType, err := buildMultipart("build speech clone form",
		filePart{field: "voice_file", filename: transcriptionFilename("", req.ReferenceMIME), data: req.ReferenceAudio},
		[]formField{
			{"input", req.Input, true},
			{"voice", req.Voice, false},
			{"response_format", req.Format, false},
			{"speed", speed, false},
		})
	if err != nil {
		return nil, err
	}
	audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, upPath, m.id, formType, body, maxAudioResponseBytes)
	if err != nil {
		return nil, err
	}
	if len(audioBytes) == 0 {
		return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech clone response contained no audio"}
	}
	mimeType := audioResultMIME(contentType, audioBytes)
	if mimeType == "" {
		return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
			Message: fmt.Sprintf("speech clone response is not audio (Content-Type %q): %s", contentType, truncateForError(audioBytes))}
	}
	return &audio.SpeechResult{Audio: audioBytes, MIME: mimeType}, nil
}

// speechMIME resolves the result MIME type: the response Content-Type when it
// is a concrete audio type, else a mapping from the requested format, else
// audio/mpeg (the OpenAI endpoint's default container is mp3).
func speechMIME(contentType, format string) string {
	if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
		return mt
	}
	switch strings.ToLower(strings.TrimSpace(format)) {
	case "wav":
		return "audio/wav"
	case "opus":
		return "audio/ogg"
	case "aac":
		return "audio/aac"
	case "flac":
		return "audio/flac"
	default: // "", "mp3", and anything unrecognized
		return "audio/mpeg"
	}
}

// audioResultMIME resolves an audio body's MIME type: the response
// Content-Type when it is a concrete audio type, else content sniffing
// (RIFF/WAVE, Ogg and friends), else "" — the caller treats undetectable as
// an upstream error, mirroring videoMIME. Sniffed "audio/wave" is normalized
// to the conventional "audio/wav", and Ogg's container type
// "application/ogg" to "audio/ogg". Shared by the clone, sfx, and enhance
// surfaces, which all answer raw audio bodies.
func audioResultMIME(contentType string, data []byte) string {
	if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
		return mt
	}
	switch mt := http.DetectContentType(data); {
	case mt == "audio/wave":
		return "audio/wav"
	case mt == "application/ogg":
		return "audio/ogg"
	case strings.HasPrefix(mt, "audio/"):
		return mt
	}
	return ""
}

// TranscriptionModel implements audio.TranscriptionProvider, binding a
// speech-to-text model served by llama-swap (routed to a whisper.cpp-style
// upstream).
func (p *Provider) TranscriptionModel(id string, opts ...audio.TranscriptionModelOption) (audio.TranscriptionModel, error) {
	if err := p.requireBaseURL(); err != nil {
		return nil, err
	}
	_ = audio.ApplyTranscriptionModelOptions(opts)
	return &transcriptionModel{p: p, id: id}, nil
}

type transcriptionModel struct {
	p  *Provider
	id string
}

// Transcribe implements audio.TranscriptionModel via POST
// {base}/v1/audio/transcriptions (multipart/form-data — llama-swap routes by
// the `model` form field).
func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.TranscriptionRequest, opts ...audio.TranscriptionOption) (*audio.TranscriptionResult, error) {
	req = req.Apply(opts...)
	if len(req.Audio) == 0 {
		return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported)
	}

	// Translation is a per-request bool form field on whisper.cpp's server
	// (server.cpp:534, verified 2026-07-16). Its language DEFAULT is "en",
	// which would make translate a no-op — so an explicit language hint
	// wins, but an unset one is forced to auto-detection.
	translate := ""
	language := req.Language
	if req.Translate {
		translate = "true"
		if language == "" {
			language = "auto"
		}
	}
	buf, formType, err := buildMultipart("build transcription form",
		filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
		[]formField{
			{"model", m.id, true},
			{"response_format", "json", true},
			{"language", language, false},
			{"prompt", req.Prompt, false},
			{"translate", translate, false},
		})
	if err != nil {
		return nil, err
	}

	raw, _, err := m.p.doRaw(ctx, http.MethodPost, "/v1/audio/transcriptions", m.id, formType, buf, maxResponseBytes)
	if err != nil {
		return nil, err
	}
	var out struct {
		Text string `json:"text"`
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		return nil, fmt.Errorf("llama-swap: decode transcription response: %w", err)
	}
	return &audio.TranscriptionResult{Text: out.Text, Raw: json.RawMessage(raw)}, nil
}

// transcriptionFilename picks the multipart filename hint: the caller's
// (sanitized — upload metadata is untrusted and CR/LF would inject multipart
// headers), else one derived from the MIME subtype ("audio.mp3"), else
// "audio". MIME parameters ("audio/ogg; codecs=opus") are stripped before
// matching.
func transcriptionFilename(filename, mimeType string) string {
	if name := sanitizeFilename(filename); name != "" {
		return name
	}
	mt := strings.ToLower(strings.TrimSpace(mimeType))
	if parsed, _, err := mime.ParseMediaType(mt); err == nil {
		mt = parsed
	}
	switch mt {
	case "audio/mpeg", "audio/mp3":
		return "audio.mp3"
	case "audio/wav", "audio/x-wav", "audio/wave":
		return "audio.wav"
	case "audio/ogg":
		return "audio.ogg"
	case "audio/opus":
		return "audio.opus"
	case "audio/flac", "audio/x-flac":
		return "audio.flac"
	case "audio/mp4", "audio/m4a", "audio/x-m4a":
		return "audio.m4a"
	case "audio/webm", "video/webm":
		return "audio.webm"
	default:
		return "audio"
	}
}

// sanitizeFilename strips characters that would corrupt or inject into the
// multipart Content-Disposition header, or smuggle directory structure to
// the receiving side. Quotes and backslashes are escaped by mime/multipart
// itself, but a file-writing shim decodes them right back — so CR/LF/NUL
// and both path separators are dropped: an upload-metadata filename must
// never traverse ("../x", "a/b", "C:\x").
func sanitizeFilename(name string) string {
	name = strings.NewReplacer("\r", "", "\n", "", "\x00", "", "/", "", "\\", "").Replace(name)
	return strings.TrimSpace(name)
}

// ListVoices returns the voices a TTS model offers (GET
// /v1/audio/voices?model=...). The decode is tolerant: upstreams answer with
// either a bare string list or a list of {id|name} objects.
func (p *Provider) ListVoices(ctx context.Context, model string) ([]string, error) {
	if model == "" {
		return nil, fmt.Errorf("llama-swap: ListVoices requires a model id")
	}
	raw, _, err := p.doRaw(ctx, http.MethodGet, "/v1/audio/voices?model="+url.QueryEscape(model), model, "", nil, maxResponseBytes)
	if err != nil {
		return nil, err
	}
	return parseVoices(raw)
}

// parseVoices extracts voice names from the various shapes upstreams use:
// {"voices":[...]} or {"data":[...]} envelopes (or a bare array), holding
// either strings or objects keyed by id/name/voice_id.
func parseVoices(raw []byte) ([]string, error) {
	var env struct {
		Voices json.RawMessage `json:"voices"`
		Data   json.RawMessage `json:"data"`
	}
	list := json.RawMessage(raw)
	if err := json.Unmarshal(raw, &env); err == nil {
		if len(env.Voices) > 0 {
			list = env.Voices
		} else if len(env.Data) > 0 {
			list = env.Data
		}
	}

	var names []string
	if err := json.Unmarshal(list, &names); err == nil {
		return names, nil
	}
	// Not a plain string list. The failed decode above may have partially
	// populated names (Unmarshal appends zero values before erroring on an
	// element type mismatch) — start fresh for the object shape.
	names = nil
	var objs []struct {
		ID      string `json:"id"`
		Name    string `json:"name"`
		VoiceID string `json:"voice_id"`
	}
	if err := json.Unmarshal(list, &objs); err == nil {
		for _, o := range objs {
			switch {
			case o.ID != "":
				names = append(names, o.ID)
			case o.Name != "":
				names = append(names, o.Name)
			case o.VoiceID != "":
				names = append(names, o.VoiceID)
			}
		}
		return names, nil
	}
	return nil, fmt.Errorf("llama-swap: unrecognized voices payload shape")
}

// doRaw performs a request to a llama-swap endpoint and returns the raw
// response body and its Content-Type — the sibling of doJSON for endpoints
// whose success payload is not JSON (audio/video bytes) or whose shape
// varies. contentType sets the request Content-Type when body is non-nil.
// A response larger than maxBytes is an error, never a silent truncation.
func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, string, error) {
	data, hdr, err := p.doRawHeaders(ctx, method, path, model, contentType, body, maxBytes)
	if err != nil {
		return nil, "", err
	}
	return data, hdr.Get("Content-Type"), nil
}

// doRawHeaders is doRaw with the WHOLE response header rather than just
// Content-Type. Only the face swap needs it — the shim reports whether the
// likeness actually transferred in X-Swap-Report, and that answer would be
// thrown away by a function that keeps one header — so doRaw stays the
// signature 25 other call sites use and delegates here. Two bodies would be
// two places for the size cap and the status check to drift apart.
func (p *Provider) doRawHeaders(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, http.Header, error) {
	if err := p.requireBaseURL(); err != nil {
		return nil, nil, err
	}
	req, err := p.newRequest(ctx, method, path, contentType, body)
	if err != nil {
		return nil, nil, err
	}
	resp, err := p.client.Do(req)
	if err != nil {
		return nil, nil, fmt.Errorf("llama-swap: do request: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode/100 != 2 {
		return nil, nil, p.apiError(resp, model)
	}
	data, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
	if err != nil {
		return nil, nil, fmt.Errorf("llama-swap: read response: %w", err)
	}
	if int64(len(data)) > maxBytes {
		return nil, nil, fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes)
	}
	return data, resp.Header, nil
}