majordomo/provider/llamaswap/audio.go

package llamaswap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"net/http"
	"net/url"
	"strings"

	majaudio "gitea.stevedudenhoeffer.com/steve/majordomo/audio"
	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)

// SpeechModel implements audio.SpeechProvider, binding a text-to-speech model
// served by llama-swap (routed to a kokoro/chatterbox-style OpenAI-compatible
// upstream). The id is passed through verbatim and selects which upstream
// llama-swap loads.
func (p *Provider) SpeechModel(id string, opts ...majaudio.SpeechModelOption) (majaudio.SpeechModel, error) {
	if p.baseURL == "" {
		return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name)
	}
	_ = majaudio.ApplySpeechModelOptions(opts)
	return &speechModel{p: p, id: id}, nil
}

type speechModel struct {
	p  *Provider
	id string
}

// speechRequest is the OpenAI /v1/audio/speech request shape. llama-swap
// routes by the `model` field in the body.
type speechRequest struct {
	Model          string  `json:"model"`
	Input          string  `json:"input"`
	Voice          string  `json:"voice,omitempty"`
	ResponseFormat string  `json:"response_format,omitempty"`
	Speed          float64 `json:"speed,omitempty"`
}

// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech.
func (m *speechModel) Speak(ctx context.Context, req majaudio.SpeechRequest, opts ...majaudio.SpeechOption) (*majaudio.SpeechResult, error) {
	req = req.Apply(opts...)
	if strings.TrimSpace(req.Input) == "" {
		return nil, fmt.Errorf("%w: speech synthesis requires input text", llm.ErrUnsupported)
	}
	wire := speechRequest{
		Model:          m.id,
		Input:          req.Input,
		Voice:          req.Voice,
		ResponseFormat: req.Format,
		Speed:          req.Speed,
	}
	body, err := json.Marshal(wire)
	if err != nil {
		return nil, fmt.Errorf("llama-swap: encode speech request: %w", err)
	}
	audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/audio/speech", m.id, "application/json", bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	if len(audioBytes) == 0 {
		return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech response contained no audio"}
	}
	return &majaudio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil
}

// speechMIME resolves the result MIME type: the response Content-Type when it
// is a concrete audio type, else a mapping from the requested format, else
// audio/mpeg (the OpenAI endpoint's default container is mp3).
func speechMIME(contentType, format string) string {
	if mt, _, err := mime.ParseMediaType(contentType); err == nil {
		if strings.HasPrefix(mt, "audio/") || strings.HasPrefix(mt, "video/") {
			return mt
		}
	}
	switch strings.ToLower(strings.TrimSpace(format)) {
	case "", "mp3":
		return "audio/mpeg"
	case "wav":
		return "audio/wav"
	case "opus":
		return "audio/ogg"
	case "aac":
		return "audio/aac"
	case "flac":
		return "audio/flac"
	case "pcm":
		return "audio/pcm"
	default:
		return "audio/mpeg"
	}
}

// TranscriptionModel implements audio.TranscriptionProvider, binding a
// speech-to-text model served by llama-swap (routed to a whisper.cpp-style
// upstream).
func (p *Provider) TranscriptionModel(id string, opts ...majaudio.TranscriptionModelOption) (majaudio.TranscriptionModel, error) {
	if p.baseURL == "" {
		return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name)
	}
	_ = majaudio.ApplyTranscriptionModelOptions(opts)
	return &transcriptionModel{p: p, id: id}, nil
}

type transcriptionModel struct {
	p  *Provider
	id string
}

// Transcribe implements audio.TranscriptionModel via POST
// {base}/v1/audio/transcriptions (multipart/form-data — llama-swap routes by
// the `model` form field).
func (m *transcriptionModel) Transcribe(ctx context.Context, req majaudio.TranscriptionRequest, opts ...majaudio.TranscriptionOption) (*majaudio.TranscriptionResult, error) {
	req = req.Apply(opts...)
	if len(req.Audio) == 0 {
		return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported)
	}

	var buf bytes.Buffer
	w := multipart.NewWriter(&buf)
	fw, err := w.CreateFormFile("file", transcriptionFilename(req))
	if err != nil {
		return nil, fmt.Errorf("llama-swap: build transcription form: %w", err)
	}
	if _, err := fw.Write(req.Audio); err != nil {
		return nil, fmt.Errorf("llama-swap: build transcription form: %w", err)
	}
	fields := map[string]string{
		"model":           m.id,
		"language":        req.Language,
		"prompt":          req.Prompt,
		"response_format": "json",
	}
	for k, v := range fields {
		if v == "" {
			continue
		}
		if err := w.WriteField(k, v); err != nil {
			return nil, fmt.Errorf("llama-swap: build transcription form: %w", err)
		}
	}
	if err := w.Close(); err != nil {
		return nil, fmt.Errorf("llama-swap: build transcription form: %w", err)
	}

	raw, _, err := m.p.doRaw(ctx, http.MethodPost, "/v1/audio/transcriptions", m.id, w.FormDataContentType(), &buf)
	if err != nil {
		return nil, err
	}
	var out struct {
		Text string `json:"text"`
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		return nil, fmt.Errorf("llama-swap: decode transcription response: %w", err)
	}
	return &majaudio.TranscriptionResult{Text: out.Text, Raw: json.RawMessage(raw)}, nil
}

// transcriptionFilename picks the multipart filename hint: the caller's, else
// one derived from the MIME subtype ("audio.mp3"), else "audio".
func transcriptionFilename(req majaudio.TranscriptionRequest) string {
	if req.Filename != "" {
		return req.Filename
	}
	switch strings.ToLower(req.MIME) {
	case "audio/mpeg", "audio/mp3":
		return "audio.mp3"
	case "audio/wav", "audio/x-wav", "audio/wave":
		return "audio.wav"
	case "audio/ogg", "audio/opus":
		return "audio.ogg"
	case "audio/flac", "audio/x-flac":
		return "audio.flac"
	case "audio/mp4", "audio/m4a", "audio/x-m4a":
		return "audio.m4a"
	case "audio/webm", "video/webm":
		return "audio.webm"
	default:
		return "audio"
	}
}

// ListVoices returns the voices a TTS model offers (GET
// /v1/audio/voices?model=...). The decode is tolerant: upstreams answer with
// either a bare string list or a list of {id|name} objects.
func (p *Provider) ListVoices(ctx context.Context, model string) ([]string, error) {
	if model == "" {
		return nil, fmt.Errorf("llama-swap: ListVoices requires a model id")
	}
	raw, _, err := p.doRaw(ctx, http.MethodGet, "/v1/audio/voices?model="+url.QueryEscape(model), model, "", nil)
	if err != nil {
		return nil, err
	}
	return parseVoices(raw)
}

// parseVoices extracts voice names from the various shapes upstreams use:
// {"voices":[...]} or {"data":[...]} envelopes (or a bare array), holding
// either strings or objects keyed by id/name/voice_id.
func parseVoices(raw []byte) ([]string, error) {
	var env struct {
		Voices json.RawMessage `json:"voices"`
		Data   json.RawMessage `json:"data"`
	}
	list := json.RawMessage(raw)
	if err := json.Unmarshal(raw, &env); err == nil {
		if len(env.Voices) > 0 {
			list = env.Voices
		} else if len(env.Data) > 0 {
			list = env.Data
		}
	}

	var names []string
	if err := json.Unmarshal(list, &names); err == nil {
		return names, nil
	}
	// A failed decode above may have partially populated names — start fresh.
	names = nil
	var objs []struct {
		ID      string `json:"id"`
		Name    string `json:"name"`
		VoiceID string `json:"voice_id"`
	}
	if err := json.Unmarshal(list, &objs); err == nil {
		for _, o := range objs {
			switch {
			case o.ID != "":
				names = append(names, o.ID)
			case o.Name != "":
				names = append(names, o.Name)
			case o.VoiceID != "":
				names = append(names, o.VoiceID)
			}
		}
		return names, nil
	}
	return nil, fmt.Errorf("llama-swap: unrecognized voices payload shape")
}

// doRaw performs a request to a llama-swap endpoint and returns the raw
// response body and its Content-Type — the sibling of doJSON for endpoints
// whose success payload is not JSON (audio bytes) or whose shape varies.
// contentType sets the request Content-Type when body is non-nil.
func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader) ([]byte, string, error) {
	if p.baseURL == "" {
		return nil, "", fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name)
	}
	req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, body)
	if err != nil {
		return nil, "", fmt.Errorf("llama-swap: build request: %w", err)
	}
	if body != nil && contentType != "" {
		req.Header.Set("Content-Type", contentType)
	}
	if p.token != "" {
		req.Header.Set("Authorization", "Bearer "+p.token)
	}
	resp, err := p.client.Do(req)
	if err != nil {
		return nil, "", fmt.Errorf("llama-swap: do request: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode/100 != 2 {
		return nil, "", p.apiError(resp, model)
	}
	data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes))
	if err != nil {
		return nil, "", fmt.Errorf("llama-swap: read response: %w", err)
	}
	return data, resp.Header.Get("Content-Type"), nil
}