feat: audio surfaces (TTS + transcription), imagegen.Editor, llamaswap health probe
- New leaf package `audio` (ADR-0017): SpeechModel/SpeechProvider and TranscriptionModel/TranscriptionProvider with imagegen conventions (zero value = backend default, functional options + Apply, bytes in/out, never URLs). Root re-exports added. - imagegen.Editor (ADR-0018): optional image-to-image interface — EditRequest carries the generation knobs plus Init image and denoising Strength; separate interface so existing Models keep compiling. - provider/llamaswap implements all of it: POST /v1/audio/speech (JSON, raw-audio response, MIME from Content-Type with format fallback), POST /v1/audio/transcriptions (multipart, response_format=json), ListVoices (GET /v1/audio/voices?model=, tolerant of string-list and object-list shapes), POST /sdapi/v1/img2img (txt2img wire + init_images/denoising_strength, shared image decode), and Health(ctx) (GET /health) — a cheap liveness probe for often-offline hosts. - Hermetic httptest coverage for every new wire shape and validation path; README sections + support-matrix footnote updated in the same commit (also corrects the stale /v1/images/generations claim — the image path has been SDAPI since the seed fix). First consumer: mort's llamaswap media tool cluster (status / image / TTS / STT agent tools against the netherstorm host). Co-Authored-By: Claude Fable 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01AXQxVhXBw8PwFAtsVrXSmj
This commit is contained in:
@@ -85,8 +85,13 @@ func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts ..
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/txt2img", m.id, &wire, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeImages(m.p.name, m.id, &resp)
|
||||
}
|
||||
|
||||
out := &imagegen.Result{Raw: &resp}
|
||||
// decodeImages converts an SDAPI response's base64 images into an
|
||||
// imagegen.Result, erroring when nothing decodable came back.
|
||||
func decodeImages(provider, model string, resp *txt2imgResponse) (*imagegen.Result, error) {
|
||||
out := &imagegen.Result{Raw: resp}
|
||||
for i, b64 := range resp.Images {
|
||||
if b64 == "" {
|
||||
continue
|
||||
@@ -99,14 +104,70 @@ func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts ..
|
||||
}
|
||||
if len(out.Images) == 0 {
|
||||
return nil, &llm.APIError{
|
||||
Provider: m.p.name,
|
||||
Model: m.id,
|
||||
Provider: provider,
|
||||
Model: model,
|
||||
Message: "image response contained no images",
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// img2imgRequest is the stable-diffusion.cpp sd-server A1111 request shape
|
||||
// (POST /sdapi/v1/img2img): txt2img's fields plus the init image(s) and
|
||||
// denoising strength. Same endpoint-family choice as txt2img — the OpenAI
|
||||
// /v1/images/edits route is multipart and drops `seed` on this sd-server
|
||||
// build, while the SDAPI shape reuses doJSON and keeps seed parity.
|
||||
type img2imgRequest struct {
|
||||
txt2imgRequest
|
||||
InitImages []string `json:"init_images"`
|
||||
DenoisingStrength *float64 `json:"denoising_strength,omitempty"`
|
||||
}
|
||||
|
||||
// Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img.
|
||||
func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ...imagegen.EditOption) (*imagegen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Prompt) == "" {
|
||||
return nil, fmt.Errorf("%w: image edit requires a prompt", llm.ErrUnsupported)
|
||||
}
|
||||
if len(req.Init.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: image edit requires an init image", llm.ErrUnsupported)
|
||||
}
|
||||
if req.N < 0 {
|
||||
return nil, fmt.Errorf("%w: image count N must be >= 0, got %d", llm.ErrUnsupported, req.N)
|
||||
}
|
||||
if req.Strength != nil && (*req.Strength < 0 || *req.Strength > 1) {
|
||||
return nil, fmt.Errorf("%w: edit strength must be in [0,1], got %g", llm.ErrUnsupported, *req.Strength)
|
||||
}
|
||||
|
||||
width, height, err := parseSize(req.Size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
|
||||
}
|
||||
|
||||
wire := img2imgRequest{
|
||||
txt2imgRequest: txt2imgRequest{
|
||||
Model: m.id,
|
||||
Prompt: req.Prompt,
|
||||
NegativePrompt: req.NegativePrompt,
|
||||
Seed: req.Seed,
|
||||
Steps: req.Steps,
|
||||
CFGScale: req.CFGScale,
|
||||
Width: width,
|
||||
Height: height,
|
||||
SampleMethod: req.Sampler,
|
||||
BatchCount: req.N,
|
||||
},
|
||||
InitImages: []string{base64.StdEncoding.EncodeToString(req.Init.Data)},
|
||||
DenoisingStrength: req.Strength,
|
||||
}
|
||||
|
||||
var resp txt2imgResponse
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/img2img", m.id, &wire, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeImages(m.p.name, m.id, &resp)
|
||||
}
|
||||
|
||||
// parseSize splits a "WxH" string into width/height pointers. "" yields
|
||||
// (nil, nil) so the model's own default resolution applies.
|
||||
func parseSize(size string) (*int, *int, error) {
|
||||
|
||||
Reference in New Issue
Block a user