Compare commits
24 Commits
b2487a1a37
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 900317af4e | |||
| 776ef6fda9 | |||
| 89210346b8 | |||
| 4629c6af0f | |||
| 5ed18abb94 | |||
| 9c0ac1d60b | |||
| 434d721b99 | |||
| 6abb399f5b | |||
| fe44a6da26 | |||
| a213c18263 | |||
| 88d3fc3279 | |||
| 531fe1922e | |||
| a744cdc335 | |||
| 8b924700fb | |||
| 70b7aebd86 | |||
| 52bb910f4d | |||
| d71aca4c3a | |||
| 51f5ea0d2b | |||
| a457e76ac7 | |||
| 78a1d1c3bb | |||
| aa25b2c334 | |||
| 2b35f1741c | |||
| 98a2164aba | |||
| de2b2f0f28 |
@@ -1,12 +1,8 @@
|
|||||||
# Gadfly — agentic adversarial PR reviewer (https://gitea.stevedudenhoeffer.com/steve/gadfly).
|
# Gadfly adversarial review — subscribes to steve/gadfly's reusable workflow and
|
||||||
#
|
# INHERITS its default swarm. This stub holds only the triggers, the actor gate,
|
||||||
# Runs the published Gadfly image (pinned to an immutable :sha- tag — act_runner
|
# secret forwarding, and the allow-list; the swarm config (models, lenses,
|
||||||
# caches :latest, and this build is what carries foreman provider-type support)
|
# concurrency, timeouts) lives centrally in gadfly's review-reusable.yml so it is
|
||||||
# as a specialist swarm and posts
|
# tuned in ONE place. Advisory only — never blocks a merge.
|
||||||
# ONE consolidated review comment as gitea-actions. Advisory only — never blocks a
|
|
||||||
# merge. This reviews majordomo PRs with 9 ollama-cloud models + the M5 Mac
|
|
||||||
# (3-lens suite). Gadfly is a simple system — findings are advisory; always
|
|
||||||
# double-check before acting.
|
|
||||||
|
|
||||||
name: Adversarial Review (Gadfly)
|
name: Adversarial Review (Gadfly)
|
||||||
|
|
||||||
@@ -33,60 +29,26 @@ concurrency:
|
|||||||
jobs:
|
jobs:
|
||||||
review:
|
review:
|
||||||
# Security: only trusted users may trigger a secret-bearing run via a PR
|
# Security: only trusted users may trigger a secret-bearing run via a PR
|
||||||
# comment (pull_request + workflow_dispatch are already trusted). Mirrors
|
# comment (pull_request + workflow_dispatch are already trusted). Mirrors the
|
||||||
# GADFLY_ALLOWED_USERS, the in-container belt-and-suspenders check.
|
# allowed_users input below (the in-container belt-and-suspenders check) — both
|
||||||
|
# lists must stay in sync; a workflow if: can't read a workflow_call input.
|
||||||
if: >-
|
if: >-
|
||||||
github.event_name != 'issue_comment'
|
github.event_name != 'issue_comment'
|
||||||
|| (github.event.issue.pull_request
|
|| (github.event.issue.pull_request
|
||||||
&& (github.actor == 'steve'
|
&& (github.actor == 'steve'
|
||||||
|| github.actor == 'fizi'
|
|| github.actor == 'fizi'
|
||||||
|| github.actor == 'dazed'))
|
|| github.actor == 'dazed'))
|
||||||
runs-on: ubuntu-latest
|
# Tracks gadfly's v1 release tag — a curated pointer re-moved on each release
|
||||||
# Fleet: 9 cloud (lens fan-out) + the M5 Mac via foreman. The slow local
|
# (unlike @main, which moves on every push). Central swarm tuning propagates
|
||||||
# lane dominates wall time, so allow plenty of headroom. (M1 was dropped —
|
# here automatically; the tradeoff vs a full sha pin is that v1 is mutable.
|
||||||
# consistently slow for zero real findings.)
|
uses: steve/gadfly/.gitea/workflows/review-reusable.yml@5007597cf921dc3f0a83c708878facfe65fd8e8b
|
||||||
timeout-minutes: 90
|
# Least privilege: forward only the review secrets (not `secrets: inherit`,
|
||||||
steps:
|
# which would expose every repo secret). GITEA_TOKEN is the automatic token.
|
||||||
- uses: docker://gitea.stevedudenhoeffer.com/steve/gadfly:sha-d7f364d
|
secrets:
|
||||||
env:
|
OLLAMA_CLOUD_API_KEY: ${{ secrets.OLLAMA_CLOUD_API_KEY }}
|
||||||
GITEA_API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
|
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
GADFLY_FINDINGS_URL: ${{ secrets.GADFLY_FINDINGS_URL }}
|
||||||
OLLAMA_CLOUD_API_KEY: ${{ secrets.OLLAMA_CLOUD_API_KEY }}
|
GADFLY_FINDINGS_TOKEN: ${{ secrets.GADFLY_FINDINGS_TOKEN }}
|
||||||
# Local Mac, reached through its foreman queue (native Ollama on the
|
with:
|
||||||
# wire). GADFLY_ENDPOINT_M5 registers provider "m5", a foreman-preset
|
# Consumer-specific allow-list; everything else is inherited.
|
||||||
# Ollama client at the secret's URL, of the form:
|
allowed_users: "steve,fizi,dazed"
|
||||||
# foreman|https://<foreman-host>|<token>
|
|
||||||
# Needs an image with foreman provider-type support (this one). If the Mac
|
|
||||||
# is offline that model's comment shows an error and the others still post.
|
|
||||||
# (Gitea secrets aren't auto-exposed — map each explicitly.)
|
|
||||||
GADFLY_ENDPOINT_M5: ${{ secrets.GADFLY_ENDPOINT_M5 }}
|
|
||||||
# Fleet: 9 cloud + M5 Max. Cloud concurrency lives in the LENSES: cloud
|
|
||||||
# models run a few at a time (ollama-cloud=3) with their 3 lenses
|
|
||||||
# concurrent (LENS ollama-cloud=3) so comments land sooner; the Mac runs
|
|
||||||
# one model, lenses serial (its foreman queue serializes anyway). Both
|
|
||||||
# provider lanes run parallel.
|
|
||||||
GADFLY_MODELS: "minimax-m3:cloud,glm-5.2:cloud,glm-5.1:cloud,kimi-k2.7-code:cloud,deepseek-v4-pro:cloud,nemotron-3-super:cloud,gpt-oss:120b-cloud,qwen3-coder:480b-cloud,gemma4:cloud,m5/qwen3.6:35b-mlx"
|
|
||||||
GADFLY_PROVIDER_CONCURRENCY: "ollama-cloud=3,m5=1"
|
|
||||||
GADFLY_PROVIDER_LENS_CONCURRENCY: "ollama-cloud=3"
|
|
||||||
# Default => the 3-lens suite (security, correctness, error-handling).
|
|
||||||
# Set the repo var GADFLY_SPECIALISTS to override (csv / "all" / "auto").
|
|
||||||
GADFLY_SPECIALISTS: ${{ vars.GADFLY_SPECIALISTS || 'security,correctness,error-handling' }}
|
|
||||||
# Per-lens deadline + bounded steps so the slow local models stay sane.
|
|
||||||
GADFLY_TIMEOUT_SECS: "600"
|
|
||||||
GADFLY_MAX_STEPS: "14"
|
|
||||||
# Allow-list for the comment trigger (mirrors the job-level if: guard).
|
|
||||||
GADFLY_ALLOWED_USERS: "steve,fizi,dazed"
|
|
||||||
# --- findings telemetry: POST runs + findings to the gadfly-reports store ---
|
|
||||||
# Advisory & off unless GADFLY_FINDINGS_URL is set; failures only log to
|
|
||||||
# stderr and never affect the review. GADFLY_REPO / GADFLY_PR are derived
|
|
||||||
# in-container; the URL + token are user-scope secrets.
|
|
||||||
GADFLY_FINDINGS_URL: ${{ secrets.GADFLY_FINDINGS_URL }}
|
|
||||||
GADFLY_FINDINGS_TOKEN: ${{ secrets.GADFLY_FINDINGS_TOKEN }}
|
|
||||||
# --- event context (leave as-is) ---
|
|
||||||
EVENT_NAME: ${{ github.event_name }}
|
|
||||||
PR: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}
|
|
||||||
PR_BRANCH: ${{ github.head_ref }}
|
|
||||||
IS_DRAFT: ${{ github.event.pull_request.draft }}
|
|
||||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
|
||||||
COMMENT_ID: ${{ github.event.comment.id }}
|
|
||||||
ACTOR: ${{ github.actor }}
|
|
||||||
|
|||||||
@@ -34,6 +34,11 @@ majordomo Registry, Parse, env-DSN loading, chain executor, re-exports
|
|||||||
Tool/Toolbox, Capabilities, Stream, Model, Provider, errors
|
Tool/Toolbox, Capabilities, Stream, Model, Provider, errors
|
||||||
imagegen/ canonical text-to-image contract: Request/Result/Model/
|
imagegen/ canonical text-to-image contract: Request/Result/Model/
|
||||||
Provider (separate from llm; Image = llm.ImagePart) (ADR-0016)
|
Provider (separate from llm; Image = llm.ImagePart) (ADR-0016)
|
||||||
|
audio/ canonical speech contracts: SpeechModel (TTS) +
|
||||||
|
TranscriptionModel (STT), same conventions (ADR-0017)
|
||||||
|
videogen/ canonical video-generation contract: one Model for
|
||||||
|
text-to-video + image-to-video (InitImage), single-clip
|
||||||
|
Result (Image = llm.ImagePart) (ADR-0019)
|
||||||
health/ clock-injected health tracker (bench/backoff)
|
health/ clock-injected health tracker (bench/backoff)
|
||||||
media/ image normalization to target capabilities (sniff real
|
media/ image normalization to target capabilities (sniff real
|
||||||
format, downscale, transcode, byte ladder; ErrUnsupported
|
format, downscale, transcode, byte ladder; ErrUnsupported
|
||||||
@@ -44,7 +49,8 @@ majordomo Registry, Parse, env-DSN loading, chain executor, re-exports
|
|||||||
provider/ollama/ one native /api/chat client serving the ollama,
|
provider/ollama/ one native /api/chat client serving the ollama,
|
||||||
ollama-cloud, and foreman built-ins via presets
|
ollama-cloud, and foreman built-ins via presets
|
||||||
provider/llamaswap/ llama-swap proxy: chat delegates to provider/openai,
|
provider/llamaswap/ llama-swap proxy: chat delegates to provider/openai,
|
||||||
plus management methods + imagegen image client (ADR-0015)
|
plus management methods + imagegen/audio/videogen
|
||||||
|
clients (ADR-0015..0019)
|
||||||
provider/google/ Gemini on google.golang.org/genai (the one approved
|
provider/google/ Gemini on google.golang.org/genai (the one approved
|
||||||
dependency; lazy client, raw-JSON-schema tools,
|
dependency; lazy client, raw-JSON-schema tools,
|
||||||
ThinkingLevel reasoning, iter.Pull2 streaming)
|
ThinkingLevel reasoning, iter.Pull2 streaming)
|
||||||
@@ -79,12 +85,13 @@ alias := bare token (no slash), expands INLINE, recursively, cycle-checked
|
|||||||
`LLM_<NAME>=scheme://[token@]host[/path]` — e.g.
|
`LLM_<NAME>=scheme://[token@]host[/path]` — e.g.
|
||||||
`LLM_M5=foreman://token@foreman-m5.example` defines provider `m5`; then
|
`LLM_M5=foreman://token@foreman-m5.example` defines provider `m5`; then
|
||||||
`m5/qwen3:30b` works in Parse, chains, and aliases. Scheme ∈ {foreman,
|
`m5/qwen3:30b` works in Parse, chains, and aliases. Scheme ∈ {foreman,
|
||||||
ollama, ollama-cloud, openai, anthropic, google, gemini, llama-swap} ∪
|
ollama, ollama-cloud, openai, anthropic, google, gemini, llama-swap,
|
||||||
RegisterScheme. Token = credential; base URL = `https://host` always —
|
llama-swaps} ∪ RegisterScheme. Token = credential; base URL = `https://host`
|
||||||
**except `llama-swap`, which builds `http://host` (local-first; ADR-0015).**
|
always — **except `llama-swap`, which builds `http://host` (local-first);
|
||||||
`New()` scans the process env eagerly; unknown names also resolve lazily at
|
`llama-swaps` is its TLS twin (`https://host`), mirroring redis/rediss
|
||||||
Parse time (`my-prov` → `LLM_MY_PROV`). Malformed entries fail on use, not at
|
(ADR-0015).** `New()` scans the process env eagerly; unknown names also resolve
|
||||||
startup.
|
lazily at Parse time (`my-prov` → `LLM_MY_PROV`). Malformed entries fail on use,
|
||||||
|
not at startup.
|
||||||
|
|
||||||
## Health & failover (ADR-0006, ADR-0008)
|
## Health & failover (ADR-0006, ADR-0008)
|
||||||
|
|
||||||
@@ -141,9 +148,11 @@ Ship work through PRs and let Gadfly review it before merge:
|
|||||||
|
|
||||||
- **Push to a PR, never straight to `main`.** Branch, push, open a PR.
|
- **Push to a PR, never straight to `main`.** Branch, push, open a PR.
|
||||||
`.gitea/workflows/adversarial-review.yml` runs Gadfly (the standalone
|
`.gitea/workflows/adversarial-review.yml` runs Gadfly (the standalone
|
||||||
agentic adversarial reviewer) — a fleet of 9 ollama-cloud models +
|
agentic adversarial reviewer) by subscribing to gadfly's reusable workflow
|
||||||
the M5 Mac via foreman, each running the 3-lens suite (security,
|
and inheriting its default swarm — 3 cloud models + the Claude Code engine
|
||||||
correctness, error-handling). Advisory only; it never blocks the merge.
|
(sonnet/opus/opus:max), each running the 5-lens suite (security, correctness,
|
||||||
|
maintainability, performance, error-handling). The swarm is tuned centrally
|
||||||
|
in gadfly, not here. Advisory only; it never blocks the merge.
|
||||||
- **Wait for Gadfly to finish, then read its output.** Don't merge while the
|
- **Wait for Gadfly to finish, then read its output.** Don't merge while the
|
||||||
review is still running. Each model posts one consolidated comment; weigh
|
review is still running. Each model posts one consolidated comment; weigh
|
||||||
every finding on its merits and fix the real ones (Gadfly is a simple
|
every finding on its merits and fix the real ones (Gadfly is a simple
|
||||||
|
|||||||
@@ -159,21 +159,25 @@ m, _ := reg.Parse("m5/qwen3:30b,m1/qwen3:30b,thinking")
|
|||||||
```
|
```
|
||||||
|
|
||||||
DSN format: `scheme://[token@]host[/path]`, scheme ∈ `foreman`, `ollama`,
|
DSN format: `scheme://[token@]host[/path]`, scheme ∈ `foreman`, `ollama`,
|
||||||
`ollama-cloud`, `openai`, `anthropic`, `google`/`gemini`, `llama-swap`, or any
|
`ollama-cloud`, `openai`, `anthropic`, `google`/`gemini`, `llama-swap`,
|
||||||
scheme you add with `RegisterScheme`. The token is the credential (bearer token
|
`llama-swaps`, or any scheme you add with `RegisterScheme`. The token is the
|
||||||
/ API key); the base URL is always `https://host[/path]` — except `llama-swap`,
|
credential (bearer token / API key); the base URL is always `https://host[/path]`
|
||||||
which builds `http://host[:port]` since it's local-first. `New()` loads `LLM_*`
|
— except `llama-swap`, which builds `http://host[:port]` since it's local-first
|
||||||
vars eagerly; unknown provider names also resolve lazily at Parse time
|
(`llama-swaps` is the TLS twin → `https://host`, mirroring redis/rediss). `New()`
|
||||||
(`my-prov/x` → `LLM_MY_PROV`).
|
loads `LLM_*` vars eagerly; unknown provider names also resolve lazily at Parse
|
||||||
|
time (`my-prov/x` → `LLM_MY_PROV`).
|
||||||
|
|
||||||
```
|
```
|
||||||
LLM_LS=llama-swap://token@box.local:8080 # then "ls/qwen3:14b" parses
|
LLM_LS=llama-swap://token@box.local:8080 # http → "ls/qwen3:14b" parses
|
||||||
|
LLM_LS=llama-swaps://token@swap.example.com # https → TLS-fronted instance
|
||||||
```
|
```
|
||||||
|
|
||||||
[llama-swap](https://github.com/mostlygeek/llama-swap) is a model-swapping proxy
|
[llama-swap](https://github.com/mostlygeek/llama-swap) is a model-swapping proxy
|
||||||
over llama.cpp. Its chat API is OpenAI-compatible (majordomo reuses the openai
|
over llama.cpp. Its chat API is OpenAI-compatible (majordomo reuses the openai
|
||||||
client), and the `*llamaswap.Provider` adds management methods
|
client), and the `*llamaswap.Provider` adds management methods
|
||||||
(`ListModels`/`Running`/`Unload`) plus image generation (see below). A cold
|
(`ListModels`/`Running`/`Unload`/`ListVoices`), a cheap liveness probe
|
||||||
|
(`Health`, GET /health — the proxy answers without touching a model), image
|
||||||
|
generation and editing, and speech synthesis/transcription (see below). A cold
|
||||||
model swap can take many seconds — bound calls with a context deadline, not a
|
model swap can take many seconds — bound calls with a context deadline, not a
|
||||||
client timeout.
|
client timeout.
|
||||||
|
|
||||||
@@ -209,8 +213,9 @@ resp, err := m.Generate(ctx, majordomo.Request{
|
|||||||
Text-to-image is a separate contract (`imagegen`) from chat, because it shares
|
Text-to-image is a separate contract (`imagegen`) from chat, because it shares
|
||||||
none of the message/tool/stream machinery. Generated images come back as
|
none of the message/tool/stream machinery. Generated images come back as
|
||||||
`llm.ImagePart`, so they drop straight back into a chat turn. The first backend
|
`llm.ImagePart`, so they drop straight back into a chat turn. The first backend
|
||||||
is llama-swap (OpenAI `/v1/images/generations` → a stable-diffusion.cpp
|
is llama-swap (the A1111-style `/sdapi/v1/txt2img` on a stable-diffusion.cpp
|
||||||
upstream).
|
upstream — chosen over OpenAI `/v1/images/generations` because that route
|
||||||
|
ignores `seed` there).
|
||||||
|
|
||||||
```go
|
```go
|
||||||
ls := llamaswap.New(llamaswap.WithBaseURL("http://box.local:8080"))
|
ls := llamaswap.New(llamaswap.WithBaseURL("http://box.local:8080"))
|
||||||
@@ -222,9 +227,63 @@ res, err := im.Generate(ctx, imagegen.Request{Prompt: "a red bicycle"},
|
|||||||
// majordomo.UserParts(majordomo.Text("describe this"), res.Images[0])
|
// majordomo.UserParts(majordomo.Text("describe this"), res.Images[0])
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Image-to-image editing is the optional `imagegen.Editor` interface (ADR-0018)
|
||||||
|
— llama-swap's image models implement it via `/sdapi/v1/img2img`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
ed := im.(imagegen.Editor)
|
||||||
|
res, err := ed.Edit(ctx, imagegen.EditRequest{
|
||||||
|
Prompt: "make it night",
|
||||||
|
Init: res.Images[0], // any llm.ImagePart
|
||||||
|
}, imagegen.WithEditStrength(0.6)) // 0..1: how far to depart from Init
|
||||||
|
```
|
||||||
|
|
||||||
`*llamaswap.Provider` also exposes management methods: `ListModels` (what
|
`*llamaswap.Provider` also exposes management methods: `ListModels` (what
|
||||||
llama-swap can serve), `Running` (what's loaded), and `Unload` (free a model).
|
llama-swap can serve), `Running` (what's loaded), and `Unload` (free a model).
|
||||||
|
|
||||||
|
## Speech: synthesis + transcription
|
||||||
|
|
||||||
|
Text-to-speech and speech-to-text live in the `audio` package (ADR-0017),
|
||||||
|
mirroring imagegen: small `SpeechModel`/`TranscriptionModel` contracts,
|
||||||
|
zero values mean backend defaults, bytes in/out (never URLs). First backend:
|
||||||
|
llama-swap (OpenAI `/v1/audio/speech` + `/v1/audio/transcriptions` routed to
|
||||||
|
kokoro/whisper.cpp-style upstreams).
|
||||||
|
|
||||||
|
```go
|
||||||
|
ls := llamaswap.New(llamaswap.WithBaseURL("http://box.local:8080"))
|
||||||
|
|
||||||
|
sm, _ := ls.SpeechModel("kokoro")
|
||||||
|
speech, err := sm.Speak(ctx, audio.SpeechRequest{Input: "hello world"},
|
||||||
|
audio.WithVoice("af_heart"), audio.WithFormat("mp3"))
|
||||||
|
// speech.Audio ([]byte) + speech.MIME ("audio/mpeg")
|
||||||
|
|
||||||
|
tm, _ := ls.TranscriptionModel("whisper-large-v3-turbo")
|
||||||
|
tr, err := tm.Transcribe(ctx, audio.TranscriptionRequest{
|
||||||
|
Audio: speech.Audio, MIME: speech.MIME,
|
||||||
|
})
|
||||||
|
// tr.Text
|
||||||
|
|
||||||
|
voices, err := ls.ListVoices(ctx, "kokoro") // []string of voice ids
|
||||||
|
```
|
||||||
|
|
||||||
|
## Video: text-to-video + image-to-video
|
||||||
|
|
||||||
|
Video generation lives in the `videogen` package (ADR-0019), mirroring
|
||||||
|
imagegen/audio: one small `Model` contract, zero values mean backend
|
||||||
|
defaults, bytes in/out. Text-to-video and image-to-video are one surface —
|
||||||
|
a nil `InitImage` is a pure text prompt; setting it conditions generation
|
||||||
|
on that frame (hybrid checkpoints like Wan 2.2 TI2V serve both). First
|
||||||
|
backend: llama-swap (blocking `/v1/videos/sync`, vLLM-Omni style — the
|
||||||
|
response body is the encoded clip, so `Result` carries a single `Video`).
|
||||||
|
Generation runs for minutes; bound the call with a context deadline.
|
||||||
|
|
||||||
|
```go
|
||||||
|
vm, _ := ls.VideoModel("videogen-wan22-5b")
|
||||||
|
res, err := vm.Generate(ctx, videogen.Request{Prompt: "a cat surfing"},
|
||||||
|
videogen.WithSize("1280x704"), videogen.WithNumFrames(81))
|
||||||
|
// res.Video.Data ([]byte) + res.Video.MIME ("video/mp4")
|
||||||
|
```
|
||||||
|
|
||||||
## Tool calls
|
## Tool calls
|
||||||
|
|
||||||
```go
|
```go
|
||||||
@@ -356,8 +415,10 @@ response as a single delta plus final event.
|
|||||||
² llama-swap's chat is OpenAI-compatible and reuses the openai client, so these
|
² llama-swap's chat is OpenAI-compatible and reuses the openai client, so these
|
||||||
capabilities are present at the client level; whether a given call succeeds
|
capabilities are present at the client level; whether a given call succeeds
|
||||||
depends on the llama.cpp model llama-swap loads. llama-swap also provides
|
depends on the llama.cpp model llama-swap loads. llama-swap also provides
|
||||||
**image generation** (a separate `imagegen` axis, not shown above) and
|
**image generation + editing** (`imagegen`), **speech synthesis +
|
||||||
management methods on `*llamaswap.Provider`.
|
transcription** (`audio`), **video generation** (`videogen`) — separate
|
||||||
|
axes, not shown above — plus a `Health`
|
||||||
|
probe and management methods on `*llamaswap.Provider`.
|
||||||
|
|
||||||
Notes: Ollama has no native tool_choice — `"none"` drops the tools;
|
Notes: Ollama has no native tool_choice — `"none"` drops the tools;
|
||||||
`"required"`/named choices are best-effort ignored there. Ollama Cloud
|
`"required"`/named choices are best-effort ignored there. Ollama Cloud
|
||||||
|
|||||||
+116
-18
@@ -14,21 +14,55 @@ import (
|
|||||||
// their answer to the final, tool-free turn. But some models — notably several
|
// their answer to the final, tool-free turn. But some models — notably several
|
||||||
// open-weight ones — "front-load" their full answer into an earlier turn that
|
// open-weight ones — "front-load" their full answer into an earlier turn that
|
||||||
// ALSO calls a tool (e.g. answer text alongside a citation call), then close
|
// ALSO calls a tool (e.g. answer text alongside a citation call), then close
|
||||||
// with a trivial pointer such as "(Already answered above.)". Returning only
|
// with a degenerate terminal turn that is not itself the answer. Two shapes are
|
||||||
// the terminal text would discard the real answer, which is still present
|
// recovered from the transcript (zero extra model calls):
|
||||||
// earlier in the transcript. When the terminal text is weak (empty, or a short
|
//
|
||||||
// back-reference) fall back to the last substantive assistant content in msgs.
|
// - a trivial back-reference ("(Already answered above.)", "see above", …):
|
||||||
|
// the real answer sits earlier, so recover it and DISCARD the worthless
|
||||||
|
// closer.
|
||||||
|
// - a sources/citations-only addendum ("Sources: [x](…), [y](…)"): the model
|
||||||
|
// front-loaded the prose answer and closed with just its citations (the
|
||||||
|
// glm-5.2 "cite" pattern behind mort issue #1418). The citations are real,
|
||||||
|
// useful content — unlike a back-reference — so recover the prior answer and
|
||||||
|
// KEEP the citations, appended below it.
|
||||||
|
//
|
||||||
|
// A citations addendum is tested first and wins over the back-reference test (a
|
||||||
|
// short terminal can be both), so its links are never discarded. When the
|
||||||
|
// terminal text stands on its own it is returned unchanged; when it is
|
||||||
|
// degenerate but nothing better can be recovered, it is returned as-is (a bare
|
||||||
|
// sources list still beats nothing).
|
||||||
//
|
//
|
||||||
// msgs must already include the terminal assistant message as its last element
|
// msgs must already include the terminal assistant message as its last element
|
||||||
// (the loop appends it before calling this); terminal is that message's text.
|
// (the loop appends it before calling this); terminal is that message's text.
|
||||||
func finalOutput(msgs []llm.Message, terminal string) string {
|
func finalOutput(msgs []llm.Message, terminal string) string {
|
||||||
if !isWeakFinal(terminal) {
|
citations := isCitationsOnly(terminal)
|
||||||
|
if !citations && !isWeakFinal(terminal) {
|
||||||
return terminal
|
return terminal
|
||||||
}
|
}
|
||||||
if rec, ok := lastSubstantiveAssistantText(msgs, terminal); ok {
|
rec, ok := lastSubstantiveAssistantText(msgs, terminal, citations)
|
||||||
return rec
|
if !ok {
|
||||||
|
return terminal
|
||||||
}
|
}
|
||||||
return terminal
|
if citations {
|
||||||
|
// Preserve the citations addendum below the recovered answer, unless the
|
||||||
|
// recovered turn already carries it (guards against a duplicate sources
|
||||||
|
// block when the front-loaded turn included its own citations). The
|
||||||
|
// containment test ignores <url> angle-bracket wrappers so a turn that
|
||||||
|
// listed the same sources unwrapped still suppresses the duplicate.
|
||||||
|
if tail := strings.TrimSpace(terminal); !strings.Contains(stripURLAngles(rec), stripURLAngles(tail)) {
|
||||||
|
return rec + "\n\n" + tail
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
// stripURLAngles removes the <…> wrappers Discord uses to suppress link embeds,
|
||||||
|
// so the citations dedup compares URLs regardless of that formatting delta.
|
||||||
|
func stripURLAngles(s string) string {
|
||||||
|
if !strings.ContainsAny(s, "<>") {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return strings.NewReplacer("<", "", ">", "").Replace(s)
|
||||||
}
|
}
|
||||||
|
|
||||||
// backRefRe matches a terminal turn that merely points back to an earlier
|
// backRefRe matches a terminal turn that merely points back to an earlier
|
||||||
@@ -40,19 +74,46 @@ var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(sai
|
|||||||
// check...") so a preamble is never mistaken for the answer during recovery.
|
// check...") so a preamble is never mistaken for the answer during recovery.
|
||||||
var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|sure[,. ]|okay[,. ]|on it|checking)`)
|
var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|sure[,. ]|okay[,. ]|on it|checking)`)
|
||||||
|
|
||||||
|
// citationLabelRe matches a terminal turn that OPENS with a sources/citations
|
||||||
|
// heading — the shape a model produces when it front-loads its answer into an
|
||||||
|
// earlier tool-call turn and closes with only its sources. Leading markdown
|
||||||
|
// emphasis (*, _), list (-, +, *), block-quote (>), and ATX-heading (#) markers
|
||||||
|
// — with their whitespace, since \s is in the class — are tolerated before the
|
||||||
|
// label, as is closing emphasis (** / __) plus whitespace between the label and
|
||||||
|
// the colon/dash separator. Anchored at ^ so a normal answer that merely
|
||||||
|
// mentions "sources" mid-sentence, or ends with a "Sources:" section AFTER its
|
||||||
|
// prose, is never matched.
|
||||||
|
var citationLabelRe = regexp.MustCompile(`(?i)^[\s>#*_+-]*(sources?|references?|citations?|works cited|further reading)\b[\s*_]*[::\-—]`)
|
||||||
|
|
||||||
|
// linkRe matches a whole markdown link "[label](url)" or a bare URL. Used both
|
||||||
|
// to require that a citations terminal carries at least one link and to strip
|
||||||
|
// links out when measuring how citation-dominated the terminal is.
|
||||||
|
var linkRe = regexp.MustCompile(`\[[^\]]*\]\([^)]*\)|https?://\S+`)
|
||||||
|
|
||||||
|
// citationResidueCutset is trimmed from the ends of a citations terminal's
|
||||||
|
// non-link remainder before measuring it — list bullets, separators, and the
|
||||||
|
// short per-source annotations models add in parentheses.
|
||||||
|
const citationResidueCutset = " \t\r\n,;.:|·•*_()[]—-"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// weakFinalMaxChars bounds how long a back-reference closer can be. A
|
// weakFinalMaxChars bounds how long a back-reference closer can be. A
|
||||||
// genuine final answer that merely contains "as I said" mid-sentence is
|
// genuine final answer that merely contains "as I said" mid-sentence is
|
||||||
// longer than this, so it is never treated as weak.
|
// longer than this, so it is never treated as weak.
|
||||||
weakFinalMaxChars = 120
|
weakFinalMaxChars = 120
|
||||||
// recoverMinChars: a prior assistant turn this long is treated as a real
|
// recoverMinChars: a prior assistant turn this long is treated as a real
|
||||||
// answer regardless of how it opens.
|
// answer regardless of how it opens (the preamble filter is not applied at
|
||||||
|
// this length — see isSubstantiveAnswer).
|
||||||
recoverMinChars = 200
|
recoverMinChars = 200
|
||||||
// recoverFloorChars / recoverRatio gate the borderline band: a shorter
|
// recoverFloorChars / recoverRatio gate the borderline band: a shorter
|
||||||
// prior turn must still clearly dwarf the (very short) terminal and not
|
// prior turn must clear the floor and — unless the terminal is a citations
|
||||||
// look like a preamble.
|
// addendum, which is not a rival answer — also clearly dwarf the (very
|
||||||
|
// short) terminal. See isSubstantiveAnswer.
|
||||||
recoverFloorChars = 80
|
recoverFloorChars = 80
|
||||||
recoverRatio = 3
|
recoverRatio = 3
|
||||||
|
// citationDominatedDivisor: a citations terminal's non-link remainder must
|
||||||
|
// be at most len/N of the whole, so a prose answer that merely opens with
|
||||||
|
// "Source:" and cites a URL mid-sentence is not mistaken for a bare list.
|
||||||
|
citationDominatedDivisor = 3
|
||||||
)
|
)
|
||||||
|
|
||||||
// isWeakFinal reports whether a terminal turn's text fails to stand on its own
|
// isWeakFinal reports whether a terminal turn's text fails to stand on its own
|
||||||
@@ -65,10 +126,37 @@ func isWeakFinal(s string) bool {
|
|||||||
return len(t) <= weakFinalMaxChars && backRefRe.MatchString(t)
|
return len(t) <= weakFinalMaxChars && backRefRe.MatchString(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isCitationsOnly reports whether a terminal turn is essentially just a
|
||||||
|
// sources/citations addendum: it OPENS with a citations heading, carries at
|
||||||
|
// least one link, and — once the heading and links are removed — is dominated
|
||||||
|
// by that citation structure (only list punctuation and short per-source
|
||||||
|
// annotations remain). The dominance check is what separates a bare sources
|
||||||
|
// list (recover the front-loaded answer, keep the links) from a real prose
|
||||||
|
// answer that merely opens with "Source:" and references a URL mid-sentence
|
||||||
|
// (leave it as the answer). Unlike a back-reference closer the links are worth
|
||||||
|
// keeping, so finalOutput appends them to the recovered answer.
|
||||||
|
//
|
||||||
|
// A citations terminal whose sources are bare domains (no scheme, no markdown
|
||||||
|
// link) is intentionally out of scope — there is no reliable link signal, so it
|
||||||
|
// is left as-is rather than risk misclassifying prose.
|
||||||
|
func isCitationsOnly(s string) bool {
|
||||||
|
t := strings.TrimSpace(s)
|
||||||
|
if !citationLabelRe.MatchString(t) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
body := citationLabelRe.ReplaceAllString(t, "")
|
||||||
|
if !linkRe.MatchString(body) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
residue := strings.Trim(linkRe.ReplaceAllString(body, ""), citationResidueCutset)
|
||||||
|
return len(residue) <= len(t)/citationDominatedDivisor
|
||||||
|
}
|
||||||
|
|
||||||
// lastSubstantiveAssistantText scans msgs newest→oldest (skipping the terminal
|
// lastSubstantiveAssistantText scans msgs newest→oldest (skipping the terminal
|
||||||
// turn and empty tool-only turns) for the most recent assistant turn whose text
|
// turn and empty tool-only turns) for the most recent assistant turn whose text
|
||||||
// reads like a real answer. Returns ("", false) when nothing qualifies.
|
// reads like a real answer. citations selects the recovery bar (see
|
||||||
func lastSubstantiveAssistantText(msgs []llm.Message, terminal string) (string, bool) {
|
// isSubstantiveAnswer). Returns ("", false) when nothing qualifies.
|
||||||
|
func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations bool) (string, bool) {
|
||||||
tt := strings.TrimSpace(terminal)
|
tt := strings.TrimSpace(terminal)
|
||||||
for i := len(msgs) - 1; i >= 0; i-- {
|
for i := len(msgs) - 1; i >= 0; i-- {
|
||||||
m := msgs[i]
|
m := msgs[i]
|
||||||
@@ -79,7 +167,7 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string) (string,
|
|||||||
if txt == "" || txt == tt {
|
if txt == "" || txt == tt {
|
||||||
continue // the terminal turn itself, or an empty tool-only turn
|
continue // the terminal turn itself, or an empty tool-only turn
|
||||||
}
|
}
|
||||||
if isSubstantiveAnswer(txt, tt) {
|
if isSubstantiveAnswer(txt, tt, citations) {
|
||||||
return txt, true
|
return txt, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -88,11 +176,21 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string) (string,
|
|||||||
|
|
||||||
// isSubstantiveAnswer reports whether txt (a prior assistant turn) reads like a
|
// isSubstantiveAnswer reports whether txt (a prior assistant turn) reads like a
|
||||||
// real answer rather than a preamble, relative to the terminal text.
|
// real answer rather than a preamble, relative to the terminal text.
|
||||||
func isSubstantiveAnswer(txt, terminal string) bool {
|
//
|
||||||
|
// A sufficiently long turn (>= recoverMinChars) is accepted unconditionally: a
|
||||||
|
// multi-hundred-char turn is an answer even when it opens conversationally
|
||||||
|
// ("Sure, here's…", "Let me explain: …"), so the preamble filter is NOT applied
|
||||||
|
// to it — applying it there would drop a legitimate long front-loaded answer.
|
||||||
|
// Only in the borderline band does a turn have to clear a floor, not read like a
|
||||||
|
// short planning preamble ("Let me look that up…"), and — unless the terminal is
|
||||||
|
// a citations addendum (not a rival answer, so its length is irrelevant) — also
|
||||||
|
// clearly dwarf the terminal.
|
||||||
|
func isSubstantiveAnswer(txt, terminal string, citations bool) bool {
|
||||||
if len(txt) >= recoverMinChars {
|
if len(txt) >= recoverMinChars {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return len(txt) >= recoverFloorChars &&
|
if len(txt) < recoverFloorChars || preambleRe.MatchString(txt) {
|
||||||
len(txt) >= recoverRatio*len(terminal) &&
|
return false
|
||||||
!preambleRe.MatchString(txt)
|
}
|
||||||
|
return citations || len(txt) >= recoverRatio*len(terminal)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,6 +36,42 @@ func TestIsWeakFinal(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsCitationsOnly(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
in string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"sources-md-links", "Sources: [pcprice.watch](https://pcprice.watch/x), [ebay](https://ebay.com/1).", true},
|
||||||
|
{"lowercase-bare-url", "sources: see https://example.com/a", true},
|
||||||
|
{"bold-label-colon-inside", "**Sources:** [a](https://a), [b](https://b)", true},
|
||||||
|
{"bold-label-colon-outside", "**Sources**: [a](https://a), [b](https://b)", true}, // colon after the closing **
|
||||||
|
{"references-dash", "References — [a](https://a)", true},
|
||||||
|
{"citations-label", "Citations: https://x/y", true},
|
||||||
|
{"leading-list-marker", "- Sources: [a](https://a)", true},
|
||||||
|
{"atx-heading", "## Sources: [a](https://a), [b](https://b)", true}, // ATX heading marker + its trailing space
|
||||||
|
{"further-reading", "Further reading: https://example.com/deep-dive", true},
|
||||||
|
{"annotated-multi-source", "Sources: [pcprice.watch](https://a) (tracker), [eBay](https://b) (sold), [bestvaluegpu](https://c) (retail), [resaleprices](https://d) (asking).", true}, // the reported issue-1418 shape
|
||||||
|
{"backref-plus-links-is-citations", "References: as noted above, [pcprice.watch](https://pcprice.watch/x).", true}, // a back-ref phrase inside a real sources list is still citations
|
||||||
|
|
||||||
|
{"empty", "", false},
|
||||||
|
{"label-but-no-link", "Source: internal analysis, no URL here", false},
|
||||||
|
{"prose-then-sources", "It sells for ~$2,700. Sources: [a](https://a)", false}, // answer first → not a pure addendum
|
||||||
|
{"source-led-prose-answer", "Source: According to https://cdc.gov the flu vaccine is 40-60% effective, and the CDC recommends annual vaccination for everyone over six months old.", false}, // a prose answer that merely opens with a "Source:" label
|
||||||
|
{"mentions-sources-midsentence", "The sources of the leak were never confirmed.", false},
|
||||||
|
{"link-without-label", "Here is the link you asked for: [a](https://a)", false},
|
||||||
|
{"bare-domains-out-of-scope", "Sources: pcprice.watch (used ~$200), ebay.com (sold listings)", false}, // bare domains: no scheme or markdown link to key on
|
||||||
|
{"crisp-number", "42", false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
if got := isCitationsOnly(c.in); got != c.want {
|
||||||
|
t.Errorf("isCitationsOnly(%q) = %v, want %v", c.in, got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func asst(text string, tools ...llm.ToolCall) llm.Message {
|
func asst(text string, tools ...llm.ToolCall) llm.Message {
|
||||||
m := llm.Message{Role: llm.RoleAssistant}
|
m := llm.Message{Role: llm.RoleAssistant}
|
||||||
if text != "" {
|
if text != "" {
|
||||||
@@ -48,6 +84,25 @@ func asst(text string, tools ...llm.ToolCall) llm.Message {
|
|||||||
func TestFinalOutput(t *testing.T) {
|
func TestFinalOutput(t *testing.T) {
|
||||||
cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}
|
cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}
|
||||||
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
|
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
|
||||||
|
// A sources/citations-only terminal — the glm-5.2 "cite" shape behind mort
|
||||||
|
// issue #1418: the prose answer was front-loaded into the tool-call turn and
|
||||||
|
// the terminal turn carried only the citations.
|
||||||
|
sources := "Sources: [pcprice.watch](https://pcprice.watch/x), [ebay](https://www.ebay.com/itm/1)."
|
||||||
|
answerWithSources := longAnswer + "\n\n" + sources
|
||||||
|
// A concise (>80, <200 byte) front-loaded answer + a long citations terminal:
|
||||||
|
// the ratio arm can't be met against the long terminal, so citations mode
|
||||||
|
// must fall back to the floor.
|
||||||
|
conciseAnswer := "It sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle."
|
||||||
|
longSources := "Sources: [pcprice.watch](https://pcprice.watch/gpu/rtx5090) (tracker), [ebay](https://www.ebay.com/sch/rtx5090) (sold), [newegg](https://newegg.com/rtx5090) (retail), [pcpartpicker](https://pcpartpicker.com/rtx5090) (history)."
|
||||||
|
// A substantive answer that merely OPENS with "Source:" (not a bare list).
|
||||||
|
sourceLedAnswer := "Source: https://nvd.nist.gov/vuln/detail/CVE-2024-1234 — this is the authoritative NVD entry for the vulnerability, rated CVSS 9.8 critical."
|
||||||
|
// A borderline-band (80–200 byte) turn that opens like a planning preamble:
|
||||||
|
// it clears the floor, but the preamble filter still vetoes it (the filter
|
||||||
|
// applies only in the borderline band; a >=200-byte turn is accepted as-is).
|
||||||
|
preambleTurn := "Let me look that up across a few different sites and then compile the full comparison for you here."
|
||||||
|
// A >=200-byte real answer that merely OPENS with a conversational word
|
||||||
|
// ("Sure,"). The preamble filter must NOT veto it (gadfly regression guard).
|
||||||
|
longConversationalAnswer := "Sure, here's the rundown: it currently sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle, with the sealed Founders Edition commanding the top of that range while used AIB cards go a bit lower."
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -110,6 +165,97 @@ func TestFinalOutput(t *testing.T) {
|
|||||||
terminal: "(see above)",
|
terminal: "(see above)",
|
||||||
want: "(see above)", // preamble excluded; falls back to terminal
|
want: "(see above)", // preamble excluded; falls back to terminal
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "citations-only terminal recovers front-loaded answer and keeps sources",
|
||||||
|
msgs: []llm.Message{
|
||||||
|
llm.UserText("q?"),
|
||||||
|
asst(longAnswer, cite...),
|
||||||
|
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||||
|
asst(sources),
|
||||||
|
},
|
||||||
|
terminal: sources,
|
||||||
|
want: answerWithSources, // answer recovered, citations appended
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "citations-only terminal but only a preamble prior: keeps the sources",
|
||||||
|
msgs: []llm.Message{
|
||||||
|
llm.UserText("q?"),
|
||||||
|
asst("Let me gather the sources.", cite...),
|
||||||
|
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||||
|
asst(sources),
|
||||||
|
},
|
||||||
|
terminal: sources,
|
||||||
|
want: sources, // nothing substantive to recover → keep the addendum
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "citations already in the recovered answer are not duplicated",
|
||||||
|
msgs: []llm.Message{
|
||||||
|
llm.UserText("q?"),
|
||||||
|
asst(answerWithSources, cite...),
|
||||||
|
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||||
|
asst(sources),
|
||||||
|
},
|
||||||
|
terminal: sources,
|
||||||
|
want: answerWithSources, // recovered turn already carries the sources
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// #1418 persisted for CONCISE answers: a <200-char front-loaded
|
||||||
|
// answer must still be recovered against a long citations terminal
|
||||||
|
// (the ratio arm is skipped in citations mode).
|
||||||
|
name: "concise front-loaded answer recovered against a long citations terminal",
|
||||||
|
msgs: []llm.Message{
|
||||||
|
llm.UserText("q?"),
|
||||||
|
asst(conciseAnswer, cite...),
|
||||||
|
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||||
|
asst(longSources),
|
||||||
|
},
|
||||||
|
terminal: longSources,
|
||||||
|
want: conciseAnswer + "\n\n" + longSources,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// A substantive answer that merely OPENS with "Source:" and cites a
|
||||||
|
// URL mid-sentence is NOT a citations addendum — return it verbatim,
|
||||||
|
// never prepend the prior planning turn.
|
||||||
|
name: "source-led substantive answer is not hijacked by a prior turn",
|
||||||
|
msgs: []llm.Message{
|
||||||
|
llm.UserText("what's the authoritative URL?"),
|
||||||
|
asst("I'll look up the CVE in the NVD database, cross-reference the vendor advisory, and confirm the canonical URL before I answer.", cite...),
|
||||||
|
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||||
|
asst(sourceLedAnswer),
|
||||||
|
},
|
||||||
|
terminal: sourceLedAnswer,
|
||||||
|
want: sourceLedAnswer,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// A borderline-length turn that opens like a preamble is vetoed
|
||||||
|
// during recovery; the older real answer is recovered instead. (A
|
||||||
|
// >=200-byte turn would be accepted verbatim — see the next case.)
|
||||||
|
name: "borderline preamble is skipped; older real answer recovered",
|
||||||
|
msgs: []llm.Message{
|
||||||
|
llm.UserText("q?"),
|
||||||
|
asst(conciseAnswer, cite...),
|
||||||
|
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||||
|
asst(preambleTurn, cite...),
|
||||||
|
llm.ToolResultsMessage(llm.ToolResult{ID: "c2", Name: "cite", Content: "ok"}),
|
||||||
|
asst(sources),
|
||||||
|
},
|
||||||
|
terminal: sources,
|
||||||
|
want: conciseAnswer + "\n\n" + sources,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Guards the gadfly regression: a LONG (>=200-byte) front-loaded
|
||||||
|
// answer that merely opens with a conversational word ("Sure, …")
|
||||||
|
// must still be recovered — the preamble filter must not veto it.
|
||||||
|
name: "long answer opening with a conversational word is still recovered",
|
||||||
|
msgs: []llm.Message{
|
||||||
|
llm.UserText("q?"),
|
||||||
|
asst(longConversationalAnswer, cite...),
|
||||||
|
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||||
|
asst(sources),
|
||||||
|
},
|
||||||
|
terminal: sources,
|
||||||
|
want: longConversationalAnswer + "\n\n" + sources,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
@@ -179,3 +325,36 @@ func TestRun_HealthyTerminalUnchanged(t *testing.T) {
|
|||||||
t.Errorf("Output = %q, want terminal answer unchanged", res.Output)
|
t.Errorf("Output = %q, want terminal answer unchanged", res.Output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRun_RecoversFrontLoadedAnswerWithCitations reproduces mort issue #1418
|
||||||
|
// end-to-end: the model front-loads the prose answer into the tool-call turn
|
||||||
|
// and closes with a sources-only terminal turn. The delivered output must be
|
||||||
|
// the recovered answer with the citations appended (not the bare sources list),
|
||||||
|
// with no extra model call.
|
||||||
|
func TestRun_RecoversFrontLoadedAnswerWithCitations(t *testing.T) {
|
||||||
|
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6))
|
||||||
|
sources := "Sources: [docs](https://example.com/docs), [pricing](https://example.com/pricing)."
|
||||||
|
fp := fake.New("fp")
|
||||||
|
fp.Enqueue("test-model",
|
||||||
|
fake.ReplyWith(llm.Response{
|
||||||
|
Parts: []llm.Part{llm.Text(longAnswer)},
|
||||||
|
ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
|
||||||
|
FinishReason: llm.FinishToolCalls,
|
||||||
|
Usage: llm.Usage{InputTokens: 10, OutputTokens: 5},
|
||||||
|
}),
|
||||||
|
fake.Reply(sources),
|
||||||
|
)
|
||||||
|
|
||||||
|
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||||||
|
res, err := a.Run(context.Background(), "is there a meet time limit?")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
want := longAnswer + "\n\n" + sources
|
||||||
|
if res.Output != want {
|
||||||
|
t.Errorf("Output = %q, want recovered answer + citations %q", res.Output, want)
|
||||||
|
}
|
||||||
|
if n := len(fp.Calls()); n != 2 {
|
||||||
|
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+186
@@ -0,0 +1,186 @@
|
|||||||
|
// Package audio is majordomo's canonical speech surface: text-to-speech
|
||||||
|
// (SpeechModel) and audio transcription (TranscriptionModel). Like imagegen,
|
||||||
|
// it is a deliberately separate contract from the llm package — synthesis and
|
||||||
|
// transcription share none of the chat message/tool/stream machinery, so they
|
||||||
|
// get their own small Provider/Model interfaces rather than overloading
|
||||||
|
// llm.Model (ADR-0017).
|
||||||
|
//
|
||||||
|
// Zero values mean "backend default" throughout, mirroring imagegen: an empty
|
||||||
|
// Voice uses the model's default voice, an empty Format the backend's default
|
||||||
|
// container, a zero Speed the natural rate.
|
||||||
|
//
|
||||||
|
// The first implementation is provider/llamaswap, which targets the OpenAI
|
||||||
|
// /v1/audio/speech and /v1/audio/transcriptions endpoints routed to
|
||||||
|
// kokoro/whisper.cpp-style upstreams.
|
||||||
|
package audio
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// SpeechRequest is a text-to-speech request.
|
||||||
|
type SpeechRequest struct {
|
||||||
|
// Input is the text to speak.
|
||||||
|
Input string
|
||||||
|
|
||||||
|
// Voice selects the voice; "" = the model's default voice.
|
||||||
|
Voice string
|
||||||
|
|
||||||
|
// Format is the audio container ("mp3", "wav", "opus", ...);
|
||||||
|
// "" = backend default.
|
||||||
|
Format string
|
||||||
|
|
||||||
|
// Speed is the playback-rate multiplier; 0 = backend default (1.0).
|
||||||
|
Speed float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// SpeechResult is the canonical synthesis result: raw audio bytes plus the
|
||||||
|
// MIME type reported (or implied) by the backend.
|
||||||
|
type SpeechResult struct {
|
||||||
|
// Audio is the encoded audio.
|
||||||
|
Audio []byte
|
||||||
|
|
||||||
|
// MIME is the audio MIME type, e.g. "audio/mpeg".
|
||||||
|
MIME string
|
||||||
|
|
||||||
|
// Raw is the provider-native response object, an escape hatch for
|
||||||
|
// provider-specific fields. May be nil; never required for normal use.
|
||||||
|
Raw any
|
||||||
|
}
|
||||||
|
|
||||||
|
// SpeechOption mutates a SpeechRequest before it is sent. Options passed to
|
||||||
|
// Speak are applied to a copy, so a request value can be reused.
|
||||||
|
type SpeechOption func(*SpeechRequest)
|
||||||
|
|
||||||
|
// WithVoice selects the voice.
|
||||||
|
func WithVoice(v string) SpeechOption { return func(r *SpeechRequest) { r.Voice = v } }
|
||||||
|
|
||||||
|
// WithFormat sets the audio container format (e.g. "mp3", "wav").
|
||||||
|
func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Format = f } }
|
||||||
|
|
||||||
|
// WithSpeed sets the playback-rate multiplier.
|
||||||
|
func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } }
|
||||||
|
|
||||||
|
// Apply returns a copy of the request with all options applied. Providers
|
||||||
|
// call this once at the top of Speak.
|
||||||
|
func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest {
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&r)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// SpeechModel synthesizes speech from text.
|
||||||
|
type SpeechModel interface {
|
||||||
|
// Speak renders the request's input text as audio.
|
||||||
|
Speak(ctx context.Context, req SpeechRequest, opts ...SpeechOption) (*SpeechResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SpeechModelOption configures a SpeechModel at construction time. Reserved
|
||||||
|
// for future per-model settings; present so the interface is
|
||||||
|
// forward-compatible (mirrors imagegen.ModelOption).
|
||||||
|
type SpeechModelOption func(*SpeechModelConfig)
|
||||||
|
|
||||||
|
// SpeechModelConfig carries per-model construction settings.
|
||||||
|
type SpeechModelConfig struct{}
|
||||||
|
|
||||||
|
// ApplySpeechModelOptions folds options into a config.
|
||||||
|
func ApplySpeechModelOptions(opts []SpeechModelOption) SpeechModelConfig {
|
||||||
|
var cfg SpeechModelConfig
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&cfg)
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
// SpeechProvider mints speech models bound to one backend.
|
||||||
|
type SpeechProvider interface {
|
||||||
|
// Name is the registry identifier for the provider.
|
||||||
|
Name() string
|
||||||
|
|
||||||
|
// SpeechModel returns a SpeechModel bound to the given id (passed through
|
||||||
|
// to the backend verbatim; no catalog validation).
|
||||||
|
SpeechModel(id string, opts ...SpeechModelOption) (SpeechModel, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranscriptionRequest is a speech-to-text request. Audio is carried as bytes
|
||||||
|
// (never a URL), mirroring llm.ImagePart's bytes-only contract.
|
||||||
|
type TranscriptionRequest struct {
|
||||||
|
// Audio is the encoded audio to transcribe.
|
||||||
|
Audio []byte
|
||||||
|
|
||||||
|
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
|
||||||
|
// sniff it.
|
||||||
|
MIME string
|
||||||
|
|
||||||
|
// Filename is the multipart filename hint some backends key their format
|
||||||
|
// detection on; "" derives one from MIME ("audio.mp3") or falls back to
|
||||||
|
// "audio".
|
||||||
|
Filename string
|
||||||
|
|
||||||
|
// Language is a BCP-47/ISO-639 hint (e.g. "en"); "" = auto-detect.
|
||||||
|
Language string
|
||||||
|
|
||||||
|
// Prompt is optional context or vocabulary to bias decoding; "" = none.
|
||||||
|
Prompt string
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranscriptionResult is the canonical transcription result.
|
||||||
|
type TranscriptionResult struct {
|
||||||
|
// Text is the transcript.
|
||||||
|
Text string
|
||||||
|
|
||||||
|
// Raw is the provider-native response object. May be nil.
|
||||||
|
Raw any
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranscriptionOption mutates a TranscriptionRequest before it is sent.
|
||||||
|
type TranscriptionOption func(*TranscriptionRequest)
|
||||||
|
|
||||||
|
// WithLanguage sets the language hint (e.g. "en").
|
||||||
|
func WithLanguage(l string) TranscriptionOption {
|
||||||
|
return func(r *TranscriptionRequest) { r.Language = l }
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithPrompt sets the decoding context/vocabulary hint.
|
||||||
|
func WithPrompt(p string) TranscriptionOption {
|
||||||
|
return func(r *TranscriptionRequest) { r.Prompt = p }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply returns a copy of the request with all options applied.
|
||||||
|
func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest {
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&r)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranscriptionModel transcribes audio to text.
|
||||||
|
type TranscriptionModel interface {
|
||||||
|
// Transcribe converts the request's audio into text.
|
||||||
|
Transcribe(ctx context.Context, req TranscriptionRequest, opts ...TranscriptionOption) (*TranscriptionResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranscriptionModelOption configures a TranscriptionModel at construction
|
||||||
|
// time. Reserved for future per-model settings.
|
||||||
|
type TranscriptionModelOption func(*TranscriptionModelConfig)
|
||||||
|
|
||||||
|
// TranscriptionModelConfig carries per-model construction settings.
|
||||||
|
type TranscriptionModelConfig struct{}
|
||||||
|
|
||||||
|
// ApplyTranscriptionModelOptions folds options into a config.
|
||||||
|
func ApplyTranscriptionModelOptions(opts []TranscriptionModelOption) TranscriptionModelConfig {
|
||||||
|
var cfg TranscriptionModelConfig
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&cfg)
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranscriptionProvider mints transcription models bound to one backend.
|
||||||
|
type TranscriptionProvider interface {
|
||||||
|
// Name is the registry identifier for the provider.
|
||||||
|
Name() string
|
||||||
|
|
||||||
|
// TranscriptionModel returns a TranscriptionModel bound to the given id
|
||||||
|
// (passed through to the backend verbatim; no catalog validation).
|
||||||
|
TranscriptionModel(id string, opts ...TranscriptionModelOption) (TranscriptionModel, error)
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package audio
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestSpeechRequestApply(t *testing.T) {
|
||||||
|
base := SpeechRequest{Input: "hello"}
|
||||||
|
got := base.Apply(WithVoice("af_heart"), WithFormat("wav"), WithSpeed(1.5))
|
||||||
|
|
||||||
|
if got.Input != "hello" {
|
||||||
|
t.Errorf("Input = %q, want %q", got.Input, "hello")
|
||||||
|
}
|
||||||
|
if got.Voice != "af_heart" || got.Format != "wav" || got.Speed != 1.5 {
|
||||||
|
t.Errorf("got = %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply must not mutate the receiver (options apply to a copy).
|
||||||
|
if base.Voice != "" || base.Format != "" || base.Speed != 0 {
|
||||||
|
t.Errorf("base mutated: %+v", base)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscriptionRequestApply(t *testing.T) {
|
||||||
|
base := TranscriptionRequest{Audio: []byte{1, 2, 3}}
|
||||||
|
got := base.Apply(WithLanguage("en"), WithPrompt("names"))
|
||||||
|
|
||||||
|
if got.Language != "en" || got.Prompt != "names" {
|
||||||
|
t.Errorf("got = %+v", got)
|
||||||
|
}
|
||||||
|
if base.Language != "" || base.Prompt != "" {
|
||||||
|
t.Errorf("base mutated: %+v", base)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyModelOptions(t *testing.T) {
|
||||||
|
// No options yet; just verify they return usable zero configs.
|
||||||
|
_ = ApplySpeechModelOptions(nil)
|
||||||
|
_ = ApplyTranscriptionModelOptions(nil)
|
||||||
|
}
|
||||||
+21
-11
@@ -20,6 +20,12 @@ const (
|
|||||||
ProviderOllamaCloud = "ollama-cloud"
|
ProviderOllamaCloud = "ollama-cloud"
|
||||||
ProviderForeman = "foreman"
|
ProviderForeman = "foreman"
|
||||||
ProviderLlamaSwap = "llama-swap"
|
ProviderLlamaSwap = "llama-swap"
|
||||||
|
// ProviderLlamaSwapTLS is the DSN scheme for a TLS-fronted llama-swap
|
||||||
|
// (https base URL). It is a scheme only, not a default built-in provider
|
||||||
|
// name. Why a separate scheme rather than auto-detecting: a DSN carries no
|
||||||
|
// reliable signal for http vs https, so the choice is explicit
|
||||||
|
// (llama-swap = http local-first, llama-swaps = https), mirroring rediss.
|
||||||
|
ProviderLlamaSwapTLS = "llama-swaps"
|
||||||
)
|
)
|
||||||
|
|
||||||
// registerBuiltins installs the built-in providers and env-DSN scheme
|
// registerBuiltins installs the built-in providers and env-DSN scheme
|
||||||
@@ -70,24 +76,28 @@ func registerBuiltins(r *Registry, httpClient *http.Client) {
|
|||||||
|
|
||||||
// llama-swap: OpenAI-compatible chat + image generation + management
|
// llama-swap: OpenAI-compatible chat + image generation + management
|
||||||
// endpoints over a model-swapping proxy. Chat reuses the openai client
|
// endpoints over a model-swapping proxy. Chat reuses the openai client
|
||||||
// (provider/llamaswap delegates); the DSN builds an http:// base URL
|
// (provider/llamaswap delegates). Two schemes: "llama-swap" builds an
|
||||||
// because llama-swap is local-first (TLS-fronted instances can use the
|
// http:// base URL (local-first default), "llama-swaps" builds https://
|
||||||
// openai:// scheme for chat). The no-DSN built-in errors on use with a
|
// for a TLS-fronted instance (mirrors redis/rediss). The no-DSN built-in
|
||||||
// clear message, mirroring foreman.
|
// errors on use with a clear message, mirroring foreman.
|
||||||
llamaSwapOpts := func(extra ...llamaswap.Option) []llamaswap.Option {
|
llamaSwapOpts := func(extra ...llamaswap.Option) []llamaswap.Option {
|
||||||
if httpClient != nil {
|
if httpClient != nil {
|
||||||
extra = append(extra, llamaswap.WithHTTPClient(httpClient))
|
extra = append(extra, llamaswap.WithHTTPClient(httpClient))
|
||||||
}
|
}
|
||||||
return extra
|
return extra
|
||||||
}
|
}
|
||||||
r.providers[ProviderLlamaSwap] = llamaswap.New(llamaSwapOpts(llamaswap.WithName(ProviderLlamaSwap))...)
|
llamaSwapScheme := func(urlScheme string) SchemeFactory {
|
||||||
r.schemes[ProviderLlamaSwap] = func(name string, dsn DSN) (llm.Provider, error) {
|
return func(name string, dsn DSN) (llm.Provider, error) {
|
||||||
return llamaswap.New(llamaSwapOpts(
|
return llamaswap.New(llamaSwapOpts(
|
||||||
llamaswap.WithName(name),
|
llamaswap.WithName(name),
|
||||||
llamaswap.WithBaseURL("http://"+dsn.Host),
|
llamaswap.WithBaseURL(urlScheme+"://"+dsn.Host),
|
||||||
llamaswap.WithToken(dsn.Token),
|
llamaswap.WithToken(dsn.Token),
|
||||||
)...), nil
|
)...), nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
r.providers[ProviderLlamaSwap] = llamaswap.New(llamaSwapOpts(llamaswap.WithName(ProviderLlamaSwap))...)
|
||||||
|
r.schemes[ProviderLlamaSwap] = llamaSwapScheme("http")
|
||||||
|
r.schemes[ProviderLlamaSwapTLS] = llamaSwapScheme("https")
|
||||||
|
|
||||||
// Anthropic and Anthropic-compatible endpoints.
|
// Anthropic and Anthropic-compatible endpoints.
|
||||||
anthropicOpts := func(extra ...anthropic.Option) []anthropic.Option {
|
anthropicOpts := func(extra ...anthropic.Option) []anthropic.Option {
|
||||||
|
|||||||
@@ -51,6 +51,28 @@ func TestLlamaSwapScheme(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestLlamaSwapsScheme: the "llama-swaps" scheme builds an https base URL for a
|
||||||
|
// TLS-fronted instance (vs "llama-swap" which is http local-first).
|
||||||
|
func TestLlamaSwapsScheme(t *testing.T) {
|
||||||
|
r := newTestRegistry(t)
|
||||||
|
if err := r.LoadEnv(map[string]string{
|
||||||
|
"LLM_LST": "llama-swaps://tok@swap.example.com",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("LoadEnv: %v", err)
|
||||||
|
}
|
||||||
|
p, ok := r.Provider("lst")
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("provider \"lst\" not registered")
|
||||||
|
}
|
||||||
|
lp, ok := p.(*llamaswap.Provider)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("provider is %T, want *llamaswap.Provider", p)
|
||||||
|
}
|
||||||
|
if want := "https://swap.example.com"; lp.BaseURL() != want {
|
||||||
|
t.Errorf("baseURL = %q, want %q", lp.BaseURL(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestLlamaSwapBuiltinNoURL: the no-DSN built-in resolves but errors clearly on
|
// TestLlamaSwapBuiltinNoURL: the no-DSN built-in resolves but errors clearly on
|
||||||
// use (mirrors foreman), rather than silently hitting a wrong host.
|
// use (mirrors foreman), rather than silently hitting a wrong host.
|
||||||
func TestLlamaSwapBuiltinNoURL(t *testing.T) {
|
func TestLlamaSwapBuiltinNoURL(t *testing.T) {
|
||||||
|
|||||||
@@ -38,11 +38,13 @@ features leak into the canonical API).
|
|||||||
`Unload` (POST `/api/models/unload[/:model]`). A small `doJSON` helper shares
|
`Unload` (POST `/api/models/unload[/:model]`). A small `doJSON` helper shares
|
||||||
bearer auth + error mapping; non-2xx → `*llm.APIError` (so `llm.Classify`
|
bearer auth + error mapping; non-2xx → `*llm.APIError` (so `llm.Classify`
|
||||||
applies), transport errors wrapped raw.
|
applies), transport errors wrapped raw.
|
||||||
- DSN: the `llama-swap` scheme builds an **http://** base URL from the host
|
- DSN: two schemes share one factory. `llama-swap` builds an **http://** base
|
||||||
(llama-swap is local-first), deliberately *not* the DSN's https-always
|
URL from the host (llama-swap is local-first), deliberately *not* the DSN's
|
||||||
`BaseURL()`. A TLS-fronted instance can use the `openai://` scheme for chat.
|
https-always `BaseURL()`; `llama-swaps` builds **https://** for a TLS-fronted
|
||||||
A no-DSN built-in `llama-swap` provider registers but errors on use (mirrors
|
instance (mirrors redis/rediss). Why a second scheme rather than auto-detect:
|
||||||
foreman).
|
a DSN carries no reliable http-vs-https signal, so the choice stays explicit.
|
||||||
|
Only `llama-swap` registers a no-DSN built-in provider (errors on use, mirrors
|
||||||
|
foreman); `llama-swaps` is a scheme only.
|
||||||
- Image generation is implemented here too, against the new `imagegen`
|
- Image generation is implemented here too, against the new `imagegen`
|
||||||
interface (see ADR-0016).
|
interface (see ADR-0016).
|
||||||
|
|
||||||
|
|||||||
@@ -42,3 +42,22 @@ asked for "a new ai image interface as opposed to llm".
|
|||||||
callers (additive fields/options).
|
callers (additive fields/options).
|
||||||
- No health/failover for image models yet; if needed it can be added as a
|
- No health/failover for image models yet; if needed it can be added as a
|
||||||
separate chain type rather than retrofitting the chat chain.
|
separate chain type rather than retrofitting the chat chain.
|
||||||
|
|
||||||
|
## Update — optional per-request settings
|
||||||
|
|
||||||
|
`Request` gained additive optional overrides — `Steps *int`, `CFGScale *float64`,
|
||||||
|
`NegativePrompt string`, `Sampler string`, `Seed *int64` — with mirror options
|
||||||
|
(`WithSteps`, …). nil/"" means "leave the backend's per-model default", so the v1
|
||||||
|
contract is unchanged for callers that don't set them. `provider/llamaswap`
|
||||||
|
forwards them to sd-server as `steps`/`cfg_scale`/`negative_prompt`/`sample_method`/
|
||||||
|
`seed` (omitempty). This realizes the "seeds/steps … additive fields" note above;
|
||||||
|
img2img/masks/streaming remain deferred.
|
||||||
|
|
||||||
|
## Update — A1111 txt2img endpoint (seed support)
|
||||||
|
|
||||||
|
`provider/llamaswap` now POSTs to sd-server's **`/sdapi/v1/txt2img`** (A1111)
|
||||||
|
instead of the OpenAI `/v1/images/generations`. That OpenAI endpoint **ignores
|
||||||
|
`seed`** on the stable-diffusion.cpp build we run — every render of a prompt is
|
||||||
|
byte-identical, so a batch of N collapses to one image. `/sdapi/v1/txt2img`
|
||||||
|
honours `seed`, restoring real per-render variety. llama-swap still routes by
|
||||||
|
the `model` field in the body; `Size` is split into `width`/`height`.
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# ADR-0017: audio — canonical speech synthesis + transcription interfaces
|
||||||
|
|
||||||
|
**Status:** Accepted — 2026-07-11
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
mort is growing agent tools that speak (TTS) and transcribe audio through a
|
||||||
|
llama-swap host whose upstreams expose the OpenAI `/v1/audio/speech` and
|
||||||
|
`/v1/audio/transcriptions` endpoints (kokoro, chatterbox, whisper.cpp). Like
|
||||||
|
image generation before it (ADR-0016), speech shares none of the chat
|
||||||
|
contract's message/tool/stream machinery, and majordomo had no speech surface
|
||||||
|
— an earlier migration doc explicitly scoped transcription out of the llm
|
||||||
|
package. The same reasoning that produced `imagegen` applies.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
- One new canonical **leaf package `audio`** holding both directions —
|
||||||
|
synthesis and transcription — rather than two packages; they are one
|
||||||
|
modality and will share future types (voice metadata, audio formats).
|
||||||
|
Root re-exports mirror imagegen (`SpeechModel`, `SpeechProvider`,
|
||||||
|
`SpeechRequest`, `SpeechResult`, `TranscriptionModel`, ...).
|
||||||
|
- Minimal v1 surface, imagegen conventions throughout (zero value = backend
|
||||||
|
default, functional options + `Apply`, `Raw any` escape hatch):
|
||||||
|
- `SpeechRequest{ Input; Voice; Format; Speed }` →
|
||||||
|
`SpeechResult{ Audio []byte; MIME string; Raw }`;
|
||||||
|
`SpeechModel.Speak(ctx, req, ...opts)`;
|
||||||
|
`SpeechProvider.SpeechModel(id, ...)`.
|
||||||
|
- `TranscriptionRequest{ Audio []byte; MIME; Filename; Language; Prompt }` →
|
||||||
|
`TranscriptionResult{ Text string; Raw }`;
|
||||||
|
`TranscriptionModel.Transcribe(ctx, req, ...opts)`;
|
||||||
|
`TranscriptionProvider.TranscriptionModel(id, ...)`.
|
||||||
|
- **Bytes in/out, never URLs** — mirrors `llm.ImagePart`'s bytes-only
|
||||||
|
contract; fetching is the caller's concern.
|
||||||
|
- Providers are split (`SpeechProvider` vs `TranscriptionProvider`) so a
|
||||||
|
backend can implement either half; llamaswap implements both.
|
||||||
|
- First implementation: `provider/llamaswap` — `/v1/audio/speech` (JSON body,
|
||||||
|
raw audio response; MIME from Content-Type with a format-based fallback),
|
||||||
|
`/v1/audio/transcriptions` (multipart, `response_format=json`), plus
|
||||||
|
`ListVoices(ctx, model)` (GET `/v1/audio/voices?model=`, tolerant of the
|
||||||
|
string-list and object-list shapes upstreams use) as a llamaswap management
|
||||||
|
method, not part of the canonical contract.
|
||||||
|
- Out of scope for v1 (designed-for, deferred): streaming synthesis,
|
||||||
|
word-level timestamps/segments, translation, voice cloning inputs, and
|
||||||
|
registry-level DSN resolution for audio models.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Speech is provider-agnostic from day one; an OpenAI or Google speech
|
||||||
|
backend implements the same interfaces.
|
||||||
|
- `SpeechResult.Audio` is a plain byte slice, so results flow into any file
|
||||||
|
store or attachment pipeline without a majordomo dependency.
|
||||||
|
- The `audio` package name is the modality, not the direction; if music/sfx
|
||||||
|
generation ever lands it has a home.
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# ADR-0018: imagegen.Editor — image-to-image as a separate optional interface
|
||||||
|
|
||||||
|
**Status:** Accepted — 2026-07-11
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
ADR-0016 shipped text-to-image and explicitly deferred img2img. mort's new
|
||||||
|
llama-swap media tools need "edit this image under this prompt"
|
||||||
|
(image-to-image with a denoising strength). Two shape questions: does Edit
|
||||||
|
belong on `imagegen.Model`, and which llama-swap endpoint carries it —
|
||||||
|
OpenAI-style `/v1/images/edits` (multipart) or A1111-style `/sdapi/v1/img2img`
|
||||||
|
(JSON)?
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
- **`Editor` is a separate, optional interface** (`Edit(ctx, EditRequest,
|
||||||
|
...EditOption) (*Result, error)`), not a new method on `Model`. Existing
|
||||||
|
`Model` implementations keep compiling; callers type-assert
|
||||||
|
(`m.(imagegen.Editor)`) or require the capability explicitly.
|
||||||
|
- `EditRequest` = the generation knobs (prompt, N, size, steps, cfg, negative
|
||||||
|
prompt, sampler, seed) plus `Init Image` (required) and `Strength *float64`
|
||||||
|
(denoising strength in [0,1]; nil = backend default). Same option/Apply
|
||||||
|
conventions; result type is the shared `Result`.
|
||||||
|
- llama-swap implements it via **`/sdapi/v1/img2img`**, not
|
||||||
|
`/v1/images/edits`: the same sd-server build that ignores `seed` on the
|
||||||
|
OpenAI images route (the reason txt2img went SDAPI in ADR-0016's
|
||||||
|
implementation) applies; the JSON shape is txt2img's plus
|
||||||
|
`init_images: ["<b64>"]` + `denoising_strength`, so it reuses `doJSON`
|
||||||
|
verbatim, where the OpenAI route is multipart.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Backends that can't edit simply don't implement `Editor`; no stub methods.
|
||||||
|
- The init image travels base64-inline in JSON (~33% overhead) — acceptable at
|
||||||
|
chat-image sizes; a future backend needing multipart can still satisfy the
|
||||||
|
same interface.
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# ADR-0019: videogen — canonical video-generation surface
|
||||||
|
|
||||||
|
**Status:** Accepted — 2026-07-12
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
mort is adding local video generation (Wan 2.2 / LTX-class models on a 24GB
|
||||||
|
GPU) behind the same llama-swap instance that serves imagegen and audio. Like
|
||||||
|
those modalities, video generation shares none of the chat machinery, so it
|
||||||
|
needs its own small contract package (the ADR-0016/0017 pattern). Three shape
|
||||||
|
questions: one interface for text-to-video and image-to-video or two, batch or
|
||||||
|
single result, and which wire endpoint the llama-swap provider targets.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
- **New `videogen/` package** with the established conventions: `Request` /
|
||||||
|
`Result` / `Option` + `Apply`, `Model` / `ModelOption` / `Provider`, zero
|
||||||
|
values mean backend default, `Image = llm.ImagePart`.
|
||||||
|
- **Text-to-video and image-to-video are one surface.** `Request.InitImage
|
||||||
|
*Image` (nil = pure text-to-video) instead of an imagegen-style separate
|
||||||
|
`Editor` interface: hybrid checkpoints (Wan 2.2 TI2V) serve both modes from
|
||||||
|
the same model and endpoint, so a second interface would duplicate the
|
||||||
|
request shape for no dispatch benefit.
|
||||||
|
- **`Result` carries a single `Video`, not a batch.** The blocking sync
|
||||||
|
endpoint answers with the encoded clip as the response body — one request,
|
||||||
|
one clip. Batching multi-minute generations behind one HTTP request is the
|
||||||
|
wrong shape; if batch ever matters it arrives with an async job surface,
|
||||||
|
not by widening this one.
|
||||||
|
- **provider/llamaswap targets `POST {base}/v1/videos/sync`**
|
||||||
|
(multipart/form-data, llama-swap routes by the `model` form field; the
|
||||||
|
response body is the video). This is vLLM-Omni's blocking videos endpoint,
|
||||||
|
and the steve/llama-swap fork dispatches it as a model route. Parameter
|
||||||
|
names follow vLLM-Omni (`num_frames`, `fps`, `num_inference_steps`,
|
||||||
|
`guidance_scale`); the conditioning frame is an `input_reference` file part
|
||||||
|
per OpenAI's videos API. Optional fields stay off the wire so per-model
|
||||||
|
launch-flag defaults apply — the imagegen convention.
|
||||||
|
- **No polling in v1.** The async `POST /v1/videos` + `GET /v1/videos/{id}`
|
||||||
|
job flow is deliberately not wrapped: callers (mort's skill tools) already
|
||||||
|
run synchronous-with-generous-timeout and bound the call with a context
|
||||||
|
deadline. An async `Job` surface is a compatible later addition.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- A `videogen.Video` is its own type (`Data []byte`, `MIME string`); there is
|
||||||
|
no `llm.VideoPart`, and no chat-side video-input support is implied.
|
||||||
|
- Duration is expressed as `NumFrames` + `FPS` (the diffusion-native knobs),
|
||||||
|
not seconds; callers wanting seconds convert at their edge.
|
||||||
|
- Any upstream exposing the same `/v1/videos/sync` shape (e.g. a ComfyUI
|
||||||
|
shim) works unchanged; the contract does not name an engine.
|
||||||
@@ -20,3 +20,6 @@ One decision per file, append-only; supersede rather than rewrite.
|
|||||||
| [0014](0014-conversion-driven-extensions.md) | Conversion-driven extensions (resolvers, typed tools, hooks, ops controls) | Accepted |
|
| [0014](0014-conversion-driven-extensions.md) | Conversion-driven extensions (resolvers, typed tools, hooks, ops controls) | Accepted |
|
||||||
| [0015](0015-llama-swap-provider.md) | llama-swap provider — reuse openai for chat, tailored management + image | Accepted |
|
| [0015](0015-llama-swap-provider.md) | llama-swap provider — reuse openai for chat, tailored management + image | Accepted |
|
||||||
| [0016](0016-imagegen-interface.md) | imagegen — a canonical text-to-image interface | Accepted |
|
| [0016](0016-imagegen-interface.md) | imagegen — a canonical text-to-image interface | Accepted |
|
||||||
|
| [0017](0017-audio-interfaces.md) | audio — canonical speech synthesis + transcription interfaces | Accepted |
|
||||||
|
| [0018](0018-imagegen-editor.md) | imagegen.Editor — image-to-image as a separate optional interface | Accepted |
|
||||||
|
| [0019](0019-videogen-interface.md) | videogen — canonical video-generation surface | Accepted |
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package imagegen
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// EditRequest is an image-to-image (edit) request: a prompt applied to an
|
||||||
|
// initial image. As with Request, zero values mean "backend default"
|
||||||
|
// (ADR-0018).
|
||||||
|
type EditRequest struct {
|
||||||
|
// Prompt is the text description of the desired edit.
|
||||||
|
Prompt string
|
||||||
|
|
||||||
|
// Init is the initial image the edit starts from. Required.
|
||||||
|
Init Image
|
||||||
|
|
||||||
|
// Strength is the denoising strength in [0,1] — how far the result may
|
||||||
|
// depart from Init (0 = return the input, 1 = ignore it); nil = backend
|
||||||
|
// default.
|
||||||
|
Strength *float64
|
||||||
|
|
||||||
|
// N is the number of images to generate; 0 = provider default.
|
||||||
|
N int
|
||||||
|
|
||||||
|
// Size is the requested resolution, e.g. "1024x1024"; "" = provider
|
||||||
|
// default (usually the init image's own resolution).
|
||||||
|
Size string
|
||||||
|
|
||||||
|
// Steps is the number of diffusion steps; nil = backend default.
|
||||||
|
Steps *int
|
||||||
|
|
||||||
|
// CFGScale is the classifier-free-guidance scale; nil = backend default.
|
||||||
|
CFGScale *float64
|
||||||
|
|
||||||
|
// NegativePrompt steers generation away from concepts; "" = none.
|
||||||
|
NegativePrompt string
|
||||||
|
|
||||||
|
// Sampler selects the sampling method (e.g. "euler", "euler_a");
|
||||||
|
// "" = backend default.
|
||||||
|
Sampler string
|
||||||
|
|
||||||
|
// Seed fixes the RNG seed for reproducible output; nil = random.
|
||||||
|
Seed *int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// EditOption mutates an EditRequest before it is sent. Options passed to Edit
|
||||||
|
// are applied to a copy of the request, so an EditRequest value can be reused.
|
||||||
|
type EditOption func(*EditRequest)
|
||||||
|
|
||||||
|
// WithEditStrength sets the denoising strength in [0,1].
|
||||||
|
func WithEditStrength(s float64) EditOption { return func(r *EditRequest) { r.Strength = &s } }
|
||||||
|
|
||||||
|
// WithEditN sets the number of images to generate.
|
||||||
|
func WithEditN(n int) EditOption { return func(r *EditRequest) { r.N = n } }
|
||||||
|
|
||||||
|
// WithEditSize sets the requested resolution (e.g. "1024x1024").
|
||||||
|
func WithEditSize(size string) EditOption { return func(r *EditRequest) { r.Size = size } }
|
||||||
|
|
||||||
|
// WithEditSteps overrides the number of diffusion steps.
|
||||||
|
func WithEditSteps(n int) EditOption { return func(r *EditRequest) { r.Steps = &n } }
|
||||||
|
|
||||||
|
// WithEditCFGScale overrides the classifier-free-guidance scale.
|
||||||
|
func WithEditCFGScale(s float64) EditOption { return func(r *EditRequest) { r.CFGScale = &s } }
|
||||||
|
|
||||||
|
// WithEditNegativePrompt sets a negative prompt.
|
||||||
|
func WithEditNegativePrompt(s string) EditOption {
|
||||||
|
return func(r *EditRequest) { r.NegativePrompt = s }
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithEditSampler overrides the sampling method.
|
||||||
|
func WithEditSampler(s string) EditOption { return func(r *EditRequest) { r.Sampler = s } }
|
||||||
|
|
||||||
|
// WithEditSeed fixes the RNG seed for reproducible output.
|
||||||
|
func WithEditSeed(seed int64) EditOption { return func(r *EditRequest) { r.Seed = &seed } }
|
||||||
|
|
||||||
|
// Apply returns a copy of the request with all options applied. Providers
|
||||||
|
// call this once at the top of Edit.
|
||||||
|
func (r EditRequest) Apply(opts ...EditOption) EditRequest {
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&r)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// Editor is the image-to-image surface. It is a separate, optional interface
|
||||||
|
// rather than a method on Model so existing Model implementations keep
|
||||||
|
// compiling; callers type-assert (`m.(imagegen.Editor)`) or require it
|
||||||
|
// explicitly.
|
||||||
|
type Editor interface {
|
||||||
|
// Edit produces one or more images derived from the request's init image
|
||||||
|
// under the request's prompt.
|
||||||
|
Edit(ctx context.Context, req EditRequest, opts ...EditOption) (*Result, error)
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package imagegen
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestEditRequestApply(t *testing.T) {
|
||||||
|
base := EditRequest{Prompt: "make it night", Init: Image{MIME: "image/png", Data: []byte{1}}}
|
||||||
|
got := base.Apply(WithEditStrength(0.7), WithEditN(2), WithEditSeed(42))
|
||||||
|
|
||||||
|
if got.Prompt != "make it night" || len(got.Init.Data) != 1 {
|
||||||
|
t.Errorf("got = %+v", got)
|
||||||
|
}
|
||||||
|
if got.Strength == nil || *got.Strength != 0.7 {
|
||||||
|
t.Errorf("Strength = %v, want 0.7", got.Strength)
|
||||||
|
}
|
||||||
|
if got.N != 2 {
|
||||||
|
t.Errorf("N = %d, want 2", got.N)
|
||||||
|
}
|
||||||
|
if got.Seed == nil || *got.Seed != 42 {
|
||||||
|
t.Errorf("Seed = %v, want 42", got.Seed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply must not mutate the receiver (options apply to a copy).
|
||||||
|
if base.Strength != nil || base.N != 0 || base.Seed != nil {
|
||||||
|
t.Errorf("base mutated: %+v", base)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -38,6 +38,29 @@ type Request struct {
|
|||||||
// Size is the requested resolution, e.g. "512x512" or "1024x1024";
|
// Size is the requested resolution, e.g. "512x512" or "1024x1024";
|
||||||
// "" = provider default.
|
// "" = provider default.
|
||||||
Size string
|
Size string
|
||||||
|
|
||||||
|
// The fields below are optional per-request overrides. Their zero value
|
||||||
|
// (nil pointer or empty string) means "leave the backend's own default" —
|
||||||
|
// for stable-diffusion.cpp that is the per-model default baked into the
|
||||||
|
// llama-swap launch flags. A caller overrides only what it explicitly sets.
|
||||||
|
|
||||||
|
// Steps is the number of diffusion steps; nil = backend default.
|
||||||
|
Steps *int
|
||||||
|
|
||||||
|
// CFGScale is the classifier-free-guidance scale; nil = backend default.
|
||||||
|
// Architecture-sensitive (SDXL likes ~7, Flux wants 1), so prefer leaving
|
||||||
|
// it nil unless the caller knows the target model.
|
||||||
|
CFGScale *float64
|
||||||
|
|
||||||
|
// NegativePrompt steers generation away from concepts; "" = none.
|
||||||
|
NegativePrompt string
|
||||||
|
|
||||||
|
// Sampler selects the sampling method (e.g. "euler", "euler_a");
|
||||||
|
// "" = backend default.
|
||||||
|
Sampler string
|
||||||
|
|
||||||
|
// Seed fixes the RNG seed for reproducible output; nil = random.
|
||||||
|
Seed *int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Result is the canonical image-generation result.
|
// Result is the canonical image-generation result.
|
||||||
@@ -60,6 +83,21 @@ func WithN(n int) Option { return func(r *Request) { r.N = n } }
|
|||||||
// WithSize sets the requested resolution (e.g. "1024x1024").
|
// WithSize sets the requested resolution (e.g. "1024x1024").
|
||||||
func WithSize(size string) Option { return func(r *Request) { r.Size = size } }
|
func WithSize(size string) Option { return func(r *Request) { r.Size = size } }
|
||||||
|
|
||||||
|
// WithSteps overrides the number of diffusion steps.
|
||||||
|
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
|
||||||
|
|
||||||
|
// WithCFGScale overrides the classifier-free-guidance scale.
|
||||||
|
func WithCFGScale(s float64) Option { return func(r *Request) { r.CFGScale = &s } }
|
||||||
|
|
||||||
|
// WithNegativePrompt sets a negative prompt.
|
||||||
|
func WithNegativePrompt(s string) Option { return func(r *Request) { r.NegativePrompt = s } }
|
||||||
|
|
||||||
|
// WithSampler overrides the sampling method (e.g. "euler", "euler_a").
|
||||||
|
func WithSampler(s string) Option { return func(r *Request) { r.Sampler = s } }
|
||||||
|
|
||||||
|
// WithSeed fixes the RNG seed for reproducible output.
|
||||||
|
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
|
||||||
|
|
||||||
// Apply returns a copy of the request with all options applied. Providers call
|
// Apply returns a copy of the request with all options applied. Providers call
|
||||||
// this once at the top of Generate.
|
// this once at the top of Generate.
|
||||||
func (r Request) Apply(opts ...Option) Request {
|
func (r Request) Apply(opts ...Option) Request {
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||||
)
|
)
|
||||||
@@ -67,6 +68,26 @@ type (
|
|||||||
ImageResult = imagegen.Result
|
ImageResult = imagegen.Result
|
||||||
ImageOption = imagegen.Option
|
ImageOption = imagegen.Option
|
||||||
ImageModelOption = imagegen.ModelOption
|
ImageModelOption = imagegen.ModelOption
|
||||||
|
ImageEditor = imagegen.Editor
|
||||||
|
ImageEditRequest = imagegen.EditRequest
|
||||||
|
ImageEditOption = imagegen.EditOption
|
||||||
|
)
|
||||||
|
|
||||||
|
// Re-exported canonical speech types. See the audio package for
|
||||||
|
// documentation. Speech synthesis and transcription are separate contracts
|
||||||
|
// from llm, mirroring imagegen (ADR-0017); the first backend is
|
||||||
|
// provider/llamaswap.
|
||||||
|
type (
|
||||||
|
SpeechModel = audio.SpeechModel
|
||||||
|
SpeechProvider = audio.SpeechProvider
|
||||||
|
SpeechRequest = audio.SpeechRequest
|
||||||
|
SpeechResult = audio.SpeechResult
|
||||||
|
SpeechOption = audio.SpeechOption
|
||||||
|
TranscriptionModel = audio.TranscriptionModel
|
||||||
|
TranscriptionProvider = audio.TranscriptionProvider
|
||||||
|
TranscriptionRequest = audio.TranscriptionRequest
|
||||||
|
TranscriptionResult = audio.TranscriptionResult
|
||||||
|
TranscriptionOption = audio.TranscriptionOption
|
||||||
)
|
)
|
||||||
|
|
||||||
// Re-exported role and finish-reason constants.
|
// Re-exported role and finish-reason constants.
|
||||||
|
|||||||
+41
-15
@@ -5,10 +5,16 @@
|
|||||||
// already satisfies the target's llm.Capabilities. Images that do not fit
|
// already satisfies the target's llm.Capabilities. Images that do not fit
|
||||||
// are decoded, downscaled (never upscaled), and re-encoded into an allowed
|
// are decoded, downscaled (never upscaled), and re-encoded into an allowed
|
||||||
// format and byte budget. Anything that cannot honestly be made to fit —
|
// format and byte budget. Anything that cannot honestly be made to fit —
|
||||||
// undecodable formats, impossible byte budgets, too many images, images for
|
// undecodable formats, impossible byte budgets, images for a text-only
|
||||||
// a text-only target — fails with an error wrapping llm.ErrUnsupported so a
|
// target — fails with an error wrapping llm.ErrUnsupported so a failover
|
||||||
// failover chain can advance to a more capable target without a health
|
// chain can advance to a more capable target without a health penalty.
|
||||||
// penalty.
|
//
|
||||||
|
// Over-count is the exception: a request carrying more images than
|
||||||
|
// MaxImagesPerReq does NOT fail — the oldest images are replaced with a short
|
||||||
|
// text placeholder and the most-recent MaxImagesPerReq are kept, because a hard
|
||||||
|
// refuse exhausts a chain whose targets share the same cap (e.g. an agent loop
|
||||||
|
// accumulating a preview image per iteration). MaxImagesPerReq remains the
|
||||||
|
// per-model knob (0 = no image support).
|
||||||
//
|
//
|
||||||
// Why a separate package: every provider would otherwise duplicate the same
|
// Why a separate package: every provider would otherwise duplicate the same
|
||||||
// decode/scale/encode pipeline. Providers keep only a cheap capability
|
// decode/scale/encode pipeline. Providers keep only a cheap capability
|
||||||
@@ -52,15 +58,21 @@ func Normalize(req llm.Request, caps llm.Capabilities) (llm.Request, error) {
|
|||||||
if !caps.SupportsImages() {
|
if !caps.SupportsImages() {
|
||||||
return llm.Request{}, fmt.Errorf("media: %w: target does not accept image input (request carries %d image(s))", llm.ErrUnsupported, total)
|
return llm.Request{}, fmt.Errorf("media: %w: target does not accept image input (request carries %d image(s))", llm.ErrUnsupported, total)
|
||||||
}
|
}
|
||||||
// Why error instead of dropping the overflow: silently removing an image
|
// Over-cap images are elided in the same copy-on-write pass below: the
|
||||||
// changes the question the caller asked; the honest move is to refuse and
|
// OLDEST excess are replaced with a placeholder and the most-recent
|
||||||
// let a chain try a roomier target.
|
// MaxImagesPerReq kept (see the package doc for why we elide rather than
|
||||||
|
// refuse). toElide is how many of the first images, front-to-back, to drop.
|
||||||
|
toElide := 0
|
||||||
if total > caps.MaxImagesPerReq {
|
if total > caps.MaxImagesPerReq {
|
||||||
return llm.Request{}, fmt.Errorf("media: %w: request carries %d images, target allows at most %d per request", llm.ErrUnsupported, total, caps.MaxImagesPerReq)
|
toElide = total - caps.MaxImagesPerReq
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Single copy-on-write pass: for each image, the first toElide become a text
|
||||||
|
// placeholder; the rest are size-normalized against caps. The Messages slice
|
||||||
|
// and an affected message's Parts slice are copied at most once.
|
||||||
out := req
|
out := req
|
||||||
copiedMessages := false
|
copiedMessages := false
|
||||||
|
seen := 0
|
||||||
for mi := range req.Messages {
|
for mi := range req.Messages {
|
||||||
copiedParts := false
|
copiedParts := false
|
||||||
for pi, part := range req.Messages[mi].Parts {
|
for pi, part := range req.Messages[mi].Parts {
|
||||||
@@ -68,13 +80,22 @@ func Normalize(req llm.Request, caps llm.Capabilities) (llm.Request, error) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
norm, changed, err := normalizeImage(ip, caps)
|
seen++
|
||||||
if err != nil {
|
|
||||||
return llm.Request{}, fmt.Errorf("media: message %d, part %d: %w", mi, pi, err)
|
var replacement llm.Part
|
||||||
}
|
if seen <= toElide {
|
||||||
if !changed {
|
replacement = llm.Text(imageOverflowPlaceholder)
|
||||||
continue
|
} else {
|
||||||
|
norm, changed, err := normalizeImage(ip, caps)
|
||||||
|
if err != nil {
|
||||||
|
return llm.Request{}, fmt.Errorf("media: message %d, part %d: %w", mi, pi, err)
|
||||||
|
}
|
||||||
|
if !changed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
replacement = norm
|
||||||
}
|
}
|
||||||
|
|
||||||
if !copiedMessages {
|
if !copiedMessages {
|
||||||
out.Messages = make([]llm.Message, len(req.Messages))
|
out.Messages = make([]llm.Message, len(req.Messages))
|
||||||
copy(out.Messages, req.Messages)
|
copy(out.Messages, req.Messages)
|
||||||
@@ -86,12 +107,17 @@ func Normalize(req llm.Request, caps llm.Capabilities) (llm.Request, error) {
|
|||||||
out.Messages[mi].Parts = parts
|
out.Messages[mi].Parts = parts
|
||||||
copiedParts = true
|
copiedParts = true
|
||||||
}
|
}
|
||||||
out.Messages[mi].Parts[pi] = norm
|
out.Messages[mi].Parts[pi] = replacement
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// imageOverflowPlaceholder replaces an image elided to fit a target's
|
||||||
|
// per-request image cap. It keeps the message turn intact and tells the model
|
||||||
|
// an earlier image was omitted rather than silently changing the conversation.
|
||||||
|
const imageOverflowPlaceholder = "[earlier image omitted to fit this model's per-request image limit]"
|
||||||
|
|
||||||
// Info reports an image part's sniffed format ("jpeg", "png", "gif", or
|
// Info reports an image part's sniffed format ("jpeg", "png", "gif", or
|
||||||
// "webp") and pixel dimensions. It is a cheap metadata read — the pixels are
|
// "webp") and pixel dimensions. It is a cheap metadata read — the pixels are
|
||||||
// never decoded. webp is recognized by signature but not decodable with the
|
// never decoded. webp is recognized by signature but not decodable with the
|
||||||
|
|||||||
+39
-9
@@ -149,18 +149,48 @@ func TestNormalizeImagesUnsupported(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNormalizeTooManyImages(t *testing.T) {
|
func TestNormalizeOverCount(t *testing.T) {
|
||||||
img := llm.Image("image/png", encPNG(t, gradient(4, 4)))
|
// 3 distinguishable images across 2 messages; cap = 2. Over-count no longer
|
||||||
|
// errors — the OLDEST image is replaced with a placeholder and the most-recent
|
||||||
|
// two (the relevant ones in an iterative run) are kept, in order.
|
||||||
|
a := llm.Image("image/png", encPNG(t, gradient(2, 2))).(llm.ImagePart)
|
||||||
|
b := llm.Image("image/png", encPNG(t, gradient(4, 4))).(llm.ImagePart)
|
||||||
|
c := llm.Image("image/png", encPNG(t, gradient(8, 8))).(llm.ImagePart)
|
||||||
req := llm.Request{Messages: []llm.Message{
|
req := llm.Request{Messages: []llm.Message{
|
||||||
llm.UserParts(img, img),
|
llm.UserParts(a, b),
|
||||||
llm.UserParts(img),
|
llm.UserParts(c),
|
||||||
}}
|
}}
|
||||||
_, err := Normalize(req, llm.Capabilities{MaxImagesPerReq: 2})
|
caps := llm.Capabilities{MaxImagesPerReq: 2, MaxImageDimension: 64, MaxImageBytes: 1 << 20, AllowedImageMIME: []string{"image/png"}}
|
||||||
if !errors.Is(err, llm.ErrUnsupported) {
|
out, err := Normalize(req, caps)
|
||||||
t.Fatalf("err = %v, want ErrUnsupported", err)
|
if err != nil {
|
||||||
|
t.Fatalf("over-count should not error: %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "3 images") || !strings.Contains(err.Error(), "at most 2") {
|
var imgs []llm.ImagePart
|
||||||
t.Errorf("err message %q lacks the counts", err)
|
placeholders := 0
|
||||||
|
for _, m := range out.Messages {
|
||||||
|
for _, p := range m.Parts {
|
||||||
|
switch v := p.(type) {
|
||||||
|
case llm.ImagePart:
|
||||||
|
imgs = append(imgs, v)
|
||||||
|
case llm.TextPart:
|
||||||
|
if v.Text == imageOverflowPlaceholder {
|
||||||
|
placeholders++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The exact survivors are the most-recent two, in order: b then c (a elided).
|
||||||
|
if len(imgs) != 2 || !bytes.Equal(imgs[0].Data, b.Data) || !bytes.Equal(imgs[1].Data, c.Data) {
|
||||||
|
t.Fatalf("kept %d images; want exactly [b, c] (the most-recent two)", len(imgs))
|
||||||
|
}
|
||||||
|
if placeholders != 1 {
|
||||||
|
t.Errorf("placeholders = %d, want 1 for the elided oldest image", placeholders)
|
||||||
|
}
|
||||||
|
// Input request untouched (copy-on-write): the first part is still image a,
|
||||||
|
// not a placeholder — a len check alone wouldn't catch in-place substitution.
|
||||||
|
first, ok := req.Messages[0].Parts[0].(llm.ImagePart)
|
||||||
|
if !ok || !bytes.Equal(first.Data, a.Data) {
|
||||||
|
t.Errorf("input request was mutated; first part = %+v", req.Messages[0].Parts[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+23
@@ -1,5 +1,12 @@
|
|||||||
# progress
|
# progress
|
||||||
|
|
||||||
|
## 2026-06-27 — llama-swaps (TLS) DSN scheme
|
||||||
|
|
||||||
|
Follow-up to the llama-swap provider: added the `llama-swaps` DSN scheme (https
|
||||||
|
base URL) alongside `llama-swap` (http, local-first), mirroring redis/rediss, so
|
||||||
|
a TLS-fronted instance is first-class instead of being pushed to the `openai://`
|
||||||
|
scheme. Scheme-only (no default built-in); shares one factory in builtin.go.
|
||||||
|
|
||||||
## 2026-06-27 — llama-swap provider + canonical image-gen interface
|
## 2026-06-27 — llama-swap provider + canonical image-gen interface
|
||||||
|
|
||||||
**Landed (ADR-0015, ADR-0016).** New `provider/llamaswap`: chat **delegates to
|
**Landed (ADR-0015, ADR-0016).** New `provider/llamaswap`: chat **delegates to
|
||||||
@@ -241,3 +248,19 @@ alias-in-chain failover, permanent-policy override) and wires anything the
|
|||||||
tests flush out.
|
tests flush out.
|
||||||
|
|
||||||
**Next:** Phase 2 — exhaustive health/chain test matrix.
|
**Next:** Phase 2 — exhaustive health/chain test matrix.
|
||||||
|
|
||||||
|
## 2026-07-11 — audio surfaces + image editing (ADR-0017, ADR-0018)
|
||||||
|
|
||||||
|
- New leaf package `audio`: `SpeechModel`/`SpeechProvider` (TTS) and
|
||||||
|
`TranscriptionModel`/`TranscriptionProvider` (STT), imagegen conventions
|
||||||
|
(zero value = backend default, options + Apply, bytes in/out). Root
|
||||||
|
re-exports added.
|
||||||
|
- `imagegen.Editor` — optional img2img interface (`EditRequest` with
|
||||||
|
`Init Image` + `Strength`); shared `Result`.
|
||||||
|
- provider/llamaswap implements all of it: `/v1/audio/speech`,
|
||||||
|
`/v1/audio/transcriptions` (multipart), `ListVoices`
|
||||||
|
(`/v1/audio/voices?model=`, shape-tolerant), `/sdapi/v1/img2img`
|
||||||
|
(txt2img wire + init_images/denoising_strength, shared decode), and a
|
||||||
|
cheap `Health(ctx)` probe (GET /health) for often-offline hosts.
|
||||||
|
- Hermetic httptest coverage for every new wire shape + validation errors.
|
||||||
|
- Consumer: mort's llamaswap media tool cluster (status/image/TTS/STT tools).
|
||||||
|
|||||||
@@ -0,0 +1,286 @@
|
|||||||
|
package llamaswap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime"
|
||||||
|
"mime/multipart"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SpeechModel implements audio.SpeechProvider, binding a text-to-speech model
|
||||||
|
// served by llama-swap (routed to a kokoro/chatterbox-style OpenAI-compatible
|
||||||
|
// upstream). The id is passed through verbatim and selects which upstream
|
||||||
|
// llama-swap loads.
|
||||||
|
func (p *Provider) SpeechModel(id string, opts ...audio.SpeechModelOption) (audio.SpeechModel, error) {
|
||||||
|
if err := p.requireBaseURL(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = audio.ApplySpeechModelOptions(opts)
|
||||||
|
return &speechModel{p: p, id: id}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type speechModel struct {
|
||||||
|
p *Provider
|
||||||
|
id string
|
||||||
|
}
|
||||||
|
|
||||||
|
// speechRequest is the OpenAI /v1/audio/speech request shape. llama-swap
|
||||||
|
// routes by the `model` field in the body.
|
||||||
|
type speechRequest struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Input string `json:"input"`
|
||||||
|
Voice string `json:"voice,omitempty"`
|
||||||
|
ResponseFormat string `json:"response_format,omitempty"`
|
||||||
|
Speed float64 `json:"speed,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech.
|
||||||
|
func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) {
|
||||||
|
req = req.Apply(opts...)
|
||||||
|
if strings.TrimSpace(req.Input) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: speech synthesis requires input text", llm.ErrUnsupported)
|
||||||
|
}
|
||||||
|
if req.Speed < 0 {
|
||||||
|
return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed)
|
||||||
|
}
|
||||||
|
wire := speechRequest{
|
||||||
|
Model: m.id,
|
||||||
|
Input: req.Input,
|
||||||
|
Voice: req.Voice,
|
||||||
|
ResponseFormat: req.Format,
|
||||||
|
Speed: req.Speed,
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(wire)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: encode speech request: %w", err)
|
||||||
|
}
|
||||||
|
audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/audio/speech", m.id, "application/json", bytes.NewReader(body), maxResponseBytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(audioBytes) == 0 {
|
||||||
|
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech response contained no audio"}
|
||||||
|
}
|
||||||
|
return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// speechMIME resolves the result MIME type: the response Content-Type when it
|
||||||
|
// is a concrete audio type, else a mapping from the requested format, else
|
||||||
|
// audio/mpeg (the OpenAI endpoint's default container is mp3).
|
||||||
|
func speechMIME(contentType, format string) string {
|
||||||
|
if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
|
||||||
|
return mt
|
||||||
|
}
|
||||||
|
switch strings.ToLower(strings.TrimSpace(format)) {
|
||||||
|
case "wav":
|
||||||
|
return "audio/wav"
|
||||||
|
case "opus":
|
||||||
|
return "audio/ogg"
|
||||||
|
case "aac":
|
||||||
|
return "audio/aac"
|
||||||
|
case "flac":
|
||||||
|
return "audio/flac"
|
||||||
|
default: // "", "mp3", and anything unrecognized
|
||||||
|
return "audio/mpeg"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranscriptionModel implements audio.TranscriptionProvider, binding a
|
||||||
|
// speech-to-text model served by llama-swap (routed to a whisper.cpp-style
|
||||||
|
// upstream).
|
||||||
|
func (p *Provider) TranscriptionModel(id string, opts ...audio.TranscriptionModelOption) (audio.TranscriptionModel, error) {
|
||||||
|
if err := p.requireBaseURL(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = audio.ApplyTranscriptionModelOptions(opts)
|
||||||
|
return &transcriptionModel{p: p, id: id}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type transcriptionModel struct {
|
||||||
|
p *Provider
|
||||||
|
id string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Transcribe implements audio.TranscriptionModel via POST
|
||||||
|
// {base}/v1/audio/transcriptions (multipart/form-data — llama-swap routes by
|
||||||
|
// the `model` form field).
|
||||||
|
func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.TranscriptionRequest, opts ...audio.TranscriptionOption) (*audio.TranscriptionResult, error) {
|
||||||
|
req = req.Apply(opts...)
|
||||||
|
if len(req.Audio) == 0 {
|
||||||
|
return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
w := multipart.NewWriter(&buf)
|
||||||
|
fw, err := w.CreateFormFile("file", transcriptionFilename(req))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: build transcription form: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := fw.Write(req.Audio); err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: build transcription form: %w", err)
|
||||||
|
}
|
||||||
|
if err := writeFormFields(w, "build transcription form", []formField{
|
||||||
|
{"model", m.id, true},
|
||||||
|
{"response_format", "json", true},
|
||||||
|
{"language", req.Language, false},
|
||||||
|
{"prompt", req.Prompt, false},
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: build transcription form: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, _, err := m.p.doRaw(ctx, http.MethodPost, "/v1/audio/transcriptions", m.id, w.FormDataContentType(), &buf, maxResponseBytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &out); err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: decode transcription response: %w", err)
|
||||||
|
}
|
||||||
|
return &audio.TranscriptionResult{Text: out.Text, Raw: json.RawMessage(raw)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// transcriptionFilename picks the multipart filename hint: the caller's
|
||||||
|
// (sanitized — upload metadata is untrusted and CR/LF would inject multipart
|
||||||
|
// headers), else one derived from the MIME subtype ("audio.mp3"), else
|
||||||
|
// "audio". MIME parameters ("audio/ogg; codecs=opus") are stripped before
|
||||||
|
// matching.
|
||||||
|
func transcriptionFilename(req audio.TranscriptionRequest) string {
|
||||||
|
if name := sanitizeFilename(req.Filename); name != "" {
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
mt := strings.ToLower(strings.TrimSpace(req.MIME))
|
||||||
|
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
||||||
|
mt = parsed
|
||||||
|
}
|
||||||
|
switch mt {
|
||||||
|
case "audio/mpeg", "audio/mp3":
|
||||||
|
return "audio.mp3"
|
||||||
|
case "audio/wav", "audio/x-wav", "audio/wave":
|
||||||
|
return "audio.wav"
|
||||||
|
case "audio/ogg":
|
||||||
|
return "audio.ogg"
|
||||||
|
case "audio/opus":
|
||||||
|
return "audio.opus"
|
||||||
|
case "audio/flac", "audio/x-flac":
|
||||||
|
return "audio.flac"
|
||||||
|
case "audio/mp4", "audio/m4a", "audio/x-m4a":
|
||||||
|
return "audio.m4a"
|
||||||
|
case "audio/webm", "video/webm":
|
||||||
|
return "audio.webm"
|
||||||
|
default:
|
||||||
|
return "audio"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sanitizeFilename strips characters that would corrupt or inject into the
|
||||||
|
// multipart Content-Disposition header. Quotes and backslashes are escaped
|
||||||
|
// by mime/multipart itself; CR/LF are not — they must go.
|
||||||
|
func sanitizeFilename(name string) string {
|
||||||
|
name = strings.NewReplacer("\r", "", "\n", "").Replace(name)
|
||||||
|
return strings.TrimSpace(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListVoices returns the voices a TTS model offers (GET
|
||||||
|
// /v1/audio/voices?model=...). The decode is tolerant: upstreams answer with
|
||||||
|
// either a bare string list or a list of {id|name} objects.
|
||||||
|
func (p *Provider) ListVoices(ctx context.Context, model string) ([]string, error) {
|
||||||
|
if model == "" {
|
||||||
|
return nil, fmt.Errorf("llama-swap: ListVoices requires a model id")
|
||||||
|
}
|
||||||
|
raw, _, err := p.doRaw(ctx, http.MethodGet, "/v1/audio/voices?model="+url.QueryEscape(model), model, "", nil, maxResponseBytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return parseVoices(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseVoices extracts voice names from the various shapes upstreams use:
|
||||||
|
// {"voices":[...]} or {"data":[...]} envelopes (or a bare array), holding
|
||||||
|
// either strings or objects keyed by id/name/voice_id.
|
||||||
|
func parseVoices(raw []byte) ([]string, error) {
|
||||||
|
var env struct {
|
||||||
|
Voices json.RawMessage `json:"voices"`
|
||||||
|
Data json.RawMessage `json:"data"`
|
||||||
|
}
|
||||||
|
list := json.RawMessage(raw)
|
||||||
|
if err := json.Unmarshal(raw, &env); err == nil {
|
||||||
|
if len(env.Voices) > 0 {
|
||||||
|
list = env.Voices
|
||||||
|
} else if len(env.Data) > 0 {
|
||||||
|
list = env.Data
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var names []string
|
||||||
|
if err := json.Unmarshal(list, &names); err == nil {
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
// Not a plain string list. The failed decode above may have partially
|
||||||
|
// populated names (Unmarshal appends zero values before erroring on an
|
||||||
|
// element type mismatch) — start fresh for the object shape.
|
||||||
|
names = nil
|
||||||
|
var objs []struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
VoiceID string `json:"voice_id"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(list, &objs); err == nil {
|
||||||
|
for _, o := range objs {
|
||||||
|
switch {
|
||||||
|
case o.ID != "":
|
||||||
|
names = append(names, o.ID)
|
||||||
|
case o.Name != "":
|
||||||
|
names = append(names, o.Name)
|
||||||
|
case o.VoiceID != "":
|
||||||
|
names = append(names, o.VoiceID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("llama-swap: unrecognized voices payload shape")
|
||||||
|
}
|
||||||
|
|
||||||
|
// doRaw performs a request to a llama-swap endpoint and returns the raw
|
||||||
|
// response body and its Content-Type — the sibling of doJSON for endpoints
|
||||||
|
// whose success payload is not JSON (audio/video bytes) or whose shape
|
||||||
|
// varies. contentType sets the request Content-Type when body is non-nil.
|
||||||
|
// A response larger than maxBytes is an error, never a silent truncation.
|
||||||
|
func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, string, error) {
|
||||||
|
if err := p.requireBaseURL(); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
req, err := p.newRequest(ctx, method, path, contentType, body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
resp, err := p.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("llama-swap: do request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode/100 != 2 {
|
||||||
|
return nil, "", p.apiError(resp, model)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("llama-swap: read response: %w", err)
|
||||||
|
}
|
||||||
|
if int64(len(data)) > maxBytes {
|
||||||
|
return nil, "", fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes)
|
||||||
|
}
|
||||||
|
return data, resp.Header.Get("Content-Type"), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,247 @@
|
|||||||
|
package llamaswap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSpeak(t *testing.T) {
|
||||||
|
var gotBody map[string]any
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/audio/speech" {
|
||||||
|
t.Errorf("path = %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
if r.Header.Get("Authorization") != "Bearer tok" {
|
||||||
|
t.Errorf("auth = %q", r.Header.Get("Authorization"))
|
||||||
|
}
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||||
|
w.Header().Set("Content-Type", "audio/mpeg")
|
||||||
|
_, _ = w.Write([]byte("MP3BYTES"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithToken("tok"), WithHTTPClient(srv.Client()))
|
||||||
|
sm, err := p.SpeechModel("kokoro")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SpeechModel: %v", err)
|
||||||
|
}
|
||||||
|
res, err := sm.Speak(context.Background(),
|
||||||
|
audio.SpeechRequest{Input: "hello world"},
|
||||||
|
audio.WithVoice("af_heart"), audio.WithFormat("mp3"), audio.WithSpeed(1.2),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Speak: %v", err)
|
||||||
|
}
|
||||||
|
if string(res.Audio) != "MP3BYTES" || res.MIME != "audio/mpeg" {
|
||||||
|
t.Errorf("result = %q %q", res.Audio, res.MIME)
|
||||||
|
}
|
||||||
|
want := map[string]any{"model": "kokoro", "input": "hello world", "voice": "af_heart", "response_format": "mp3", "speed": 1.2}
|
||||||
|
for k, w := range want {
|
||||||
|
if gotBody[k] != w {
|
||||||
|
t.Errorf("%s = %v, want %v", k, gotBody[k], w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpeakDefaultsOmitted(t *testing.T) {
|
||||||
|
var gotBody map[string]any
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||||
|
_, _ = w.Write([]byte("x"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
sm, _ := p.SpeechModel("kokoro")
|
||||||
|
res, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "hi"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Speak: %v", err)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"voice", "response_format", "speed"} {
|
||||||
|
if v, ok := gotBody[k]; ok {
|
||||||
|
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// No usable Content-Type and no requested format → mp3 default.
|
||||||
|
if res.MIME != "audio/mpeg" {
|
||||||
|
t.Errorf("MIME = %q, want audio/mpeg fallback", res.MIME)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpeakEmptyInput(t *testing.T) {
|
||||||
|
p := New(WithBaseURL("http://example.invalid"))
|
||||||
|
sm, _ := p.SpeechModel("kokoro")
|
||||||
|
if _, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: " "}); !errors.Is(err, llm.ErrUnsupported) {
|
||||||
|
t.Errorf("err = %v, want ErrUnsupported", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpeechMIME(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
contentType, format, want string
|
||||||
|
}{
|
||||||
|
{"audio/ogg; codecs=opus", "mp3", "audio/ogg"}, // concrete header wins
|
||||||
|
{"application/octet-stream", "wav", "audio/wav"},
|
||||||
|
{"", "", "audio/mpeg"},
|
||||||
|
{"", "opus", "audio/ogg"},
|
||||||
|
{"text/plain", "flac", "audio/flac"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := speechMIME(tc.contentType, tc.format); got != tc.want {
|
||||||
|
t.Errorf("speechMIME(%q, %q) = %q, want %q", tc.contentType, tc.format, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscribe(t *testing.T) {
|
||||||
|
var gotFields map[string]string
|
||||||
|
var gotFile []byte
|
||||||
|
var gotFilename string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/audio/transcriptions" {
|
||||||
|
t.Errorf("path = %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||||
|
t.Fatalf("parse multipart: %v", err)
|
||||||
|
}
|
||||||
|
gotFields = map[string]string{}
|
||||||
|
for k, v := range r.MultipartForm.Value {
|
||||||
|
gotFields[k] = v[0]
|
||||||
|
}
|
||||||
|
f, hdr, err := r.FormFile("file")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("form file: %v", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
gotFile, _ = io.ReadAll(f)
|
||||||
|
gotFilename = hdr.Filename
|
||||||
|
_, _ = w.Write([]byte(`{"text":"hello there"}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
tm, err := p.TranscriptionModel("whisper")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("TranscriptionModel: %v", err)
|
||||||
|
}
|
||||||
|
res, err := tm.Transcribe(context.Background(),
|
||||||
|
audio.TranscriptionRequest{Audio: []byte("AUDIO"), MIME: "audio/mpeg"},
|
||||||
|
audio.WithLanguage("en"), audio.WithPrompt("robot names"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Transcribe: %v", err)
|
||||||
|
}
|
||||||
|
if res.Text != "hello there" {
|
||||||
|
t.Errorf("text = %q", res.Text)
|
||||||
|
}
|
||||||
|
if string(gotFile) != "AUDIO" || gotFilename != "audio.mp3" {
|
||||||
|
t.Errorf("file = %q name = %q", gotFile, gotFilename)
|
||||||
|
}
|
||||||
|
want := map[string]string{"model": "whisper", "language": "en", "prompt": "robot names", "response_format": "json"}
|
||||||
|
for k, w := range want {
|
||||||
|
if gotFields[k] != w {
|
||||||
|
t.Errorf("%s = %q, want %q", k, gotFields[k], w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscribeEmptyAudio(t *testing.T) {
|
||||||
|
p := New(WithBaseURL("http://example.invalid"))
|
||||||
|
tm, _ := p.TranscriptionModel("whisper")
|
||||||
|
if _, err := tm.Transcribe(context.Background(), audio.TranscriptionRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||||
|
t.Errorf("err = %v, want ErrUnsupported", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListVoices(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/v1/audio/voices" {
|
||||||
|
t.Errorf("path = %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
if got := r.URL.Query().Get("model"); got != "kokoro" {
|
||||||
|
t.Errorf("model = %q", got)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"voices":["af_heart","af_bella"]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
voices, err := p.ListVoices(context.Background(), "kokoro")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListVoices: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(voices, []string{"af_heart", "af_bella"}) {
|
||||||
|
t.Errorf("voices = %v", voices)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := p.ListVoices(context.Background(), ""); err == nil {
|
||||||
|
t.Error("empty model: want error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseVoicesShapes(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"envelope strings", `{"voices":["a","b"]}`, []string{"a", "b"}},
|
||||||
|
{"bare array", `["a","b"]`, []string{"a", "b"}},
|
||||||
|
{"objects by id", `{"voices":[{"id":"a"},{"id":"b"}]}`, []string{"a", "b"}},
|
||||||
|
{"objects by name", `{"data":[{"name":"a"},{"voice_id":"b"}]}`, []string{"a", "b"}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
got, err := parseVoices([]byte(tc.raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s: %v", tc.name, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, tc.want) {
|
||||||
|
t.Errorf("%s: got %v, want %v", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := parseVoices([]byte(`"just a string"`)); err == nil {
|
||||||
|
t.Error("unparseable shape: want error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioAPIError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
_, _ = w.Write([]byte(`{"error":{"message":"model is loading"}}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
sm, _ := p.SpeechModel("kokoro")
|
||||||
|
_, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "x"})
|
||||||
|
var apiErr *llm.APIError
|
||||||
|
if !errors.As(err, &apiErr) {
|
||||||
|
t.Fatalf("err = %T %v, want *llm.APIError", err, err)
|
||||||
|
}
|
||||||
|
if apiErr.Status != http.StatusServiceUnavailable || apiErr.Message != "model is loading" || apiErr.Model != "kokoro" {
|
||||||
|
t.Errorf("apiErr = %+v", apiErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioNoBaseURL(t *testing.T) {
|
||||||
|
p := New()
|
||||||
|
if _, err := p.SpeechModel("kokoro"); err == nil {
|
||||||
|
t.Error("SpeechModel: want error without base URL")
|
||||||
|
}
|
||||||
|
if _, err := p.TranscriptionModel("whisper"); err == nil {
|
||||||
|
t.Error("TranscriptionModel: want error without base URL")
|
||||||
|
}
|
||||||
|
if _, err := p.ListVoices(context.Background(), "kokoro"); err == nil {
|
||||||
|
t.Error("ListVoices: want error without base URL")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package llamaswap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func editInit(t *testing.T) imagegen.Image {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode fixture: %v", err)
|
||||||
|
}
|
||||||
|
return imagegen.Image{MIME: "image/png", Data: raw}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImageEdit(t *testing.T) {
|
||||||
|
var gotBody map[string]any
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/sdapi/v1/img2img" {
|
||||||
|
t.Errorf("path = %q", r.URL.Path)
|
||||||
|
}
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||||
|
_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
im, _ := p.ImageModel("sd")
|
||||||
|
ed, ok := im.(imagegen.Editor)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("imageModel does not implement imagegen.Editor")
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := ed.Edit(context.Background(),
|
||||||
|
imagegen.EditRequest{Prompt: "make it night", Init: editInit(t)},
|
||||||
|
imagegen.WithEditStrength(0.6),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Edit: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Images) != 1 || res.Images[0].MIME != "image/png" {
|
||||||
|
t.Fatalf("images = %+v", res.Images)
|
||||||
|
}
|
||||||
|
|
||||||
|
if gotBody["model"] != "sd" || gotBody["prompt"] != "make it night" {
|
||||||
|
t.Errorf("model/prompt = %v/%v", gotBody["model"], gotBody["prompt"])
|
||||||
|
}
|
||||||
|
inits, ok := gotBody["init_images"].([]any)
|
||||||
|
if !ok || len(inits) != 1 || inits[0] != onePixelPNG {
|
||||||
|
t.Errorf("init_images = %v, want the b64 fixture", gotBody["init_images"])
|
||||||
|
}
|
||||||
|
if gotBody["denoising_strength"] != 0.6 {
|
||||||
|
t.Errorf("denoising_strength = %v, want 0.6", gotBody["denoising_strength"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImageEditOmitsUnsetOverrides(t *testing.T) {
|
||||||
|
var gotBody map[string]any
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||||
|
_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
im, _ := p.ImageModel("sd")
|
||||||
|
ed := im.(imagegen.Editor)
|
||||||
|
if _, err := ed.Edit(context.Background(), imagegen.EditRequest{Prompt: "x", Init: editInit(t)}); err != nil {
|
||||||
|
t.Fatalf("Edit: %v", err)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"denoising_strength", "steps", "cfg_scale", "negative_prompt", "sample_method", "seed", "width", "height"} {
|
||||||
|
if v, ok := gotBody[k]; ok {
|
||||||
|
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImageEditValidation(t *testing.T) {
|
||||||
|
p := New(WithBaseURL("http://example.invalid"))
|
||||||
|
im, _ := p.ImageModel("sd")
|
||||||
|
ed := im.(imagegen.Editor)
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
req imagegen.EditRequest
|
||||||
|
}{
|
||||||
|
{"empty prompt", imagegen.EditRequest{Prompt: " ", Init: imagegen.Image{Data: []byte{1}}}},
|
||||||
|
{"missing init", imagegen.EditRequest{Prompt: "x"}},
|
||||||
|
{"negative N", imagegen.EditRequest{Prompt: "x", Init: imagegen.Image{Data: []byte{1}}, N: -1}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if _, err := ed.Edit(context.Background(), tc.req); !errors.Is(err, llm.ErrUnsupported) {
|
||||||
|
t.Errorf("%s: err = %v, want ErrUnsupported", tc.name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bad := 1.5
|
||||||
|
if _, err := ed.Edit(context.Background(), imagegen.EditRequest{Prompt: "x", Init: imagegen.Image{Data: []byte{1}}, Strength: &bad}); !errors.Is(err, llm.ErrUnsupported) {
|
||||||
|
t.Errorf("out-of-range strength: err = %v, want ErrUnsupported", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package llamaswap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Health reports whether the llama-swap instance is reachable (GET
|
||||||
|
// {base}/health, which answers "OK" without touching any model). A nil error
|
||||||
|
// means the proxy itself is up — it says nothing about how long a subsequent
|
||||||
|
// request will take (a cold model swap can still block for minutes). Bound
|
||||||
|
// the probe with a short context deadline; the client has no timeout by
|
||||||
|
// design.
|
||||||
|
//
|
||||||
|
// Failure taxonomy matches the rest of the package: transport failures wrap
|
||||||
|
// the raw net error; a reachable-but-unhealthy status comes back as
|
||||||
|
// *llm.APIError, so callers can errors.As-distinguish the two.
|
||||||
|
func (p *Provider) Health(ctx context.Context) error {
|
||||||
|
if err := p.requireBaseURL(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req, err := p.newRequest(ctx, http.MethodGet, "/health", "", nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp, err := p.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("llama-swap: health probe: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<10))
|
||||||
|
if resp.StatusCode/100 != 2 {
|
||||||
|
return &llm.APIError{
|
||||||
|
Provider: p.name,
|
||||||
|
Status: resp.StatusCode,
|
||||||
|
Message: "health endpoint returned a non-2xx status",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package llamaswap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHealth(t *testing.T) {
|
||||||
|
var gotPath, gotAuth string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
gotPath = r.URL.Path
|
||||||
|
gotAuth = r.Header.Get("Authorization")
|
||||||
|
_, _ = w.Write([]byte("OK"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithToken("tok"), WithHTTPClient(srv.Client()))
|
||||||
|
if err := p.Health(context.Background()); err != nil {
|
||||||
|
t.Fatalf("Health: %v", err)
|
||||||
|
}
|
||||||
|
if gotPath != "/health" || gotAuth != "Bearer tok" {
|
||||||
|
t.Errorf("path/auth = %q/%q", gotPath, gotAuth)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthUnhealthyStatus(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusBadGateway)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
err := p.Health(context.Background())
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "502") {
|
||||||
|
t.Errorf("err = %v, want status-502 error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthUnreachable(t *testing.T) {
|
||||||
|
// A closed server: transport error, not an HTTP status.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||||
|
url := srv.URL
|
||||||
|
srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(url))
|
||||||
|
if err := p.Health(context.Background()); err == nil {
|
||||||
|
t.Error("want error for unreachable host")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthNoBaseURL(t *testing.T) {
|
||||||
|
p := New()
|
||||||
|
if err := p.Health(context.Background()); err == nil {
|
||||||
|
t.Error("want error without base URL")
|
||||||
|
}
|
||||||
|
}
|
||||||
+127
-48
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||||
@@ -15,8 +16,8 @@ import (
|
|||||||
// served by llama-swap (routed to a stable-diffusion.cpp upstream). The id is
|
// served by llama-swap (routed to a stable-diffusion.cpp upstream). The id is
|
||||||
// passed through verbatim and selects which upstream llama-swap loads.
|
// passed through verbatim and selects which upstream llama-swap loads.
|
||||||
func (p *Provider) ImageModel(id string, opts ...imagegen.ModelOption) (imagegen.Model, error) {
|
func (p *Provider) ImageModel(id string, opts ...imagegen.ModelOption) (imagegen.Model, error) {
|
||||||
if p.baseURL == "" {
|
if err := p.requireBaseURL(); err != nil {
|
||||||
return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name)
|
return nil, err
|
||||||
}
|
}
|
||||||
_ = imagegen.ApplyModelOptions(opts)
|
_ = imagegen.ApplyModelOptions(opts)
|
||||||
return &imageModel{p: p, id: id}, nil
|
return &imageModel{p: p, id: id}, nil
|
||||||
@@ -27,60 +28,82 @@ type imageModel struct {
|
|||||||
id string
|
id string
|
||||||
}
|
}
|
||||||
|
|
||||||
// imageRequest is the OpenAI /v1/images/generations request shape. We always
|
// txt2imgRequest is the stable-diffusion.cpp sd-server A1111 request shape
|
||||||
// request b64_json so the bytes come back inline (no second fetch).
|
// (POST /sdapi/v1/txt2img). We use this endpoint rather than the OpenAI
|
||||||
type imageRequest struct {
|
// /v1/images/generations one because that endpoint IGNORES `seed` on this
|
||||||
Model string `json:"model"`
|
// sd-server build — every render of a given prompt comes back byte-identical,
|
||||||
Prompt string `json:"prompt"`
|
// so a batch of N collapses to one image. /sdapi/v1/txt2img honours `seed`,
|
||||||
N int `json:"n,omitempty"`
|
// giving real variety. llama-swap still routes by the `model` field in the
|
||||||
Size string `json:"size,omitempty"`
|
// body. Optional fields are pointers/omitempty so an unset value falls back to
|
||||||
ResponseFormat string `json:"response_format"`
|
// the model's baked default (the per-model --steps/--cfg-scale/etc. flags).
|
||||||
|
type txt2imgRequest struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Prompt string `json:"prompt"`
|
||||||
|
NegativePrompt string `json:"negative_prompt,omitempty"`
|
||||||
|
Seed *int64 `json:"seed,omitempty"`
|
||||||
|
Steps *int `json:"steps,omitempty"`
|
||||||
|
CFGScale *float64 `json:"cfg_scale,omitempty"`
|
||||||
|
Width *int `json:"width,omitempty"`
|
||||||
|
Height *int `json:"height,omitempty"`
|
||||||
|
SampleMethod string `json:"sample_method,omitempty"`
|
||||||
|
BatchCount int `json:"batch_count,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type imageResponse struct {
|
type txt2imgResponse struct {
|
||||||
Created int64 `json:"created"`
|
Images []string `json:"images"`
|
||||||
Data []struct {
|
|
||||||
B64JSON string `json:"b64_json"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
} `json:"data"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate implements imagegen.Model via POST {base}/v1/images/generations.
|
// sdWire validates the generation knobs shared by Generate and Edit and
|
||||||
|
// builds the common txt2img wire fields. verb labels validation errors.
|
||||||
|
func (m *imageModel) sdWire(verb, prompt, negativePrompt, sampler, size string, seed *int64, steps *int, cfgScale *float64, n int) (txt2imgRequest, error) {
|
||||||
|
if strings.TrimSpace(prompt) == "" {
|
||||||
|
return txt2imgRequest{}, fmt.Errorf("%w: image %s requires a prompt", llm.ErrUnsupported, verb)
|
||||||
|
}
|
||||||
|
if n < 0 {
|
||||||
|
return txt2imgRequest{}, fmt.Errorf("%w: image count N must be >= 0, got %d", llm.ErrUnsupported, n)
|
||||||
|
}
|
||||||
|
width, height, err := parseSize(size)
|
||||||
|
if err != nil {
|
||||||
|
return txt2imgRequest{}, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
|
||||||
|
}
|
||||||
|
return txt2imgRequest{
|
||||||
|
Model: m.id,
|
||||||
|
Prompt: prompt,
|
||||||
|
NegativePrompt: negativePrompt,
|
||||||
|
Seed: seed,
|
||||||
|
Steps: steps,
|
||||||
|
CFGScale: cfgScale,
|
||||||
|
Width: width,
|
||||||
|
Height: height,
|
||||||
|
SampleMethod: sampler,
|
||||||
|
BatchCount: n,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate implements imagegen.Model via POST {base}/sdapi/v1/txt2img.
|
||||||
func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts ...imagegen.Option) (*imagegen.Result, error) {
|
func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts ...imagegen.Option) (*imagegen.Result, error) {
|
||||||
req = req.Apply(opts...)
|
req = req.Apply(opts...)
|
||||||
if strings.TrimSpace(req.Prompt) == "" {
|
wire, err := m.sdWire("generation", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N)
|
||||||
return nil, fmt.Errorf("%w: image generation requires a prompt", llm.ErrUnsupported)
|
if err != nil {
|
||||||
}
|
|
||||||
if req.N < 0 {
|
|
||||||
return nil, fmt.Errorf("%w: image count N must be >= 0, got %d", llm.ErrUnsupported, req.N)
|
|
||||||
}
|
|
||||||
|
|
||||||
wire := imageRequest{
|
|
||||||
Model: m.id,
|
|
||||||
Prompt: req.Prompt,
|
|
||||||
N: req.N,
|
|
||||||
Size: req.Size,
|
|
||||||
ResponseFormat: "b64_json",
|
|
||||||
}
|
|
||||||
|
|
||||||
var resp imageResponse
|
|
||||||
if err := m.p.doJSON(ctx, http.MethodPost, "/v1/images/generations", m.id, &wire, &resp); err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
out := &imagegen.Result{Raw: &resp}
|
var resp txt2imgResponse
|
||||||
for i, d := range resp.Data {
|
if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/txt2img", m.id, &wire, &resp); err != nil {
|
||||||
if d.B64JSON == "" {
|
return nil, err
|
||||||
// Why error rather than skip: a url-only entry means the backend
|
}
|
||||||
// ignored response_format; we don't fetch remote content (mirrors
|
return decodeImages(m.p.name, m.id, &resp)
|
||||||
// llm.ImagePart's bytes-only contract), so surface it.
|
}
|
||||||
return nil, &llm.APIError{
|
|
||||||
Provider: m.p.name,
|
// decodeImages converts an SDAPI response's base64 images into an
|
||||||
Model: m.id,
|
// imagegen.Result, erroring when nothing decodable came back.
|
||||||
Message: fmt.Sprintf("image %d returned no inline b64_json data", i),
|
func decodeImages(provider, model string, resp *txt2imgResponse) (*imagegen.Result, error) {
|
||||||
}
|
out := &imagegen.Result{Raw: resp}
|
||||||
|
for i, b64 := range resp.Images {
|
||||||
|
if b64 == "" {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
raw, err := base64.StdEncoding.DecodeString(d.B64JSON)
|
raw, err := base64.StdEncoding.DecodeString(b64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("llama-swap: decode image %d: %w", i, err)
|
return nil, fmt.Errorf("llama-swap: decode image %d: %w", i, err)
|
||||||
}
|
}
|
||||||
@@ -88,14 +111,70 @@ func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts ..
|
|||||||
}
|
}
|
||||||
if len(out.Images) == 0 {
|
if len(out.Images) == 0 {
|
||||||
return nil, &llm.APIError{
|
return nil, &llm.APIError{
|
||||||
Provider: m.p.name,
|
Provider: provider,
|
||||||
Model: m.id,
|
Model: model,
|
||||||
Message: "image response contained no images",
|
Message: "image response contained no images",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// img2imgRequest is the stable-diffusion.cpp sd-server A1111 request shape
|
||||||
|
// (POST /sdapi/v1/img2img): txt2img's fields plus the init image(s) and
|
||||||
|
// denoising strength. Same endpoint-family choice as txt2img — the OpenAI
|
||||||
|
// /v1/images/edits route is multipart and drops `seed` on this sd-server
|
||||||
|
// build, while the SDAPI shape reuses doJSON and keeps seed parity.
|
||||||
|
type img2imgRequest struct {
|
||||||
|
txt2imgRequest
|
||||||
|
InitImages []string `json:"init_images"`
|
||||||
|
DenoisingStrength *float64 `json:"denoising_strength,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img.
|
||||||
|
func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ...imagegen.EditOption) (*imagegen.Result, error) {
|
||||||
|
req = req.Apply(opts...)
|
||||||
|
if len(req.Init.Data) == 0 {
|
||||||
|
return nil, fmt.Errorf("%w: image edit requires an init image", llm.ErrUnsupported)
|
||||||
|
}
|
||||||
|
if req.Strength != nil && (*req.Strength < 0 || *req.Strength > 1) {
|
||||||
|
return nil, fmt.Errorf("%w: edit strength must be in [0,1], got %g", llm.ErrUnsupported, *req.Strength)
|
||||||
|
}
|
||||||
|
base, err := m.sdWire("edit", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
wire := img2imgRequest{
|
||||||
|
txt2imgRequest: base,
|
||||||
|
InitImages: []string{base64.StdEncoding.EncodeToString(req.Init.Data)},
|
||||||
|
DenoisingStrength: req.Strength,
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp txt2imgResponse
|
||||||
|
if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/img2img", m.id, &wire, &resp); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return decodeImages(m.p.name, m.id, &resp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseSize splits a "WxH" string into width/height pointers. "" yields
|
||||||
|
// (nil, nil) so the model's own default resolution applies.
|
||||||
|
func parseSize(size string) (*int, *int, error) {
|
||||||
|
size = strings.TrimSpace(size)
|
||||||
|
if size == "" {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
parts := strings.SplitN(strings.ToLower(size), "x", 2)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
return nil, nil, fmt.Errorf("invalid size %q (want WxH)", size)
|
||||||
|
}
|
||||||
|
w, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||||
|
h, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
|
||||||
|
if err1 != nil || err2 != nil || w <= 0 || h <= 0 {
|
||||||
|
return nil, nil, fmt.Errorf("invalid size %q (want WxH)", size)
|
||||||
|
}
|
||||||
|
return &w, &h, nil
|
||||||
|
}
|
||||||
|
|
||||||
// sniffImageMIME identifies the image format from its leading bytes, defaulting
|
// sniffImageMIME identifies the image format from its leading bytes, defaulting
|
||||||
// to image/png (stable-diffusion.cpp emits PNG) when detection is inconclusive.
|
// to image/png (stable-diffusion.cpp emits PNG) when detection is inconclusive.
|
||||||
func sniffImageMIME(data []byte) string {
|
func sniffImageMIME(data []byte) string {
|
||||||
|
|||||||
@@ -9,9 +9,11 @@
|
|||||||
// package adds beyond a bare OpenAI-compat endpoint is the "tailored" surface:
|
// package adds beyond a bare OpenAI-compat endpoint is the "tailored" surface:
|
||||||
//
|
//
|
||||||
// - llama-swap management endpoints exposed as concrete methods — ListModels
|
// - llama-swap management endpoints exposed as concrete methods — ListModels
|
||||||
// (GET /v1/models), Running (GET /running), Unload (POST /api/models/unload)
|
// (GET /v1/models), Running (GET /running), Unload (POST /api/models/unload),
|
||||||
// — which have no place on the canonical llm.Provider interface;
|
// ListVoices (GET /v1/audio/voices), Health (GET /health) — which have no
|
||||||
// - image generation via the imagegen interface (see image.go); and
|
// place on the canonical llm.Provider interface;
|
||||||
|
// - image generation + editing via the imagegen interfaces (see image.go);
|
||||||
|
// - speech synthesis + transcription via the audio interfaces (see audio.go); and
|
||||||
// - swap-aware defaults: the HTTP client carries NO timeout, because the
|
// - swap-aware defaults: the HTTP client carries NO timeout, because the
|
||||||
// first request to an unloaded model blocks while llama-swap spawns the
|
// first request to an unloaded model blocks while llama-swap spawns the
|
||||||
// upstream (its healthCheckTimeout is at least 15s). Bound a call with a
|
// upstream (its healthCheckTimeout is at least 15s). Bound a call with a
|
||||||
@@ -28,6 +30,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"mime"
|
||||||
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -43,6 +47,12 @@ const DefaultName = "llama-swap"
|
|||||||
// can't make a decode allocate without limit.
|
// can't make a decode allocate without limit.
|
||||||
const maxResponseBytes = 64 << 20
|
const maxResponseBytes = 64 << 20
|
||||||
|
|
||||||
|
// maxVideoResponseBytes caps the /v1/videos/sync body — the response IS an
|
||||||
|
// encoded clip, which legitimately dwarfs any JSON/audio payload (a long
|
||||||
|
// high-bitrate generation can pass 64MB). Still bounded so a buggy upstream
|
||||||
|
// can't allocate without limit.
|
||||||
|
const maxVideoResponseBytes = 512 << 20
|
||||||
|
|
||||||
// Provider is a llama-swap client. It satisfies llm.Provider (chat, delegated
|
// Provider is a llama-swap client. It satisfies llm.Provider (chat, delegated
|
||||||
// to provider/openai) and imagegen.Provider (image generation), and exposes
|
// to provider/openai) and imagegen.Provider (image generation), and exposes
|
||||||
// llama-swap's management endpoints as concrete methods.
|
// llama-swap's management endpoints as concrete methods.
|
||||||
@@ -100,8 +110,8 @@ func (p *Provider) BaseURL() string { return p.baseURL }
|
|||||||
// endpoint, delegating to provider/openai. The id is passed through verbatim
|
// endpoint, delegating to provider/openai. The id is passed through verbatim
|
||||||
// and selects which upstream llama-swap loads.
|
// and selects which upstream llama-swap loads.
|
||||||
func (p *Provider) Model(id string, opts ...llm.ModelOption) (llm.Model, error) {
|
func (p *Provider) Model(id string, opts ...llm.ModelOption) (llm.Model, error) {
|
||||||
if p.baseURL == "" {
|
if err := p.requireBaseURL(); err != nil {
|
||||||
return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name)
|
return nil, err
|
||||||
}
|
}
|
||||||
return p.chatProvider().Model(id, opts...)
|
return p.chatProvider().Model(id, opts...)
|
||||||
}
|
}
|
||||||
@@ -178,7 +188,32 @@ func (p *Provider) Unload(ctx context.Context, model string) error {
|
|||||||
return p.doJSON(ctx, http.MethodPost, path, "", nil, nil)
|
return p.doJSON(ctx, http.MethodPost, path, "", nil, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- shared HTTP helper for management + image endpoints ---
|
// --- shared HTTP helpers for management + image + audio endpoints ---
|
||||||
|
|
||||||
|
// requireBaseURL is the shared guard for every entry point: construction
|
||||||
|
// never fails (see New), so a missing base URL surfaces here, at use time.
|
||||||
|
func (p *Provider) requireBaseURL() error {
|
||||||
|
if p.baseURL == "" {
|
||||||
|
return fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRequest builds an authenticated request relative to baseURL.
|
||||||
|
// contentType is applied only when a body is present.
|
||||||
|
func (p *Provider) newRequest(ctx context.Context, method, path, contentType string, body io.Reader) (*http.Request, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: build request: %w", err)
|
||||||
|
}
|
||||||
|
if body != nil && contentType != "" {
|
||||||
|
req.Header.Set("Content-Type", contentType)
|
||||||
|
}
|
||||||
|
if p.token != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+p.token)
|
||||||
|
}
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
// doJSON performs a request to a llama-swap endpoint relative to baseURL,
|
// doJSON performs a request to a llama-swap endpoint relative to baseURL,
|
||||||
// optionally encoding body and decoding into out (either may be nil). model
|
// optionally encoding body and decoding into out (either may be nil). model
|
||||||
@@ -186,8 +221,8 @@ func (p *Provider) Unload(ctx context.Context, model string) error {
|
|||||||
// model-specific). Transport failures are wrapped raw so llm.Classify still
|
// model-specific). Transport failures are wrapped raw so llm.Classify still
|
||||||
// sees the underlying net error; non-2xx responses become *llm.APIError.
|
// sees the underlying net error; non-2xx responses become *llm.APIError.
|
||||||
func (p *Provider) doJSON(ctx context.Context, method, path, model string, body, out any) error {
|
func (p *Provider) doJSON(ctx context.Context, method, path, model string, body, out any) error {
|
||||||
if p.baseURL == "" {
|
if err := p.requireBaseURL(); err != nil {
|
||||||
return fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name)
|
return err
|
||||||
}
|
}
|
||||||
var rdr io.Reader
|
var rdr io.Reader
|
||||||
if body != nil {
|
if body != nil {
|
||||||
@@ -197,15 +232,9 @@ func (p *Provider) doJSON(ctx context.Context, method, path, model string, body,
|
|||||||
}
|
}
|
||||||
rdr = bytes.NewReader(b)
|
rdr = bytes.NewReader(b)
|
||||||
}
|
}
|
||||||
req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, rdr)
|
req, err := p.newRequest(ctx, method, path, "application/json", rdr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("llama-swap: build request: %w", err)
|
return err
|
||||||
}
|
|
||||||
if body != nil {
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
}
|
|
||||||
if p.token != "" {
|
|
||||||
req.Header.Set("Authorization", "Bearer "+p.token)
|
|
||||||
}
|
}
|
||||||
resp, err := p.client.Do(req)
|
resp, err := p.client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -259,3 +288,34 @@ func (p *Provider) apiError(resp *http.Response, model string) error {
|
|||||||
e.Message = strings.TrimSpace(string(body))
|
e.Message = strings.TrimSpace(string(body))
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// formField is one multipart form entry for the media endpoints. Required
|
||||||
|
// fields always go on the wire (an empty model id should fail loudly
|
||||||
|
// upstream, not silently vanish); optional ones only when set.
|
||||||
|
type formField struct {
|
||||||
|
key, value string
|
||||||
|
required bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeFormFields appends fields to a multipart writer, skipping unset
|
||||||
|
// optional entries. wrap labels errors ("build transcription form", ...).
|
||||||
|
func writeFormFields(w *multipart.Writer, wrap string, fields []formField) error {
|
||||||
|
for _, f := range fields {
|
||||||
|
if !f.required && f.value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := w.WriteField(f.key, f.value); err != nil {
|
||||||
|
return fmt.Errorf("llama-swap: %s: %w", wrap, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mimeFromContentType returns the parsed media type when it matches prefix
|
||||||
|
// ("audio/", "video/"), else "".
|
||||||
|
func mimeFromContentType(contentType, prefix string) string {
|
||||||
|
if mt, _, err := mime.ParseMediaType(contentType); err == nil && strings.HasPrefix(mt, prefix) {
|
||||||
|
return mt
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|||||||
@@ -166,11 +166,11 @@ func TestRunningRaw(t *testing.T) {
|
|||||||
func TestImageGenerate(t *testing.T) {
|
func TestImageGenerate(t *testing.T) {
|
||||||
var gotBody map[string]any
|
var gotBody map[string]any
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.URL.Path != "/v1/images/generations" {
|
if r.URL.Path != "/sdapi/v1/txt2img" {
|
||||||
t.Errorf("path = %q", r.URL.Path)
|
t.Errorf("path = %q", r.URL.Path)
|
||||||
}
|
}
|
||||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||||
_, _ = w.Write([]byte(`{"created":1,"data":[{"b64_json":"` + onePixelPNG + `"}]}`))
|
_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
|
||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
@@ -192,12 +192,51 @@ func TestImageGenerate(t *testing.T) {
|
|||||||
if len(res.Images[0].Data) == 0 {
|
if len(res.Images[0].Data) == 0 {
|
||||||
t.Error("decoded image has no bytes")
|
t.Error("decoded image has no bytes")
|
||||||
}
|
}
|
||||||
// response_format must be forced to b64_json, and options applied.
|
// Size is split into width/height ints for the A1111 endpoint.
|
||||||
if gotBody["response_format"] != "b64_json" {
|
if gotBody["width"] != float64(512) || gotBody["height"] != float64(512) {
|
||||||
t.Errorf("response_format = %v, want b64_json", gotBody["response_format"])
|
t.Errorf("width/height = %v/%v, want 512/512", gotBody["width"], gotBody["height"])
|
||||||
}
|
}
|
||||||
if gotBody["size"] != "512x512" {
|
}
|
||||||
t.Errorf("size = %v, want 512x512", gotBody["size"])
|
|
||||||
|
func TestImageGenerateSettings(t *testing.T) {
|
||||||
|
var gotBody map[string]any
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||||
|
_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
im, _ := p.ImageModel("sd")
|
||||||
|
|
||||||
|
// Unset overrides must be omitted entirely so sd-server keeps its own
|
||||||
|
// per-model defaults.
|
||||||
|
if _, err := im.Generate(context.Background(), imagegen.Request{Prompt: "x"}); err != nil {
|
||||||
|
t.Fatalf("Generate: %v", err)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"steps", "cfg_scale", "negative_prompt", "sample_method", "seed"} {
|
||||||
|
if v, ok := gotBody[k]; ok {
|
||||||
|
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set overrides are forwarded with the sd-server-friendly field names.
|
||||||
|
gotBody = nil
|
||||||
|
_, err := im.Generate(context.Background(), imagegen.Request{Prompt: "x"},
|
||||||
|
imagegen.WithSteps(8),
|
||||||
|
imagegen.WithCFGScale(3.5),
|
||||||
|
imagegen.WithNegativePrompt("blurry"),
|
||||||
|
imagegen.WithSampler("euler"),
|
||||||
|
imagegen.WithSeed(42),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate: %v", err)
|
||||||
|
}
|
||||||
|
want := map[string]any{"steps": float64(8), "cfg_scale": 3.5, "negative_prompt": "blurry", "sample_method": "euler", "seed": float64(42)}
|
||||||
|
for k, w := range want {
|
||||||
|
if gotBody[k] != w {
|
||||||
|
t.Errorf("%s = %v, want %v", k, gotBody[k], w)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,184 @@
|
|||||||
|
package llamaswap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"mime"
|
||||||
|
"mime/multipart"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||||
|
)
|
||||||
|
|
||||||
|
// VideoModel implements videogen.Provider, binding a video-generation model
|
||||||
|
// served by llama-swap (routed to a vLLM-Omni-style upstream, or any shim
|
||||||
|
// exposing the same /v1/videos/sync shape). The id is passed through verbatim
|
||||||
|
// and selects which upstream llama-swap loads.
|
||||||
|
func (p *Provider) VideoModel(id string, opts ...videogen.ModelOption) (videogen.Model, error) {
|
||||||
|
if err := p.requireBaseURL(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = videogen.ApplyModelOptions(opts)
|
||||||
|
return &videoModel{p: p, id: id}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type videoModel struct {
|
||||||
|
p *Provider
|
||||||
|
id string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate implements videogen.Model via POST {base}/v1/videos/sync
|
||||||
|
// (multipart/form-data — llama-swap routes by the `model` form field). The
|
||||||
|
// blocking sync endpoint answers with the encoded video itself, so the
|
||||||
|
// response body is the result; there is no job id to poll. Generation runs
|
||||||
|
// for minutes — the provider client carries no timeout by design, and callers
|
||||||
|
// bound the call with a context deadline.
|
||||||
|
//
|
||||||
|
// Parameter names follow vLLM-Omni's videos API (num_frames, fps,
|
||||||
|
// num_inference_steps, guidance_scale); the conditioning frame is sent as an
|
||||||
|
// `input_reference` file part, following OpenAI's videos API. Upstreams
|
||||||
|
// ignore fields they don't understand, and optional fields stay off the wire
|
||||||
|
// entirely so the model's own defaults apply.
|
||||||
|
func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) {
|
||||||
|
req = req.Apply(opts...)
|
||||||
|
if strings.TrimSpace(req.Prompt) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: video generation requires a prompt", llm.ErrUnsupported)
|
||||||
|
}
|
||||||
|
if req.NumFrames < 0 {
|
||||||
|
return nil, fmt.Errorf("%w: video frame count must be >= 0, got %d", llm.ErrUnsupported, req.NumFrames)
|
||||||
|
}
|
||||||
|
if req.FPS < 0 {
|
||||||
|
return nil, fmt.Errorf("%w: video fps must be >= 0, got %d", llm.ErrUnsupported, req.FPS)
|
||||||
|
}
|
||||||
|
if req.InitImage != nil && len(req.InitImage.Data) == 0 {
|
||||||
|
return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported)
|
||||||
|
}
|
||||||
|
width, height, err := parseSize(req.Size)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
w := multipart.NewWriter(&buf)
|
||||||
|
if err := writeFormFields(w, "build video form", []formField{
|
||||||
|
{"model", m.id, true},
|
||||||
|
{"prompt", req.Prompt, true},
|
||||||
|
{"negative_prompt", req.NegativePrompt, false},
|
||||||
|
// Resolution rides the wire twice: width/height (vLLM-Omni's
|
||||||
|
// names) AND the equivalent OpenAI-style size string, since
|
||||||
|
// upstreams silently ignore fields they don't understand and the
|
||||||
|
// values can never disagree.
|
||||||
|
{"width", formatInt(width), false},
|
||||||
|
{"height", formatInt(height), false},
|
||||||
|
{"size", strings.TrimSpace(req.Size), false},
|
||||||
|
{"num_frames", formatNonZero(req.NumFrames), false},
|
||||||
|
{"fps", formatNonZero(req.FPS), false},
|
||||||
|
{"num_inference_steps", formatInt(req.Steps), false},
|
||||||
|
{"guidance_scale", formatFloat(req.GuidanceScale), false},
|
||||||
|
{"seed", formatInt64(req.Seed), false},
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if req.InitImage != nil {
|
||||||
|
fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := fw.Write(req.InitImage.Data); err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
videoBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/videos/sync", m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(videoBytes) == 0 {
|
||||||
|
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video response contained no video"}
|
||||||
|
}
|
||||||
|
mimeType := videoMIME(contentType, videoBytes)
|
||||||
|
if mimeType == "" {
|
||||||
|
// A 2xx body that is neither declared nor sniffable as video is a
|
||||||
|
// misconfigured upstream (a JSON job envelope, an HTML error page
|
||||||
|
// behind a proxy) — fail loud rather than hand back garbage as a
|
||||||
|
// playable clip.
|
||||||
|
return nil, &llm.APIError{
|
||||||
|
Provider: m.p.name,
|
||||||
|
Model: m.id,
|
||||||
|
Message: fmt.Sprintf("video response is not a video (content-type %q)", contentType),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: mimeType}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// videoMIME resolves the result MIME type: the response Content-Type when it
|
||||||
|
// is a concrete video type, else content sniffing (mp4/webm magic bytes),
|
||||||
|
// else "" — the caller treats undetectable as an upstream error, unlike the
|
||||||
|
// audio path where the request's format param implies the container.
|
||||||
|
func videoMIME(contentType string, data []byte) string {
|
||||||
|
if mt := mimeFromContentType(contentType, "video/"); mt != "" {
|
||||||
|
return mt
|
||||||
|
}
|
||||||
|
if mt := http.DetectContentType(data); strings.HasPrefix(mt, "video/") {
|
||||||
|
return mt
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// initImageFilename picks the multipart filename hint for the conditioning
|
||||||
|
// frame from its MIME subtype. The name is provider-chosen (never
|
||||||
|
// caller-supplied), so no sanitization is needed.
|
||||||
|
func initImageFilename(mimeType string) string {
|
||||||
|
mt := strings.ToLower(strings.TrimSpace(mimeType))
|
||||||
|
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
||||||
|
mt = parsed
|
||||||
|
}
|
||||||
|
switch mt {
|
||||||
|
case "image/jpeg", "image/jpg":
|
||||||
|
return "frame.jpg"
|
||||||
|
case "image/webp":
|
||||||
|
return "frame.webp"
|
||||||
|
default: // unknown MIME — PNG is the safe hint
|
||||||
|
return "frame.png"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatInt renders an optional int pointer for a form field; nil = "" (omit).
|
||||||
|
func formatInt(v *int) string {
|
||||||
|
if v == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strconv.Itoa(*v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatInt64 renders an optional int64 pointer for a form field; nil = "" (omit).
|
||||||
|
func formatInt64(v *int64) string {
|
||||||
|
if v == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strconv.FormatInt(*v, 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatFloat renders an optional float pointer for a form field; nil = "" (omit).
|
||||||
|
func formatFloat(v *float64) string {
|
||||||
|
if v == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strconv.FormatFloat(*v, 'g', -1, 64)
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatNonZero renders a non-negative int for a form field; 0 = "" (omit,
|
||||||
|
// backend default).
|
||||||
|
func formatNonZero(v int) string {
|
||||||
|
if v == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strconv.Itoa(v)
|
||||||
|
}
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
package llamaswap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestVideoGenerate(t *testing.T) {
|
||||||
|
var gotPath, gotContentType string
|
||||||
|
var gotForm map[string]string
|
||||||
|
var gotFrame []byte
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
gotPath = r.URL.Path
|
||||||
|
gotContentType = r.Header.Get("Content-Type")
|
||||||
|
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||||
|
t.Errorf("parse form: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
gotForm = map[string]string{}
|
||||||
|
for k, v := range r.MultipartForm.Value {
|
||||||
|
gotForm[k] = v[0]
|
||||||
|
}
|
||||||
|
if f, _, err := r.FormFile("input_reference"); err == nil {
|
||||||
|
gotFrame, _ = io.ReadAll(f)
|
||||||
|
f.Close()
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "video/mp4")
|
||||||
|
_, _ = w.Write([]byte("fake-mp4-bytes"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
vm, err := p.VideoModel("videogen-wan")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("VideoModel: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
frame, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||||
|
res, err := vm.Generate(context.Background(),
|
||||||
|
videogen.Request{Prompt: "a cat surfing", InitImage: &videogen.Image{MIME: "image/png", Data: frame}},
|
||||||
|
videogen.WithSize("1280x704"),
|
||||||
|
videogen.WithNumFrames(81),
|
||||||
|
videogen.WithFPS(16),
|
||||||
|
videogen.WithSteps(4),
|
||||||
|
videogen.WithGuidanceScale(1.0),
|
||||||
|
videogen.WithNegativePrompt("blurry"),
|
||||||
|
videogen.WithSeed(42),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate: %v", err)
|
||||||
|
}
|
||||||
|
if string(res.Video.Data) != "fake-mp4-bytes" || res.Video.MIME != "video/mp4" {
|
||||||
|
t.Fatalf("video = %d bytes, MIME %q", len(res.Video.Data), res.Video.MIME)
|
||||||
|
}
|
||||||
|
|
||||||
|
if gotPath != "/v1/videos/sync" {
|
||||||
|
t.Errorf("path = %q", gotPath)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(gotContentType, "multipart/form-data") {
|
||||||
|
t.Errorf("content-type = %q", gotContentType)
|
||||||
|
}
|
||||||
|
want := map[string]string{
|
||||||
|
"model": "videogen-wan",
|
||||||
|
"prompt": "a cat surfing",
|
||||||
|
"negative_prompt": "blurry",
|
||||||
|
"width": "1280",
|
||||||
|
"height": "704",
|
||||||
|
"num_frames": "81",
|
||||||
|
"fps": "16",
|
||||||
|
"num_inference_steps": "4",
|
||||||
|
"guidance_scale": "1",
|
||||||
|
"seed": "42",
|
||||||
|
}
|
||||||
|
for k, v := range want {
|
||||||
|
if gotForm[k] != v {
|
||||||
|
t.Errorf("form[%q] = %q, want %q", k, gotForm[k], v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if string(gotFrame) != string(frame) {
|
||||||
|
t.Errorf("input_reference = %d bytes, want %d", len(gotFrame), len(frame))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoGenerateOmitsUnsetOverrides(t *testing.T) {
|
||||||
|
var gotForm map[string][]string
|
||||||
|
var hadFrame bool
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = r.ParseMultipartForm(32 << 20)
|
||||||
|
gotForm = r.MultipartForm.Value
|
||||||
|
_, _, err := r.FormFile("input_reference")
|
||||||
|
hadFrame = err == nil
|
||||||
|
w.Header().Set("Content-Type", "video/mp4")
|
||||||
|
_, _ = w.Write([]byte("v"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
vm, _ := p.VideoModel("videogen-wan")
|
||||||
|
if _, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"}); err != nil {
|
||||||
|
t.Fatalf("Generate: %v", err)
|
||||||
|
}
|
||||||
|
for _, k := range []string{"negative_prompt", "width", "height", "num_frames", "fps", "num_inference_steps", "guidance_scale", "seed"} {
|
||||||
|
if _, ok := gotForm[k]; ok {
|
||||||
|
t.Errorf("form field %q sent, want omitted", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hadFrame {
|
||||||
|
t.Error("input_reference sent, want omitted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoGenerateValidation(t *testing.T) {
|
||||||
|
p := New(WithBaseURL("http://unused.invalid"))
|
||||||
|
vm, _ := p.VideoModel("videogen-wan")
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
req videogen.Request
|
||||||
|
}{
|
||||||
|
{"empty prompt", videogen.Request{}},
|
||||||
|
{"negative frames", videogen.Request{Prompt: "x", NumFrames: -1}},
|
||||||
|
{"negative fps", videogen.Request{Prompt: "x", FPS: -1}},
|
||||||
|
{"empty init image", videogen.Request{Prompt: "x", InitImage: &videogen.Image{}}},
|
||||||
|
{"bad size", videogen.Request{Prompt: "x", Size: "banana"}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := vm.Generate(context.Background(), tc.req)
|
||||||
|
if !errors.Is(err, llm.ErrUnsupported) {
|
||||||
|
t.Fatalf("err = %v, want ErrUnsupported", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoGenerateUpstreamError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, `{"error":{"message":"boom"}}`, http.StatusInternalServerError)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
vm, _ := p.VideoModel("videogen-wan")
|
||||||
|
_, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"})
|
||||||
|
var apiErr *llm.APIError
|
||||||
|
if !errors.As(err, &apiErr) {
|
||||||
|
t.Fatalf("err = %v, want *llm.APIError", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoGenerateEmptyResponse(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "video/mp4")
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
vm, _ := p.VideoModel("videogen-wan")
|
||||||
|
_, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"})
|
||||||
|
var apiErr *llm.APIError
|
||||||
|
if !errors.As(err, &apiErr) {
|
||||||
|
t.Fatalf("err = %v, want *llm.APIError for empty body", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoModelRequiresBaseURL(t *testing.T) {
|
||||||
|
p := New()
|
||||||
|
if _, err := p.VideoModel("videogen-wan"); err == nil {
|
||||||
|
t.Fatal("VideoModel with no base URL should error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoMIME(t *testing.T) {
|
||||||
|
// A minimal ISO-BMFF prefix http.DetectContentType sniffs as video/mp4
|
||||||
|
// (the "mp4" brand prefix must appear inside the declared ftyp box).
|
||||||
|
mp4Magic := append([]byte{0, 0, 0, 20}, []byte("ftypmp42\x00\x00\x00\x00mp42")...)
|
||||||
|
cases := []struct {
|
||||||
|
contentType string
|
||||||
|
data []byte
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"video/webm", []byte("x"), "video/webm"},
|
||||||
|
{"video/mp4; charset=binary", []byte("x"), "video/mp4"},
|
||||||
|
{"application/octet-stream", mp4Magic, "video/mp4"},
|
||||||
|
// Neither declared nor sniffable as video → "" (Generate errors).
|
||||||
|
{"application/octet-stream", []byte(`{"id":"job-1"}`), ""},
|
||||||
|
{"", []byte("x"), ""},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := videoMIME(tc.contentType, tc.data); got != tc.want {
|
||||||
|
t.Errorf("videoMIME(%q, %q) = %q, want %q", tc.contentType, tc.data, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoGenerateNonVideoBodyErrors(t *testing.T) {
|
||||||
|
// A stock async /v1/videos handler mounted at the sync path (or an HTML
|
||||||
|
// error page behind a proxy) answers 200 with a non-video body — that
|
||||||
|
// must be an error, never a "successful" garbage clip.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"id":"job-1","status":"queued"}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||||
|
vm, _ := p.VideoModel("videogen-wan")
|
||||||
|
_, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"})
|
||||||
|
var apiErr *llm.APIError
|
||||||
|
if !errors.As(err, &apiErr) {
|
||||||
|
t.Fatalf("err = %v, want *llm.APIError for non-video 2xx body", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(apiErr.Message, "not a video") {
|
||||||
|
t.Errorf("message = %q, want mention of non-video body", apiErr.Message)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
// Package videogen is majordomo's canonical video-generation surface. Like
|
||||||
|
// imagegen and audio, it is a deliberately separate contract from the llm
|
||||||
|
// package: video generation shares none of the chat message/tool/stream
|
||||||
|
// machinery, so it gets its own small Provider/Model interface rather than
|
||||||
|
// overloading llm.Model (ADR-0019).
|
||||||
|
//
|
||||||
|
// Zero values mean "backend default" throughout, mirroring imagegen: an empty
|
||||||
|
// Size leaves the backend's default resolution, zero NumFrames/FPS the
|
||||||
|
// backend's default clip length and rate.
|
||||||
|
//
|
||||||
|
// Text-to-video and image-to-video are one surface: a Request with a nil
|
||||||
|
// InitImage is a pure text prompt, a non-nil InitImage conditions generation
|
||||||
|
// on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same
|
||||||
|
// checkpoint, so unlike imagegen there is no separate Editor-style interface.
|
||||||
|
//
|
||||||
|
// The first implementation is provider/llamaswap, which targets the blocking
|
||||||
|
// OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is
|
||||||
|
// the encoded video itself, so one request yields exactly one clip — Result
|
||||||
|
// carries a single Video, not a batch.
|
||||||
|
package videogen
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Image is a conditioning input frame (bytes + MIME). Aliased to
|
||||||
|
// llm.ImagePart so chat-sourced images feed image-to-video without
|
||||||
|
// conversion, mirroring imagegen.Image.
|
||||||
|
type Image = llm.ImagePart
|
||||||
|
|
||||||
|
// Video is one generated video: raw encoded bytes plus a MIME type
|
||||||
|
// (e.g. "video/mp4").
|
||||||
|
type Video struct {
|
||||||
|
// Data is the encoded video container.
|
||||||
|
Data []byte
|
||||||
|
|
||||||
|
// MIME is the video MIME type, e.g. "video/mp4".
|
||||||
|
MIME string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request is a video generation request. Zero values mean "backend default" —
|
||||||
|
// for llama-swap-served models that is the per-model default baked into the
|
||||||
|
// upstream launch flags. A caller overrides only what it explicitly sets.
|
||||||
|
type Request struct {
|
||||||
|
// Prompt is the text description of the video to generate.
|
||||||
|
Prompt string
|
||||||
|
|
||||||
|
// InitImage conditions generation on a starting frame (image-to-video);
|
||||||
|
// nil = pure text-to-video.
|
||||||
|
InitImage *Image
|
||||||
|
|
||||||
|
// Size is the requested resolution, e.g. "1280x704"; "" = backend default.
|
||||||
|
Size string
|
||||||
|
|
||||||
|
// NumFrames is the clip length in frames; 0 = backend default.
|
||||||
|
NumFrames int
|
||||||
|
|
||||||
|
// FPS is the frame rate of the generated clip; 0 = backend default.
|
||||||
|
FPS int
|
||||||
|
|
||||||
|
// Steps is the number of diffusion steps; nil = backend default.
|
||||||
|
Steps *int
|
||||||
|
|
||||||
|
// GuidanceScale is the guidance strength; nil = backend default.
|
||||||
|
// Architecture-sensitive (distilled models want low or none), so prefer
|
||||||
|
// leaving it nil unless the caller knows the target model.
|
||||||
|
GuidanceScale *float64
|
||||||
|
|
||||||
|
// NegativePrompt steers generation away from concepts; "" = none.
|
||||||
|
NegativePrompt string
|
||||||
|
|
||||||
|
// Seed fixes the RNG seed for reproducible output; nil = random.
|
||||||
|
Seed *int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result is the canonical video-generation result.
|
||||||
|
type Result struct {
|
||||||
|
// Video is the generated clip.
|
||||||
|
Video Video
|
||||||
|
|
||||||
|
// Raw is the provider-native response object, an escape hatch for
|
||||||
|
// provider-specific fields. May be nil; never required for normal use.
|
||||||
|
Raw any
|
||||||
|
}
|
||||||
|
|
||||||
|
// Option mutates a Request before it is sent. Options passed to Generate are
|
||||||
|
// applied to a copy of the request, so a Request value can be reused.
|
||||||
|
type Option func(*Request)
|
||||||
|
|
||||||
|
// WithInitImage conditions generation on a starting frame (image-to-video).
|
||||||
|
func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } }
|
||||||
|
|
||||||
|
// WithSize sets the requested resolution (e.g. "1280x704").
|
||||||
|
func WithSize(size string) Option { return func(r *Request) { r.Size = size } }
|
||||||
|
|
||||||
|
// WithNumFrames sets the clip length in frames.
|
||||||
|
func WithNumFrames(n int) Option { return func(r *Request) { r.NumFrames = n } }
|
||||||
|
|
||||||
|
// WithFPS sets the frame rate of the generated clip.
|
||||||
|
func WithFPS(fps int) Option { return func(r *Request) { r.FPS = fps } }
|
||||||
|
|
||||||
|
// WithSteps overrides the number of diffusion steps.
|
||||||
|
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
|
||||||
|
|
||||||
|
// WithGuidanceScale overrides the guidance strength.
|
||||||
|
func WithGuidanceScale(s float64) Option { return func(r *Request) { r.GuidanceScale = &s } }
|
||||||
|
|
||||||
|
// WithNegativePrompt sets a negative prompt.
|
||||||
|
func WithNegativePrompt(s string) Option { return func(r *Request) { r.NegativePrompt = s } }
|
||||||
|
|
||||||
|
// WithSeed fixes the RNG seed for reproducible output.
|
||||||
|
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
|
||||||
|
|
||||||
|
// Apply returns a copy of the request with all options applied. Providers call
|
||||||
|
// this once at the top of Generate.
|
||||||
|
func (r Request) Apply(opts ...Option) Request {
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&r)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// Model generates a video clip from a text prompt and optional conditioning
|
||||||
|
// frame. It is intentionally narrower than llm.Model — no Stream, no
|
||||||
|
// Capabilities, no tool calls.
|
||||||
|
type Model interface {
|
||||||
|
// Generate produces one clip for the request. Generation is slow
|
||||||
|
// (minutes on consumer hardware) and the call blocks until the clip is
|
||||||
|
// ready; callers bound it with a context deadline.
|
||||||
|
Generate(ctx context.Context, req Request, opts ...Option) (*Result, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ModelOption configures a Model at construction time (Provider.VideoModel).
|
||||||
|
// Reserved for future per-model settings; present now so the interface is
|
||||||
|
// forward-compatible.
|
||||||
|
type ModelOption func(*ModelConfig)
|
||||||
|
|
||||||
|
// ModelConfig carries per-model construction settings.
|
||||||
|
type ModelConfig struct{}
|
||||||
|
|
||||||
|
// ApplyModelOptions folds options into a config.
|
||||||
|
func ApplyModelOptions(opts []ModelOption) ModelConfig {
|
||||||
|
var cfg ModelConfig
|
||||||
|
for _, opt := range opts {
|
||||||
|
opt(&cfg)
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
// Provider mints video Models bound to one backend. It mirrors llm.Provider
|
||||||
|
// but for video generation.
|
||||||
|
type Provider interface {
|
||||||
|
// Name is the registry identifier for the provider.
|
||||||
|
Name() string
|
||||||
|
|
||||||
|
// VideoModel returns a Model bound to the given id (passed through to the
|
||||||
|
// backend verbatim; no catalog validation).
|
||||||
|
VideoModel(id string, opts ...ModelOption) (Model, error)
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package videogen
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestApplyDoesNotMutateOriginal(t *testing.T) {
|
||||||
|
orig := Request{Prompt: "a cat"}
|
||||||
|
got := orig.Apply(
|
||||||
|
WithSize("1280x704"),
|
||||||
|
WithNumFrames(81),
|
||||||
|
WithFPS(16),
|
||||||
|
WithSteps(4),
|
||||||
|
WithGuidanceScale(1.0),
|
||||||
|
WithNegativePrompt("blurry"),
|
||||||
|
WithSeed(42),
|
||||||
|
WithInitImage(Image{MIME: "image/png", Data: []byte{1}}),
|
||||||
|
)
|
||||||
|
|
||||||
|
if orig.Size != "" || orig.NumFrames != 0 || orig.FPS != 0 || orig.Steps != nil ||
|
||||||
|
orig.GuidanceScale != nil || orig.NegativePrompt != "" || orig.Seed != nil || orig.InitImage != nil {
|
||||||
|
t.Fatalf("original mutated: %+v", orig)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got.Size != "1280x704" || got.NumFrames != 81 || got.FPS != 16 {
|
||||||
|
t.Errorf("size/frames/fps = %q/%d/%d", got.Size, got.NumFrames, got.FPS)
|
||||||
|
}
|
||||||
|
if got.Steps == nil || *got.Steps != 4 {
|
||||||
|
t.Errorf("steps = %v", got.Steps)
|
||||||
|
}
|
||||||
|
if got.GuidanceScale == nil || *got.GuidanceScale != 1.0 {
|
||||||
|
t.Errorf("guidance = %v", got.GuidanceScale)
|
||||||
|
}
|
||||||
|
if got.NegativePrompt != "blurry" {
|
||||||
|
t.Errorf("negative = %q", got.NegativePrompt)
|
||||||
|
}
|
||||||
|
if got.Seed == nil || *got.Seed != 42 {
|
||||||
|
t.Errorf("seed = %v", got.Seed)
|
||||||
|
}
|
||||||
|
if got.InitImage == nil || got.InitImage.MIME != "image/png" {
|
||||||
|
t.Errorf("init image = %+v", got.InitImage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyModelOptions(t *testing.T) {
|
||||||
|
called := false
|
||||||
|
ApplyModelOptions([]ModelOption{func(*ModelConfig) { called = true }})
|
||||||
|
if !called {
|
||||||
|
t.Fatal("model option not applied")
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user