Compare commits
51
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f837115a55 | ||
|
|
f8ced9c629 | ||
|
|
0760cf96d4 | ||
|
|
8670ed22be | ||
|
|
f1f2b653c3 | ||
|
|
31d6b59356 | ||
|
|
02cd561eaf | ||
|
|
e779169416 | ||
|
|
5994d96921 | ||
|
|
588e092465 | ||
|
|
dbc96898ab | ||
|
|
44fcfbb273 | ||
|
|
203895696c | ||
|
|
1bbbdaa1e5 | ||
|
|
21b4775d16 | ||
|
|
127966bb3a | ||
|
|
0bd14e01b3 | ||
|
|
6995a8dee1 | ||
|
|
ae2615ca68 | ||
|
|
ff832cb6b5 | ||
|
|
372bf826aa | ||
|
|
0ff90d80f6 | ||
|
|
316a430116 | ||
|
|
2c70d32fd4 | ||
|
|
a941f5ff4a | ||
|
|
9922166d7a | ||
|
|
68bf7157d3 | ||
|
|
54b295efc3 | ||
|
|
c5f84b95d8 | ||
|
|
fcbb01b729 | ||
|
|
2bfffff47a | ||
|
|
a9c864d3f7 | ||
|
|
95147f7582 | ||
|
|
036406b221 | ||
|
|
2660693132 | ||
|
|
56b5b000a6 | ||
|
|
31bccf6e19 | ||
|
|
966ea16166 | ||
|
|
5f175ecf82 | ||
|
|
b3a172a053 | ||
|
|
e6987f54b2 | ||
|
|
cd43009672 | ||
|
|
499ee16222 | ||
|
|
a07cba25dc | ||
|
|
a4cf9202cd | ||
|
|
19972b1864 | ||
|
|
fd8d56c3c1 | ||
|
|
89a0b2bdd3 | ||
|
|
148069f6ef | ||
|
|
cf2d83f157 | ||
|
|
cacce61ddc |
@@ -6,6 +6,8 @@ OLLAMA_API_KEY=your-ollama-cloud-key-here
|
||||
|
||||
# Built-in provider keys (each optional; only needed for the providers you use).
|
||||
#OPENAI_API_KEY=sk-...
|
||||
#KIMI_API_KEY=sk-... # Moonshot AI (Kimi); provider name "kimi"
|
||||
#QWEN_API_KEY=sk-... # Alibaba Model Studio (Qwen); provider name "qwen"
|
||||
#ANTHROPIC_API_KEY=sk-ant-...
|
||||
#GOOGLE_API_KEY=...
|
||||
|
||||
|
||||
@@ -38,10 +38,11 @@ jobs:
|
||||
&& (github.actor == 'steve'
|
||||
|| github.actor == 'fizi'
|
||||
|| github.actor == 'dazed'))
|
||||
# Tracks gadfly's v1 release tag — a curated pointer re-moved on each release
|
||||
# (unlike @main, which moves on every push). Central swarm tuning propagates
|
||||
# here automatically; the tradeoff vs a full sha pin is that v1 is mutable.
|
||||
uses: steve/gadfly/.gitea/workflows/review-reusable.yml@5007597cf921dc3f0a83c708878facfe65fd8e8b
|
||||
# Pinned to an immutable gadfly commit (not @v1): our act_runners are long-lived
|
||||
# and cache the reusable-workflow ref, so a moved v1 tag keeps resolving to the
|
||||
# stale cached copy. A unique sha forces a cache miss → fresh fetch. Bump this
|
||||
# sha to adopt central swarm changes.
|
||||
uses: steve/gadfly/.gitea/workflows/review-reusable.yml@c9dab69d143cb614c1840a5b06d6ffc358f4752d
|
||||
# Least privilege: forward only the review secrets (not `secrets: inherit`,
|
||||
# which would expose every repo secret). GITEA_TOKEN is the automatic token.
|
||||
secrets:
|
||||
@@ -52,3 +53,6 @@ jobs:
|
||||
with:
|
||||
# Consumer-specific allow-list; everything else is inherited.
|
||||
allowed_users: "steve,fizi,dazed"
|
||||
# Gitea >= 1.27 does not propagate dispatch inputs into a called workflow's
|
||||
# github.event — thread the PR number explicitly (empty on non-dispatch events).
|
||||
pr_number: ${{ github.event.inputs.pr_number }}
|
||||
|
||||
@@ -121,6 +121,8 @@ Chains are health-tracked per target:
|
||||
| Provider | Spec name | Key env var | Default endpoint |
|
||||
|----------|-----------|-------------|------------------|
|
||||
| OpenAI (+compatible) | `openai` | `OPENAI_API_KEY` | https://api.openai.com/v1 |
|
||||
| Kimi (Moonshot AI) | `kimi` | `KIMI_API_KEY` | https://api.moonshot.ai/v1 |
|
||||
| Qwen (Alibaba) | `qwen` | `QWEN_API_KEY` | https://dashscope-intl.aliyuncs.com/compatible-mode/v1 |
|
||||
| Anthropic (+compatible) | `anthropic` | `ANTHROPIC_API_KEY` | https://api.anthropic.com |
|
||||
| Google (Gemini) | `google` | `GOOGLE_API_KEY` / `GEMINI_API_KEY` | Gemini API (official SDK) |
|
||||
| Ollama Cloud | `ollama-cloud` | `OLLAMA_API_KEY` | https://ollama.com |
|
||||
@@ -128,6 +130,24 @@ Chains are health-tracked per target:
|
||||
| foreman | `foreman` | — (token via DSN) | requires an LLM_* DSN or `ollama.Foreman(url, token)` |
|
||||
| llama-swap | `llama-swap` | — (token via DSN) | requires an LLM_* DSN or `llamaswap.New(...)` |
|
||||
|
||||
Kimi is Moonshot AI's OpenAI-compatible Chat Completions endpoint, so it reuses
|
||||
the openai client (like llama-swap). The `kimi` built-in defaults to the
|
||||
international endpoint; reach the China endpoint (or any other host) with a
|
||||
`kimi://` DSN, e.g. `LLM_KCN=kimi://[email protected]/v1`.
|
||||
|
||||
Qwen is the same shape: Alibaba Model Studio's OpenAI-compatible mode, reusing
|
||||
the openai client. The `qwen` built-in defaults to the international
|
||||
(Singapore) host; reach the China host or a workspace-scoped regional one with
|
||||
a `qwen://` DSN, e.g.
|
||||
`LLM_QCN=qwen://[email protected]/compatible-mode/v1`. Model Studio
|
||||
also fronts the same models with an Anthropic-compatible `/v1/messages` shim —
|
||||
majordomo does **not** use it, because on that surface `reasoning_effort` is
|
||||
dropped, `Request.Schema` stops being enforced, and cached-token accounting
|
||||
disappears; see [ADR-0027](docs/adr/0027-qwen-builtin.md). Two Alibaba-side
|
||||
quirks are worth knowing: thinking is on by default for some models (e.g.
|
||||
`qwen3.7-plus`), and the Qwen3 open-source models require streaming while
|
||||
thinking, so buffered `Generate` calls want a Max/Plus model.
|
||||
|
||||
OpenAI-compatible / Anthropic-compatible endpoints: construct the provider
|
||||
with a name and base URL and register it —
|
||||
|
||||
@@ -159,7 +179,7 @@ m, _ := reg.Parse("m5/qwen3:30b,m1/qwen3:30b,thinking")
|
||||
```
|
||||
|
||||
DSN format: `scheme://[token@]host[/path]`, scheme ∈ `foreman`, `ollama`,
|
||||
`ollama-cloud`, `openai`, `anthropic`, `google`/`gemini`, `llama-swap`,
|
||||
`ollama-cloud`, `openai`, `kimi`, `qwen`, `anthropic`, `google`/`gemini`, `llama-swap`,
|
||||
`llama-swaps`, or any scheme you add with `RegisterScheme`. The token is the
|
||||
credential (bearer token / API key); the base URL is always `https://host[/path]`
|
||||
— except `llama-swap`, which builds `http://host[:port]` since it's local-first
|
||||
@@ -266,17 +286,30 @@ tr, err := tm.Transcribe(ctx, audio.TranscriptionRequest{
|
||||
voices, err := ls.ListVoices(ctx, "kokoro") // []string of voice ids
|
||||
```
|
||||
|
||||
## Video: text-to-video + image-to-video
|
||||
## Video: text-to-video, image-to-video, first-last-frame
|
||||
|
||||
Video generation lives in the `videogen` package (ADR-0019), mirroring
|
||||
imagegen/audio: one small `Model` contract, zero values mean backend
|
||||
defaults, bytes in/out. Text-to-video and image-to-video are one surface —
|
||||
a nil `InitImage` is a pure text prompt; setting it conditions generation
|
||||
on that frame (hybrid checkpoints like Wan 2.2 TI2V serve both). First
|
||||
backend: llama-swap (blocking `/v1/videos/sync`, vLLM-Omni style — the
|
||||
defaults, bytes in/out. All modes are one surface, selected by which
|
||||
keyframes are set rather than by a mode flag:
|
||||
|
||||
| `InitImage` | `LastImage` | mode |
|
||||
|---|---|---|
|
||||
| nil | nil | text-to-video |
|
||||
| set | nil | image-to-video (hybrid checkpoints like Wan 2.2 TI2V serve both) |
|
||||
| set | set | first-last-frame — both ends pinned |
|
||||
| nil | set | pin the destination, model invents the approach |
|
||||
|
||||
First backend: llama-swap (blocking `/v1/videos/sync`, vLLM-Omni style — the
|
||||
response body is the encoded clip, so `Result` carries a single `Video`).
|
||||
Generation runs for minutes; bound the call with a context deadline.
|
||||
|
||||
**`LastImage` support is per-model and cannot be detected.** A backend that
|
||||
does not understand a trailing keyframe ignores the part and returns an
|
||||
ordinary clip — indistinguishable from success. There is no capability bit,
|
||||
because the contract has no way to learn one, so a caller depending on the
|
||||
pin must establish support out of band.
|
||||
|
||||
```go
|
||||
vm, _ := ls.VideoModel("videogen-wan22-5b")
|
||||
res, err := vm.Generate(ctx, videogen.Request{Prompt: "a cat surfing"},
|
||||
@@ -400,6 +433,8 @@ to build one.
|
||||
| Provider | Resolve/Parse | Chat | Streaming | Tools | Structured | Images | Env DSN |
|
||||
|----------------------|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
|
||||
| OpenAI (+compatible) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Kimi (Moonshot AI) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅³ | ✅ |
|
||||
| Qwen (Alibaba) | ✅ | ✅ | ✅ | ✅ | ✅⁴ | ✅⁴ | ✅ |
|
||||
| Anthropic (+compat) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Google (Gemini) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Ollama Cloud | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
@@ -420,6 +455,17 @@ transcription** (`audio`), **video generation** (`videogen`) — separate
|
||||
axes, not shown above — plus a `Health`
|
||||
probe and management methods on `*llamaswap.Provider`.
|
||||
|
||||
³ Kimi reuses the openai client, so image *inputs* are supported at the client
|
||||
level; whether a call succeeds depends on the Moonshot model — only the vision
|
||||
variants (e.g. `moonshot-v1-8k-vision-preview`) accept images.
|
||||
|
||||
⁴ Qwen also reuses the openai client (ADR-0027), so both columns are present at
|
||||
the client level and gated by the Model Studio model you name: `json_schema`
|
||||
structured output is on the Max/Plus families, image inputs on the `qwen-vl-*`
|
||||
/ `qwen3-vl-*` models. `reasoning_effort` rides through as a top-level field —
|
||||
one reason the built-in speaks OpenAI-compat rather than Model Studio's
|
||||
Anthropic-compat shim.
|
||||
|
||||
Notes: Ollama has no native tool_choice — `"none"` drops the tools;
|
||||
`"required"`/named choices are best-effort ignored there. Ollama Cloud
|
||||
ignores the `format` field (verified live), so the provider also states
|
||||
|
||||
+56
-13
@@ -125,10 +125,12 @@ func WithCompactor(fn func(ctx context.Context, msgs []llm.Message) ([]llm.Messa
|
||||
}
|
||||
|
||||
// WithToolErrorLimits installs loop guards: maxConsecutiveErrors bounds
|
||||
// successive steps whose tool results were ALL errors, and
|
||||
// maxSameCallRepeats bounds identical (name + arguments) tool calls within
|
||||
// one run. Either guard tripping ends the run with ErrToolLoop and the
|
||||
// partial result. Zero disables a guard.
|
||||
// successive steps whose tool results were ALL errors, and maxSameCallRepeats
|
||||
// bounds identical (name + arguments) tool calls that ALSO return an unchanged
|
||||
// result within one run — a call whose result keeps advancing (e.g. polling a
|
||||
// long-running background job) is progress and never trips this guard. Either
|
||||
// guard tripping ends the run with ErrToolLoop and the partial result. Zero
|
||||
// disables a guard.
|
||||
func WithToolErrorLimits(maxConsecutiveErrors, maxSameCallRepeats int) Option {
|
||||
return func(a *Agent) {
|
||||
a.maxConsecutiveToolErrors = maxConsecutiveErrors
|
||||
@@ -251,6 +253,15 @@ func (a *Agent) mergedTools() (map[string]llm.Tool, []llm.Tool, error) {
|
||||
// Run executes the loop: send the conversation; while the model requests
|
||||
// tools, execute them and feed results back; stop on a final answer,
|
||||
// MaxSteps, or an unrecoverable model error.
|
||||
// repeatState is the per-signature bookkeeping for the progress-aware same-call
|
||||
// guard: count is the run-length of consecutive identical calls that returned
|
||||
// lastResult (the previous call's encoded result). A changed result resets count
|
||||
// to 1. See the guard block in Run.
|
||||
type repeatState struct {
|
||||
count int
|
||||
lastResult string
|
||||
}
|
||||
|
||||
func (a *Agent) Run(ctx context.Context, input string, opts ...RunOption) (*Result, error) {
|
||||
var rc runConfig
|
||||
for _, opt := range opts {
|
||||
@@ -274,9 +285,12 @@ func (a *Agent) Run(ctx context.Context, input string, opts ...RunOption) (*Resu
|
||||
reqOpts := append(append([]llm.Option(nil), a.reqOpts...), rc.reqOpts...)
|
||||
system := a.systemPrompt()
|
||||
|
||||
// Loop-guard state (WithToolErrorLimits).
|
||||
// Loop-guard state (WithToolErrorLimits). repeatStates tracks, per identical
|
||||
// (name+arguments) signature, the run-length of consecutive calls that
|
||||
// returned the same result and that last result — see the same-call guard
|
||||
// below.
|
||||
consecutiveErrorSteps := 0
|
||||
callCounts := make(map[string]int)
|
||||
repeatStates := make(map[string]*repeatState)
|
||||
|
||||
maxSteps := func() int {
|
||||
if a.maxStepsFunc != nil {
|
||||
@@ -330,13 +344,6 @@ func (a *Agent) Run(ctx context.Context, input string, opts ...RunOption) (*Resu
|
||||
result.Messages = msgs
|
||||
return result, err
|
||||
}
|
||||
if a.maxSameCallRepeats > 0 {
|
||||
sig := call.Name + "\x00" + string(call.Arguments)
|
||||
callCounts[sig]++
|
||||
if callCounts[sig] > a.maxSameCallRepeats {
|
||||
repeatTripped = call.Name
|
||||
}
|
||||
}
|
||||
tool, ok := byName[call.Name]
|
||||
if !ok {
|
||||
results = append(results, llm.ToolResult{
|
||||
@@ -356,6 +363,42 @@ func (a *Agent) Run(ctx context.Context, input string, opts ...RunOption) (*Resu
|
||||
a.notify(rc, step)
|
||||
msgs = append(msgs, llm.ToolResultsMessage(results...))
|
||||
|
||||
// Same-call repeat guard (progress-aware). An identical (name+arguments)
|
||||
// call only counts toward the loop trip when its result is unchanged from
|
||||
// the previous identical call: a call whose result keeps advancing —
|
||||
// canonically polling a long-running background job — is progress and
|
||||
// resets its count, while a genuinely stuck call returning the same output
|
||||
// trips once it exceeds the ceiling. Result equality is exact-string on
|
||||
// the full encoded content, chosen to err toward NOT tripping: a hung job
|
||||
// whose poll still reports a ticking field is left to MaxRuntime / the
|
||||
// job's own ceiling rather than risking a false kill of real progress.
|
||||
// results[i] pairs with resp.ToolCalls[i] — every call appends exactly one
|
||||
// result (unknown tools append an error result before continue) and the
|
||||
// only early exit above is a full return on ctx cancellation.
|
||||
if a.maxSameCallRepeats > 0 {
|
||||
for i, call := range resp.ToolCalls {
|
||||
sig := call.Name + "\x00" + string(call.Arguments)
|
||||
resKey := results[i].Content
|
||||
if results[i].IsError {
|
||||
resKey = "e\x00" + resKey
|
||||
}
|
||||
st := repeatStates[sig]
|
||||
if st == nil {
|
||||
st = &repeatState{}
|
||||
repeatStates[sig] = st
|
||||
}
|
||||
if st.count > 0 && st.lastResult == resKey {
|
||||
st.count++
|
||||
} else {
|
||||
st.count = 1
|
||||
}
|
||||
st.lastResult = resKey
|
||||
if st.count > a.maxSameCallRepeats {
|
||||
repeatTripped = call.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if repeatTripped != "" {
|
||||
result.Messages = msgs
|
||||
return result, fmt.Errorf("%w: %q called identically more than %d times",
|
||||
|
||||
+145
-25
@@ -14,8 +14,8 @@ import (
|
||||
// their answer to the final, tool-free turn. But some models — notably several
|
||||
// open-weight ones — "front-load" their full answer into an earlier turn that
|
||||
// ALSO calls a tool (e.g. answer text alongside a citation call), then close
|
||||
// with a degenerate terminal turn that is not itself the answer. Two shapes are
|
||||
// recovered from the transcript (zero extra model calls):
|
||||
// with a degenerate terminal turn that is not itself the answer. Three shapes
|
||||
// are recovered from the transcript (zero extra model calls):
|
||||
//
|
||||
// - a trivial back-reference ("(Already answered above.)", "see above", …):
|
||||
// the real answer sits earlier, so recover it and DISCARD the worthless
|
||||
@@ -25,25 +25,44 @@ import (
|
||||
// glm-5.2 "cite" pattern behind mort issue #1418). The citations are real,
|
||||
// useful content — unlike a back-reference — so recover the prior answer and
|
||||
// KEEP the citations, appended below it.
|
||||
// - a bookkeeping closer ("Citations are logged. Short version: …"): the
|
||||
// model acknowledged the citation round and compressed the answer it had
|
||||
// already written into a one-liner (mort run b3cb9ee9 — a 2,089-char answer
|
||||
// shrank to a 153-byte closer at delivery). The compression is strictly
|
||||
// poorer than the front-loaded answer, so recover the prior turn and
|
||||
// DISCARD the closer — but only when the prior turn clearly dwarfs it,
|
||||
// because unlike a back-reference this closer DOES carry answer content
|
||||
// (see modeSummary).
|
||||
//
|
||||
// A citations addendum is tested first and wins over the back-reference test (a
|
||||
// short terminal can be both), so its links are never discarded. When the
|
||||
// terminal text stands on its own it is returned unchanged; when it is
|
||||
// degenerate but nothing better can be recovered, it is returned as-is (a bare
|
||||
// sources list still beats nothing).
|
||||
// A citations addendum is tested first and wins over the other two (a short
|
||||
// terminal can match more than one shape), so its links are never discarded.
|
||||
// The back-reference test wins over the summary-closer test: a terminal
|
||||
// matching both ("Citations are logged. As I said above…") carries no answer
|
||||
// content of its own, so the looser back-ref recovery bar — not the summary
|
||||
// closer's dwarf ratio — is the right one. When the terminal text stands
|
||||
// on its own it is returned unchanged; when it is degenerate but nothing
|
||||
// better can be recovered, it is returned as-is (a compressed answer still
|
||||
// beats nothing).
|
||||
//
|
||||
// msgs must already include the terminal assistant message as its last element
|
||||
// (the loop appends it before calling this); terminal is that message's text.
|
||||
func finalOutput(msgs []llm.Message, terminal string) string {
|
||||
citations := isCitationsOnly(terminal)
|
||||
if !citations && !isWeakFinal(terminal) {
|
||||
mode := modeBackRef
|
||||
switch {
|
||||
case isCitationsOnly(terminal):
|
||||
mode = modeCitations
|
||||
case isWeakFinal(terminal):
|
||||
mode = modeBackRef
|
||||
case isSummaryCloser(terminal):
|
||||
mode = modeSummary
|
||||
default:
|
||||
return terminal
|
||||
}
|
||||
rec, ok := lastSubstantiveAssistantText(msgs, terminal, citations)
|
||||
rec, ok := lastSubstantiveAssistantText(msgs, terminal, mode)
|
||||
if !ok {
|
||||
return terminal
|
||||
}
|
||||
if citations {
|
||||
if mode == modeCitations {
|
||||
// Preserve the citations addendum below the recovered answer, unless the
|
||||
// recovered turn already carries it (guards against a duplicate sources
|
||||
// block when the front-loaded turn included its own citations). The
|
||||
@@ -65,11 +84,64 @@ func stripURLAngles(s string) string {
|
||||
return strings.NewReplacer("<", "", ">", "").Replace(s)
|
||||
}
|
||||
|
||||
// recoveryMode selects the bar a prior assistant turn must clear to replace
|
||||
// the terminal turn (see isSubstantiveAnswer) and what finalOutput does with
|
||||
// the terminal once recovery succeeds.
|
||||
type recoveryMode int
|
||||
|
||||
const (
|
||||
// modeBackRef: the terminal is empty or a pure back-reference — worthless
|
||||
// on its own, so any real prior answer replaces it and it is discarded.
|
||||
modeBackRef recoveryMode = iota
|
||||
// modeCitations: the terminal is a sources-only addendum — not a rival
|
||||
// answer, so the dwarf ratio is skipped and the addendum is kept, appended
|
||||
// below the recovered answer.
|
||||
modeCitations
|
||||
// modeSummary: the terminal acknowledges the citation round and may carry
|
||||
// a short compression of the front-loaded answer. Unlike a back-reference
|
||||
// it DOES contain answer content, so it is only replaced when a prior turn
|
||||
// clearly dwarfs it — the ratio is mandatory at every length, the recovery
|
||||
// scan stops at the most recent user message (a compression can only be of
|
||||
// THIS turn's answer; never resurrect one from an earlier question), and
|
||||
// the closer is discarded (its content is a strict subset of what it
|
||||
// replaced).
|
||||
modeSummary
|
||||
)
|
||||
|
||||
// backRefRe matches a terminal turn that merely points back to an earlier
|
||||
// message instead of stating the answer ("(Already answered above.)",
|
||||
// "see above", "as I said", ...).
|
||||
var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(said|mentioned|stated|noted)|answered (that )?above|per my (previous|earlier))`)
|
||||
|
||||
// summaryCloserRe matches a terminal turn that OPENS with a bookkeeping
|
||||
// acknowledgment of the citation round — "Citations are logged.", "Sources
|
||||
// cited.", "Logged the citations." — the shape a model produces when it
|
||||
// front-loaded its answer into an earlier cite-call turn and closes by
|
||||
// acknowledging the tool results, often followed by a "Short version: …"
|
||||
// compression of the answer it already wrote. The ack clause must end at a
|
||||
// sentence terminator ([.!]) DIRECTLY after the verb: "The citations are
|
||||
// recorded in the court transcript…" is a real answer about citations, not
|
||||
// bookkeeping, and must never match. A compression marker without the ack
|
||||
// ("Short version: no.") is deliberately out of scope — a user who asked for
|
||||
// brevity would be answered with exactly that shape, and misclassifying it
|
||||
// would hijack a legitimate answer; an unmatched closer merely keeps today's
|
||||
// behavior (fail closed). Assembled from named fragments so the alternations
|
||||
// stay legible and extendable.
|
||||
const (
|
||||
summaryPreface = `((done|all set|ok(ay)?)[\s,.!:—-]+)?` // optional "Done —" style opener
|
||||
summaryNouns = `(citations?|sources?|references?|claims?)`
|
||||
// "all" appears here AND in summaryArticle on purpose: as a quantifier
|
||||
// between noun and verb ("Citations all logged.") and as a determiner
|
||||
// before the noun ("All claims cited.", "Logged all the citations.").
|
||||
summaryCopulas = `((are|were|have\s+been|all)\s+)*`
|
||||
summaryVerbs = `(logged|recorded|cited|saved|noted|captured|filed)`
|
||||
summaryArticle = `((all|the)\s+)*` // star, not ?: "Logged all the citations."
|
||||
)
|
||||
|
||||
var summaryCloserRe = regexp.MustCompile(`(?i)^` + leadingMarkers + summaryPreface +
|
||||
`(` + summaryArticle + summaryNouns + `\s+` + summaryCopulas + summaryVerbs +
|
||||
`|logged\s+` + summaryArticle + summaryNouns + `)[.!]`)
|
||||
|
||||
// preambleRe matches intent-announcing prefixes ("Let me search...", "I'll
|
||||
// check...") so a preamble is never mistaken for the answer during recovery.
|
||||
var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|sure[,. ]|okay[,. ]|on it|checking)`)
|
||||
@@ -83,7 +155,15 @@ var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|
|
||||
// the colon/dash separator. Anchored at ^ so a normal answer that merely
|
||||
// mentions "sources" mid-sentence, or ends with a "Sources:" section AFTER its
|
||||
// prose, is never matched.
|
||||
var citationLabelRe = regexp.MustCompile(`(?i)^[\s>#*_+-]*(sources?|references?|citations?|works cited|further reading)\b[\s*_]*[::\-—]`)
|
||||
var citationLabelRe = regexp.MustCompile(`(?i)^` + leadingMarkers +
|
||||
`(sources?|references?|citations?|works cited|further reading)\b[\s*_]*[::\-—]`)
|
||||
|
||||
// leadingMarkers tolerates markdown noise before a label: emphasis (*, _),
|
||||
// list (-, +, *), block-quote (>), and ATX-heading (#) markers, with their
|
||||
// whitespace. Shared by citationLabelRe and summaryCloserRe so the two
|
||||
// classifiers cannot drift apart (the first draft of the summary class
|
||||
// dropped '+' by hand-copying this set).
|
||||
const leadingMarkers = `[\s>#*_+-]*`
|
||||
|
||||
// linkRe matches a whole markdown link "[label](url)" or a bare URL. Used both
|
||||
// to require that a citations terminal carries at least one link and to strip
|
||||
@@ -114,6 +194,12 @@ const (
|
||||
// be at most len/N of the whole, so a prose answer that merely opens with
|
||||
// "Source:" and cites a URL mid-sentence is not mistaken for a bare list.
|
||||
citationDominatedDivisor = 3
|
||||
// summaryCloserMaxChars bounds a summary closer: room for the ack sentence
|
||||
// plus a couple of compression sentences (the b3cb9ee9 closer was 153
|
||||
// bytes — Go len(), which is what every threshold here compares). Beyond
|
||||
// this the "short version" is substantial enough that replacing it risks
|
||||
// losing content the front-loaded turn never had.
|
||||
summaryCloserMaxChars = 300
|
||||
)
|
||||
|
||||
// isWeakFinal reports whether a terminal turn's text fails to stand on its own
|
||||
@@ -152,14 +238,39 @@ func isCitationsOnly(s string) bool {
|
||||
return len(residue) <= len(t)/citationDominatedDivisor
|
||||
}
|
||||
|
||||
// isSummaryCloser reports whether a terminal turn is a bookkeeping closer: it
|
||||
// opens with a complete "citations are logged"-style ack sentence (see
|
||||
// summaryCloserRe) and is short enough that whatever follows the ack can only
|
||||
// be a compression of an earlier, fuller answer. Whether that fuller answer
|
||||
// actually exists is modeSummary's job — the dwarf ratio in
|
||||
// isSubstantiveAnswer keeps a matching closer in place when nothing earlier
|
||||
// clearly outweighs it.
|
||||
func isSummaryCloser(s string) bool {
|
||||
t := strings.TrimSpace(s)
|
||||
if t == "" || len(t) > summaryCloserMaxChars {
|
||||
return false
|
||||
}
|
||||
return summaryCloserRe.MatchString(t)
|
||||
}
|
||||
|
||||
// lastSubstantiveAssistantText scans msgs newest→oldest (skipping the terminal
|
||||
// turn and empty tool-only turns) for the most recent assistant turn whose text
|
||||
// reads like a real answer. citations selects the recovery bar (see
|
||||
// reads like a real answer. mode selects the recovery bar (see
|
||||
// isSubstantiveAnswer). Returns ("", false) when nothing qualifies.
|
||||
func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations bool) (string, bool) {
|
||||
func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, mode recoveryMode) (string, bool) {
|
||||
tt := strings.TrimSpace(terminal)
|
||||
for i := len(msgs) - 1; i >= 0; i-- {
|
||||
m := msgs[i]
|
||||
if mode == modeSummary && m.Role == llm.RoleUser {
|
||||
// A summary closer compresses THIS turn's front-loaded answer, so
|
||||
// the scan must not cross into an earlier question: once the dwarf
|
||||
// ratio has rejected the current turn's text, walking further back
|
||||
// would resurrect a stale answer to a DIFFERENT question — strictly
|
||||
// worse than keeping the closer. (A mid-run steer message is also a
|
||||
// user-role boundary; recovery then fails closed, which is fine.)
|
||||
// The other modes keep their historical unbounded scan.
|
||||
break
|
||||
}
|
||||
if m.Role != llm.RoleAssistant {
|
||||
continue
|
||||
}
|
||||
@@ -167,7 +278,7 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations
|
||||
if txt == "" || txt == tt {
|
||||
continue // the terminal turn itself, or an empty tool-only turn
|
||||
}
|
||||
if isSubstantiveAnswer(txt, tt, citations) {
|
||||
if isSubstantiveAnswer(txt, tt, mode) {
|
||||
return txt, true
|
||||
}
|
||||
}
|
||||
@@ -177,20 +288,29 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations
|
||||
// isSubstantiveAnswer reports whether txt (a prior assistant turn) reads like a
|
||||
// real answer rather than a preamble, relative to the terminal text.
|
||||
//
|
||||
// A sufficiently long turn (>= recoverMinChars) is accepted unconditionally: a
|
||||
// multi-hundred-char turn is an answer even when it opens conversationally
|
||||
// ("Sure, here's…", "Let me explain: …"), so the preamble filter is NOT applied
|
||||
// to it — applying it there would drop a legitimate long front-loaded answer.
|
||||
// Only in the borderline band does a turn have to clear a floor, not read like a
|
||||
// short planning preamble ("Let me look that up…"), and — unless the terminal is
|
||||
// a citations addendum (not a rival answer, so its length is irrelevant) — also
|
||||
// clearly dwarf the terminal.
|
||||
func isSubstantiveAnswer(txt, terminal string, citations bool) bool {
|
||||
// modeSummary demands the dwarf ratio FIRST, at every length: a summary closer
|
||||
// carries a real (compressed) answer, so replacing it is only justified when
|
||||
// the prior turn is clearly the fuller original it was compressed from.
|
||||
//
|
||||
// A sufficiently long turn (>= recoverMinChars) is otherwise accepted
|
||||
// unconditionally: a multi-hundred-char turn is an answer even when it opens
|
||||
// conversationally ("Sure, here's…", "Let me explain: …"), so the preamble
|
||||
// filter is NOT applied to it — applying it there would drop a legitimate long
|
||||
// front-loaded answer. Only in the borderline band does a turn have to clear a
|
||||
// floor, not read like a short planning preamble ("Let me look that up…"),
|
||||
// and — for modeBackRef only — also clearly dwarf the terminal (a citations
|
||||
// addendum is not a rival answer, so its length is irrelevant; a summary
|
||||
// closer already proved the ratio above).
|
||||
func isSubstantiveAnswer(txt, terminal string, mode recoveryMode) bool {
|
||||
dwarfs := len(txt) >= recoverRatio*len(terminal)
|
||||
if mode == modeSummary && !dwarfs {
|
||||
return false
|
||||
}
|
||||
if len(txt) >= recoverMinChars {
|
||||
return true
|
||||
}
|
||||
if len(txt) < recoverFloorChars || preambleRe.MatchString(txt) {
|
||||
return false
|
||||
}
|
||||
return citations || len(txt) >= recoverRatio*len(terminal)
|
||||
return mode != modeBackRef || dwarfs
|
||||
}
|
||||
|
||||
+179
-1
@@ -72,6 +72,46 @@ func TestIsCitationsOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// b3cb9ee9Closer is the verbatim terminal turn from mort run b3cb9ee9: a
|
||||
// 2,089-char answer was front-loaded into the cite-call turn and this 153-byte
|
||||
// compression (151 runes — the em dash is 3 bytes, and byte length is what the
|
||||
// thresholds compare) was all that got delivered.
|
||||
const b3cb9ee9Closer = "Citations are logged. Short version: the bulk of that ~$64M was AIPAC and dark-money super PACs, not the party committees — and it still wasn't enough."
|
||||
|
||||
func TestIsSummaryCloser(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"b3cb9ee9-verbatim", b3cb9ee9Closer, true},
|
||||
{"ack-only", "Citations are logged.", true},
|
||||
{"ack-no-copula", "Citations logged.", true},
|
||||
{"claims-cited", "All claims cited.", true},
|
||||
{"verb-first", "Logged the citations.", true},
|
||||
{"done-prefix", "Done — citations logged.", true},
|
||||
{"ack-then-tldr", "Sources have been recorded! TL;DR: the GPU was the bottleneck.", true},
|
||||
{"references-noted", "References noted. In short: yes, it ships Tuesday.", true},
|
||||
{"plus-list-marker", "+ Citations are logged.", true},
|
||||
{"logged-all-the", "Logged all the citations.", true},
|
||||
|
||||
{"empty", "", false},
|
||||
{"ack-continues-midsentence", "The citations are recorded in the court transcript, which shows the filing dates.", false},
|
||||
{"ack-verb-then-clause", "Citations are logged in Zotero whenever you click the save button.", false},
|
||||
{"compression-without-ack", "Short version: yes.", false}, // deliberately out of scope
|
||||
{"mentions-citations-midsentence", "The paper's citations are what got it retracted.", false},
|
||||
{"crisp-number", "42", false},
|
||||
{"over-cap", "Citations are logged. " + strings.Repeat("The long version has many more details worth keeping. ", 6), false}, // >300: too substantial to replace
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := isSummaryCloser(c.in); got != c.want {
|
||||
t.Errorf("isSummaryCloser(%q) = %v, want %v", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func asst(text string, tools ...llm.ToolCall) llm.Message {
|
||||
m := llm.Message{Role: llm.RoleAssistant}
|
||||
if text != "" {
|
||||
@@ -83,7 +123,8 @@ func asst(text string, tools ...llm.ToolCall) llm.Message {
|
||||
|
||||
func TestFinalOutput(t *testing.T) {
|
||||
cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}
|
||||
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
|
||||
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
|
||||
hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12)) // >3x the b3cb9ee9 closer
|
||||
// A sources/citations-only terminal — the glm-5.2 "cite" shape behind mort
|
||||
// issue #1418: the prose answer was front-loaded into the tool-call turn and
|
||||
// the terminal turn carried only the citations.
|
||||
@@ -103,6 +144,9 @@ func TestFinalOutput(t *testing.T) {
|
||||
// A >=200-byte real answer that merely OPENS with a conversational word
|
||||
// ("Sure,"). The preamble filter must NOT veto it (gadfly regression guard).
|
||||
longConversationalAnswer := "Sure, here's the rundown: it currently sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle, with the sealed Founders Edition commanding the top of that range while used AIB cards go a bit lower."
|
||||
// Matches BOTH the summary ack and backRefRe, within the 120-byte weak cap,
|
||||
// and long enough (>~92 bytes) that longAnswer would fail the summary bar.
|
||||
bothMatchCloser := "Citations are logged. As I mentioned above, the full detail on the money sources is in my earlier message."
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -256,6 +300,109 @@ func TestFinalOutput(t *testing.T) {
|
||||
terminal: sources,
|
||||
want: longConversationalAnswer + "\n\n" + sources,
|
||||
},
|
||||
{
|
||||
// The b3cb9ee9 shape: full answer front-loaded into the cite turn,
|
||||
// then a summary closer. The closer is discarded — its content is a
|
||||
// strict compression of the recovered answer.
|
||||
name: "summary closer discarded when the front-loaded answer dwarfs it",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("where did the $64M come from?"),
|
||||
asst(hugeAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(b3cb9ee9Closer),
|
||||
},
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: hugeAnswer,
|
||||
},
|
||||
{
|
||||
// The dwarf ratio is mandatory for a summary closer at EVERY length:
|
||||
// a prior turn that is longer but not clearly the fuller original
|
||||
// (here ~275 chars vs a 151-char closer, under the 3x bar) must not
|
||||
// displace a closer that carries real answer content.
|
||||
name: "summary closer kept when the prior turn does not dwarf it",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(b3cb9ee9Closer),
|
||||
},
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: b3cb9ee9Closer,
|
||||
},
|
||||
{
|
||||
// An ack-only closer ("Citations are logged.") is tiny, so even a
|
||||
// modest front-loaded answer clears the ratio and replaces it.
|
||||
name: "ack-only summary closer recovered over a modest answer",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst("Citations are logged."),
|
||||
},
|
||||
terminal: "Citations are logged.",
|
||||
want: longAnswer,
|
||||
},
|
||||
{
|
||||
name: "summary closer with only a preamble prior keeps the closer",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst("Let me gather the numbers.", cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(b3cb9ee9Closer),
|
||||
},
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: b3cb9ee9Closer,
|
||||
},
|
||||
{
|
||||
// The modeSummary scan must stop at the most recent user message.
|
||||
// Here the current turn's answer sits in the 1x-3x band (rejected
|
||||
// by the ratio) while a dwarfing answer to a DIFFERENT question
|
||||
// sits in history — resurrecting it would be strictly worse than
|
||||
// keeping the closer.
|
||||
name: "summary closer never resurrects a stale answer across the user boundary",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("earlier, unrelated question?"),
|
||||
asst(hugeAnswer),
|
||||
llm.UserText("q?"),
|
||||
asst(conciseAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(b3cb9ee9Closer),
|
||||
},
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: b3cb9ee9Closer,
|
||||
},
|
||||
{
|
||||
// The boundary must not break the legitimate multi-turn case: the
|
||||
// dwarfing front-loaded answer in THIS turn's window is recovered
|
||||
// even with history behind it.
|
||||
name: "summary closer recovery still works with history present",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("earlier, unrelated question?"),
|
||||
asst(longAnswer),
|
||||
llm.UserText("q?"),
|
||||
asst(hugeAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(b3cb9ee9Closer),
|
||||
},
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: hugeAnswer,
|
||||
},
|
||||
{
|
||||
// A closer matching BOTH the ack shape and a back-reference
|
||||
// carries no answer content, so the back-ref test must win and the
|
||||
// ordinary recovery bar apply — under the summary bar this
|
||||
// ~106-byte terminal would demand a ~318-byte prior and wrongly
|
||||
// keep the closer over longAnswer.
|
||||
name: "back-reference wins over the summary ack when both match",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(bothMatchCloser),
|
||||
},
|
||||
terminal: bothMatchCloser,
|
||||
want: longAnswer,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -326,6 +473,37 @@ func TestRun_HealthyTerminalUnchanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_RecoversFrontLoadedAnswerOverSummaryCloser reproduces mort run
|
||||
// b3cb9ee9 end-to-end: the model front-loads its full answer into the
|
||||
// cite-call turn, the cite results come back, and the terminal turn is only a
|
||||
// bookkeeping ack plus a one-line compression. The delivered output must be
|
||||
// the front-loaded answer, with no extra model call.
|
||||
func TestRun_RecoversFrontLoadedAnswerOverSummaryCloser(t *testing.T) {
|
||||
hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12))
|
||||
fp := fake.New("fp")
|
||||
fp.Enqueue("test-model",
|
||||
fake.ReplyWith(llm.Response{
|
||||
Parts: []llm.Part{llm.Text(hugeAnswer)},
|
||||
ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
|
||||
FinishReason: llm.FinishToolCalls,
|
||||
Usage: llm.Usage{InputTokens: 10, OutputTokens: 5},
|
||||
}),
|
||||
fake.Reply(b3cb9ee9Closer),
|
||||
)
|
||||
|
||||
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||||
res, err := a.Run(context.Background(), "where did the $64M come from?")
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if res.Output != hugeAnswer {
|
||||
t.Errorf("Output = %q, want recovered front-loaded answer", res.Output)
|
||||
}
|
||||
if n := len(fp.Calls()); n != 2 {
|
||||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_RecoversFrontLoadedAnswerWithCitations reproduces mort issue #1418
|
||||
// end-to-end: the model front-loads the prose answer into the tool-call turn
|
||||
// and closes with a sources-only terminal turn. The delivered output must be
|
||||
|
||||
@@ -173,3 +173,51 @@ func TestSameCallRepeatGuard(t *testing.T) {
|
||||
t.Errorf("varied calls must not trip the guard: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSameCallRepeatGuardProgressAware: identical (name+args) calls whose
|
||||
// RESULT keeps changing — canonically polling a long-running background job
|
||||
// whose progress advances — do not trip the repeat guard even well past the
|
||||
// limit; but identical calls returning an unchanged result still trip it.
|
||||
func TestSameCallRepeatGuardProgressAware(t *testing.T) {
|
||||
// A poll tool that advances every call, so identical args yield a
|
||||
// different result each time.
|
||||
calls := 0
|
||||
polling := llm.NewToolbox("jobs", llm.Tool{
|
||||
Name: "poll",
|
||||
Handler: func(context.Context, json.RawMessage) (any, error) {
|
||||
calls++
|
||||
return map[string]any{"status": "running", "elapsed": calls}, nil
|
||||
},
|
||||
})
|
||||
n := 0
|
||||
fp := fake.New("fp", fake.WithDefault(func(string, llm.Request) fake.Step {
|
||||
n++
|
||||
if n > 6 { // six identical polls, well past the limit of 3
|
||||
return fake.Reply("done")
|
||||
}
|
||||
return toolCallReply("c", "poll", `{"job":"x"}`)
|
||||
}))
|
||||
a := New(newModel(t, fp), "", WithToolbox(polling), WithToolErrorLimits(0, 3), WithMaxSteps(20))
|
||||
res, err := a.Run(context.Background(), "go")
|
||||
if err != nil {
|
||||
t.Fatalf("advancing-result polls must not trip the guard: %v", err)
|
||||
}
|
||||
if res.Output != "done" {
|
||||
t.Errorf("output = %q, want run to complete after polling", res.Output)
|
||||
}
|
||||
|
||||
// A call that returns an UNCHANGED result each time still trips the guard.
|
||||
frozen := llm.NewToolbox("jobs", llm.Tool{
|
||||
Name: "poll",
|
||||
Handler: func(context.Context, json.RawMessage) (any, error) {
|
||||
return map[string]any{"status": "running"}, nil // never advances
|
||||
},
|
||||
})
|
||||
fp2 := fake.New("fp", fake.WithDefault(func(string, llm.Request) fake.Step {
|
||||
return toolCallReply("c", "poll", `{"job":"x"}`)
|
||||
}))
|
||||
a2 := New(newModel(t, fp2), "", WithToolbox(frozen), WithToolErrorLimits(0, 3), WithMaxSteps(20))
|
||||
if _, err := a2.Run(context.Background(), "go"); !errors.Is(err, ErrToolLoop) {
|
||||
t.Fatalf("frozen identical result must still trip the guard: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,17 @@ type SpeechRequest struct {
|
||||
|
||||
// Speed is the playback-rate multiplier; 0 = backend default (1.0).
|
||||
Speed float64
|
||||
|
||||
// ReferenceAudio is a short voice sample for zero-shot voice cloning
|
||||
// (chatterbox style): when set, the model speaks Input in the sampled
|
||||
// voice instead of a named Voice. nil = normal synthesis. Backends
|
||||
// without cloning support must reject a reference-carrying request
|
||||
// rather than silently ignoring it.
|
||||
ReferenceAudio []byte
|
||||
|
||||
// ReferenceMIME is the reference audio's MIME type (e.g. "audio/wav");
|
||||
// "" = let the backend sniff it.
|
||||
ReferenceMIME string
|
||||
}
|
||||
|
||||
// SpeechResult is the canonical synthesis result: raw audio bytes plus the
|
||||
@@ -59,6 +70,11 @@ func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Forma
|
||||
// WithSpeed sets the playback-rate multiplier.
|
||||
func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } }
|
||||
|
||||
// WithReferenceAudio provides a voice sample for zero-shot voice cloning.
|
||||
func WithReferenceAudio(data []byte, mime string) SpeechOption {
|
||||
return func(r *SpeechRequest) { r.ReferenceAudio, r.ReferenceMIME = data, mime }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied. Providers
|
||||
// call this once at the top of Speak.
|
||||
func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest {
|
||||
@@ -121,6 +137,13 @@ type TranscriptionRequest struct {
|
||||
|
||||
// Prompt is optional context or vocabulary to bias decoding; "" = none.
|
||||
Prompt string
|
||||
|
||||
// Translate requests an English translation of the speech instead of a
|
||||
// same-language transcript (whisper's translate task). When set and no
|
||||
// Language is given, providers must force source-language auto-detection
|
||||
// — a backend whose default language is "en" would otherwise skip
|
||||
// translation entirely.
|
||||
Translate bool
|
||||
}
|
||||
|
||||
// TranscriptionResult is the canonical transcription result.
|
||||
@@ -145,6 +168,11 @@ func WithPrompt(p string) TranscriptionOption {
|
||||
return func(r *TranscriptionRequest) { r.Prompt = p }
|
||||
}
|
||||
|
||||
// WithTranslate requests an English translation instead of a transcript.
|
||||
func WithTranslate() TranscriptionOption {
|
||||
return func(r *TranscriptionRequest) { r.Translate = true }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest {
|
||||
for _, opt := range opts {
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package audio
|
||||
|
||||
import "context"
|
||||
|
||||
// EnhancementRequest asks a speech-enhancement backend (DeepFilterNet style)
|
||||
// to denoise a recording. Audio is carried as bytes (never a URL), mirroring
|
||||
// TranscriptionRequest (ADR-0024).
|
||||
type EnhancementRequest struct {
|
||||
// Audio is the encoded audio to enhance.
|
||||
Audio []byte
|
||||
|
||||
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
|
||||
// sniff it.
|
||||
MIME string
|
||||
|
||||
// Filename is the multipart filename hint some backends key their format
|
||||
// detection on; "" derives one from MIME or falls back to "audio".
|
||||
Filename string
|
||||
}
|
||||
|
||||
// EnhancementOption mutates an EnhancementRequest before it is sent.
|
||||
// Reserved for future request settings (the reference backend takes no
|
||||
// parameters).
|
||||
type EnhancementOption func(*EnhancementRequest)
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r EnhancementRequest) Apply(opts ...EnhancementOption) EnhancementRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// SpeechEnhancer denoises speech recordings. The result reuses SpeechResult
|
||||
// (audio bytes + MIME + Raw) — enhancement is audio-in/audio-out exactly
|
||||
// like synthesis.
|
||||
type SpeechEnhancer interface {
|
||||
// Enhance returns the denoised audio.
|
||||
Enhance(ctx context.Context, req EnhancementRequest, opts ...EnhancementOption) (*SpeechResult, error)
|
||||
}
|
||||
|
||||
// SpeechEnhancerModelOption configures a SpeechEnhancer at construction time.
|
||||
// Reserved for future per-model settings.
|
||||
type SpeechEnhancerModelOption func(*SpeechEnhancerModelConfig)
|
||||
|
||||
// SpeechEnhancerModelConfig carries per-model construction settings.
|
||||
type SpeechEnhancerModelConfig struct{}
|
||||
|
||||
// ApplySpeechEnhancerModelOptions folds options into a config.
|
||||
func ApplySpeechEnhancerModelOptions(opts []SpeechEnhancerModelOption) SpeechEnhancerModelConfig {
|
||||
var cfg SpeechEnhancerModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SpeechEnhancementProvider mints SpeechEnhancers bound to one backend.
|
||||
type SpeechEnhancementProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// SpeechEnhancerModel returns a SpeechEnhancer bound to the given id
|
||||
// (passed through to the backend verbatim; no catalog validation).
|
||||
SpeechEnhancerModel(id string, opts ...SpeechEnhancerModelOption) (SpeechEnhancer, error)
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package audio
|
||||
|
||||
import "context"
|
||||
|
||||
// StemSeparationRequest asks a source-separation backend (Demucs style) to
|
||||
// split a mix into stems. Audio is carried as bytes (never a URL), mirroring
|
||||
// TranscriptionRequest. Zero values mean "backend default" (ADR-0024).
|
||||
type StemSeparationRequest struct {
|
||||
// Audio is the encoded mix to separate.
|
||||
Audio []byte
|
||||
|
||||
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
|
||||
// sniff it.
|
||||
MIME string
|
||||
|
||||
// Filename is the multipart filename hint some backends key their format
|
||||
// detection on; "" derives one from MIME or falls back to "audio".
|
||||
Filename string
|
||||
|
||||
// Mode selects the split: "two" (vocals + accompaniment) or "four"
|
||||
// (vocals/drums/bass/other); "" = backend default (four).
|
||||
Mode string
|
||||
|
||||
// Model selects the separator's internal network where the backend
|
||||
// offers several (Demucs: "htdemucs", "htdemucs_ft"); "" = backend
|
||||
// default. This is NOT the provider model id — that is fixed when the
|
||||
// StemSeparator is minted (mirrors BackgroundRemovalRequest.Net).
|
||||
Model string
|
||||
|
||||
// Format is the per-stem audio container ("mp3" or "wav");
|
||||
// "" = backend default.
|
||||
Format string
|
||||
}
|
||||
|
||||
// Stem is one separated source.
|
||||
type Stem struct {
|
||||
// Name is the stem's name ("vocals", "drums", "bass", "other",
|
||||
// "no_vocals", ...), taken from the backend's own labelling.
|
||||
Name string
|
||||
|
||||
// Audio is the encoded stem.
|
||||
Audio []byte
|
||||
|
||||
// MIME is the stem's audio MIME type, e.g. "audio/mpeg".
|
||||
MIME string
|
||||
}
|
||||
|
||||
// StemSeparationResult is the canonical separation result.
|
||||
type StemSeparationResult struct {
|
||||
// Stems are the separated sources, in the order the backend returned
|
||||
// them.
|
||||
Stems []Stem
|
||||
}
|
||||
|
||||
// StemSeparationOption mutates a StemSeparationRequest before it is sent.
|
||||
type StemSeparationOption func(*StemSeparationRequest)
|
||||
|
||||
// WithStemMode selects the split ("two" or "four").
|
||||
func WithStemMode(m string) StemSeparationOption {
|
||||
return func(r *StemSeparationRequest) { r.Mode = m }
|
||||
}
|
||||
|
||||
// WithStemModel selects the separator's internal network.
|
||||
func WithStemModel(m string) StemSeparationOption {
|
||||
return func(r *StemSeparationRequest) { r.Model = m }
|
||||
}
|
||||
|
||||
// WithStemFormat sets the per-stem audio container ("mp3", "wav").
|
||||
func WithStemFormat(f string) StemSeparationOption {
|
||||
return func(r *StemSeparationRequest) { r.Format = f }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r StemSeparationRequest) Apply(opts ...StemSeparationOption) StemSeparationRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// StemSeparator splits a mix into stems.
|
||||
type StemSeparator interface {
|
||||
// SeparateStems returns the separated sources. Separation is CPU-bound
|
||||
// and slow (minutes for a full song); bound the call with a context
|
||||
// deadline.
|
||||
SeparateStems(ctx context.Context, req StemSeparationRequest, opts ...StemSeparationOption) (*StemSeparationResult, error)
|
||||
}
|
||||
|
||||
// StemSeparatorModelOption configures a StemSeparator at construction time.
|
||||
// Reserved for future per-model settings.
|
||||
type StemSeparatorModelOption func(*StemSeparatorModelConfig)
|
||||
|
||||
// StemSeparatorModelConfig carries per-model construction settings.
|
||||
type StemSeparatorModelConfig struct{}
|
||||
|
||||
// ApplyStemSeparatorModelOptions folds options into a config.
|
||||
func ApplyStemSeparatorModelOptions(opts []StemSeparatorModelOption) StemSeparatorModelConfig {
|
||||
var cfg StemSeparatorModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// StemSeparationProvider mints StemSeparators bound to one backend.
|
||||
type StemSeparationProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// StemSeparatorModel returns a StemSeparator bound to the given id
|
||||
// (passed through to the backend verbatim; no catalog validation).
|
||||
StemSeparatorModel(id string, opts ...StemSeparatorModelOption) (StemSeparator, error)
|
||||
}
|
||||
+78
-1
@@ -13,7 +13,17 @@ import (
|
||||
|
||||
// Built-in provider names.
|
||||
const (
|
||||
ProviderOpenAI = "openai"
|
||||
ProviderOpenAI = "openai"
|
||||
// ProviderKimi is Moonshot AI's Kimi models over their OpenAI-compatible
|
||||
// Chat Completions endpoint. Reuses the openai client (like llama-swap);
|
||||
// keyed by KIMI_API_KEY, default base URL kimiBaseURL.
|
||||
ProviderKimi = "kimi"
|
||||
// ProviderQwen is Alibaba's Qwen models over Model Studio's
|
||||
// OpenAI-compatible Chat Completions endpoint. Reuses the openai client
|
||||
// (like kimi and llama-swap); keyed by QWEN_API_KEY, default base URL
|
||||
// qwenBaseURL. ADR-0027 records why the OpenAI surface and not the
|
||||
// Anthropic-compatible one Model Studio also exposes.
|
||||
ProviderQwen = "qwen"
|
||||
ProviderAnthropic = "anthropic"
|
||||
ProviderGoogle = "google"
|
||||
ProviderOllama = "ollama"
|
||||
@@ -28,6 +38,59 @@ const (
|
||||
ProviderLlamaSwapTLS = "llama-swaps"
|
||||
)
|
||||
|
||||
// kimiBaseURL is Moonshot AI's international OpenAI-compatible endpoint. The
|
||||
// China endpoint (api.moonshot.cn/v1) is reachable via a kimi:// LLM_* DSN.
|
||||
const kimiBaseURL = "https://api.moonshot.ai/v1"
|
||||
|
||||
// qwenBaseURL is Alibaba Model Studio's international (Singapore) endpoint in
|
||||
// OpenAI-compatible mode. The China endpoint
|
||||
// (dashscope.aliyuncs.com/compatible-mode/v1) and any regional host are
|
||||
// reachable via a qwen:// LLM_* DSN.
|
||||
const qwenBaseURL = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
|
||||
|
||||
// openaiCompatScheme builds the DSN factory shared by every built-in that is
|
||||
// "the openai client pointed somewhere else" (kimi, qwen, ...). The provider
|
||||
// is named after the LLM_<NAME> var that defined it, takes its credential from
|
||||
// the DSN token — not the built-in's own env var, which does nothing for a
|
||||
// DSN-defined provider — and so names that same LLM_<NAME> var in the
|
||||
// missing-key hint, matching the lazy-resolution key form in providerFor.
|
||||
//
|
||||
// wrap is the caller's option-decorator (it injects the registry's HTTP
|
||||
// client), so a DSN provider is built exactly like the eager built-ins.
|
||||
func openaiCompatScheme(wrap func(...openai.Option) []openai.Option) SchemeFactory {
|
||||
return func(name string, dsn DSN) (llm.Provider, error) {
|
||||
return openai.New(wrap(
|
||||
openai.WithName(name),
|
||||
openai.WithBaseURL(dsn.BaseURL()),
|
||||
openai.WithAPIKey(dsn.Token),
|
||||
openai.WithAPIKeyName(envKeyForProvider(name)),
|
||||
)...), nil
|
||||
}
|
||||
}
|
||||
|
||||
// registerOpenAICompatBuiltin installs BOTH halves of an OpenAI-compat
|
||||
// built-in: the eager provider under name (credential from keyEnv) and the
|
||||
// matching name:// DSN scheme. Why both in one call: the two halves are a pair
|
||||
// — a built-in whose scheme is missing resolves as a spec but not from an
|
||||
// LLM_* DSN, and the credential rules below have to hold identically in each.
|
||||
// Adding the next one is a single line rather than six lines to copy.
|
||||
//
|
||||
// The two credential rules, holding by construction for every caller:
|
||||
// - WithAPIKey is passed UNCONDITIONALLY, even when the lookup comes back
|
||||
// empty. openai.New defaults its key to OPENAI_API_KEY, so anything less
|
||||
// lets an unset keyEnv silently authenticate as OpenAI.
|
||||
// - WithAPIKeyName makes the synthetic-401 hint name keyEnv, so a keyless
|
||||
// call tells the operator the variable that actually fixes it.
|
||||
func registerOpenAICompatBuiltin(r *Registry, wrap func(...openai.Option) []openai.Option, name, baseURL, keyEnv string) {
|
||||
r.providers[name] = openai.New(wrap(
|
||||
openai.WithName(name),
|
||||
openai.WithBaseURL(baseURL),
|
||||
openai.WithAPIKey(r.envLookup(keyEnv)),
|
||||
openai.WithAPIKeyName(keyEnv),
|
||||
)...)
|
||||
r.schemes[name] = openaiCompatScheme(wrap)
|
||||
}
|
||||
|
||||
// registerBuiltins installs the built-in providers and env-DSN scheme
|
||||
// factories into a fresh registry. httpClient, when non-nil, is used by
|
||||
// every provider and factory the registry itself constructs.
|
||||
@@ -74,6 +137,20 @@ func registerBuiltins(r *Registry, httpClient *http.Client) {
|
||||
)...), nil
|
||||
}
|
||||
|
||||
// Third-party endpoints that ARE the openai client at another base URL —
|
||||
// no new package, mirroring llama-swap's chat path. Each gets the eager
|
||||
// built-in plus its name:// DSN scheme, and the credential rules hold by
|
||||
// construction (see registerOpenAICompatBuiltin).
|
||||
//
|
||||
// kimi (ADR-0026): Moonshot's international endpoint; China host via
|
||||
// kimi://[email protected]/v1.
|
||||
registerOpenAICompatBuiltin(r, openaiOpts, ProviderKimi, kimiBaseURL, "KIMI_API_KEY")
|
||||
// qwen (ADR-0027): Alibaba Model Studio's international host. Model Studio
|
||||
// also exposes an Anthropic-compatible endpoint; the ADR records why the
|
||||
// OpenAI one is the built-in. China / workspace-scoped regional hosts via
|
||||
// qwen://[email protected]/compatible-mode/v1.
|
||||
registerOpenAICompatBuiltin(r, openaiOpts, ProviderQwen, qwenBaseURL, "QWEN_API_KEY")
|
||||
|
||||
// llama-swap: OpenAI-compatible chat + image generation + management
|
||||
// endpoints over a model-swapping proxy. Chat reuses the openai client
|
||||
// (provider/llamaswap delegates). Two schemes: "llama-swap" builds an
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
package majordomo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// Shared fixtures and the shared contract for the built-ins that are "the
|
||||
// openai client pointed somewhere else" (kimi, qwen, ...). They live here
|
||||
// rather than in any one provider's test file so a new OpenAI-compat built-in
|
||||
// has nothing to copy — the same reason registerOpenAICompatBuiltin exists on
|
||||
// the production side.
|
||||
|
||||
// chatCompletionOK is a minimal valid Chat Completions body, so Generate
|
||||
// returns a non-empty response (an empty one would trigger failover, not a
|
||||
// clean pass).
|
||||
const chatCompletionOK = `{"id":"c1","object":"chat.completion","choices":[` +
|
||||
`{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`
|
||||
|
||||
// captureRT records the last request (and the bytes of its body) and returns a
|
||||
// canned response without touching the network, so these tests stay hermetic
|
||||
// while still exercising the real openai client the built-ins reuse: base URL,
|
||||
// auth header, and the JSON actually put on the wire.
|
||||
type captureRT struct {
|
||||
req *http.Request
|
||||
reqBody []byte
|
||||
body string
|
||||
}
|
||||
|
||||
func (c *captureRT) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
c.req = r
|
||||
// Drain and close the request body: a RoundTripper owns it, and those
|
||||
// bytes are what wire-shape assertions read.
|
||||
c.reqBody = nil
|
||||
if r.Body != nil {
|
||||
c.reqBody, _ = io.ReadAll(r.Body)
|
||||
_ = r.Body.Close()
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: io.NopCloser(strings.NewReader(c.body)),
|
||||
Header: make(http.Header),
|
||||
Request: r,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// singleKeyEnv builds a WithEnvLookup function that knows exactly one variable
|
||||
// and returns "" for everything else. The empty default has teeth: a built-in
|
||||
// that reached for any other variable name gets nothing, so the request 401s
|
||||
// and the test fails rather than quietly authenticating off the wrong key.
|
||||
func singleKeyEnv(key, value string) func(string) string {
|
||||
return func(k string) string {
|
||||
if k == key {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// openAICompatBuiltin describes one built-in for the shared contract below.
|
||||
// Adding an OpenAI-compat built-in means adding a row here — not copying a
|
||||
// test file, which is how kimi's and qwen's suites became near-identical.
|
||||
type openAICompatBuiltin struct {
|
||||
name string // registry name and spec prefix
|
||||
keyEnv string // the credential variable this built-in reads
|
||||
model string // a current model id for that endpoint
|
||||
wantURL string // chat-completions URL the default endpoint must produce
|
||||
|
||||
// The name:// DSN case: an alternate host (regional/China endpoint)
|
||||
// reached through an LLM_<dsnVar> definition.
|
||||
dsnVar string
|
||||
dsnHost string
|
||||
wantDSNURL string
|
||||
}
|
||||
|
||||
var openAICompatBuiltins = []openAICompatBuiltin{
|
||||
{
|
||||
name: ProviderKimi,
|
||||
keyEnv: "KIMI_API_KEY",
|
||||
model: "kimi-k2-0711-preview",
|
||||
wantURL: "https://api.moonshot.ai/v1/chat/completions",
|
||||
dsnVar: "LLM_KCN",
|
||||
dsnHost: "api.moonshot.cn/v1",
|
||||
wantDSNURL: "https://api.moonshot.cn/v1/chat/completions",
|
||||
},
|
||||
{
|
||||
name: ProviderQwen,
|
||||
keyEnv: "QWEN_API_KEY",
|
||||
model: "qwen3.8-max",
|
||||
wantURL: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions",
|
||||
dsnVar: "LLM_QCN",
|
||||
dsnHost: "dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
wantDSNURL: "https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions",
|
||||
},
|
||||
}
|
||||
|
||||
// TestOpenAICompatBuiltins is the whole contract an OpenAI-compat built-in
|
||||
// owes, asserted identically for every one of them: it resolves in Parse and
|
||||
// targets its own endpoint with its own key; a missing key fails closed naming
|
||||
// the right variable and never reaching the network; its name:// DSN reaches
|
||||
// any other host on the DSN token; and a keyless DSN names the LLM_<NAME> that
|
||||
// actually fixes it rather than the built-in's variable, which does nothing
|
||||
// for a DSN-defined provider.
|
||||
func TestOpenAICompatBuiltins(t *testing.T) {
|
||||
for _, tc := range openAICompatBuiltins {
|
||||
t.Run(tc.name+"/builtin", func(t *testing.T) {
|
||||
rt := &captureRT{body: chatCompletionOK}
|
||||
secret := tc.name + "-secret"
|
||||
r := newTestRegistry(t,
|
||||
WithEnvLookup(singleKeyEnv(tc.keyEnv, secret)),
|
||||
WithHTTPClient(&http.Client{Transport: rt}),
|
||||
)
|
||||
|
||||
if p, ok := r.Provider(tc.name); !ok {
|
||||
t.Fatalf("built-in %q not registered", tc.name)
|
||||
} else if p.Name() != tc.name {
|
||||
t.Errorf("name = %q, want %q", p.Name(), tc.name)
|
||||
}
|
||||
|
||||
spec := tc.name + "/" + tc.model
|
||||
m, err := r.Parse(spec)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse(%q): %v", spec, err)
|
||||
}
|
||||
if got := targetsOf(t, m); len(got) != 1 || got[0] != spec {
|
||||
t.Fatalf("targets = %v, want [%q]", got, spec)
|
||||
}
|
||||
|
||||
if _, err := m.Generate(context.Background(), llm.Request{Messages: []llm.Message{llm.UserText("hi")}}); err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if rt.req == nil {
|
||||
t.Fatal("no request captured")
|
||||
}
|
||||
if rt.req.URL.String() != tc.wantURL {
|
||||
t.Errorf("URL = %q, want %q", rt.req.URL.String(), tc.wantURL)
|
||||
}
|
||||
if want := "Bearer " + secret; rt.req.Header.Get("Authorization") != want {
|
||||
t.Errorf("Authorization = %q, want %q", rt.req.Header.Get("Authorization"), want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(tc.name+"/builtin missing key", func(t *testing.T) {
|
||||
rt := &captureRT{body: chatCompletionOK}
|
||||
r := newTestRegistry(t, WithHTTPClient(&http.Client{Transport: rt}))
|
||||
|
||||
m, err := r.Parse(tc.name + "/" + tc.model)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse: %v", err)
|
||||
}
|
||||
_, err = m.Generate(context.Background(), llm.Request{Messages: []llm.Message{llm.UserText("hi")}})
|
||||
apiErr, ok := errors.AsType[*llm.APIError](err)
|
||||
if !ok {
|
||||
t.Fatalf("err = %v (%T), want *llm.APIError", err, err)
|
||||
}
|
||||
if apiErr.Status != http.StatusUnauthorized || apiErr.Code != "missing_api_key" {
|
||||
t.Errorf("Status/Code = %d/%q, want 401/missing_api_key", apiErr.Status, apiErr.Code)
|
||||
}
|
||||
if !strings.Contains(apiErr.Message, tc.keyEnv) {
|
||||
t.Errorf("message = %q, want it to name %s", apiErr.Message, tc.keyEnv)
|
||||
}
|
||||
// The load-bearing half: openai.New defaults its key to
|
||||
// OPENAI_API_KEY, so a built-in that stopped passing WithAPIKey
|
||||
// unconditionally would authenticate as OpenAI instead of failing.
|
||||
if strings.Contains(apiErr.Message, "OPENAI_API_KEY") {
|
||||
t.Errorf("message = %q, must not name OPENAI_API_KEY", apiErr.Message)
|
||||
}
|
||||
if rt.req != nil {
|
||||
t.Error("network was hit despite missing key")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(tc.name+"/dsn scheme", func(t *testing.T) {
|
||||
rt := &captureRT{body: chatCompletionOK}
|
||||
r := newTestRegistry(t, WithHTTPClient(&http.Client{Transport: rt}))
|
||||
if err := r.LoadEnv(map[string]string{
|
||||
tc.dsnVar: tc.name + "://tok@" + tc.dsnHost,
|
||||
}); err != nil {
|
||||
t.Fatalf("LoadEnv: %v", err)
|
||||
}
|
||||
|
||||
dsnName := strings.ToLower(strings.TrimPrefix(tc.dsnVar, "LLM_"))
|
||||
m, err := r.Parse(dsnName + "/" + tc.model)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse: %v", err)
|
||||
}
|
||||
if _, err := m.Generate(context.Background(), llm.Request{Messages: []llm.Message{llm.UserText("hi")}}); err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if rt.req == nil {
|
||||
t.Fatal("no request captured")
|
||||
}
|
||||
if rt.req.URL.String() != tc.wantDSNURL {
|
||||
t.Errorf("URL = %q, want %q", rt.req.URL.String(), tc.wantDSNURL)
|
||||
}
|
||||
if want := "Bearer tok"; rt.req.Header.Get("Authorization") != want {
|
||||
t.Errorf("Authorization = %q, want %q", rt.req.Header.Get("Authorization"), want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(tc.name+"/dsn scheme missing token", func(t *testing.T) {
|
||||
rt := &captureRT{body: chatCompletionOK}
|
||||
r := newTestRegistry(t, WithHTTPClient(&http.Client{Transport: rt}))
|
||||
if err := r.LoadEnv(map[string]string{
|
||||
tc.dsnVar: tc.name + "://" + tc.dsnHost, // no token
|
||||
}); err != nil {
|
||||
t.Fatalf("LoadEnv: %v", err)
|
||||
}
|
||||
|
||||
dsnName := strings.ToLower(strings.TrimPrefix(tc.dsnVar, "LLM_"))
|
||||
m, err := r.Parse(dsnName + "/" + tc.model)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse: %v", err)
|
||||
}
|
||||
_, err = m.Generate(context.Background(), llm.Request{Messages: []llm.Message{llm.UserText("hi")}})
|
||||
apiErr, ok := errors.AsType[*llm.APIError](err)
|
||||
if !ok {
|
||||
t.Fatalf("err = %v (%T), want *llm.APIError", err, err)
|
||||
}
|
||||
// A keyless DSN is fixed by adding a token to that DSN, so the
|
||||
// hint must name the defining variable — never the built-in's own
|
||||
// key, which does nothing for a DSN-defined provider.
|
||||
if !strings.Contains(apiErr.Message, tc.dsnVar) {
|
||||
t.Errorf("message = %q, want it to name %s", apiErr.Message, tc.dsnVar)
|
||||
}
|
||||
if strings.Contains(apiErr.Message, tc.keyEnv) {
|
||||
t.Errorf("message = %q, must not name %s for a DSN provider", apiErr.Message, tc.keyEnv)
|
||||
}
|
||||
if rt.req != nil {
|
||||
t.Error("network was hit despite missing token")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package majordomo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// The contract qwen shares with every other OpenAI-compat built-in (endpoint,
|
||||
// credential isolation, its qwen:// DSN) is asserted by the table in
|
||||
// builtin_openaicompat_test.go. What remains here is qwen-specific: the
|
||||
// reverse-leak direction, and the wire claim ADR-0027 turns on.
|
||||
|
||||
// TestQwenBuiltinKeyDoesNotLeakToOpenAI: QWEN_API_KEY is the qwen built-in's
|
||||
// credential and nothing else's. Why this direction too: the shared table's
|
||||
// missing-key case only proves qwen never borrows OPENAI_API_KEY; this proves
|
||||
// the reverse — a registry that can see QWEN_API_KEY must not hand it to the
|
||||
// openai built-in, which would send an Alibaba key to api.openai.com.
|
||||
func TestQwenBuiltinKeyDoesNotLeakToOpenAI(t *testing.T) {
|
||||
// Set before newTestRegistry: the openai built-in reads OPENAI_API_KEY at
|
||||
// construction. Giving it a real key is what keeps this test honest — a
|
||||
// keyless openai target would 401 before any request, and the assertion
|
||||
// below would pass without a single byte reaching the wire.
|
||||
t.Setenv("OPENAI_API_KEY", "openai-secret")
|
||||
|
||||
rt := &captureRT{body: chatCompletionOK}
|
||||
r := newTestRegistry(t,
|
||||
WithEnvLookup(singleKeyEnv("QWEN_API_KEY", "qwen-secret")),
|
||||
WithHTTPClient(&http.Client{Transport: rt}),
|
||||
)
|
||||
|
||||
m, err := r.Parse("openai/gpt-4o-mini")
|
||||
if err != nil {
|
||||
t.Fatalf("Parse: %v", err)
|
||||
}
|
||||
if _, err := m.Generate(context.Background(), llm.Request{Messages: []llm.Message{llm.UserText("hi")}}); err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if rt.req == nil {
|
||||
t.Fatal("no request captured")
|
||||
}
|
||||
if want := "Bearer openai-secret"; rt.req.Header.Get("Authorization") != want {
|
||||
t.Errorf("Authorization = %q, want %q — the qwen credential must not reach the openai built-in",
|
||||
rt.req.Header.Get("Authorization"), want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQwenReasoningEffortReachesWire is the load-bearing test for ADR-0027's
|
||||
// central claim: Model Studio's OpenAI-compatible surface takes reasoning as a
|
||||
// top-level "reasoning_effort" body field, which the openai client already
|
||||
// sends — so llm.WithReasoningEffort survives the trip on qwen with no
|
||||
// qwen-specific code. Routing qwen through the anthropic client instead would
|
||||
// drop it silently (provider/anthropic ignores ReasoningEffort by design), and
|
||||
// that difference would be invisible without asserting on the wire body.
|
||||
func TestQwenReasoningEffortReachesWire(t *testing.T) {
|
||||
rt := &captureRT{body: chatCompletionOK}
|
||||
r := newTestRegistry(t,
|
||||
WithEnvLookup(singleKeyEnv("QWEN_API_KEY", "qwen-secret")),
|
||||
WithHTTPClient(&http.Client{Transport: rt}),
|
||||
)
|
||||
|
||||
m, err := r.Parse("qwen/qwen3.8-max")
|
||||
if err != nil {
|
||||
t.Fatalf("Parse: %v", err)
|
||||
}
|
||||
_, err = m.Generate(context.Background(), llm.Request{
|
||||
Messages: []llm.Message{llm.UserText("hi")},
|
||||
ReasoningEffort: "high",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if rt.reqBody == nil {
|
||||
t.Fatal("no request body captured")
|
||||
}
|
||||
var sent map[string]any
|
||||
if err := json.Unmarshal(rt.reqBody, &sent); err != nil {
|
||||
t.Fatalf("decode request body: %v", err)
|
||||
}
|
||||
if got := sent["reasoning_effort"]; got != "high" {
|
||||
t.Errorf("reasoning_effort = %v, want %q (body: %s)", got, "high", rt.reqBody)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
# ADR-0021: musicgen interface (blocking Generate over an async job queue)
|
||||
|
||||
Status: Accepted (2026-07-12)
|
||||
|
||||
## Context
|
||||
|
||||
The llama-swap host gained ACE-Step 1.5 (full songs with vocals, seconds per
|
||||
clip warm on the reference GPU). Its API is an async job queue —
|
||||
`POST /release_task` → poll `POST /query_result` → `GET` the result file —
|
||||
unlike every other majordomo media backend, which is one blocking call.
|
||||
|
||||
## Decision
|
||||
|
||||
- New `musicgen` leaf package, ADR-0016→0020 conventions: `Request{Prompt,
|
||||
Lyrics, DurationSeconds, Format, Steps, Seed}`, `Result{Audio{Data, MIME},
|
||||
Raw}`, functional options, zero value = backend default, bytes-only.
|
||||
- **`Model.Generate` blocks, polling internally** (2s interval, ctx-bounded).
|
||||
The one-call contract is the package's value: callers budget the whole job
|
||||
with a context deadline exactly like imagegen/videogen, and the async
|
||||
mechanics stay a provider detail. An async job surface (mirroring the
|
||||
deliberately-deferred videogen one, ADR-0019) can come later if a caller
|
||||
ever needs progress.
|
||||
- provider/llamaswap reaches ACE-Step through the `/upstream/<model>/`
|
||||
passthrough (ADR-0020). Envelope parsing is tolerant (`data` wrapper or
|
||||
bare array; `result` arrives as a double-encoded JSON string) and the
|
||||
result-file URL is routed back through the same upstream. `audio_duration`
|
||||
is the v1 param name — an unknown field degrades to default clip length
|
||||
upstream, never an error; verify at smoke.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Music jobs occupy the exclusive GPU group like video; callers own the
|
||||
timeout budget (mort keeps the tool timeout under its agent ceiling).
|
||||
- The double-encoded `result` string and envelope shapes are pinned by the
|
||||
netherstorm image build; host smoke tests are the drift defence.
|
||||
@@ -0,0 +1,41 @@
|
||||
# ADR-0022: embeddings + rerank interface
|
||||
|
||||
Status: Accepted (2026-07-12)
|
||||
|
||||
## Context
|
||||
|
||||
majordomo had no embedding or reranking surface at all. The llama-swap host
|
||||
now runs two persistent CPU-only llama-server members (Qwen3-Embedding-0.6B
|
||||
via `/v1/embeddings`, bge-reranker-v2-m3 via `/v1/rerank`), and mort wants a
|
||||
reranking stage in memory retrieval with embedding-backed retrieval as a
|
||||
later step.
|
||||
|
||||
## Decision
|
||||
|
||||
- New `embeddings` leaf package with TWO half-surfaces, split like audio's
|
||||
Speech/Transcription: `EmbedModel`/`EmbedProvider` and
|
||||
`RerankModel`/`RerankProvider`. They are separate mints because on the
|
||||
reference host they are two DIFFERENT server instances — llama-server with
|
||||
`--embeddings` and `--rerank` together returns all-zero embeddings
|
||||
(llama.cpp #20085) — and because rerankers are cross-encoders, not
|
||||
embedders.
|
||||
- `EmbedResult.Vectors [][]float32` in input order (provider must order by
|
||||
the response's `index`, never trust wire order). No `dimensions` param:
|
||||
llama-server doesn't implement it; Matryoshka truncation is caller-side.
|
||||
- `InstructedQuery(task, query)` helper encodes the instruction-aware
|
||||
asymmetry (queries wrapped, documents bare) so call sites can't silently
|
||||
degrade retrieval by forgetting the prefix.
|
||||
- `RerankResult` sorted by descending score; parser reads ONLY
|
||||
`results[].index` and `results[].relevance_score` because llama-server
|
||||
documents the shape as subject to change. Scores are model-specific —
|
||||
comparable within one response only.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Callers get vectors/scores with strict validation (count mismatch, index
|
||||
out of range, empty vector are hard errors — a silently missing vector is
|
||||
a retrieval bug factory).
|
||||
- llama-server's rerank scoring has open correctness issues for some models
|
||||
(llama.cpp #16407); consumers must validate against a fixture before
|
||||
trusting scores in production (mort gates its memory-rerank convar on
|
||||
exactly that).
|
||||
@@ -0,0 +1,59 @@
|
||||
# ADR-0023: Wave-3 image + document surfaces (segmentation, colorize, face restore, OCR)
|
||||
|
||||
Status: Accepted (2026-07-16)
|
||||
|
||||
## Context
|
||||
|
||||
The llama-swap host is gaining four wave-3 capabilities (spec:
|
||||
mort docs/specs/2026-07-16-llamaswap-wave3.md): promptable segmentation
|
||||
(GroundingDINO + SAM 2.1), photo colorization (DDColor), face restoration
|
||||
(GFPGAN), and document OCR (Surya). All ride the `/upstream/<model>/<path>`
|
||||
passthrough (ADR-0020); none of their native APIs are OpenAI-shaped.
|
||||
|
||||
## Decision
|
||||
|
||||
1. **Grow `imagegen` with three optional interfaces**, ADR-0016→0022
|
||||
conventions (functional options, zero value = backend default, bytes-only
|
||||
I/O, provider mints model):
|
||||
- `imagegen.Segmenter` / `SegmentationProvider` — `SegmentationRequest
|
||||
{Image, Prompt, Threshold}` → one grayscale mask image, WHITE = the
|
||||
prompted region. Same polarity as `EditRequest.Mask` (white = repaint),
|
||||
so the mask feeds inpainting directly; cutouts are derived client-side
|
||||
(mask-as-alpha), one host call serving both. The client always sends
|
||||
`output=mask` — the shim's `cutout`/`boxes` modes are not exposed.
|
||||
- `imagegen.Colorizer` / `ColorizeProvider` — `ColorizeRequest{Image}`,
|
||||
no knobs (the reference backend takes none; options reserved).
|
||||
- `imagegen.FaceRestorer` / `FaceRestoreProvider` —
|
||||
`FaceRestoreRequest{Image, Upscale}` (1 or 2, 0 = backend default).
|
||||
2. **New `ocr` leaf package** rather than a method on an existing surface:
|
||||
OCR is document-shaped (multi-page, PDFs), not image-generation-shaped.
|
||||
`Request{Document, MIME, Filename, Languages, MaxPages}` →
|
||||
`Result{Text, Pages []Page{Number, Text}, Raw}`. `Result.Text` is the
|
||||
pages joined with a blank line; the per-line bbox/confidence/layout
|
||||
detail stays in `Raw` (json.RawMessage) — exact text is the contract,
|
||||
geometry is the escape hatch.
|
||||
3. **provider/llamaswap wire shapes** (pinned by the netherstorm image
|
||||
builds; host smoke tests are the drift defence):
|
||||
- `POST /upstream/<id>/v1/segment` multipart `file`,`prompt`
|
||||
[,`threshold`],`output=mask` → mask PNG.
|
||||
- `POST /upstream/<id>/v1/colorize` multipart `file` → PNG.
|
||||
- `POST /upstream/<id>/v1/restore_faces` multipart `file`[,`upscale`] → PNG.
|
||||
- `POST /upstream/<id>/v1/ocr` multipart `file`[,`langs` (comma-joined),
|
||||
`max_pages`] → JSON `{pages:[{number,text,lines,layout}]}`. The decode
|
||||
is tolerant: a page without aggregate `text` joins its line texts; a
|
||||
missing page `number` defaults to position. Zero pages is an error
|
||||
(a blank page still arrives as a page), mirroring the "no transcript"
|
||||
honesty rule.
|
||||
4. **Binary success bodies are validated before wrapping** (ADR-0020 rule):
|
||||
the three image surfaces reuse `singleImageResult` (positive evidence of
|
||||
image-ness required), OCR requires decodable JSON.
|
||||
|
||||
## Consequences
|
||||
|
||||
- imagegen grows from five to eight optional surfaces; consumers
|
||||
type-assert or use the provider methods directly, as before.
|
||||
- `ocr` is the seventh leaf media package (imagegen, audio, videogen,
|
||||
meshgen, musicgen, embeddings, ocr); the conventions have held across all
|
||||
of them.
|
||||
- PDF handling lives host-side (the shim rasterizes via pypdfium2);
|
||||
majordomo ships bytes and never needs a PDF dependency.
|
||||
@@ -0,0 +1,65 @@
|
||||
# ADR-0024: Wave-3 audio surfaces (stems, SFX, speech enhance, voice clone, translate)
|
||||
|
||||
Status: Accepted (2026-07-16)
|
||||
|
||||
## Context
|
||||
|
||||
The llama-swap host is gaining wave-3 audio capabilities (spec: mort
|
||||
docs/specs/2026-07-16-llamaswap-wave3.md): Demucs stem separation and
|
||||
DeepFilterNet speech enhancement (both on the CPU `audioutils` shim), Stable
|
||||
Audio Open sound effects (`sfxgen`), plus two upgrades to existing models —
|
||||
chatterbox's stateless voice-clone route and whisper.cpp's per-request
|
||||
translate flag (both verified against the live images 2026-07-16).
|
||||
|
||||
## Decision
|
||||
|
||||
1. **`audio.StemSeparator` / `StemSeparationProvider`** —
|
||||
`StemSeparationRequest{Audio, MIME, Filename, Mode, Model, Format}` →
|
||||
`StemSeparationResult{Stems []Stem{Name, Audio, MIME}}`.
|
||||
- `Mode` is the caller-facing split: `"two"` (vocals + accompaniment,
|
||||
sent as Demucs' `two_stems=vocals`) or `"four"`; `""` = backend
|
||||
default. `Model` selects the Demucs variant (`htdemucs`/`htdemucs_ft`),
|
||||
mirroring `BackgroundRemovalRequest.Net`.
|
||||
- **The wire format is a ZIP** (`POST /upstream/<id>/v1/stems`): four WAV
|
||||
stems would blow any JSON-of-base64 budget. Entry name → stem name,
|
||||
extension → MIME; entries may sit under a per-model directory. Unpacking
|
||||
is bounded per entry (zip-bomb guard) and a non-zip 2xx body fails loud.
|
||||
2. **`SFXModel` reuses `musicgen`** — a sound effect is a short audio clip
|
||||
from a text prompt; only the provider method differs. The sfxgen route
|
||||
(`POST /upstream/<id>/v1/sfx`, JSON `{prompt, seconds, steps?, cfg_scale?,
|
||||
seed?}`) is SYNCHRONOUS (WAV body), unlike ACE-Step's job queue.
|
||||
`musicgen.Request` gains `CFGScale *float64` for it; lyrics and non-wav
|
||||
formats are rejected (the model can't honor them). The ~11s model ceiling
|
||||
is the backend's to enforce.
|
||||
3. **`audio.SpeechEnhancer` / `SpeechEnhancementProvider`** —
|
||||
`EnhancementRequest{Audio, MIME, Filename}` →
|
||||
`POST /upstream/<id>/v1/enhance` → WAV. The result reuses `SpeechResult`;
|
||||
audio-in/audio-out needs no new result type.
|
||||
4. **Voice cloning is a `SpeechRequest` field, not a new surface** —
|
||||
`ReferenceAudio []byte` + `ReferenceMIME`. When set, the llamaswap
|
||||
speech model switches from JSON `/v1/audio/speech` to multipart
|
||||
`POST /upstream/<id>/v1/audio/speech/upload` (fields `input` +
|
||||
`voice_file`; verified live: stateless per-request cloning, no voice
|
||||
library). Voice/format/speed still ride when set; the clone route's MIME
|
||||
fallback is wav (not the JSON route's mp3). Backends without cloning must
|
||||
reject a reference-carrying request rather than silently ignore it.
|
||||
5. **Translation is a `TranscriptionRequest` bool** — `Translate` maps to
|
||||
whisper.cpp's `translate=true` form field (server.cpp:534). Because that
|
||||
server's language DEFAULT is `en` (which silently skips translation), the
|
||||
provider forces `language=auto` when translating without an explicit
|
||||
language hint; an explicit hint wins.
|
||||
6. **Binary success bodies are validated before wrapping** (ADR-0020 rule):
|
||||
sfx/enhance require positive evidence of audio-ness (declared audio/*
|
||||
Content-Type or sniffed RIFF/WAVE), stems require a parseable zip with
|
||||
at least one stem.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `audio` grows from three surfaces to five; the SFX surface adds zero new
|
||||
types (musicgen reuse) — one format→MIME table and one multipart builder
|
||||
keep serving every audio endpoint.
|
||||
- The clone-route switch means one `SpeechModel` can answer over two wire
|
||||
shapes; tests pin both routes so a regression can't silently drop cloning.
|
||||
- `two_stems` is hardwired to vocals: "isolate X vs the rest" for other
|
||||
sources is a backend capability not exposed in v1 (add a field when a
|
||||
caller needs it, not before).
|
||||
@@ -0,0 +1,67 @@
|
||||
# ADR-0025: Wave-3 video surfaces (lipsync, video matte, video upscale, chain jobs)
|
||||
|
||||
Status: Accepted (2026-07-16)
|
||||
|
||||
## Context
|
||||
|
||||
The llama-swap host is gaining wave-3 video capabilities (spec: mort
|
||||
docs/specs/2026-07-16-llamaswap-wave3.md): SadTalker talking heads, Robust
|
||||
Video Matting and per-frame Real-ESRGAN on the mediautils shim, and a
|
||||
videoutils orchestrator that generates LONG videos as a chain of i2v
|
||||
segments (generate → extract last frame → continue → concat → optional RIFE
|
||||
smoothing), exposed as an async job API following the ACE-Step precedent.
|
||||
|
||||
## Decision
|
||||
|
||||
1. **Three new optional `videogen` interfaces**, ADR-0016→0024 conventions:
|
||||
- `videogen.LipSyncer` / `LipsyncProvider` — `LipsyncRequest{Image,
|
||||
Audio, AudioMIME, AudioFilename, Still, Enhance, Preprocess}` →
|
||||
`POST /upstream/<id>/v1/talking_head` (multipart `image` + `audio`
|
||||
file parts + optional `still`/`enhance`/`preprocess` fields) → mp4.
|
||||
Sync and minutes-slow (Hunyuan precedent); context deadline is the
|
||||
budget.
|
||||
- `videogen.VideoBackgroundRemover` / `VideoBackgroundRemovalProvider` —
|
||||
`VideoBackgroundRemovalRequest{Video, MIME, Filename, Output}` →
|
||||
`POST /upstream/<id>/v1/video/matte`. `Output` ∈
|
||||
`greenscreen_mp4` (universally playable) | `alpha_webm` (true
|
||||
transparency); "" = backend default.
|
||||
- `videogen.VideoUpscaler` / `VideoUpscaleProvider` —
|
||||
`VideoUpscaleRequest{Video, MIME, Filename, Scale}` (2 or 4) →
|
||||
`POST /upstream/<id>/v1/video/upscale` → mp4.
|
||||
2. **The chain client is deliberately ASYNC** (`videogen.Chainer` /
|
||||
`ChainerProvider`), unlike musicgen's blocking Generate (ADR-0021): a
|
||||
chain holds the GPU through multiple model swaps for many minutes, and
|
||||
the caller must be able to poll progress AND fetch completed segments
|
||||
after a mid-chain failure — partial delivery is mandatory (never discard
|
||||
multi-minute GPU output), which a blocking one-call contract cannot
|
||||
express.
|
||||
- `SubmitChain(ctx, ChainRequest{Segments[{Prompt,Seconds}], InitImage,
|
||||
SmoothJoins, Size}) (jobID, error)` — JSON
|
||||
`POST /upstream/<id>/v1/video/chain`, init image as base64
|
||||
`init_image_b64` (JSON submit, not multipart, per the pinned host
|
||||
contract).
|
||||
- `ChainStatus(ctx, jobID) (*ChainJob{Status, Segment, Total,
|
||||
SegmentIDs, Raw})` — `GET /v1/jobs/{id}`; polling doubles as a
|
||||
liveness signal for the shim's idle TTL. Segment entries are tolerated
|
||||
as strings or `{id|segment_id}` objects.
|
||||
- `ChainResult(ctx, jobID)` / `ChainSegmentResult(ctx, jobID, n)` —
|
||||
`GET /v1/jobs/{id}/result` and `/v1/jobs/{id}/segments/{n}`.
|
||||
- Job ids are echoed server input: job paths reject ids carrying path
|
||||
structure (`/?#`, `..`), the upstreamPath smuggling rule.
|
||||
3. **Binary success bodies are validated before wrapping** (ADR-0020 rule):
|
||||
all four clip-returning calls go through a shared `singleVideoResult`
|
||||
(positive evidence of video-ness — declared video/* Content-Type or
|
||||
sniffed mp4/webm magic — so a JSON status page or proxy error can never
|
||||
become "the clip").
|
||||
|
||||
## Consequences
|
||||
|
||||
- videogen grows from two surfaces (Model, Interpolator) to six; the
|
||||
chain client is the package's first async surface — the job-API shape
|
||||
deferred in ADR-0019/0021 now exists where the workload actually
|
||||
demands it.
|
||||
- The wire shapes are pinned by the netherstorm videoutils/mediautils/
|
||||
SadTalker image builds; host smoke tests are the drift defence.
|
||||
- mort's long-video tool owns the poll loop, timeout budget, and
|
||||
partial-result envelope; majordomo only guarantees the artifacts stay
|
||||
fetchable.
|
||||
@@ -0,0 +1,60 @@
|
||||
# ADR-0026: Kimi (Moonshot AI) built-in provider
|
||||
|
||||
**Status:** Accepted — 2026-07-18
|
||||
|
||||
## Context
|
||||
|
||||
Moonshot AI's Kimi models (Kimi K2, `moonshot-v1-*`, and the vision variants)
|
||||
are served over an OpenAI-compatible Chat Completions API at
|
||||
`https://api.moonshot.ai/v1` (`https://api.moonshot.cn/v1` for China),
|
||||
authenticated with a bearer key. mort wants Kimi as a first-class failover
|
||||
tier, so `kimi/kimi-k2-...` should parse, chain, and alias out of the box with
|
||||
a dedicated `KIMI_API_KEY` env var — the same ergonomics as `openai`,
|
||||
`anthropic`, and `google`.
|
||||
|
||||
Two tensions:
|
||||
|
||||
- The wire protocol is byte-for-byte OpenAI Chat Completions, so a hand-rolled
|
||||
client would duplicate `provider/openai` for zero gain (ADR-0007 forbids it),
|
||||
exactly as ADR-0015 found for llama-swap.
|
||||
- The README's current stance is that arbitrary OpenAI-compatible endpoints
|
||||
(Groq, Together, …) are *consumer-registered*, not baked in. Blessing Kimi as
|
||||
a built-in is a deliberate, narrow exception justified by the north star:
|
||||
mort names Kimi directly in its tiers, and a built-in with `KIMI_API_KEY`
|
||||
keeps mort's config free of boilerplate `openai.New(WithName/WithBaseURL)`
|
||||
wiring.
|
||||
|
||||
## Decision
|
||||
|
||||
- **No new package.** The `kimi` built-in and `kimi://` DSN scheme both
|
||||
construct `provider/openai` pointed at the Moonshot base URL — the chat path
|
||||
inherits every openai feature/fix automatically (like llama-swap's chat).
|
||||
- The built-in reads its key through the registry's injected `envLookup`
|
||||
(`KIMI_API_KEY` only — no `MOONSHOT_API_KEY` alias, per the project owner) so
|
||||
it stays hermetically testable via `WithEnvLookup`.
|
||||
- **`WithAPIKey` is passed unconditionally, even when empty.** `openai.New`
|
||||
defaults its key to `OPENAI_API_KEY`; without an explicit override an unset
|
||||
`KIMI_API_KEY` would silently authenticate Kimi with the OpenAI key. Passing
|
||||
the (possibly empty) lookup result severs that fallthrough.
|
||||
- New `openai.WithAPIKeyName("KIMI_API_KEY")` option customizes only the
|
||||
synthetic-401 missing-key hint (default `OPENAI_API_KEY`), so a keyless kimi
|
||||
call tells the operator the *right* variable to set.
|
||||
- The default endpoint is the international host (`kimiBaseURL`). The China
|
||||
endpoint (or any other host) is reachable with a `kimi://` DSN, e.g.
|
||||
`LLM_KCN=kimi://[email protected]/v1`. The `kimi://` scheme is an
|
||||
OpenAI-compatible target labeled `kimi` with the same key-name hint; it is
|
||||
intentionally near-identical to `openai://` — its value is a clear name in
|
||||
specs and error reporting.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `kimi/<model>` is first-class in Parse, chains, aliases, and health/failover
|
||||
with no consumer wiring; model ids pass through verbatim (no catalog).
|
||||
- Chat, streaming, tools, and structured output ride the openai client. Image
|
||||
*inputs* work at the client level but only the Moonshot vision models accept
|
||||
them (matrix footnote ³).
|
||||
- `WithAPIKeyName` is a small, generally useful addition to `provider/openai`;
|
||||
the default preserves existing behavior for every other openai-compat target.
|
||||
- Blessing one third-party endpoint as a built-in sets a precedent; future ones
|
||||
should clear the same bar (a named consumer needs it in-config), not be added
|
||||
reflexively — `RegisterProvider`/`LLM_*` remain the path for the rest.
|
||||
@@ -0,0 +1,102 @@
|
||||
# ADR-0027: Qwen (Alibaba) built-in provider — OpenAI-compat, not Anthropic-compat
|
||||
|
||||
**Status:** Accepted — 2026-08-12
|
||||
|
||||
## Context
|
||||
|
||||
Alibaba's Qwen models (`qwen3.8-max`, `qwen3.7-plus`, the `qwen3-vl-*` vision
|
||||
variants, …) are served from Model Studio / DashScope, and mort wants them as a
|
||||
first-class failover tier with a dedicated `QWEN_API_KEY` — the same ergonomics
|
||||
ADR-0026 gave Kimi.
|
||||
|
||||
Unlike Kimi, Model Studio exposes the same models over **two** protocols:
|
||||
|
||||
| | OpenAI-compatible | Anthropic-compatible |
|
||||
|---|---|---|
|
||||
| Base URL | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | `https://dashscope-intl.aliyuncs.com/apps/anthropic` |
|
||||
| Endpoints | full Chat Completions surface | `/v1/messages` only (no `/v1/models`) |
|
||||
| Purpose | the documented developer API | a shim, documented around hosting Claude Code |
|
||||
|
||||
So the question this ADR answers is not "which client do we reuse" but
|
||||
"which of Alibaba's two wire protocols does the built-in speak".
|
||||
|
||||
## Decision
|
||||
|
||||
**The `qwen` built-in and the `qwen://` DSN scheme speak OpenAI-compat**, over
|
||||
`provider/openai` — no new package, mirroring ADR-0026 (kimi) and ADR-0015
|
||||
(llama-swap chat). Default base URL is the international host; the China host
|
||||
(`dashscope.aliyuncs.com/compatible-mode/v1`) and workspace-scoped regional
|
||||
hosts are reachable with a `qwen://` DSN.
|
||||
|
||||
Credential handling is copied from kimi verbatim, because both of its rules
|
||||
are load-bearing: `WithAPIKey` is passed unconditionally (even empty) so an
|
||||
unset `QWEN_API_KEY` can never fall through to `openai.New`'s `OPENAI_API_KEY`
|
||||
default, and `WithAPIKeyName("QWEN_API_KEY")` makes the synthetic-401 hint name
|
||||
the variable the operator actually has to set.
|
||||
|
||||
The kimi and qwen DSN factories were identical, so they now share one
|
||||
`openaiCompatScheme` helper — the next OpenAI-compat built-in gets the
|
||||
credential and key-hint rules by construction rather than by copy.
|
||||
|
||||
### Why not the Anthropic-compatible endpoint
|
||||
|
||||
Every concrete difference favors OpenAI-compat *for this codebase*:
|
||||
|
||||
- **Reasoning survives the trip.** Model Studio takes `reasoning_effort` as a
|
||||
top-level field on the OpenAI surface, which `provider/openai` already sends
|
||||
— `llm.WithReasoningEffort` works on qwen with zero qwen-specific code
|
||||
(`TestQwenReasoningEffortReachesWire` asserts it on the wire). Down the
|
||||
anthropic client it would be dropped in silence: `provider/anthropic`
|
||||
deliberately ignores `Request.ReasoningEffort`, because first-party Claude
|
||||
has no such knob.
|
||||
- **Structured output would regress.** `provider/anthropic` implements
|
||||
`Request.Schema` with the first-party GA `output_config.format` mechanism.
|
||||
Alibaba's shim does not implement it; a compat endpoint that ignores an
|
||||
unknown field returns unconstrained prose while still reporting success.
|
||||
The OpenAI path sends `response_format: json_schema`, which Model Studio
|
||||
supports natively on the Max/Plus families.
|
||||
- **Cache accounting already lands.** Model Studio's implicit prefix cache
|
||||
reports hits in `usage.prompt_tokens_details.cached_tokens`, which the openai
|
||||
client already maps to `llm.Usage.CacheReadTokens`. The anthropic client
|
||||
reads `cache_read_input_tokens`, a field the shim has no reason to emit.
|
||||
- **Thinking content is discarded on the anthropic path anyway.**
|
||||
`provider/anthropic` skips `thinking` blocks in both the buffered and
|
||||
streaming decoders, so the shim's headline feature — first-class
|
||||
`thinking: {type: "enabled", budget_tokens: N}` — buys majordomo nothing
|
||||
today.
|
||||
- **Smaller blast radius.** The anthropic client has no `WithAPIKeyName`
|
||||
option, so a keyless qwen would tell the operator to set `ANTHROPIC_API_KEY`;
|
||||
fixing that means changing the first-party Anthropic client to serve a
|
||||
third-party shim.
|
||||
- **It is the less-exercised surface.** The Anthropic endpoint is documented as
|
||||
Messages-only, with a temperature range that differs from Anthropic's own
|
||||
([0, 2) vs [0.0, 1.0]) — i.e. it is Qwen semantics wearing an Anthropic
|
||||
envelope, not an Anthropic-equivalent target.
|
||||
|
||||
The one thing the Anthropic surface offers that OpenAI-compat does not is
|
||||
explicit `cache_control` breakpoints reached through `Request.PromptCache`.
|
||||
That is not a reason to route Qwen through it: Model Studio's implicit cache is
|
||||
automatic and already metered, and if explicit breakpoints ever matter they
|
||||
belong in `provider/openai` (Model Studio accepts `cache_control` on content
|
||||
blocks there too), where every OpenAI-compat target would get them.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `qwen/<model>` is first-class in Parse, chains, aliases, and health/failover
|
||||
with no consumer wiring; model ids pass through verbatim (no catalog).
|
||||
- Chat, streaming, tools, structured output, reasoning effort, and cached-token
|
||||
accounting all ride the openai client and inherit its fixes.
|
||||
- Image *inputs* work at the client level, but only the `qwen-vl-*` /
|
||||
`qwen3-vl-*` models accept them (matrix footnote ⁴; ³ is kimi's).
|
||||
- Two model-side quirks are Alibaba's, not majordomo's, and are left to the
|
||||
caller rather than papered over: thinking is **on by default** on some models
|
||||
(e.g. `qwen3.7-plus`), and Qwen3 *open-source* models require streaming when
|
||||
thinking is enabled — a buffered `Generate` against one of those needs a
|
||||
model that supports non-streaming thinking (the Max/Plus families do).
|
||||
- If a future consumer genuinely needs the Anthropic surface, it is reachable
|
||||
today without library changes:
|
||||
`LLM_QWEN_ANTHROPIC=anthropic://[email protected]/apps/anthropic`
|
||||
— with the reasoning/structured-output caveats above.
|
||||
- Second third-party built-in after kimi. The ADR-0026 bar still holds: a named
|
||||
consumer needs it in-config. `RegisterProvider`/`LLM_*` remain the path for
|
||||
everything else.
|
||||
@@ -23,3 +23,11 @@ One decision per file, append-only; supersede rather than rewrite.
|
||||
| [0017](0017-audio-interfaces.md) | audio — canonical speech synthesis + transcription interfaces | Accepted |
|
||||
| [0018](0018-imagegen-editor.md) | imagegen.Editor — image-to-image as a separate optional interface | Accepted |
|
||||
| [0019](0019-videogen-interface.md) | videogen — canonical video-generation surface | Accepted |
|
||||
| [0020](0020-upstream-passthrough-media-surfaces.md) | Upstream-passthrough media surfaces (mask, upscale, background removal, interpolation, diarization, meshgen) | Accepted |
|
||||
| [0021](0021-musicgen-interface.md) | musicgen — blocking Generate over an async job queue | Accepted |
|
||||
| [0022](0022-embeddings-rerank-interface.md) | embeddings + rerank interface | Accepted |
|
||||
| [0023](0023-image-doc-surfaces.md) | Wave-3 image + document surfaces (segmentation, colorize, face restore, OCR) | Accepted |
|
||||
| [0024](0024-audio-wave3-surfaces.md) | Wave-3 audio surfaces (stems, SFX, speech enhance, voice clone, translate) | Accepted |
|
||||
| [0025](0025-videogen-wave3-surfaces.md) | Wave-3 video surfaces (lipsync, video matte, video upscale, chain jobs) | Accepted |
|
||||
| [0026](0026-kimi-builtin.md) | Kimi (Moonshot AI) built-in provider — reuse openai client, KIMI_API_KEY | Accepted |
|
||||
| [0027](0027-qwen-builtin.md) | Qwen (Alibaba) built-in provider — OpenAI-compat, not Model Studio's Anthropic-compat endpoint | Accepted |
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
// Package embeddings is majordomo's canonical text-embedding and reranking
|
||||
// surface (ADR-0022, following the ADR-0016→0021 leaf-contract lineage).
|
||||
// Two small halves, split like audio's Speech/Transcription so backends can
|
||||
// implement either:
|
||||
//
|
||||
// - EmbedModel turns texts into dense vectors (/v1/embeddings-style).
|
||||
// - RerankModel scores documents against a query with a cross-encoder
|
||||
// (/v1/rerank-style) — usually a DIFFERENT backend model than the
|
||||
// embedder, hence a separate mint.
|
||||
//
|
||||
// Instruction-aware embedders (Qwen3-Embedding et al.) want queries wrapped
|
||||
// as "Instruct: {task}\nQuery: {query}" while documents go in bare;
|
||||
// InstructedQuery encodes that so callers don't hand-roll (and silently
|
||||
// degrade retrieval) at each site.
|
||||
package embeddings
|
||||
|
||||
import "context"
|
||||
|
||||
// EmbedRequest is a batch embedding request. Inputs are embedded
|
||||
// independently; the result vector order matches the input order.
|
||||
type EmbedRequest struct {
|
||||
// Inputs are the texts to embed. Required (at least one).
|
||||
Inputs []string
|
||||
}
|
||||
|
||||
// EmbedResult is the canonical embedding result.
|
||||
type EmbedResult struct {
|
||||
// Vectors holds one embedding per input, in input order. Backends
|
||||
// normalize per their own convention (llama-server: Euclidean-normalized).
|
||||
Vectors [][]float32
|
||||
|
||||
// Raw is the provider-native response object. May be nil.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// EmbedOption mutates an EmbedRequest before it is sent. Reserved: the
|
||||
// request shape is deliberately minimal today.
|
||||
type EmbedOption func(*EmbedRequest)
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r EmbedRequest) Apply(opts ...EmbedOption) EmbedRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// InstructedQuery wraps a retrieval QUERY for instruction-aware embedding
|
||||
// models. Documents must NOT be wrapped — the asymmetry is the point, and
|
||||
// getting it wrong silently costs retrieval quality. An empty task uses the
|
||||
// generic web-search instruction the reference model was trained with.
|
||||
func InstructedQuery(task, query string) string {
|
||||
if task == "" {
|
||||
task = "Given a web search query, retrieve relevant passages that answer the query"
|
||||
}
|
||||
return "Instruct: " + task + "\nQuery: " + query
|
||||
}
|
||||
|
||||
// EmbedModel embeds texts as dense vectors.
|
||||
type EmbedModel interface {
|
||||
// Embed returns one vector per input, in input order.
|
||||
Embed(ctx context.Context, req EmbedRequest, opts ...EmbedOption) (*EmbedResult, error)
|
||||
}
|
||||
|
||||
// EmbedModelOption configures an EmbedModel at construction time. Reserved
|
||||
// for future per-model settings.
|
||||
type EmbedModelOption func(*EmbedModelConfig)
|
||||
|
||||
// EmbedModelConfig carries per-model construction settings.
|
||||
type EmbedModelConfig struct{}
|
||||
|
||||
// ApplyEmbedModelOptions folds options into a config.
|
||||
func ApplyEmbedModelOptions(opts []EmbedModelOption) EmbedModelConfig {
|
||||
var cfg EmbedModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// EmbedProvider mints embedding models bound to one backend.
|
||||
type EmbedProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// EmbedModel returns an EmbedModel bound to the given id (passed through
|
||||
// to the backend verbatim; no catalog validation).
|
||||
EmbedModel(id string, opts ...EmbedModelOption) (EmbedModel, error)
|
||||
}
|
||||
|
||||
// RerankRequest scores documents against a query.
|
||||
type RerankRequest struct {
|
||||
// Query is the search query. Required.
|
||||
Query string
|
||||
|
||||
// Documents are the candidate texts to score. Required (at least one).
|
||||
Documents []string
|
||||
|
||||
// TopN limits how many results the backend returns; 0 = all.
|
||||
TopN int
|
||||
}
|
||||
|
||||
// RerankItem is one scored document.
|
||||
type RerankItem struct {
|
||||
// Index is the document's position in the request's Documents slice.
|
||||
Index int
|
||||
|
||||
// Score is the backend's relevance score (higher = more relevant).
|
||||
// Scales are model-specific — compare within one response only.
|
||||
Score float64
|
||||
}
|
||||
|
||||
// RerankResult is the canonical rerank result, sorted by descending Score.
|
||||
type RerankResult struct {
|
||||
// Results are the scored documents (top-N when the request bounded it).
|
||||
Results []RerankItem
|
||||
|
||||
// Raw is the provider-native response object. May be nil.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// RerankOption mutates a RerankRequest before it is sent.
|
||||
type RerankOption func(*RerankRequest)
|
||||
|
||||
// WithTopN limits how many results the backend returns.
|
||||
func WithTopN(n int) RerankOption { return func(r *RerankRequest) { r.TopN = n } }
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r RerankRequest) Apply(opts ...RerankOption) RerankRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// RerankModel scores documents against a query with a cross-encoder.
|
||||
type RerankModel interface {
|
||||
// Rerank returns scored documents sorted by descending relevance.
|
||||
Rerank(ctx context.Context, req RerankRequest, opts ...RerankOption) (*RerankResult, error)
|
||||
}
|
||||
|
||||
// RerankModelOption configures a RerankModel at construction time. Reserved
|
||||
// for future per-model settings.
|
||||
type RerankModelOption func(*RerankModelConfig)
|
||||
|
||||
// RerankModelConfig carries per-model construction settings.
|
||||
type RerankModelConfig struct{}
|
||||
|
||||
// ApplyRerankModelOptions folds options into a config.
|
||||
func ApplyRerankModelOptions(opts []RerankModelOption) RerankModelConfig {
|
||||
var cfg RerankModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// RerankProvider mints rerank models bound to one backend.
|
||||
type RerankProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// RerankModel returns a RerankModel bound to the given id (passed
|
||||
// through to the backend verbatim; no catalog validation).
|
||||
RerankModel(id string, opts ...RerankModelOption) (RerankModel, error)
|
||||
}
|
||||
@@ -26,8 +26,9 @@ var ErrUnknownProvider = errors.New("unknown provider")
|
||||
// authenticated with the bearer token "test-token".
|
||||
type DSN struct {
|
||||
// Scheme selects the provider implementation: "foreman", "ollama",
|
||||
// "ollama-cloud", "openai", "anthropic", "google"/"gemini", or any
|
||||
// custom scheme registered with RegisterScheme.
|
||||
// "ollama-cloud", "openai", "kimi", "qwen", "anthropic",
|
||||
// "google"/"gemini", "llama-swap"/"llama-swaps", or any custom scheme
|
||||
// registered with RegisterScheme.
|
||||
Scheme string
|
||||
// Token is the provider secret (bearer token or API key); empty = none.
|
||||
Token string
|
||||
@@ -40,6 +41,19 @@ type DSN struct {
|
||||
// env-defined providers always speak TLS).
|
||||
func (d DSN) BaseURL() string { return "https://" + d.Host }
|
||||
|
||||
// envKeyForProvider returns the LLM_* variable that defines the provider named
|
||||
// name: "m1" → LLM_M1, "my-prov" → LLM_MY_PROV.
|
||||
//
|
||||
// This is the single definition on purpose. Two call sites need byte-identical
|
||||
// output and would drift apart in silence: lazy resolution reads this variable
|
||||
// to find an unregistered provider, and openaiCompatScheme names it in the
|
||||
// missing-key hint so a keyless DSN target tells the operator which variable to
|
||||
// set. Those two were separate copies with a comment asserting they matched —
|
||||
// a comment is not enforcement, this function is.
|
||||
func envKeyForProvider(name string) string {
|
||||
return "LLM_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
|
||||
}
|
||||
|
||||
// ParseDSN parses a raw DSN string. The algorithm matches go-llm exactly:
|
||||
// split on "://", then an optional "@" separates the token from the host;
|
||||
// trailing slashes on the host are trimmed.
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package imagegen
|
||||
|
||||
import "context"
|
||||
|
||||
// ColorizeRequest asks a colorization backend (DDColor style) to add color to
|
||||
// a grayscale/faded photo (ADR-0023).
|
||||
type ColorizeRequest struct {
|
||||
// Image is the image to colorize. Required.
|
||||
Image Image
|
||||
}
|
||||
|
||||
// ColorizeOption mutates a ColorizeRequest before it is sent. Reserved for
|
||||
// future request settings (the reference backend takes no parameters).
|
||||
type ColorizeOption func(*ColorizeRequest)
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r ColorizeRequest) Apply(opts ...ColorizeOption) ColorizeRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Colorizer adds color to grayscale images.
|
||||
type Colorizer interface {
|
||||
// Colorize returns the colorized image as a one-image Result.
|
||||
Colorize(ctx context.Context, req ColorizeRequest, opts ...ColorizeOption) (*Result, error)
|
||||
}
|
||||
|
||||
// ColorizeModelOption configures a Colorizer at construction time. Reserved
|
||||
// for future per-model settings.
|
||||
type ColorizeModelOption func(*ColorizeModelConfig)
|
||||
|
||||
// ColorizeModelConfig carries per-model construction settings.
|
||||
type ColorizeModelConfig struct{}
|
||||
|
||||
// ApplyColorizeModelOptions folds options into a config.
|
||||
func ApplyColorizeModelOptions(opts []ColorizeModelOption) ColorizeModelConfig {
|
||||
var cfg ColorizeModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// ColorizeProvider mints Colorizers bound to one backend.
|
||||
type ColorizeProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// ColorizeModel returns a Colorizer bound to the given id (passed through
|
||||
// to the backend verbatim; no catalog validation).
|
||||
ColorizeModel(id string, opts ...ColorizeModelOption) (Colorizer, error)
|
||||
}
|
||||
+31
-1
@@ -9,9 +9,32 @@ type EditRequest struct {
|
||||
// Prompt is the text description of the desired edit.
|
||||
Prompt string
|
||||
|
||||
// Init is the initial image the edit starts from. Required.
|
||||
// Init is the initial image the edit starts from. Required, EXCEPT when
|
||||
// RefImages is set — see there.
|
||||
Init Image
|
||||
|
||||
// RefImages carries reference images for INSTRUCTION-EDIT models
|
||||
// (FLUX.1 Kontext, Qwen-Image-Edit), which are a different kind of edit
|
||||
// from img2img and reach the model by a different path.
|
||||
//
|
||||
// img2img noises Init and denoises it back under the prompt: the prompt
|
||||
// describes the DESIRED IMAGE, and how much of the original survives is a
|
||||
// function of Strength. An instruction-edit model instead takes the
|
||||
// picture as conditioning and the prompt as an INSTRUCTION about it
|
||||
// ("change the sign to read OPEN"), leaving everything it was not asked
|
||||
// to touch bit-for-bit intact — no mask, no strength, no compositing.
|
||||
//
|
||||
// Sending one of these models an Init instead of a RefImage does not
|
||||
// degrade gracefully, it silently does the wrong thing: measured against
|
||||
// FLUX.1-Kontext on 2026-07-30, "change the blue rectangle to green" via
|
||||
// init_images left the rectangle blue and drifted every other region,
|
||||
// while the same prompt via a reference image turned it green and left
|
||||
// the rest of the frame numerically unchanged.
|
||||
//
|
||||
// When RefImages is non-empty, Init/Mask/Strength are IGNORED: they
|
||||
// describe a pipeline this model does not run.
|
||||
RefImages []Image
|
||||
|
||||
// Mask restricts the edit to a region (inpainting): a single-channel or
|
||||
// RGB image the same size as Init where WHITE pixels are repainted and
|
||||
// BLACK pixels are kept. Empty = whole-image edit. Backends without mask
|
||||
@@ -54,6 +77,13 @@ type EditOption func(*EditRequest)
|
||||
// WithEditMask restricts the edit to a region (white = repaint, black = keep).
|
||||
func WithEditMask(m Image) EditOption { return func(r *EditRequest) { r.Mask = m } }
|
||||
|
||||
// WithEditRefImages supplies reference images for an instruction-edit model
|
||||
// (Kontext / Qwen-Image-Edit). See EditRequest.RefImages — this selects a
|
||||
// different edit path, not a variation on img2img.
|
||||
func WithEditRefImages(imgs ...Image) EditOption {
|
||||
return func(r *EditRequest) { r.RefImages = imgs }
|
||||
}
|
||||
|
||||
// WithEditStrength sets the denoising strength in [0,1].
|
||||
func WithEditStrength(s float64) EditOption { return func(r *EditRequest) { r.Strength = &s } }
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package imagegen
|
||||
|
||||
import "context"
|
||||
|
||||
// FaceRestoreRequest asks a face-restoration backend (GFPGAN style) to repair
|
||||
// degraded faces in a photo. Zero values mean "backend default" (ADR-0023).
|
||||
type FaceRestoreRequest struct {
|
||||
// Image is the image to restore. Required.
|
||||
Image Image
|
||||
|
||||
// Upscale is the output enlargement factor (1 or 2 on the reference
|
||||
// backend); 0 = backend default.
|
||||
Upscale int
|
||||
}
|
||||
|
||||
// FaceRestoreOption mutates a FaceRestoreRequest before it is sent.
|
||||
type FaceRestoreOption func(*FaceRestoreRequest)
|
||||
|
||||
// WithFaceRestoreUpscale sets the output enlargement factor.
|
||||
func WithFaceRestoreUpscale(n int) FaceRestoreOption {
|
||||
return func(r *FaceRestoreRequest) { r.Upscale = n }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r FaceRestoreRequest) Apply(opts ...FaceRestoreOption) FaceRestoreRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// FaceRestorer repairs degraded/blurry faces in photos.
|
||||
type FaceRestorer interface {
|
||||
// RestoreFaces returns the restored image as a one-image Result.
|
||||
RestoreFaces(ctx context.Context, req FaceRestoreRequest, opts ...FaceRestoreOption) (*Result, error)
|
||||
}
|
||||
|
||||
// FaceRestoreModelOption configures a FaceRestorer at construction time.
|
||||
// Reserved for future per-model settings.
|
||||
type FaceRestoreModelOption func(*FaceRestoreModelConfig)
|
||||
|
||||
// FaceRestoreModelConfig carries per-model construction settings.
|
||||
type FaceRestoreModelConfig struct{}
|
||||
|
||||
// ApplyFaceRestoreModelOptions folds options into a config.
|
||||
func ApplyFaceRestoreModelOptions(opts []FaceRestoreModelOption) FaceRestoreModelConfig {
|
||||
var cfg FaceRestoreModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// FaceRestoreProvider mints FaceRestorers bound to one backend.
|
||||
type FaceRestoreProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// FaceRestoreModel returns a FaceRestorer bound to the given id (passed
|
||||
// through to the backend verbatim; no catalog validation).
|
||||
FaceRestoreModel(id string, opts ...FaceRestoreModelOption) (FaceRestorer, error)
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package imagegen
|
||||
|
||||
import "context"
|
||||
|
||||
// FaceSwapRequest transfers an identity from Source into Target.
|
||||
//
|
||||
// This is a DIFFERENT OPERATION from Edit, not a better-tuned one. Measured
|
||||
// against the instruction-edit models on 2026-07-31, asking a diffusion model
|
||||
// to put a specific person's face into a photo does not work by any route —
|
||||
// by name, by description, or by supplying the portrait as a reference image.
|
||||
// Face swapping is a dedicated detect/align/blend pipeline; a provider that
|
||||
// cannot do it should not pretend Edit is a substitute.
|
||||
type FaceSwapRequest struct {
|
||||
// Target is the photo to edit — the pose, expression, lighting and
|
||||
// everything outside the face are preserved from it.
|
||||
Target Image
|
||||
|
||||
// Source is a photo of the face to put in. Only the identity travels;
|
||||
// the source's own pose and expression do not.
|
||||
Source Image
|
||||
|
||||
// Index selects WHICH face in Target, in the provider's documented
|
||||
// ordering (llamaswap: left to right by box centre, as reported by
|
||||
// ListFaces). nil = the largest face, which is right for a portrait and
|
||||
// wrong for a group — enumerate first when it matters.
|
||||
Index *int
|
||||
|
||||
// All swaps every detected face and ignores Index.
|
||||
All bool
|
||||
}
|
||||
|
||||
// FaceSwapOption mutates a FaceSwapRequest before it is sent.
|
||||
type FaceSwapOption func(*FaceSwapRequest)
|
||||
|
||||
// WithFaceIndex selects which face in the target to replace.
|
||||
func WithFaceIndex(i int) FaceSwapOption { return func(r *FaceSwapRequest) { r.Index = &i } }
|
||||
|
||||
// WithAllFaces swaps every detected face.
|
||||
func WithAllFaces() FaceSwapOption { return func(r *FaceSwapRequest) { r.All = true } }
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r FaceSwapRequest) Apply(opts ...FaceSwapOption) FaceSwapRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// DetectedFace is one face located in an image, in PIXEL coordinates.
|
||||
type DetectedFace struct {
|
||||
// Index is the face's position in the provider's stable ordering, and
|
||||
// the value FaceSwapRequest.Index expects.
|
||||
Index int
|
||||
// Box is [x0, y0, x1, y1].
|
||||
Box [4]int
|
||||
// Score is the detector's confidence, 0-1.
|
||||
Score float64
|
||||
|
||||
// Yaw is how far the head is turned from camera, in degrees, or nil when
|
||||
// the provider does not report pose. Exposed on ENUMERATION, not just
|
||||
// after the fact, because it is how a caller picks a face a swap will
|
||||
// actually work on: past roughly ±45° the features carrying identity are
|
||||
// edge-on, and the result reads as a generic person however good the
|
||||
// transfer is. A bounding box cannot show this.
|
||||
Yaw *float64
|
||||
}
|
||||
|
||||
// Size returns the box dimensions. Derived rather than stored: carrying
|
||||
// width/height alongside Box is two sources of truth for one fact, and the
|
||||
// pair can disagree after any transform.
|
||||
func (f DetectedFace) Size() (w, h int) {
|
||||
return f.Box[2] - f.Box[0], f.Box[3] - f.Box[1]
|
||||
}
|
||||
|
||||
// SwappedFace is the MEASURED outcome for one face the provider replaced.
|
||||
//
|
||||
// It exists because "the call returned an image" and "the likeness
|
||||
// transferred" are different claims that look identical from outside, and a
|
||||
// caller that cannot tell them apart will go looking for another way to
|
||||
// check. The one it reaches for — asking a vision model who the result looks
|
||||
// like — is wrong in exactly the cases that matter: a VLM shown a jogger in a
|
||||
// Georgetown cap holding McDonald's cups answers "Bill Clinton" whoever's
|
||||
// face is on him, so it reports failure on a correct swap.
|
||||
type SwappedFace struct {
|
||||
// Index is the face's position in the provider's left-to-right ordering.
|
||||
Index int
|
||||
|
||||
// Width, Height are the replaced face's pixel size in the TARGET.
|
||||
// Meaningful only against ImageWidth/ImageHeight: a 138px face is large
|
||||
// in a 400px picture and nearly invisible in a 2000px one, and it is the
|
||||
// ratio, not the absolute size, that decides whether a person notices.
|
||||
Width, Height int
|
||||
|
||||
// ImageWidth, ImageHeight are the target image's dimensions, repeated on
|
||||
// every entry so a single face is self-describing without the caller
|
||||
// holding onto the rest of the response.
|
||||
ImageWidth, ImageHeight int
|
||||
|
||||
// Yaw is how far the head is turned from camera, in degrees, or nil when
|
||||
// the provider does not report pose. The best single predictor of whether
|
||||
// a swap will READ as the source person: past roughly ±45° the features
|
||||
// carrying identity are edge-on and the result looks like a generic
|
||||
// person rather than a specific one.
|
||||
Yaw *float64
|
||||
|
||||
// IdentitySimilarity is cosine similarity between the source face and the
|
||||
// face actually present in the result, 0-1, or nil when the provider
|
||||
// could not measure it. Above ~0.5 the identity transferred; a LOW value
|
||||
// is the only evidence that a swap genuinely failed.
|
||||
IdentitySimilarity *float64
|
||||
}
|
||||
|
||||
// FractionOfImage is the swapped face's width as a share of the image's, 0-1.
|
||||
// The number that predicts whether a person will SEE the change: the swap
|
||||
// that prompted all this replaced a 138px face in a 1010px-wide photo — 14%,
|
||||
// correct by every measure and invisible at a glance — while the same code on
|
||||
// a 168px face in a 385px picture (44%) is unmistakable. Returns 0 when the
|
||||
// dimensions are unknown.
|
||||
func (f SwappedFace) FractionOfImage() float64 {
|
||||
if f.ImageWidth <= 0 || f.Width <= 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(f.Width) / float64(f.ImageWidth)
|
||||
}
|
||||
|
||||
// FaceSwapper is the optional face-transfer surface. Separate interface so
|
||||
// existing providers keep compiling; callers type-assert.
|
||||
type FaceSwapper interface {
|
||||
// ListFaces enumerates the faces in an image, in the SAME ordering
|
||||
// FaceSwapRequest.Index uses. Exposed because a caller asked to change
|
||||
// "the man on the right" needs a way to name one face and to check its
|
||||
// own choice against pixel boxes.
|
||||
ListFaces(ctx context.Context, img Image) ([]DetectedFace, error)
|
||||
|
||||
// FaceSwap transfers Source's identity into Target.
|
||||
FaceSwap(ctx context.Context, req FaceSwapRequest, opts ...FaceSwapOption) (*Result, error)
|
||||
}
|
||||
@@ -68,6 +68,15 @@ type Result struct {
|
||||
// Images are the generated images, in the order the backend returned them.
|
||||
Images []Image
|
||||
|
||||
// SwappedFaces is the measured outcome of a FaceSwap, one entry per face
|
||||
// replaced. Empty for every other operation, and empty from a provider
|
||||
// that does not measure. TYPED rather than tucked into Raw: a caller has
|
||||
// to act on this — it is the only way to distinguish a swap that
|
||||
// transferred the likeness from one that returned an image and nothing
|
||||
// more — and a value reachable solely by type-asserting an `any` is one
|
||||
// nobody discovers in time to use.
|
||||
SwappedFaces []SwappedFace
|
||||
|
||||
// Raw is the provider-native response object, an escape hatch for
|
||||
// provider-specific fields. May be nil; never required for normal use.
|
||||
Raw any
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package imagegen
|
||||
|
||||
import "context"
|
||||
|
||||
// SegmentationRequest asks a promptable-segmentation backend (GroundingDINO +
|
||||
// SAM style) for the mask of a text-described region. Zero values mean
|
||||
// "backend default" (ADR-0023).
|
||||
type SegmentationRequest struct {
|
||||
// Image is the image to segment. Required.
|
||||
Image Image
|
||||
|
||||
// Prompt is the text description of the region to segment (e.g. "the red
|
||||
// car"). Required — promptless segmentation is a different capability.
|
||||
Prompt string
|
||||
|
||||
// Threshold is the detection confidence threshold in (0,1]; 0 = backend
|
||||
// default.
|
||||
Threshold float64
|
||||
}
|
||||
|
||||
// SegmentationOption mutates a SegmentationRequest before it is sent.
|
||||
type SegmentationOption func(*SegmentationRequest)
|
||||
|
||||
// WithSegmentationThreshold sets the detection confidence threshold.
|
||||
func WithSegmentationThreshold(t float64) SegmentationOption {
|
||||
return func(r *SegmentationRequest) { r.Threshold = t }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r SegmentationRequest) Apply(opts ...SegmentationOption) SegmentationRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Segmenter extracts a prompted region's mask from an image. The result is a
|
||||
// single grayscale mask image where WHITE marks the prompted region — the
|
||||
// same polarity as EditRequest.Mask (white = repaint), so it feeds inpainting
|
||||
// directly; derive a cutout client-side by applying the mask as alpha.
|
||||
type Segmenter interface {
|
||||
// Segment returns the region mask as a one-image Result.
|
||||
Segment(ctx context.Context, req SegmentationRequest, opts ...SegmentationOption) (*Result, error)
|
||||
}
|
||||
|
||||
// SegmentationModelOption configures a Segmenter at construction time.
|
||||
// Reserved for future per-model settings (mirrors ModelOption).
|
||||
type SegmentationModelOption func(*SegmentationModelConfig)
|
||||
|
||||
// SegmentationModelConfig carries per-model construction settings.
|
||||
type SegmentationModelConfig struct{}
|
||||
|
||||
// ApplySegmentationModelOptions folds options into a config.
|
||||
func ApplySegmentationModelOptions(opts []SegmentationModelOption) SegmentationModelConfig {
|
||||
var cfg SegmentationModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SegmentationProvider mints Segmenters bound to one backend.
|
||||
type SegmentationProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// SegmentationModel returns a Segmenter bound to the given id (passed
|
||||
// through to the backend verbatim; no catalog validation).
|
||||
SegmentationModel(id string, opts ...SegmentationModelOption) (Segmenter, error)
|
||||
}
|
||||
@@ -150,3 +150,23 @@ type Provider interface {
|
||||
// backend verbatim; no catalog validation).
|
||||
MeshModel(id string, opts ...ModelOption) (Model, error)
|
||||
}
|
||||
|
||||
// Converter re-encodes a mesh between containers (GLB/STL/OBJ). A separate
|
||||
// optional surface because conversion typically runs on a DIFFERENT backend
|
||||
// than generation (the reference host converts on its mediautils shim —
|
||||
// Hunyuan3D's live server always emits GLB regardless of the requested
|
||||
// format).
|
||||
type Converter interface {
|
||||
// Convert returns the mesh re-encoded in the given format.
|
||||
Convert(ctx context.Context, mesh Mesh, format string) (*Result, error)
|
||||
}
|
||||
|
||||
// ConverterProvider mints Converters bound to one backend.
|
||||
type ConverterProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// MeshConverter returns a Converter bound to the given id (passed
|
||||
// through to the backend verbatim; no catalog validation).
|
||||
MeshConverter(id string) (Converter, error)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
// Package musicgen is majordomo's canonical music-generation surface (full
|
||||
// songs from a text prompt, optionally with lyrics). Like imagegen/audio/
|
||||
// videogen/meshgen, it is a deliberately separate leaf contract from the llm
|
||||
// package (ADR-0021, following the ADR-0016→0020 lineage: functional
|
||||
// options, zero values = backend default, bytes-only I/O, Raw escape hatch).
|
||||
//
|
||||
// The first implementation is provider/llamaswap, which targets an
|
||||
// ACE-Step-1.5-style async job API reached through the /upstream
|
||||
// passthrough; Generate blocks (polling internally) so callers get the
|
||||
// imagegen-style one-call contract — bound it with a context deadline.
|
||||
package musicgen
|
||||
|
||||
import "context"
|
||||
|
||||
// Audio is one generated piece: raw encoded bytes plus a MIME type.
|
||||
type Audio struct {
|
||||
// Data is the encoded audio container.
|
||||
Data []byte
|
||||
|
||||
// MIME is the audio MIME type, e.g. "audio/mpeg".
|
||||
MIME string
|
||||
}
|
||||
|
||||
// Request is a music-generation request. Zero values mean "backend default".
|
||||
type Request struct {
|
||||
// Prompt describes the music (genre, mood, instrumentation, tempo...).
|
||||
Prompt string
|
||||
|
||||
// Lyrics are optional song lyrics for models that sing; "" =
|
||||
// instrumental or model-written lyrics, per backend behavior.
|
||||
Lyrics string
|
||||
|
||||
// DurationSeconds is the requested clip length; 0 = backend default.
|
||||
DurationSeconds int
|
||||
|
||||
// Format is the audio container ("mp3", "wav", "flac", "opus", "aac");
|
||||
// "" = backend default (mp3 on ACE-Step).
|
||||
Format string
|
||||
|
||||
// Steps is the number of inference steps; nil = backend default.
|
||||
Steps *int
|
||||
|
||||
// CFGScale is the classifier-free-guidance scale; nil = backend default.
|
||||
// Architecture-sensitive, so prefer leaving it nil unless the caller
|
||||
// knows the target model. Backends without the knob ignore it.
|
||||
CFGScale *float64
|
||||
|
||||
// Seed fixes the RNG seed for reproducible output; nil = backend
|
||||
// default (random).
|
||||
Seed *int64
|
||||
}
|
||||
|
||||
// Result is the canonical music-generation result.
|
||||
type Result struct {
|
||||
// Audio is the generated piece.
|
||||
Audio Audio
|
||||
|
||||
// Raw is the provider-native response object (e.g. the job result with
|
||||
// bpm/keyscale metadata). May be nil.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// Option mutates a Request before it is sent. Options passed to Generate are
|
||||
// applied to a copy of the request, so a Request value can be reused.
|
||||
type Option func(*Request)
|
||||
|
||||
// WithLyrics sets song lyrics.
|
||||
func WithLyrics(l string) Option { return func(r *Request) { r.Lyrics = l } }
|
||||
|
||||
// WithDuration sets the requested clip length in seconds.
|
||||
func WithDuration(seconds int) Option {
|
||||
return func(r *Request) { r.DurationSeconds = seconds }
|
||||
}
|
||||
|
||||
// WithFormat sets the audio container format.
|
||||
func WithFormat(f string) Option { return func(r *Request) { r.Format = f } }
|
||||
|
||||
// WithSteps overrides the number of inference steps.
|
||||
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
|
||||
|
||||
// WithCFGScale overrides the classifier-free-guidance scale.
|
||||
func WithCFGScale(s float64) Option { return func(r *Request) { r.CFGScale = &s } }
|
||||
|
||||
// WithSeed fixes the RNG seed.
|
||||
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
|
||||
|
||||
// Apply returns a copy of the request with all options applied. Providers
|
||||
// call this once at the top of Generate.
|
||||
func (r Request) Apply(opts ...Option) Request {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Model generates music from text.
|
||||
type Model interface {
|
||||
// Generate renders the request as one audio clip. It blocks until the
|
||||
// backend finishes (or ctx expires) even when the backend is an async
|
||||
// job queue — polling is the provider's business, not the caller's.
|
||||
Generate(ctx context.Context, req Request, opts ...Option) (*Result, error)
|
||||
}
|
||||
|
||||
// ModelOption configures a Model at construction time. Reserved for future
|
||||
// per-model settings.
|
||||
type ModelOption func(*ModelConfig)
|
||||
|
||||
// ModelConfig carries per-model construction settings.
|
||||
type ModelConfig struct{}
|
||||
|
||||
// ApplyModelOptions folds options into a config.
|
||||
func ApplyModelOptions(opts []ModelOption) ModelConfig {
|
||||
var cfg ModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// Provider mints music models bound to one backend.
|
||||
type Provider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// MusicModel returns a Model bound to the given id (passed through to
|
||||
// the backend verbatim; no catalog validation).
|
||||
MusicModel(id string, opts ...ModelOption) (Model, error)
|
||||
}
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
// Package ocr is majordomo's canonical document text-recognition surface.
|
||||
// Like imagegen/audio/videogen/musicgen, it is a deliberately separate leaf
|
||||
// contract from the llm package (ADR-0023, following the ADR-0016→0022
|
||||
// lineage: functional options, zero values = backend default, bytes-only
|
||||
// I/O, Raw escape hatch). OCR is not chat-vision: it targets dedicated
|
||||
// detection+recognition models (Surya style) that return exact per-page,
|
||||
// per-line text rather than a model's paraphrase.
|
||||
//
|
||||
// The first implementation is provider/llamaswap, which posts the document
|
||||
// to a Surya shim through the /upstream passthrough; the shim rasterizes
|
||||
// PDFs itself, so Document may be an image (png/jpg/webp) or a PDF.
|
||||
package ocr
|
||||
|
||||
import "context"
|
||||
|
||||
// Request is a text-recognition request. Zero values mean "backend default".
|
||||
type Request struct {
|
||||
// Document is the encoded document to recognize: an image (png/jpg/webp)
|
||||
// or a PDF. Required. Carried as bytes, never a URL.
|
||||
Document []byte
|
||||
|
||||
// MIME is the document MIME type (e.g. "image/png", "application/pdf");
|
||||
// "" = let the backend sniff it.
|
||||
MIME string
|
||||
|
||||
// Filename is the multipart filename hint some backends key their format
|
||||
// detection on; "" derives one from MIME ("document.pdf") or falls back
|
||||
// to "document".
|
||||
Filename string
|
||||
|
||||
// Languages are ISO-639 hints for the recognizer (e.g. "en", "de");
|
||||
// nil = backend default (auto/multilingual).
|
||||
Languages []string
|
||||
|
||||
// MaxPages caps how many pages of a multi-page document are recognized;
|
||||
// 0 = backend default (all pages, up to the backend's own ceiling).
|
||||
MaxPages int
|
||||
}
|
||||
|
||||
// Page is the recognized text of one page.
|
||||
type Page struct {
|
||||
// Number is the 1-based page number.
|
||||
Number int
|
||||
|
||||
// Text is the page's recognized text.
|
||||
Text string
|
||||
}
|
||||
|
||||
// Result is the canonical text-recognition result.
|
||||
type Result struct {
|
||||
// Text is the full recognized text, pages joined in order.
|
||||
Text string
|
||||
|
||||
// Pages are the per-page results in page order.
|
||||
Pages []Page
|
||||
|
||||
// Raw is the provider-native response object (e.g. the per-line
|
||||
// bbox/confidence/layout detail), an escape hatch for provider-specific
|
||||
// fields. May be nil; never required for normal use.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// Option mutates a Request before it is sent. Options passed to Recognize are
|
||||
// applied to a copy of the request, so a Request value can be reused.
|
||||
type Option func(*Request)
|
||||
|
||||
// WithLanguages sets the recognizer language hints.
|
||||
func WithLanguages(langs ...string) Option {
|
||||
return func(r *Request) { r.Languages = langs }
|
||||
}
|
||||
|
||||
// WithMaxPages caps how many pages are recognized.
|
||||
func WithMaxPages(n int) Option { return func(r *Request) { r.MaxPages = n } }
|
||||
|
||||
// Apply returns a copy of the request with all options applied. Providers
|
||||
// call this once at the top of Recognize.
|
||||
func (r Request) Apply(opts ...Option) Request {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Model recognizes text in documents.
|
||||
type Model interface {
|
||||
// Recognize extracts the document's text, page by page.
|
||||
Recognize(ctx context.Context, req Request, opts ...Option) (*Result, error)
|
||||
}
|
||||
|
||||
// ModelOption configures a Model at construction time. Reserved for future
|
||||
// per-model settings.
|
||||
type ModelOption func(*ModelConfig)
|
||||
|
||||
// ModelConfig carries per-model construction settings.
|
||||
type ModelConfig struct{}
|
||||
|
||||
// ApplyModelOptions folds options into a config.
|
||||
func ApplyModelOptions(opts []ModelOption) ModelConfig {
|
||||
var cfg ModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// Provider mints OCR models bound to one backend.
|
||||
type Provider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// OCRModel returns a Model bound to the given id (passed through to the
|
||||
// backend verbatim; no catalog validation).
|
||||
OCRModel(id string, opts ...ModelOption) (Model, error)
|
||||
}
|
||||
+3
-1
@@ -213,7 +213,9 @@ func TestBuiltinsResolve(t *testing.T) {
|
||||
r := newTestRegistry(t)
|
||||
// All built-in provider names resolve even before their client
|
||||
// implementations land (stub providers error only on use).
|
||||
for _, name := range []string{"openai", "anthropic", "google", "ollama", "ollama-cloud", "foreman"} {
|
||||
// Note: llama-swap is intentionally excluded — its no-URL built-in errors
|
||||
// at Model() construction (not just on use), so it can't resolve here.
|
||||
for _, name := range []string{"openai", "kimi", "anthropic", "google", "ollama", "ollama-cloud", "foreman"} {
|
||||
if _, err := r.Parse(name + "/anything"); err != nil {
|
||||
t.Errorf("Parse(%s/anything): %v", name, err)
|
||||
}
|
||||
|
||||
+57
@@ -264,3 +264,60 @@ tests flush out.
|
||||
cheap `Health(ctx)` probe (GET /health) for often-offline hosts.
|
||||
- Hermetic httptest coverage for every new wire shape + validation errors.
|
||||
- Consumer: mort's llamaswap media tool cluster (status/image/TTS/STT tools).
|
||||
|
||||
## 2026-07-18 — Kimi (Moonshot AI) built-in provider (ADR-0026)
|
||||
|
||||
- New built-in `kimi` provider + `kimi://` DSN scheme: Moonshot's
|
||||
OpenAI-compatible Chat Completions endpoint, so both reuse `provider/openai`
|
||||
(no new client, mirrors llama-swap's chat path). Default base URL
|
||||
`https://api.moonshot.ai/v1`; China endpoint via
|
||||
`LLM_KCN=kimi://[email protected]/v1`.
|
||||
- Credential is `KIMI_API_KEY` (read through the registry's injected envLookup,
|
||||
so it's hermetically testable). `WithAPIKey` is passed unconditionally so an
|
||||
unset `KIMI_API_KEY` can never fall through to the openai client's
|
||||
`OPENAI_API_KEY` default.
|
||||
- New `openai.WithAPIKeyName` option customizes the missing-key error hint
|
||||
(default `OPENAI_API_KEY`); the kimi built-in/scheme name `KIMI_API_KEY`.
|
||||
- Hermetic tests (capturing RoundTripper): built-in base URL + bearer, missing
|
||||
key names KIMI_API_KEY with no OPENAI fallthrough and no network hit, and the
|
||||
kimi:// scheme round-trips against the China host.
|
||||
- Docs kept in sync: README built-in table + DSN scheme list + support matrix
|
||||
(footnote ³), `.env.example`, ADR-0026 (+ index; also backfilled the missing
|
||||
0024/0025 index rows).
|
||||
- Consumer: mort names Kimi as a failover tier.
|
||||
|
||||
## 2026-08-12 — Qwen (Alibaba) built-in provider (ADR-0027)
|
||||
|
||||
- New built-in `qwen` provider + `qwen://` DSN scheme over Alibaba Model
|
||||
Studio's OpenAI-compatible mode, reusing `provider/openai` (no new client,
|
||||
mirrors kimi/llama-swap). Default base URL
|
||||
`https://dashscope-intl.aliyuncs.com/compatible-mode/v1`; China/regional
|
||||
hosts via `LLM_QCN=qwen://[email protected]/compatible-mode/v1`.
|
||||
- Credential is `QWEN_API_KEY` (via the registry's injected envLookup).
|
||||
`WithAPIKey` passed unconditionally so an unset key cannot fall through to
|
||||
`OPENAI_API_KEY`; `WithAPIKeyName` names `QWEN_API_KEY` in the 401 hint.
|
||||
- **Chose OpenAI-compat over Model Studio's Anthropic-compatible
|
||||
`/apps/anthropic` shim** (ADR-0027): on the anthropic client
|
||||
`ReasoningEffort` is ignored by design, `Request.Schema` rides
|
||||
`output_config.format` (which the shim does not implement), and cached-token
|
||||
accounting reads Anthropic-only usage fields. The shim is still reachable
|
||||
ad hoc via an `anthropic://` DSN.
|
||||
- `registerOpenAICompatBuiltin` installs BOTH halves of an OpenAI-compat
|
||||
built-in (eager provider + `name://` DSN scheme via the shared
|
||||
`openaiCompatScheme`), so the two credential rules — unconditional
|
||||
`WithAPIKey`, and `WithAPIKeyName` naming that same variable — hold by
|
||||
construction. kimi and qwen are one line each.
|
||||
- `envKeyForProvider` is the single definition of the `LLM_<NAME>` form,
|
||||
shared by lazy resolution (`registry.go`) and the DSN missing-key hint. They
|
||||
were separate copies with a comment asserting they matched.
|
||||
- The shared contract is ONE table (`builtin_openaicompat_test.go`), run
|
||||
identically for every OpenAI-compat built-in: endpoint + bearer, missing key
|
||||
fails closed naming its own variable with no network hit, the `name://` DSN
|
||||
reaching another host, and a keyless DSN naming `LLM_<NAME>` rather than the
|
||||
built-in's key. Adding a built-in is a table row that immediately owes all
|
||||
four; `builtin_kimi_test.go` was retired into it. Qwen-only tests: the
|
||||
reverse credential leak, and `reasoning_effort` asserted on the wire body
|
||||
(the ADR's load-bearing claim).
|
||||
- Docs in sync: README built-in table + Qwen paragraph + DSN scheme list +
|
||||
support matrix (footnote ⁴), `.env.example`, ADR-0027 (+ index).
|
||||
- Consumer: mort wants Qwen as a failover tier.
|
||||
|
||||
+114
-12
@@ -9,6 +9,7 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
@@ -42,7 +43,11 @@ type speechRequest struct {
|
||||
Speed float64 `json:"speed,omitempty"`
|
||||
}
|
||||
|
||||
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech.
|
||||
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech, or —
|
||||
// when the request carries reference audio for voice cloning — via multipart
|
||||
// POST {base}/upstream/<id>/v1/audio/speech/upload (the chatterbox clone
|
||||
// route; verified live 2026-07-16: stateless per-request cloning, fields
|
||||
// `input` + `voice_file`).
|
||||
func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Input) == "" {
|
||||
@@ -51,6 +56,9 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
|
||||
if req.Speed < 0 {
|
||||
return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed)
|
||||
}
|
||||
if len(req.ReferenceAudio) > 0 {
|
||||
return m.speakWithReference(ctx, req)
|
||||
}
|
||||
wire := speechRequest{
|
||||
Model: m.id,
|
||||
Input: req.Input,
|
||||
@@ -72,6 +80,48 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
|
||||
return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil
|
||||
}
|
||||
|
||||
// speakWithReference performs zero-shot voice cloning via the upstream
|
||||
// passthrough clone route. The reference sample rides as the `voice_file`
|
||||
// part and the text as an `input` field; voice/format/speed still go on the
|
||||
// wire when set (upstreams ignore fields they don't understand). The result
|
||||
// MIME comes from the response header or content sniffing (audioResultMIME,
|
||||
// same validation as the sfx/enhance surfaces) — a JSON soft error or an
|
||||
// HTML proxy page must never be wrapped up as audio bytes.
|
||||
func (m *speechModel) speakWithReference(ctx context.Context, req audio.SpeechRequest) (*audio.SpeechResult, error) {
|
||||
upPath, err := upstreamPath(m.id, "/v1/audio/speech/upload")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
speed := ""
|
||||
if req.Speed != 0 {
|
||||
speed = strconv.FormatFloat(req.Speed, 'g', -1, 64)
|
||||
}
|
||||
body, formType, err := buildMultipart("build speech clone form",
|
||||
filePart{field: "voice_file", filename: transcriptionFilename("", req.ReferenceMIME), data: req.ReferenceAudio},
|
||||
[]formField{
|
||||
{"input", req.Input, true},
|
||||
{"voice", req.Voice, false},
|
||||
{"response_format", req.Format, false},
|
||||
{"speed", speed, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, upPath, m.id, formType, body, maxAudioResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech clone response contained no audio"}
|
||||
}
|
||||
mimeType := audioResultMIME(contentType, audioBytes)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("speech clone response is not audio (Content-Type %q): %s", contentType, truncateForError(audioBytes))}
|
||||
}
|
||||
return &audio.SpeechResult{Audio: audioBytes, MIME: mimeType}, nil
|
||||
}
|
||||
|
||||
// speechMIME resolves the result MIME type: the response Content-Type when it
|
||||
// is a concrete audio type, else a mapping from the requested format, else
|
||||
// audio/mpeg (the OpenAI endpoint's default container is mp3).
|
||||
@@ -93,6 +143,28 @@ func speechMIME(contentType, format string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// audioResultMIME resolves an audio body's MIME type: the response
|
||||
// Content-Type when it is a concrete audio type, else content sniffing
|
||||
// (RIFF/WAVE, Ogg and friends), else "" — the caller treats undetectable as
|
||||
// an upstream error, mirroring videoMIME. Sniffed "audio/wave" is normalized
|
||||
// to the conventional "audio/wav", and Ogg's container type
|
||||
// "application/ogg" to "audio/ogg". Shared by the clone, sfx, and enhance
|
||||
// surfaces, which all answer raw audio bodies.
|
||||
func audioResultMIME(contentType string, data []byte) string {
|
||||
if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
|
||||
return mt
|
||||
}
|
||||
switch mt := http.DetectContentType(data); {
|
||||
case mt == "audio/wave":
|
||||
return "audio/wav"
|
||||
case mt == "application/ogg":
|
||||
return "audio/ogg"
|
||||
case strings.HasPrefix(mt, "audio/"):
|
||||
return mt
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TranscriptionModel implements audio.TranscriptionProvider, binding a
|
||||
// speech-to-text model served by llama-swap (routed to a whisper.cpp-style
|
||||
// upstream).
|
||||
@@ -118,13 +190,26 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.Transcrip
|
||||
return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
|
||||
// Translation is a per-request bool form field on whisper.cpp's server
|
||||
// (server.cpp:534, verified 2026-07-16). Its language DEFAULT is "en",
|
||||
// which would make translate a no-op — so an explicit language hint
|
||||
// wins, but an unset one is forced to auto-detection.
|
||||
translate := ""
|
||||
language := req.Language
|
||||
if req.Translate {
|
||||
translate = "true"
|
||||
if language == "" {
|
||||
language = "auto"
|
||||
}
|
||||
}
|
||||
buf, formType, err := buildMultipart("build transcription form",
|
||||
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
|
||||
[]formField{
|
||||
{"model", m.id, true},
|
||||
{"response_format", "json", true},
|
||||
{"language", req.Language, false},
|
||||
{"language", language, false},
|
||||
{"prompt", req.Prompt, false},
|
||||
{"translate", translate, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -177,10 +262,13 @@ func transcriptionFilename(filename, mimeType string) string {
|
||||
}
|
||||
|
||||
// sanitizeFilename strips characters that would corrupt or inject into the
|
||||
// multipart Content-Disposition header. Quotes and backslashes are escaped
|
||||
// by mime/multipart itself; CR/LF are not — they must go.
|
||||
// multipart Content-Disposition header, or smuggle directory structure to
|
||||
// the receiving side. Quotes and backslashes are escaped by mime/multipart
|
||||
// itself, but a file-writing shim decodes them right back — so CR/LF/NUL
|
||||
// and both path separators are dropped: an upload-metadata filename must
|
||||
// never traverse ("../x", "a/b", "C:\x").
|
||||
func sanitizeFilename(name string) string {
|
||||
name = strings.NewReplacer("\r", "", "\n", "").Replace(name)
|
||||
name = strings.NewReplacer("\r", "", "\n", "", "\x00", "", "/", "", "\\", "").Replace(name)
|
||||
return strings.TrimSpace(name)
|
||||
}
|
||||
|
||||
@@ -250,27 +338,41 @@ func parseVoices(raw []byte) ([]string, error) {
|
||||
// varies. contentType sets the request Content-Type when body is non-nil.
|
||||
// A response larger than maxBytes is an error, never a silent truncation.
|
||||
func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, string, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
data, hdr, err := p.doRawHeaders(ctx, method, path, model, contentType, body, maxBytes)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return data, hdr.Get("Content-Type"), nil
|
||||
}
|
||||
|
||||
// doRawHeaders is doRaw with the WHOLE response header rather than just
|
||||
// Content-Type. Only the face swap needs it — the shim reports whether the
|
||||
// likeness actually transferred in X-Swap-Report, and that answer would be
|
||||
// thrown away by a function that keeps one header — so doRaw stays the
|
||||
// signature 25 other call sites use and delegates here. Two bodies would be
|
||||
// two places for the size cap and the status check to drift apart.
|
||||
func (p *Provider) doRawHeaders(ctx context.Context, method, path, model, contentType string, body io.Reader, maxBytes int64) ([]byte, http.Header, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
req, err := p.newRequest(ctx, method, path, contentType, body)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, nil, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: do request: %w", err)
|
||||
return nil, nil, fmt.Errorf("llama-swap: do request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, "", p.apiError(resp, model)
|
||||
return nil, nil, p.apiError(resp, model)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: read response: %w", err)
|
||||
return nil, nil, fmt.Errorf("llama-swap: read response: %w", err)
|
||||
}
|
||||
if int64(len(data)) > maxBytes {
|
||||
return nil, "", fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes)
|
||||
return nil, nil, fmt.Errorf("llama-swap: response exceeds %d bytes", maxBytes)
|
||||
}
|
||||
return data, resp.Header.Get("Content-Type"), nil
|
||||
return data, resp.Header, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
func TestSpeakWithReferenceUsesCloneRoute(t *testing.T) {
|
||||
var gotPath, gotInput, gotVoice, gotFilename string
|
||||
var gotRef []byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotInput = r.FormValue("input")
|
||||
gotVoice = r.FormValue("voice")
|
||||
f, hdr, err := r.FormFile("voice_file")
|
||||
if err != nil {
|
||||
t.Fatalf("voice_file part: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
gotRef, _ = io.ReadAll(f)
|
||||
gotFilename = hdr.Filename
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SpeechModel("chatterbox")
|
||||
res, err := sm.Speak(context.Background(),
|
||||
audio.SpeechRequest{Input: "hello in my voice", Voice: "narrator"},
|
||||
audio.WithReferenceAudio([]byte("REFWAV"), "audio/wav"))
|
||||
if err != nil {
|
||||
t.Fatalf("Speak: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/chatterbox/v1/audio/speech/upload" {
|
||||
t.Errorf("path = %q, want clone route", gotPath)
|
||||
}
|
||||
if gotInput != "hello in my voice" || gotVoice != "narrator" {
|
||||
t.Errorf("input/voice = %q/%q", gotInput, gotVoice)
|
||||
}
|
||||
if string(gotRef) != "REFWAV" || gotFilename != "audio.wav" {
|
||||
t.Errorf("ref = %q name = %q", gotRef, gotFilename)
|
||||
}
|
||||
if res.MIME != "audio/wav" || len(res.Audio) == 0 {
|
||||
t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpeakWithReferenceSniffsHeaderlessWav(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
for _, k := range []string{"voice", "response_format", "speed"} {
|
||||
if v, ok := r.MultipartForm.Value[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
w.Header()["Content-Type"] = nil // no declared type at all
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SpeechModel("chatterbox")
|
||||
res, err := sm.Speak(context.Background(),
|
||||
audio.SpeechRequest{Input: "hi"},
|
||||
audio.WithReferenceAudio([]byte("REF"), ""))
|
||||
if err != nil {
|
||||
t.Fatalf("Speak: %v", err)
|
||||
}
|
||||
// Headerless RIFF sniffs audio/wave, normalized to the conventional wav.
|
||||
if res.MIME != "audio/wav" {
|
||||
t.Errorf("MIME = %q, want sniffed audio/wav", res.MIME)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpeakWithReferenceRejectsNonAudioResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// A FastAPI-style 2xx soft error must not be wrapped up as audio.
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"detail":"reference audio too short"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SpeechModel("chatterbox")
|
||||
_, err := sm.Speak(context.Background(),
|
||||
audio.SpeechRequest{Input: "hi"},
|
||||
audio.WithReferenceAudio([]byte("REF"), "audio/wav"))
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-audio clone body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFilenameStripsPathAndControlBytes(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"song.mp3": "song.mp3",
|
||||
"../../etc/passwd": "....etcpasswd",
|
||||
"a\r\nContent-Type: evil": "aContent-Type: evil",
|
||||
"..\\..\\boot.ini": "....boot.ini",
|
||||
"nul\x00byte.wav": "nulbyte.wav",
|
||||
" / ": "",
|
||||
} {
|
||||
if got := sanitizeFilename(in); got != want {
|
||||
t.Errorf("sanitizeFilename(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpeakWithoutReferenceKeepsJSONRoute(t *testing.T) {
|
||||
var gotPath, gotContentType string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
gotContentType = r.Header.Get("Content-Type")
|
||||
_, _ = w.Write([]byte("MP3"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SpeechModel("chatterbox")
|
||||
if _, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "hi"}); err != nil {
|
||||
t.Fatalf("Speak: %v", err)
|
||||
}
|
||||
if gotPath != "/v1/audio/speech" || gotContentType != "application/json" {
|
||||
t.Errorf("path/content-type = %q/%q, want JSON route", gotPath, gotContentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeTranslateForcesAutoLanguage(t *testing.T) {
|
||||
var gotTranslate, gotLanguage string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotTranslate = r.FormValue("translate")
|
||||
gotLanguage = r.FormValue("language")
|
||||
_, _ = w.Write([]byte(`{"text":"hello"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
|
||||
res, err := tm.Transcribe(context.Background(),
|
||||
audio.TranscriptionRequest{Audio: []byte("AUDIO")},
|
||||
audio.WithTranslate())
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
// whisper.cpp's server default language is "en", which would skip
|
||||
// translation — translate must force auto-detection when no explicit
|
||||
// language hint was given.
|
||||
if gotTranslate != "true" || gotLanguage != "auto" {
|
||||
t.Errorf("translate/language = %q/%q, want true/auto", gotTranslate, gotLanguage)
|
||||
}
|
||||
if res.Text != "hello" {
|
||||
t.Errorf("text = %q", res.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeTranslateKeepsExplicitLanguage(t *testing.T) {
|
||||
var gotTranslate, gotLanguage string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotTranslate = r.FormValue("translate")
|
||||
gotLanguage = r.FormValue("language")
|
||||
_, _ = w.Write([]byte(`{"text":"hello"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
|
||||
if _, err := tm.Transcribe(context.Background(),
|
||||
audio.TranscriptionRequest{Audio: []byte("AUDIO"), Language: "de", Translate: true}); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if gotTranslate != "true" || gotLanguage != "de" {
|
||||
t.Errorf("translate/language = %q/%q, want true/de", gotTranslate, gotLanguage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeWithoutTranslateOmitsFields(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
for _, k := range []string{"translate", "language"} {
|
||||
if v, ok := r.MultipartForm.Value[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"text":"hi"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
|
||||
if _, err := tm.Transcribe(context.Background(),
|
||||
audio.TranscriptionRequest{Audio: []byte("AUDIO")}); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -152,3 +152,76 @@ func TestImageEditWithoutMaskOmitsField(t *testing.T) {
|
||||
t.Error("mask field sent for unmasked edit; want omitted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageEditByReferenceUsesTxt2ImgExtraImages pins the instruction-edit
|
||||
// wire shape. It is a DIFFERENT endpoint and a DIFFERENT field from img2img,
|
||||
// and the difference is not cosmetic: measured against FLUX.1-Kontext on
|
||||
// 2026-07-30, the same prompt sent as init_images left the thing it was told
|
||||
// to change untouched and drifted everything else, while extra_images changed
|
||||
// exactly what was asked and left the rest of the frame numerically
|
||||
// unchanged. Routing a reference edit down the img2img path would look like
|
||||
// a working call and silently produce the wrong picture.
|
||||
func TestImageEditByReferenceUsesTxt2ImgExtraImages(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
im, _ := p.ImageModel("imagegen-flux-kontext")
|
||||
ed := im.(imagegen.Editor)
|
||||
|
||||
ref := editInit(t)
|
||||
if _, err := ed.Edit(context.Background(), imagegen.EditRequest{
|
||||
Prompt: "make the sign read OPEN",
|
||||
// Init/Mask/Strength are set and must be IGNORED — they describe a
|
||||
// pipeline this model does not run.
|
||||
Init: ref,
|
||||
Mask: ref,
|
||||
Strength: func() *float64 { s := 0.75; return &s }(),
|
||||
}, imagegen.WithEditRefImages(ref)); err != nil {
|
||||
t.Fatalf("reference edit: %v", err)
|
||||
}
|
||||
|
||||
if gotPath != "/sdapi/v1/txt2img" {
|
||||
t.Errorf("path = %q, want /sdapi/v1/txt2img (there is no init latent to denoise)", gotPath)
|
||||
}
|
||||
extra, ok := gotBody["extra_images"].([]any)
|
||||
if !ok || len(extra) != 1 {
|
||||
t.Fatalf("extra_images = %v, want the one reference image", gotBody["extra_images"])
|
||||
}
|
||||
if _, present := gotBody["init_images"]; present {
|
||||
t.Error("init_images must NOT be sent on the reference path — it re-noises the picture")
|
||||
}
|
||||
if _, present := gotBody["denoising_strength"]; present {
|
||||
t.Error("denoising_strength must NOT be sent on the reference path")
|
||||
}
|
||||
if _, present := gotBody["mask"]; present {
|
||||
t.Error("mask must NOT be sent on the reference path")
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageEditByReferenceRejectsEmptyRefs guards the case that would
|
||||
// otherwise silently become a plain txt2img: a reference edit whose only
|
||||
// reference carries no bytes has nothing to condition on, and rendering the
|
||||
// prompt from scratch is not what the caller asked for.
|
||||
func TestImageEditByReferenceRejectsEmptyRefs(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
im, _ := p.ImageModel("imagegen-flux-kontext")
|
||||
ed := im.(imagegen.Editor)
|
||||
|
||||
_, err := ed.Edit(context.Background(), imagegen.EditRequest{Prompt: "anything"},
|
||||
imagegen.WithEditRefImages(imagegen.Image{MIME: "image/png"}))
|
||||
if !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Fatalf("err = %v, want ErrUnsupported for an all-empty reference set", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
// embed.go implements embeddings.EmbedProvider and embeddings.RerankProvider
|
||||
// against llama-server instances behind llama-swap (ADR-0022):
|
||||
//
|
||||
// POST /v1/embeddings {model, input: [...]} (OpenAI shape)
|
||||
// POST /v1/rerank {model, query, documents, top_n} (Jina-ish shape)
|
||||
//
|
||||
// Both paths are in llama-swap's normal model-routed tables — no /upstream
|
||||
// needed. The two surfaces are minted separately because they are two
|
||||
// DIFFERENT server instances on the host: llama-server with --embeddings
|
||||
// and --rerank enabled together returns all-zero embeddings (llama.cpp
|
||||
// #20085), so the host runs one of each and the ids differ.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/embeddings"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// EmbedModel implements embeddings.EmbedProvider. The id selects which
|
||||
// upstream llama-swap loads (a persistent CPU member on the reference host,
|
||||
// so calls are cheap and never evict GPU models).
|
||||
func (p *Provider) EmbedModel(id string, opts ...embeddings.EmbedModelOption) (embeddings.EmbedModel, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = embeddings.ApplyEmbedModelOptions(opts)
|
||||
return &embedModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type embedModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Embed implements embeddings.EmbedModel via POST {base}/v1/embeddings.
|
||||
func (m *embedModel) Embed(ctx context.Context, req embeddings.EmbedRequest, opts ...embeddings.EmbedOption) (*embeddings.EmbedResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Inputs) == 0 {
|
||||
return nil, fmt.Errorf("%w: embedding requires at least one input", llm.ErrUnsupported)
|
||||
}
|
||||
for i, in := range req.Inputs {
|
||||
if strings.TrimSpace(in) == "" {
|
||||
return nil, fmt.Errorf("%w: embedding input %d is empty", llm.ErrUnsupported, i)
|
||||
}
|
||||
}
|
||||
wire := struct {
|
||||
Model string `json:"model"`
|
||||
Input []string `json:"input"`
|
||||
}{Model: m.id, Input: req.Inputs}
|
||||
|
||||
var resp struct {
|
||||
Data []struct {
|
||||
Index int `json:"index"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, "/v1/embeddings", m.id, &wire, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(resp.Data) != len(req.Inputs) {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("embeddings response has %d vectors for %d inputs", len(resp.Data), len(req.Inputs))}
|
||||
}
|
||||
// The OpenAI shape carries an index per entry; order by it rather than
|
||||
// trusting response order.
|
||||
vectors := make([][]float32, len(req.Inputs))
|
||||
for _, d := range resp.Data {
|
||||
if d.Index < 0 || d.Index >= len(vectors) || len(d.Embedding) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("embeddings response entry index %d invalid or empty", d.Index)}
|
||||
}
|
||||
if vectors[d.Index] != nil {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("embeddings response repeats index %d", d.Index)}
|
||||
}
|
||||
vectors[d.Index] = d.Embedding
|
||||
}
|
||||
for i, v := range vectors {
|
||||
if v == nil {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("embeddings response missing vector for input %d", i)}
|
||||
}
|
||||
}
|
||||
return &embeddings.EmbedResult{Vectors: vectors, Raw: &resp}, nil
|
||||
}
|
||||
|
||||
// RerankModel implements embeddings.RerankProvider.
|
||||
func (p *Provider) RerankModel(id string, opts ...embeddings.RerankModelOption) (embeddings.RerankModel, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = embeddings.ApplyRerankModelOptions(opts)
|
||||
return &rerankModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type rerankModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Rerank implements embeddings.RerankModel via POST {base}/v1/rerank. The
|
||||
// response parser reads only results[].index and results[].relevance_score —
|
||||
// llama-server documents the shape as "might change", so stay minimal.
|
||||
func (m *rerankModel) Rerank(ctx context.Context, req embeddings.RerankRequest, opts ...embeddings.RerankOption) (*embeddings.RerankResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Query) == "" {
|
||||
return nil, fmt.Errorf("%w: rerank requires a query", llm.ErrUnsupported)
|
||||
}
|
||||
if len(req.Documents) == 0 {
|
||||
return nil, fmt.Errorf("%w: rerank requires at least one document", llm.ErrUnsupported)
|
||||
}
|
||||
if req.TopN < 0 {
|
||||
return nil, fmt.Errorf("%w: rerank top_n must be >= 0, got %d", llm.ErrUnsupported, req.TopN)
|
||||
}
|
||||
wire := struct {
|
||||
Model string `json:"model"`
|
||||
Query string `json:"query"`
|
||||
Documents []string `json:"documents"`
|
||||
TopN int `json:"top_n,omitempty"`
|
||||
}{Model: m.id, Query: req.Query, Documents: req.Documents, TopN: req.TopN}
|
||||
|
||||
var resp struct {
|
||||
Results []struct {
|
||||
Index int `json:"index"`
|
||||
RelevanceScore float64 `json:"relevance_score"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, "/v1/rerank", m.id, &wire, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(resp.Results) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "rerank response contained no results"}
|
||||
}
|
||||
out := &embeddings.RerankResult{Raw: &resp}
|
||||
for _, r := range resp.Results {
|
||||
if r.Index < 0 || r.Index >= len(req.Documents) {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("rerank result index %d out of range", r.Index)}
|
||||
}
|
||||
out.Results = append(out.Results, embeddings.RerankItem{Index: r.Index, Score: r.RelevanceScore})
|
||||
}
|
||||
sort.SliceStable(out.Results, func(i, j int) bool { return out.Results[i].Score > out.Results[j].Score })
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
// enhance.go implements audio.SpeechEnhancementProvider against a
|
||||
// DeepFilterNet shim (audioutils) reached through llama-swap's /upstream
|
||||
// passthrough (ADR-0024):
|
||||
//
|
||||
// POST /upstream/<id>/v1/enhance multipart file -> WAV
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// SpeechEnhancerModel implements audio.SpeechEnhancementProvider. The id
|
||||
// selects which upstream llama-swap loads.
|
||||
func (p *Provider) SpeechEnhancerModel(id string, opts ...audio.SpeechEnhancerModelOption) (audio.SpeechEnhancer, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = audio.ApplySpeechEnhancerModelOptions(opts)
|
||||
return &speechEnhancerModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type speechEnhancerModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Enhance implements audio.SpeechEnhancer. The endpoint always answers WAV.
|
||||
func (m *speechEnhancerModel) Enhance(ctx context.Context, req audio.EnhancementRequest, opts ...audio.EnhancementOption) (*audio.SpeechResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Audio) == 0 {
|
||||
return nil, fmt.Errorf("%w: speech enhancement requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/enhance")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, contentType, err := buildMultipart("build enhance form",
|
||||
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxAudioResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "enhance response contained no audio"}
|
||||
}
|
||||
mimeType := audioResultMIME(respType, raw)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("enhance response is not audio (Content-Type %q): %s", respType, truncateForError(raw))}
|
||||
}
|
||||
return &audio.SpeechResult{Audio: raw, MIME: mimeType}, nil
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
// faceswap.go implements imagegen.FaceSwapper against the InsightFace shim
|
||||
// (buffalo_l + inswapper_128) reached through llama-swap's /upstream
|
||||
// passthrough (ADR-0024):
|
||||
//
|
||||
// POST /upstream/<id>/v1/faces multipart file -> JSON
|
||||
// POST /upstream/<id>/v1/faceswap multipart target, source -> PNG
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// maxFaceSwapResponseBytes bounds the returned PNG. Generous: the shim echoes
|
||||
// the target's dimensions, and a 4K photo round-trips as a large lossless PNG.
|
||||
const maxFaceSwapResponseBytes = 64 << 20
|
||||
|
||||
// FaceSwapModel implements the face-transfer surface. The id selects which
|
||||
// upstream llama-swap loads.
|
||||
func (p *Provider) FaceSwapModel(id string) (imagegen.FaceSwapper, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &faceSwapModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type faceSwapModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// facesResponse mirrors the shim's /v1/faces body.
|
||||
type facesResponse struct {
|
||||
Count int `json:"count"`
|
||||
Faces []struct {
|
||||
Index int `json:"index"`
|
||||
Box []int `json:"box"`
|
||||
Score float64 `json:"score"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Yaw *float64 `json:"yaw"`
|
||||
} `json:"faces"`
|
||||
}
|
||||
|
||||
// ListFaces implements imagegen.FaceSwapper.
|
||||
func (m *faceSwapModel) ListFaces(ctx context.Context, img imagegen.Image) ([]imagegen.DetectedFace, error) {
|
||||
if len(img.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: face detection requires image bytes", llm.ErrUnsupported)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/faces")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, contentType, err := buildMultipart("build faces form",
|
||||
filePart{field: "file", filename: imageFilename(img.MIME, "image"), data: img.Data}, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var parsed facesResponse
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("faces response is not JSON: %s", truncateForError(raw))}
|
||||
}
|
||||
out := make([]imagegen.DetectedFace, 0, len(parsed.Faces))
|
||||
for _, f := range parsed.Faces {
|
||||
df := imagegen.DetectedFace{Index: f.Index, Score: f.Score, Yaw: f.Yaw}
|
||||
// A short box would silently index out of range below; treat a
|
||||
// malformed entry as a protocol error rather than zero-filling it,
|
||||
// because a wrong box sends the caller at the wrong face.
|
||||
if len(f.Box) != 4 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("face %d has a %d-element box, want 4", f.Index, len(f.Box))}
|
||||
}
|
||||
copy(df.Box[:], f.Box)
|
||||
out = append(out, df)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FaceSwap implements imagegen.FaceSwapper. The endpoint always answers PNG.
|
||||
func (m *faceSwapModel) FaceSwap(ctx context.Context, req imagegen.FaceSwapRequest, opts ...imagegen.FaceSwapOption) (*imagegen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Target.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: face swap requires a target image", llm.ErrUnsupported)
|
||||
}
|
||||
if len(req.Source.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: face swap requires a source image", llm.ErrUnsupported)
|
||||
}
|
||||
// Only when it will actually be sent: under All the index is documented
|
||||
// as ignored, so rejecting a negative one there would fail a request that
|
||||
// is perfectly well formed.
|
||||
if !req.All && req.Index != nil && *req.Index < 0 {
|
||||
return nil, fmt.Errorf("%w: face index must be >= 0, got %d", llm.ErrUnsupported, *req.Index)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/faceswap")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var fields []formField
|
||||
if req.All {
|
||||
fields = append(fields, formField{key: "all", value: "true", required: true})
|
||||
} else if req.Index != nil {
|
||||
// Only sent when NOT swapping all: the shim ignores index under
|
||||
// all=true, and sending both would imply a precedence the caller
|
||||
// cannot see.
|
||||
fields = append(fields, formField{key: "index", value: strconv.Itoa(*req.Index), required: true})
|
||||
}
|
||||
|
||||
body, contentType, err := buildMultipartFiles("build faceswap form",
|
||||
[]filePart{
|
||||
{field: "target", filename: imageFilename(req.Target.MIME, "target"), data: req.Target.Data},
|
||||
{field: "source", filename: imageFilename(req.Source.MIME, "source"), data: req.Source.Data},
|
||||
}, fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respHdr, err := m.p.doRawHeaders(ctx, http.MethodPost, path, m.id, contentType, body, maxFaceSwapResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
respType := respHdr.Get("Content-Type")
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "face swap response contained no image"}
|
||||
}
|
||||
// Validate the BYTES, not the header. sniffImageMIME falls back to
|
||||
// image/png when detection is inconclusive, so trusting it here would
|
||||
// label a JSON error body as a PNG and return it as a successful image —
|
||||
// and a header check alone misses the case where the response carries no
|
||||
// Content-Type at all. The shim answers JSON on a semantic miss (no face
|
||||
// found), which is exactly the body that would sail through.
|
||||
detected := http.DetectContentType(raw)
|
||||
if !strings.HasPrefix(detected, "image/") {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("face swap response is not an image (sniffed %q, Content-Type %q): %s",
|
||||
detected, respType, truncateForError(raw))}
|
||||
}
|
||||
// Prefer the server's own label when it is an image type (it knows
|
||||
// subtypes the sniffer does not), else what the bytes actually are.
|
||||
mimeType := detected
|
||||
if hdr := mimeFromContentType(respType, "image/"); hdr != "" {
|
||||
mimeType = hdr
|
||||
}
|
||||
return &imagegen.Result{
|
||||
Images: []llm.ImagePart{{MIME: mimeType, Data: raw}},
|
||||
SwappedFaces: parseSwapReport(respHdr.Get("X-Swap-Report")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// swapReport mirrors the shim's X-Swap-Report header.
|
||||
type swapReport struct {
|
||||
Image []int `json:"image"`
|
||||
Faces []struct {
|
||||
Index int `json:"index"`
|
||||
Size []int `json:"size"`
|
||||
Yaw *float64 `json:"yaw"`
|
||||
IdentitySimilarity *float64 `json:"identity_similarity"`
|
||||
} `json:"faces"`
|
||||
}
|
||||
|
||||
// parseSwapReport decodes the measured outcome. A missing or malformed header
|
||||
// yields nil rather than an error: an older shim does not send it, and a swap
|
||||
// that produced a good image must not fail because the diagnostics alongside
|
||||
// it were unreadable.
|
||||
func parseSwapReport(header string) []imagegen.SwappedFace {
|
||||
header = strings.TrimSpace(header)
|
||||
if header == "" {
|
||||
return nil
|
||||
}
|
||||
var rep swapReport
|
||||
if err := json.Unmarshal([]byte(header), &rep); err != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]imagegen.SwappedFace, 0, len(rep.Faces))
|
||||
for _, f := range rep.Faces {
|
||||
sf := imagegen.SwappedFace{
|
||||
Index: f.Index,
|
||||
Yaw: f.Yaw,
|
||||
IdentitySimilarity: f.IdentitySimilarity,
|
||||
}
|
||||
if len(f.Size) == 2 {
|
||||
sf.Width, sf.Height = f.Size[0], f.Size[1]
|
||||
}
|
||||
if len(rep.Image) == 2 {
|
||||
sf.ImageWidth, sf.ImageHeight = rep.Image[0], rep.Image[1]
|
||||
}
|
||||
out = append(out, sf)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// imageFilename picks a multipart filename for an image part. The shim reads
|
||||
// bytes, not names, but a plausible extension keeps server-side sniffing and
|
||||
// request logs honest. base distinguishes the parts of a multi-file form
|
||||
// ("target"/"source", "frame"/"frame_last") so a log line says which one was
|
||||
// malformed — and, for the video keyframes, so a backend that stages uploads
|
||||
// by filename cannot have the second overwrite the first.
|
||||
//
|
||||
// Every caller routes through here: two copies of one extension table is how
|
||||
// they drift.
|
||||
func imageFilename(mimeType, base string) string {
|
||||
if base == "" {
|
||||
base = "image"
|
||||
}
|
||||
mt := strings.ToLower(strings.TrimSpace(mimeType))
|
||||
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
||||
mt = parsed
|
||||
}
|
||||
switch mt {
|
||||
case "image/jpeg", "image/jpg":
|
||||
return base + ".jpg"
|
||||
case "image/webp":
|
||||
return base + ".webp"
|
||||
case "image/gif":
|
||||
return base + ".gif"
|
||||
case "image/bmp":
|
||||
return base + ".bmp"
|
||||
default:
|
||||
// PNG is the safe default: every caller in this repo either sends PNG
|
||||
// or sends something the decoder identifies by magic bytes anyway.
|
||||
return base + ".png"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// parseParts pulls the multipart form a handler received.
|
||||
func parseParts(t *testing.T, r *http.Request) (files map[string][]byte, fields map[string]string) {
|
||||
t.Helper()
|
||||
_, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil {
|
||||
t.Fatalf("content type: %v", err)
|
||||
}
|
||||
mr := multipart.NewReader(r.Body, params["boundary"])
|
||||
files, fields = map[string][]byte{}, map[string]string{}
|
||||
for {
|
||||
p, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("next part: %v", err)
|
||||
}
|
||||
body, _ := io.ReadAll(p)
|
||||
if p.FileName() != "" {
|
||||
files[p.FormName()] = body
|
||||
} else {
|
||||
fields[p.FormName()] = string(body)
|
||||
}
|
||||
}
|
||||
return files, fields
|
||||
}
|
||||
|
||||
// TestFaceSwapSendsBothFiles pins the two-file wire shape. A face swap is the
|
||||
// first endpoint in this provider taking more than one file, so buildMultipart
|
||||
// grew a sibling; getting the field NAMES wrong would reach the shim as a
|
||||
// missing-argument 422 rather than anything self-explanatory.
|
||||
func TestFaceSwapSendsBothFiles(t *testing.T) {
|
||||
var gotPath string
|
||||
var files map[string][]byte
|
||||
var fields map[string]string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
files, fields = parseParts(t, r)
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
raw, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
_, _ = w.Write(raw)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, err := p.FaceSwapModel("faceswap")
|
||||
if err != nil {
|
||||
t.Fatalf("model: %v", err)
|
||||
}
|
||||
img := editInit(t)
|
||||
res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img},
|
||||
imagegen.WithFaceIndex(2))
|
||||
if err != nil {
|
||||
t.Fatalf("faceswap: %v", err)
|
||||
}
|
||||
if len(res.Images) != 1 {
|
||||
t.Fatalf("images = %d, want 1", len(res.Images))
|
||||
}
|
||||
if !strings.HasSuffix(gotPath, "/upstream/faceswap/v1/faceswap") {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
for _, want := range []string{"target", "source"} {
|
||||
if len(files[want]) == 0 {
|
||||
t.Errorf("no %q file part — the shim requires both", want)
|
||||
}
|
||||
}
|
||||
if fields["index"] != "2" {
|
||||
t.Errorf("index = %q, want 2", fields["index"])
|
||||
}
|
||||
if _, ok := fields["all"]; ok {
|
||||
t.Error("all sent alongside index — the shim ignores index under all=true, so sending both implies a precedence the caller cannot see")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFaceSwapAllSuppressesIndex: same reasoning from the other side.
|
||||
func TestFaceSwapAllSuppressesIndex(t *testing.T) {
|
||||
var fields map[string]string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, fields = parseParts(t, r)
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
raw, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
_, _ = w.Write(raw)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, _ := p.FaceSwapModel("faceswap")
|
||||
img := editInit(t)
|
||||
if _, err := m.FaceSwap(context.Background(),
|
||||
imagegen.FaceSwapRequest{Target: img, Source: img, Index: new(int), All: true}); err != nil {
|
||||
t.Fatalf("faceswap: %v", err)
|
||||
}
|
||||
if fields["all"] != "true" {
|
||||
t.Errorf("all = %q, want true", fields["all"])
|
||||
}
|
||||
if _, ok := fields["index"]; ok {
|
||||
t.Error("index sent under all=true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFaceSwapRejectsMissingImages: both are required, and the error should
|
||||
// name which one rather than surfacing a shim 422.
|
||||
func TestFaceSwapRejectsMissingImages(t *testing.T) {
|
||||
p := New(WithBaseURL("http://example.invalid"))
|
||||
m, _ := p.FaceSwapModel("faceswap")
|
||||
img := editInit(t)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
req imagegen.FaceSwapRequest
|
||||
want string
|
||||
}{
|
||||
{"no target", imagegen.FaceSwapRequest{Source: img}, "target"},
|
||||
{"no source", imagegen.FaceSwapRequest{Target: img}, "source"},
|
||||
} {
|
||||
_, err := m.FaceSwap(context.Background(), tc.req)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Errorf("%s: err = %v, want one naming %q", tc.name, err, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFaceSwapRejectsNonImageResponse: the shim answers JSON on a semantic
|
||||
// miss (no face found). Returning those bytes as an "image" would hand the
|
||||
// caller a file that is not a picture and call it success.
|
||||
func TestFaceSwapRejectsNonImageResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"detail":{"error":"no_face_in_source"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, _ := p.FaceSwapModel("faceswap")
|
||||
img := editInit(t)
|
||||
_, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img})
|
||||
if err == nil {
|
||||
t.Fatal("a JSON body was accepted as an image")
|
||||
}
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Errorf("err = %T, want *llm.APIError so callers can classify it", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no_face_in_source") {
|
||||
t.Errorf("err = %v, want it to relay the shim's reason", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListFacesParsesOrdering: the shim's left-to-right index is the contract
|
||||
// callers select against, so it must survive decoding intact.
|
||||
func TestListFacesParsesOrdering(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasSuffix(r.URL.Path, "/upstream/faceswap/v1/faces") {
|
||||
t.Errorf("path = %q", r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"count":2,"faces":[
|
||||
{"index":0,"box":[10,20,30,40],"score":0.9,"width":20,"height":20},
|
||||
{"index":1,"box":[50,20,90,60],"score":0.8,"width":40,"height":40}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, _ := p.FaceSwapModel("faceswap")
|
||||
faces, err := m.ListFaces(context.Background(), editInit(t))
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(faces) != 2 || faces[0].Index != 0 || faces[1].Index != 1 {
|
||||
t.Fatalf("faces = %+v", faces)
|
||||
}
|
||||
if faces[1].Box != [4]int{50, 20, 90, 60} {
|
||||
t.Errorf("box = %v", faces[1].Box)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListFacesRejectsShortBox: a malformed box would send a caller at the
|
||||
// wrong face, which is worse than an error.
|
||||
func TestListFacesRejectsShortBox(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"count":1,"faces":[{"index":0,"box":[1,2],"score":0.9}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, _ := p.FaceSwapModel("faceswap")
|
||||
if _, err := m.ListFaces(context.Background(), editInit(t)); err == nil {
|
||||
t.Fatal("a 2-element box was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFaceSwapRejectsHeaderlessNonImage is the regression for gadfly's
|
||||
// blocking finding on #23, agreed by both models. sniffImageMIME falls back
|
||||
// to image/png when detection is inconclusive, and the original guard only
|
||||
// looked at Content-Type — so a JSON error body sent WITHOUT a Content-Type
|
||||
// header was labelled a PNG and returned as a successful image. The shim
|
||||
// answers JSON on a semantic miss, which is precisely the body that would
|
||||
// have sailed through.
|
||||
func TestFaceSwapRejectsHeaderlessNonImage(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
// Explicitly no Content-Type — Go only sets one if we write before
|
||||
// deleting it, so clear it to model a bare upstream response.
|
||||
w.Header()["Content-Type"] = nil
|
||||
_, _ = w.Write([]byte(`{"detail":{"error":"no_face_in_target"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, _ := p.FaceSwapModel("faceswap")
|
||||
img := editInit(t)
|
||||
_, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img})
|
||||
if err == nil {
|
||||
t.Fatal("a headerless JSON body was accepted and would have been returned as image/png")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no_face_in_target") {
|
||||
t.Errorf("err = %v, want it to relay what actually came back", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFaceSwapAllowsNegativeIndexUnderAll: index is documented as ignored
|
||||
// when all=true, so validating it there would reject a well-formed request.
|
||||
func TestFaceSwapAllowsNegativeIndexUnderAll(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
raw, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
_, _ = w.Write(raw)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, _ := p.FaceSwapModel("faceswap")
|
||||
img := editInit(t)
|
||||
neg := -1
|
||||
if _, err := m.FaceSwap(context.Background(),
|
||||
imagegen.FaceSwapRequest{Target: img, Source: img, Index: &neg, All: true}); err != nil {
|
||||
t.Fatalf("negative index rejected under all=true, where it is ignored: %v", err)
|
||||
}
|
||||
// ...but still rejected when it WOULD be sent.
|
||||
if _, err := m.FaceSwap(context.Background(),
|
||||
imagegen.FaceSwapRequest{Target: img, Source: img, Index: &neg}); err == nil {
|
||||
t.Error("negative index accepted when it would actually be sent")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFaceSwapParsesSwapReport: the measured outcome is the whole reason the
|
||||
// header exists — a caller that cannot tell "the likeness transferred" from
|
||||
// "an image came back" goes and asks a vision model, which is wrong in
|
||||
// exactly the cases that matter.
|
||||
func TestFaceSwapParsesSwapReport(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
w.Header().Set("X-Swap-Report",
|
||||
`{"image":[1010,1200],"faces":[{"index":2,"size":[138,172],"yaw":-82.2,"identity_similarity":0.791}]}`)
|
||||
raw, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
_, _ = w.Write(raw)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, err := p.FaceSwapModel("faceswap")
|
||||
if err != nil {
|
||||
t.Fatalf("model: %v", err)
|
||||
}
|
||||
img := editInit(t)
|
||||
res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img})
|
||||
if err != nil {
|
||||
t.Fatalf("FaceSwap: %v", err)
|
||||
}
|
||||
if len(res.SwappedFaces) != 1 {
|
||||
t.Fatalf("SwappedFaces = %d, want 1 — the measurement was dropped", len(res.SwappedFaces))
|
||||
}
|
||||
f := res.SwappedFaces[0]
|
||||
if f.Index != 2 || f.Width != 138 || f.Height != 172 {
|
||||
t.Errorf("face = %+v, want index 2 at 138x172", f)
|
||||
}
|
||||
if f.Yaw == nil || *f.Yaw != -82.2 {
|
||||
t.Errorf("yaw = %v, want -82.2 — the pose signal is how a caller knows a profile swap will not read", f.Yaw)
|
||||
}
|
||||
if f.IdentitySimilarity == nil || *f.IdentitySimilarity != 0.791 {
|
||||
t.Errorf("identity_similarity = %v, want 0.791", f.IdentitySimilarity)
|
||||
}
|
||||
// 138/1010 — the number that says "correct, and invisible at a glance".
|
||||
if got := f.FractionOfImage(); got < 0.13 || got > 0.14 {
|
||||
t.Errorf("FractionOfImage = %.3f, want ~0.137", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFaceSwapSurvivesMissingReport: an older shim sends no header at all. A
|
||||
// swap that produced a good image must not fail because the diagnostics
|
||||
// beside it were absent or malformed.
|
||||
func TestFaceSwapSurvivesMissingReport(t *testing.T) {
|
||||
for name, hdr := range map[string]string{
|
||||
"absent": "",
|
||||
"garbage": "not json at all",
|
||||
"wrongtype": `{"faces":"nope"}`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
if hdr != "" {
|
||||
w.Header().Set("X-Swap-Report", hdr)
|
||||
}
|
||||
raw, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
_, _ = w.Write(raw)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
m, err := p.FaceSwapModel("faceswap")
|
||||
if err != nil {
|
||||
t.Fatalf("model: %v", err)
|
||||
}
|
||||
img := editInit(t)
|
||||
res, err := m.FaceSwap(context.Background(), imagegen.FaceSwapRequest{Target: img, Source: img})
|
||||
if err != nil {
|
||||
t.Fatalf("a %s report failed the whole swap: %v", name, err)
|
||||
}
|
||||
if len(res.Images) != 1 {
|
||||
t.Fatal("image lost")
|
||||
}
|
||||
if res.SwappedFaces != nil {
|
||||
t.Errorf("SwappedFaces = %+v, want nil for a %s report", res.SwappedFaces, name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -135,9 +135,30 @@ type img2imgRequest struct {
|
||||
Mask string `json:"mask,omitempty"`
|
||||
}
|
||||
|
||||
// Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img.
|
||||
// refEditRequest is the wire shape for an INSTRUCTION-EDIT model. sd-server
|
||||
// exposes reference images as `extra_images` on the shared img-gen request
|
||||
// builder (routes_sdapi.cpp lands them in gen_params.ref_images — the same
|
||||
// place the CLI's -r/--ref-image goes), and that field is read on BOTH
|
||||
// /txt2img and /img2img.
|
||||
//
|
||||
// It posts to /txt2img because there is no init latent to denoise: the
|
||||
// reference IS the conditioning, so an init image plus a denoising strength
|
||||
// would only add noise to a pipeline that does not want any. Output
|
||||
// resolution follows the reference image.
|
||||
type refEditRequest struct {
|
||||
txt2imgRequest
|
||||
ExtraImages []string `json:"extra_images"`
|
||||
}
|
||||
|
||||
// Edit implements imagegen.Editor. Two different pipelines live behind it,
|
||||
// selected by the request: RefImages routes to an instruction-edit model via
|
||||
// /sdapi/v1/txt2img + extra_images, everything else is img2img. See
|
||||
// imagegen.EditRequest.RefImages for why they are not interchangeable.
|
||||
func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ...imagegen.EditOption) (*imagegen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.RefImages) > 0 {
|
||||
return m.editByReference(ctx, req)
|
||||
}
|
||||
if len(req.Init.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: image edit requires an init image", llm.ErrUnsupported)
|
||||
}
|
||||
@@ -164,6 +185,34 @@ func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ..
|
||||
return decodeImages(m.p.name, m.id, &resp)
|
||||
}
|
||||
|
||||
// editByReference runs the instruction-edit path. Mask and Strength are
|
||||
// deliberately NOT rejected when set: a caller that hands the same
|
||||
// EditRequest to whichever model is configured should get the better result
|
||||
// on a Kontext-class model, not an error, and both fields describe a
|
||||
// pipeline that simply does not exist here.
|
||||
func (m *imageModel) editByReference(ctx context.Context, req imagegen.EditRequest) (*imagegen.Result, error) {
|
||||
base, err := m.sdWire("reference edit", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wire := refEditRequest{txt2imgRequest: base}
|
||||
for _, ref := range req.RefImages {
|
||||
if len(ref.Data) == 0 {
|
||||
continue
|
||||
}
|
||||
wire.ExtraImages = append(wire.ExtraImages, base64.StdEncoding.EncodeToString(ref.Data))
|
||||
}
|
||||
if len(wire.ExtraImages) == 0 {
|
||||
return nil, fmt.Errorf("%w: reference edit requires at least one non-empty reference image", llm.ErrUnsupported)
|
||||
}
|
||||
|
||||
var resp txt2imgResponse
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/txt2img", m.id, &wire, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeImages(m.p.name, m.id, &resp)
|
||||
}
|
||||
|
||||
// parseSize splits a "WxH" string into width/height pointers. "" yields
|
||||
// (nil, nil) so the model's own default resolution applies.
|
||||
func parseSize(size string) (*int, *int, error) {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
// lipsync.go implements videogen.LipsyncProvider against a SadTalker shim
|
||||
// reached through llama-swap's /upstream passthrough (ADR-0025):
|
||||
//
|
||||
// POST /upstream/<id>/v1/talking_head multipart image,audio[,still,enhance,preprocess]
|
||||
//
|
||||
// The response body IS the encoded clip (same contract as /v1/videos/sync),
|
||||
// hence the video-sized response cap and the same MIME resolution rules.
|
||||
// Generation is sync and slow (minutes) — bound calls with a context
|
||||
// deadline (Hunyuan precedent).
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
// LipsyncModel implements videogen.LipsyncProvider. The id selects which
|
||||
// upstream llama-swap loads.
|
||||
func (p *Provider) LipsyncModel(id string, opts ...videogen.LipsyncModelOption) (videogen.Lipsyncer, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = videogen.ApplyLipsyncModelOptions(opts)
|
||||
return &lipsyncModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type lipsyncModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Lipsync implements videogen.Lipsyncer.
|
||||
func (m *lipsyncModel) Lipsync(ctx context.Context, req videogen.LipsyncRequest, opts ...videogen.LipsyncOption) (*videogen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Image.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: lipsync requires a portrait image", llm.ErrUnsupported)
|
||||
}
|
||||
if len(req.Audio) == 0 {
|
||||
return nil, fmt.Errorf("%w: lipsync requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Preprocess != "" && req.Preprocess != "crop" && req.Preprocess != "full" {
|
||||
return nil, fmt.Errorf("%w: lipsync preprocess must be \"crop\" or \"full\", got %q", llm.ErrUnsupported, req.Preprocess)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/talking_head")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Two file parts — buildMultipart handles exactly one, so assemble by
|
||||
// hand (mirrors videoModel.Generate).
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
fw, err := w.CreateFormFile("image", imageFilename(req.Image.MIME, "frame"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
||||
}
|
||||
if _, err := fw.Write(req.Image.Data); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
||||
}
|
||||
fw, err = w.CreateFormFile("audio", transcriptionFilename(req.AudioFilename, req.AudioMIME))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
||||
}
|
||||
if _, err := fw.Write(req.Audio); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
||||
}
|
||||
still := ""
|
||||
if req.Still {
|
||||
still = "true"
|
||||
}
|
||||
enhance := ""
|
||||
if req.Enhance {
|
||||
enhance = "true"
|
||||
}
|
||||
if err := writeFormFields(w, "build lipsync form", []formField{
|
||||
{"still", still, false},
|
||||
{"enhance", enhance, false},
|
||||
{"preprocess", req.Preprocess, false},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build lipsync form: %w", err)
|
||||
}
|
||||
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, w.FormDataContentType(), &buf, maxVideoResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleVideoResult(m.p.name, m.id, "lipsync", raw, respType)
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
// webmFixture is a minimal EBML header so http.DetectContentType sniffs
|
||||
// video/webm.
|
||||
func webmFixture() []byte {
|
||||
return append([]byte{0x1A, 0x45, 0xDF, 0xA3}, make([]byte, 20)...)
|
||||
}
|
||||
|
||||
func TestLipsync(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
var gotPath, gotStill, gotEnhance, gotPreprocess string
|
||||
var gotImage, gotAudio []byte
|
||||
var gotImageName, gotAudioName string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotStill = r.FormValue("still")
|
||||
gotEnhance = r.FormValue("enhance")
|
||||
gotPreprocess = r.FormValue("preprocess")
|
||||
if f, hdr, err := r.FormFile("image"); err == nil {
|
||||
gotImage, _ = io.ReadAll(f)
|
||||
gotImageName = hdr.Filename
|
||||
f.Close()
|
||||
} else {
|
||||
t.Errorf("image part: %v", err)
|
||||
}
|
||||
if f, hdr, err := r.FormFile("audio"); err == nil {
|
||||
gotAudio, _ = io.ReadAll(f)
|
||||
gotAudioName = hdr.Filename
|
||||
f.Close()
|
||||
} else {
|
||||
t.Errorf("audio part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write(mp4Fixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ls, err := p.LipsyncModel("lipsync-sadtalker")
|
||||
if err != nil {
|
||||
t.Fatalf("LipsyncModel: %v", err)
|
||||
}
|
||||
res, err := ls.Lipsync(context.Background(),
|
||||
videogen.LipsyncRequest{
|
||||
Image: videogen.Image{MIME: "image/png", Data: png},
|
||||
Audio: []byte("SPEECH"),
|
||||
AudioMIME: "audio/wav",
|
||||
},
|
||||
videogen.WithLipsyncStill(), videogen.WithLipsyncEnhance(), videogen.WithLipsyncPreprocess("full"))
|
||||
if err != nil {
|
||||
t.Fatalf("Lipsync: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/lipsync-sadtalker/v1/talking_head" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotStill != "true" || gotEnhance != "true" || gotPreprocess != "full" {
|
||||
t.Errorf("still/enhance/preprocess = %q/%q/%q", gotStill, gotEnhance, gotPreprocess)
|
||||
}
|
||||
if string(gotImage) != string(png) || gotImageName != "frame.png" {
|
||||
t.Errorf("image bytes/name = %d bytes/%q", len(gotImage), gotImageName)
|
||||
}
|
||||
if string(gotAudio) != "SPEECH" || gotAudioName != "audio.wav" {
|
||||
t.Errorf("audio = %q name = %q", gotAudio, gotAudioName)
|
||||
}
|
||||
if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
|
||||
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLipsyncOmitsUnsetFlags(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
for _, k := range []string{"still", "enhance", "preprocess"} {
|
||||
if v, ok := r.MultipartForm.Value[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write(mp4Fixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ls, _ := p.LipsyncModel("lipsync-sadtalker")
|
||||
if _, err := ls.Lipsync(context.Background(),
|
||||
videogen.LipsyncRequest{Image: videogen.Image{Data: pngFixture(t)}, Audio: []byte("A")}); err != nil {
|
||||
t.Fatalf("Lipsync: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLipsyncRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
ls, _ := p.LipsyncModel("lipsync-sadtalker")
|
||||
if _, err := ls.Lipsync(context.Background(),
|
||||
videogen.LipsyncRequest{Audio: []byte("A")}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no image: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ls.Lipsync(context.Background(),
|
||||
videogen.LipsyncRequest{Image: videogen.Image{Data: []byte{1}}}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no audio: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ls.Lipsync(context.Background(),
|
||||
videogen.LipsyncRequest{Image: videogen.Image{Data: []byte{1}}, Audio: []byte{1}, Preprocess: "zoom"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("preprocess zoom: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLipsyncRejectsNonVideoResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte("<html>proxy error page</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ls, _ := p.LipsyncModel("lipsync-sadtalker")
|
||||
_, err := ls.Lipsync(context.Background(),
|
||||
videogen.LipsyncRequest{Image: videogen.Image{Data: pngFixture(t)}, Audio: []byte("A")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-video body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveVideoBackground(t *testing.T) {
|
||||
var gotPath, gotOutput, gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotOutput = r.FormValue("output")
|
||||
if _, hdr, err := r.FormFile("file"); err == nil {
|
||||
gotFilename = hdr.Filename
|
||||
} else {
|
||||
t.Errorf("file part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/webm")
|
||||
_, _ = w.Write(webmFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vb, err := p.VideoBackgroundRemoverModel("mediautils")
|
||||
if err != nil {
|
||||
t.Fatalf("VideoBackgroundRemoverModel: %v", err)
|
||||
}
|
||||
res, err := vb.RemoveVideoBackground(context.Background(),
|
||||
videogen.VideoBackgroundRemovalRequest{Video: mp4Fixture(), MIME: "video/mp4"},
|
||||
videogen.WithVideoBackgroundOutput("alpha_webm"))
|
||||
if err != nil {
|
||||
t.Fatalf("RemoveVideoBackground: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/mediautils/v1/video/matte" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotOutput != "alpha_webm" || gotFilename != "video.mp4" {
|
||||
t.Errorf("output/filename = %q/%q", gotOutput, gotFilename)
|
||||
}
|
||||
if res.Video.MIME != "video/webm" || len(res.Video.Data) == 0 {
|
||||
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveVideoBackgroundOmitsDefaultOutput(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
if v, ok := r.MultipartForm.Value["output"]; ok {
|
||||
t.Errorf("output field sent for default: %v; want omitted", v)
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write(mp4Fixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vb, _ := p.VideoBackgroundRemoverModel("mediautils")
|
||||
if _, err := vb.RemoveVideoBackground(context.Background(),
|
||||
videogen.VideoBackgroundRemovalRequest{Video: mp4Fixture()}); err != nil {
|
||||
t.Fatalf("RemoveVideoBackground: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveVideoBackgroundRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
vb, _ := p.VideoBackgroundRemoverModel("mediautils")
|
||||
if _, err := vb.RemoveVideoBackground(context.Background(),
|
||||
videogen.VideoBackgroundRemovalRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no video: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := vb.RemoveVideoBackground(context.Background(),
|
||||
videogen.VideoBackgroundRemovalRequest{Video: []byte{1}, Output: "gif"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("output gif: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpscaleVideo(t *testing.T) {
|
||||
var gotPath, gotScale string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotScale = r.FormValue("scale")
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write(mp4Fixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vu, err := p.VideoUpscalerModel("mediautils")
|
||||
if err != nil {
|
||||
t.Fatalf("VideoUpscalerModel: %v", err)
|
||||
}
|
||||
res, err := vu.UpscaleVideo(context.Background(),
|
||||
videogen.VideoUpscaleRequest{Video: mp4Fixture(), MIME: "video/mp4"},
|
||||
videogen.WithVideoUpscaleScale(2))
|
||||
if err != nil {
|
||||
t.Fatalf("UpscaleVideo: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/mediautils/v1/video/upscale" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotScale != "2" {
|
||||
t.Errorf("scale = %q", gotScale)
|
||||
}
|
||||
if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
|
||||
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpscaleVideoRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
vu, _ := p.VideoUpscalerModel("mediautils")
|
||||
if _, err := vu.UpscaleVideo(context.Background(), videogen.VideoUpscaleRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no video: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := vu.UpscaleVideo(context.Background(),
|
||||
videogen.VideoUpscaleRequest{Video: []byte{1}, Scale: 3}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("scale 3: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpscaleVideoRejectsNonVideoResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header()["Content-Type"] = nil // NO Content-Type at all
|
||||
_, _ = w.Write([]byte("502 bad gateway"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vu, _ := p.VideoUpscalerModel("mediautils")
|
||||
_, err := vu.UpscaleVideo(context.Background(), videogen.VideoUpscaleRequest{Video: mp4Fixture()})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for headerless non-video body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoInputFilename(t *testing.T) {
|
||||
cases := []struct {
|
||||
filename, mime, want string
|
||||
}{
|
||||
{"clip.mp4", "", "clip.mp4"},
|
||||
{"evil\r\nclip.mp4", "", "evilclip.mp4"},
|
||||
{"", "video/mp4", "video.mp4"},
|
||||
{"", "video/webm", "video.webm"},
|
||||
{"", "video/quicktime", "video.mov"},
|
||||
{"", "", "video"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := videoInputFilename(tc.filename, tc.mime); got != tc.want {
|
||||
t.Errorf("videoInputFilename(%q, %q) = %q, want %q", tc.filename, tc.mime, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,12 @@ const maxResponseBytes = 64 << 20
|
||||
// can't allocate without limit.
|
||||
const maxVideoResponseBytes = 512 << 20
|
||||
|
||||
// maxAudioResponseBytes caps bodies that ARE a single encoded audio clip
|
||||
// (voice clone, speech enhancement, sfx): a long uncompressed WAV
|
||||
// legitimately passes the 64MB JSON cap. Still bounded so a buggy upstream
|
||||
// can't allocate without limit.
|
||||
const maxAudioResponseBytes = 256 << 20
|
||||
|
||||
// Provider is a llama-swap client. It satisfies llm.Provider (chat, delegated
|
||||
// to provider/openai) and imagegen.Provider (image generation), and exposes
|
||||
// llama-swap's management endpoints as concrete methods.
|
||||
|
||||
@@ -207,12 +207,5 @@ func (m *interpolatorModel) Interpolate(ctx context.Context, req videogen.Interp
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "interpolate response contained no video"}
|
||||
}
|
||||
mimeType := videoMIME(respType, raw)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "interpolate response is not a video"}
|
||||
}
|
||||
return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil
|
||||
return singleVideoResult(m.p.name, m.id, "interpolate", raw, respType)
|
||||
}
|
||||
|
||||
@@ -238,7 +238,7 @@ func TestInterpolateRejectsBadArgs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestUpstreamPathRejectsSeparators(t *testing.T) {
|
||||
for _, bad := range []string{"", "a/b", "a?b", "a#b"} {
|
||||
for _, bad := range []string{"", "a/b", "a?b", "a#b", "a%2Fb", "%2e%2e", "a%b"} {
|
||||
if _, err := upstreamPath(bad, "/x"); err == nil {
|
||||
t.Errorf("upstreamPath(%q) succeeded; want error", bad)
|
||||
}
|
||||
|
||||
@@ -42,7 +42,11 @@ type meshModel struct {
|
||||
|
||||
// hunyuanGenerateRequest is the Hunyuan3D api_server /generate shape.
|
||||
// Optional fields are pointers/omitempty so unset values fall back to the
|
||||
// server's defaults (mirrors the sd-server wire structs).
|
||||
// server's defaults (mirrors the sd-server wire structs). Type is sent for
|
||||
// forward-compat but the LIVE server's GenerationRequest has no such field
|
||||
// and always returns GLB (verified 2026-07-14) — the response format is
|
||||
// therefore SNIFFED, and non-GLB output is the caller's conversion problem
|
||||
// (meshgen.Converter / the mediautils shim).
|
||||
type hunyuanGenerateRequest struct {
|
||||
Image string `json:"image"`
|
||||
Type string `json:"type,omitempty"`
|
||||
@@ -123,7 +127,24 @@ func (m *meshModel) Generate(ctx context.Context, req meshgen.Request, opts ...m
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: "mesh response is JSON, not mesh bytes: " + truncateForError(raw)}
|
||||
}
|
||||
return &meshgen.Result{Mesh: meshgen.Mesh{Data: raw, Format: format, MIME: mimeType}}, nil
|
||||
// Label the result by what the bytes ARE, not what was requested: the
|
||||
// live server ignores the format field entirely.
|
||||
actualFormat, actualMIME := sniffMeshFormat(raw, format, mimeType)
|
||||
return &meshgen.Result{Mesh: meshgen.Mesh{Data: raw, Format: actualFormat, MIME: actualMIME}}, nil
|
||||
}
|
||||
|
||||
// sniffMeshFormat identifies the mesh container from magic bytes, falling
|
||||
// back to the requested format only when the bytes are ambiguous (binary
|
||||
// STL has no magic).
|
||||
func sniffMeshFormat(raw []byte, requested, requestedMIME string) (string, string) {
|
||||
switch {
|
||||
case len(raw) >= 4 && string(raw[:4]) == "glTF":
|
||||
return "glb", meshFormats["glb"]
|
||||
case len(raw) >= 6 && strings.EqualFold(string(raw[:6]), "solid "):
|
||||
return "stl", meshFormats["stl"]
|
||||
default:
|
||||
return requested, requestedMIME
|
||||
}
|
||||
}
|
||||
|
||||
// truncateForError bounds a payload quoted into an error message.
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// mesh_convert.go implements meshgen.ConverterProvider against the
|
||||
// mediautils shim's POST /v1/convert_mesh (multipart file + target),
|
||||
// reached through the /upstream passthrough (ADR-0020). Exists because
|
||||
// Hunyuan3D's api_server always returns GLB — STL for the printer
|
||||
// pipeline is produced by this conversion hop.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/meshgen"
|
||||
)
|
||||
|
||||
// MeshConverter implements meshgen.ConverterProvider.
|
||||
func (p *Provider) MeshConverter(id string) (meshgen.Converter, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &meshConverter{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type meshConverter struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Convert implements meshgen.Converter.
|
||||
func (m *meshConverter) Convert(ctx context.Context, mesh meshgen.Mesh, format string) (*meshgen.Result, error) {
|
||||
if len(mesh.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: mesh conversion requires mesh bytes", llm.ErrUnsupported)
|
||||
}
|
||||
format = strings.ToLower(strings.TrimSpace(format))
|
||||
mimeType, ok := meshFormats[format]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: unsupported mesh format %q (want glb, stl, or obj)", llm.ErrUnsupported, format)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/convert_mesh")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filename := "mesh." + mesh.Format
|
||||
if mesh.Format == "" {
|
||||
filename = "mesh"
|
||||
}
|
||||
body, contentType, err := buildMultipart("build mesh-convert form",
|
||||
filePart{field: "file", filename: filename, data: mesh.Data},
|
||||
[]formField{{"target", format, true}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxMeshResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "mesh conversion returned no data"}
|
||||
}
|
||||
actualFormat, actualMIME := sniffMeshFormat(raw, format, mimeType)
|
||||
return &meshgen.Result{Mesh: meshgen.Mesh{Data: raw, Format: actualFormat, MIME: actualMIME}}, nil
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
// music.go implements musicgen.Provider against an ACE-Step-1.5-style API
|
||||
// server reached through llama-swap's /upstream passthrough (ADR-0021):
|
||||
//
|
||||
// POST /upstream/<id>/release_task {prompt, lyrics, ...} -> {task_id}
|
||||
// POST /upstream/<id>/query_result {task_id_list: [...]} -> status+result
|
||||
// GET /upstream/<id>/<result file URL> -> audio bytes
|
||||
//
|
||||
// The backend is an async job queue; Generate wraps it into the blocking
|
||||
// one-call contract by polling, so a context deadline is the caller's
|
||||
// budget for the whole job (mort's tool timeout sits well under its
|
||||
// agent-runtime ceiling for exactly this reason).
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
|
||||
)
|
||||
|
||||
// musicPollInterval is the delay between query_result polls. Long enough to
|
||||
// be polite to the queue, short enough that a ~10s xl-turbo song isn't
|
||||
// dominated by poll latency. A var so tests can shrink it.
|
||||
var musicPollInterval = 2 * time.Second
|
||||
|
||||
// musicPollMaxConsecutiveFailures bounds how many consecutive BAD polls
|
||||
// (transport error, unparseable payload, task momentarily absent) are
|
||||
// tolerated before aborting. A multi-minute GPU job must not die to one
|
||||
// blip; a genuinely broken upstream still fails within ~5 intervals.
|
||||
const musicPollMaxConsecutiveFailures = 5
|
||||
|
||||
// MusicModel implements musicgen.Provider. The id selects which upstream
|
||||
// llama-swap loads.
|
||||
func (p *Provider) MusicModel(id string, opts ...musicgen.ModelOption) (musicgen.Model, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = musicgen.ApplyModelOptions(opts)
|
||||
return &musicModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type musicModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// releaseTaskRequest is the ACE-Step POST /release_task shape. audio_duration
|
||||
// is the v1 param name; verify against the upstream ACE-Step-1.5 repo's
|
||||
// docs/en/API.md at smoke time — an
|
||||
// unknown field is ignored upstream, degrading to the default clip length,
|
||||
// never an error.
|
||||
type releaseTaskRequest struct {
|
||||
Prompt string `json:"prompt"`
|
||||
Lyrics string `json:"lyrics,omitempty"`
|
||||
AudioFormat string `json:"audio_format,omitempty"`
|
||||
TaskType string `json:"task_type"`
|
||||
AudioDuration int `json:"audio_duration,omitempty"`
|
||||
InferenceSteps *int `json:"inference_steps,omitempty"`
|
||||
Seed *int64 `json:"seed,omitempty"`
|
||||
}
|
||||
|
||||
// queryItem is one task's poll state. `result` arrives as a JSON-encoded
|
||||
// STRING (the ACE-Step API double-encodes it).
|
||||
type queryItem struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Status int `json:"status"` // 0 queued/running, 1 succeeded, 2 failed
|
||||
Result string `json:"result"`
|
||||
}
|
||||
|
||||
// musicResult is the useful subset of ACE-Step's double-encoded result
|
||||
// blob.
|
||||
type musicResult struct {
|
||||
File string `json:"file"`
|
||||
}
|
||||
|
||||
// parseMusicResult decodes the `result` string, tolerating two live-API
|
||||
// quirks (observed 2026-07-14): the payload is an ARRAY of result objects
|
||||
// (not a bare object), and string values can contain RAW control
|
||||
// characters (a literal newline in timing fields) that strict JSON
|
||||
// rejects. Control chars can only legally sit inside string values in the
|
||||
// double-encoded blob, so replacing them with spaces preserves structure.
|
||||
func parseMusicResult(blob string) (musicResult, bool) {
|
||||
sanitized := strings.Map(func(r rune) rune {
|
||||
if r < 0x20 {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, blob)
|
||||
var arr []musicResult
|
||||
if err := json.Unmarshal([]byte(sanitized), &arr); err == nil && len(arr) > 0 {
|
||||
return arr[0], true
|
||||
}
|
||||
var one musicResult
|
||||
if err := json.Unmarshal([]byte(sanitized), &one); err == nil {
|
||||
return one, true
|
||||
}
|
||||
return musicResult{}, false
|
||||
}
|
||||
|
||||
// musicFormatMIME resolves the clip MIME from the response Content-Type
|
||||
// or the requested format, reusing speechMIME's table (one format->MIME
|
||||
// map for the whole provider). wav32 is ACE-Step-specific: normalize it
|
||||
// to wav before the shared lookup.
|
||||
func musicFormatMIME(contentType, format string) string {
|
||||
format = strings.ToLower(strings.TrimSpace(format))
|
||||
if format == "wav32" {
|
||||
format = "wav"
|
||||
}
|
||||
return speechMIME(contentType, format)
|
||||
}
|
||||
|
||||
// Generate implements musicgen.Model.
|
||||
func (m *musicModel) Generate(ctx context.Context, req musicgen.Request, opts ...musicgen.Option) (*musicgen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Prompt) == "" {
|
||||
return nil, fmt.Errorf("%w: music generation requires a prompt", llm.ErrUnsupported)
|
||||
}
|
||||
if req.DurationSeconds < 0 {
|
||||
return nil, fmt.Errorf("%w: duration must be >= 0, got %d", llm.ErrUnsupported, req.DurationSeconds)
|
||||
}
|
||||
if req.Steps != nil && *req.Steps <= 0 {
|
||||
return nil, fmt.Errorf("%w: inference steps must be > 0, got %d", llm.ErrUnsupported, *req.Steps)
|
||||
}
|
||||
|
||||
taskID, err := m.releaseTask(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item, err := m.pollResult(ctx, taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.fetchResult(ctx, req.Format, item)
|
||||
}
|
||||
|
||||
// releaseTask submits the job and returns its task id.
|
||||
func (m *musicModel) releaseTask(ctx context.Context, req musicgen.Request) (string, error) {
|
||||
path, err := upstreamPath(m.id, "/release_task")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
wire := releaseTaskRequest{
|
||||
Prompt: req.Prompt,
|
||||
Lyrics: req.Lyrics,
|
||||
AudioFormat: req.Format,
|
||||
TaskType: "text2music",
|
||||
AudioDuration: req.DurationSeconds,
|
||||
InferenceSteps: req.Steps,
|
||||
Seed: req.Seed,
|
||||
}
|
||||
// Tolerant envelope: {"data": {"task_id": ...}} per the docs, with a
|
||||
// top-level fallback in case the wrapper changes.
|
||||
var resp struct {
|
||||
Data struct {
|
||||
TaskID string `json:"task_id"`
|
||||
} `json:"data"`
|
||||
TaskID string `json:"task_id"`
|
||||
}
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, path, m.id, &wire, &resp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
taskID := resp.Data.TaskID
|
||||
if taskID == "" {
|
||||
taskID = resp.TaskID
|
||||
}
|
||||
if taskID == "" {
|
||||
return "", &llm.APIError{Provider: m.p.name, Model: m.id, Message: "release_task returned no task_id"}
|
||||
}
|
||||
return taskID, nil
|
||||
}
|
||||
|
||||
// pollResult polls query_result until the task succeeds, fails, or ctx
|
||||
// expires. Transient trouble — a transport blip, a momentarily
|
||||
// unparseable payload, the task briefly absent from the response — is
|
||||
// tolerated up to musicPollMaxConsecutiveFailures in a row: a
|
||||
// multi-minute exclusive-GPU job must not die to one flaky poll. Only an
|
||||
// explicit status=2, a run of consecutive failures, or the ctx deadline
|
||||
// aborts.
|
||||
func (m *musicModel) pollResult(ctx context.Context, taskID string) (*queryItem, error) {
|
||||
path, err := upstreamPath(m.id, "/query_result")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body := map[string]any{"task_id_list": []string{taskID}}
|
||||
ticker := time.NewTicker(musicPollInterval)
|
||||
defer ticker.Stop()
|
||||
failures := 0
|
||||
var lastErr error
|
||||
for {
|
||||
item, pollErr := m.pollOnce(ctx, path, body, taskID)
|
||||
switch {
|
||||
case pollErr != nil:
|
||||
// Ctx expiry is never transient — bail with the deadline error.
|
||||
if ctx.Err() != nil {
|
||||
return nil, fmt.Errorf("llama-swap: music generation: %w", ctx.Err())
|
||||
}
|
||||
failures++
|
||||
lastErr = pollErr
|
||||
if failures >= musicPollMaxConsecutiveFailures {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("music poll failed %d times in a row: %v", failures, lastErr)}
|
||||
}
|
||||
case item.Status == 1:
|
||||
return item, nil
|
||||
case item.Status == 2:
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: "music generation failed upstream: " + truncateForError([]byte(item.Result))}
|
||||
default:
|
||||
failures = 0 // healthy queued/running poll
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("llama-swap: music generation: %w", ctx.Err())
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pollOnce performs one query_result round trip and locates the task.
|
||||
func (m *musicModel) pollOnce(ctx context.Context, path string, body any, taskID string) (*queryItem, error) {
|
||||
// Tolerant envelope: items under "data" or a bare array.
|
||||
var raw json.RawMessage
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, path, m.id, body, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return findQueryItem(raw, taskID)
|
||||
}
|
||||
|
||||
// findQueryItem digs the task's entry out of the query_result payload,
|
||||
// tolerating {"data": [...]}, {"data": {...}}, and bare-array envelopes.
|
||||
func findQueryItem(raw json.RawMessage, taskID string) (*queryItem, error) {
|
||||
var env struct {
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
candidates := raw
|
||||
if json.Unmarshal(raw, &env) == nil && len(env.Data) > 0 {
|
||||
candidates = env.Data
|
||||
}
|
||||
var items []queryItem
|
||||
if err := json.Unmarshal(candidates, &items); err != nil {
|
||||
var one queryItem
|
||||
if err := json.Unmarshal(candidates, &one); err != nil {
|
||||
return nil, fmt.Errorf("unrecognized query_result payload shape")
|
||||
}
|
||||
items = []queryItem{one}
|
||||
}
|
||||
for i := range items {
|
||||
// Single-item responses without a task_id echo are assumed to be
|
||||
// ours — we only ever poll one task; a mismatch surfaces as a
|
||||
// transient miss and is retried by the caller.
|
||||
if items[i].TaskID == taskID || (items[i].TaskID == "" && len(items) == 1) {
|
||||
return &items[i], nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("query_result did not include task %s", taskID)
|
||||
}
|
||||
|
||||
// fetchResult downloads the finished clip named by the job's result blob.
|
||||
func (m *musicModel) fetchResult(ctx context.Context, format string, item *queryItem) (*musicgen.Result, error) {
|
||||
result, ok := parseMusicResult(item.Result)
|
||||
if !ok || result.File == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: "music result blob missing file URL: " + truncateForError([]byte(item.Result))}
|
||||
}
|
||||
// The file URL is server-relative (e.g. "/v1/audio?path=..."); route it
|
||||
// back through the same upstream. upstreamPath additionally refuses
|
||||
// dot-dot/scheme smuggling in this SERVER-SUPPLIED value.
|
||||
if !strings.HasPrefix(result.File, "/") {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: "music result file URL is not server-relative: " + truncateForError([]byte(result.File))}
|
||||
}
|
||||
path, err := upstreamPath(m.id, result.File)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, contentType, err := m.p.doRaw(ctx, http.MethodGet, path, m.id, "", nil, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "music response contained no audio"}
|
||||
}
|
||||
mimeType := musicFormatMIME(contentType, format)
|
||||
return &musicgen.Result{
|
||||
Audio: musicgen.Audio{Data: raw, MIME: mimeType},
|
||||
Raw: json.RawMessage(item.Result),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/embeddings"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/meshgen"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
|
||||
)
|
||||
|
||||
// aceStepStub emulates the ACE-Step job API: one queued poll, then success.
|
||||
func aceStepStub(t *testing.T, mp3 []byte) (*httptest.Server, *atomic.Int32) {
|
||||
t.Helper()
|
||||
var polls atomic.Int32
|
||||
var gotRelease map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/upstream/musicgen-acestep/release_task":
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotRelease)
|
||||
if gotRelease["task_type"] != "text2music" {
|
||||
t.Errorf("task_type = %v", gotRelease["task_type"])
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"data": {"task_id": "t-1", "status": "queued"}}`))
|
||||
case "/upstream/musicgen-acestep/query_result":
|
||||
n := polls.Add(1)
|
||||
if n == 1 {
|
||||
_, _ = w.Write([]byte(`{"data": [{"task_id": "t-1", "status": 0, "result": ""}]}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"data": [{"task_id": "t-1", "status": 1,
|
||||
"result": "{\"file\": \"/v1/audio?path=out.mp3\", \"metas\": {\"bpm\": 120}}"}]}`))
|
||||
case "/upstream/musicgen-acestep/v1/audio":
|
||||
if got := r.URL.Query().Get("path"); got != "out.mp3" {
|
||||
t.Errorf("audio path = %q", got)
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
_, _ = w.Write(mp3)
|
||||
default:
|
||||
t.Errorf("unexpected path %q", r.URL.Path)
|
||||
w.WriteHeader(404)
|
||||
}
|
||||
}))
|
||||
return srv, &polls
|
||||
}
|
||||
|
||||
func TestMusicGenerate(t *testing.T) {
|
||||
old := musicPollInterval
|
||||
musicPollInterval = 5 * time.Millisecond
|
||||
defer func() { musicPollInterval = old }()
|
||||
|
||||
mp3 := []byte("ID3fakeaudio")
|
||||
srv, polls := aceStepStub(t, mp3)
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
mm, err := p.MusicModel("musicgen-acestep")
|
||||
if err != nil {
|
||||
t.Fatalf("MusicModel: %v", err)
|
||||
}
|
||||
// Shrink the poll interval indirectly by bounding the whole call: the
|
||||
// stub succeeds on poll #2, so a generous deadline still finishes fast.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
res, err := mm.Generate(ctx,
|
||||
musicgen.Request{Prompt: "chiptune anthem about mortbux"},
|
||||
musicgen.WithLyrics("mort mort mort"), musicgen.WithDuration(30))
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if polls.Load() < 2 {
|
||||
t.Errorf("polls = %d, want >= 2 (queued then done)", polls.Load())
|
||||
}
|
||||
if res.Audio.MIME != "audio/mpeg" || string(res.Audio.Data) != string(mp3) {
|
||||
t.Fatalf("audio = %q/%d bytes", res.Audio.MIME, len(res.Audio.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMusicGenerateUpstreamFailure(t *testing.T) {
|
||||
old := musicPollInterval
|
||||
musicPollInterval = 5 * time.Millisecond
|
||||
defer func() { musicPollInterval = old }()
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/upstream/musicgen-acestep/release_task":
|
||||
_, _ = w.Write([]byte(`{"data": {"task_id": "t-2"}}`))
|
||||
case "/upstream/musicgen-acestep/query_result":
|
||||
_, _ = w.Write([]byte(`{"data": [{"task_id": "t-2", "status": 2, "result": "{\"error\": \"OOM\"}"}]}`))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
mm, _ := p.MusicModel("musicgen-acestep")
|
||||
_, err := mm.Generate(context.Background(), musicgen.Request{Prompt: "p"})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for failed job", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMusicGenerateRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
mm, _ := p.MusicModel("musicgen-acestep")
|
||||
if _, err := mm.Generate(context.Background(), musicgen.Request{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no prompt: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbed(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/embeddings" {
|
||||
t.Errorf("path = %q", r.URL.Path)
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
// Deliberately out of order: the client must sort by index.
|
||||
_, _ = w.Write([]byte(`{"object":"list","data":[
|
||||
{"object":"embedding","index":1,"embedding":[0.3,0.4]},
|
||||
{"object":"embedding","index":0,"embedding":[0.1,0.2]}
|
||||
]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
em, err := p.EmbedModel("embed-qwen3-0.6b")
|
||||
if err != nil {
|
||||
t.Fatalf("EmbedModel: %v", err)
|
||||
}
|
||||
res, err := em.Embed(context.Background(), embeddings.EmbedRequest{Inputs: []string{"a", "b"}})
|
||||
if err != nil {
|
||||
t.Fatalf("Embed: %v", err)
|
||||
}
|
||||
if gotBody["model"] != "embed-qwen3-0.6b" {
|
||||
t.Errorf("model = %v", gotBody["model"])
|
||||
}
|
||||
if len(res.Vectors) != 2 || res.Vectors[0][0] != 0.1 || res.Vectors[1][0] != 0.3 {
|
||||
t.Fatalf("vectors = %+v (index ordering broken?)", res.Vectors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbedCountMismatchIsError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"data":[{"index":0,"embedding":[0.1]}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
em, _ := p.EmbedModel("embed-qwen3-0.6b")
|
||||
_, err := em.Embed(context.Background(), embeddings.EmbedRequest{Inputs: []string{"a", "b"}})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for count mismatch", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbedRejectsEmptyInputs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
em, _ := p.EmbedModel("embed-qwen3-0.6b")
|
||||
if _, err := em.Embed(context.Background(), embeddings.EmbedRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no inputs: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := em.Embed(context.Background(), embeddings.EmbedRequest{Inputs: []string{"a", " "}}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("blank input: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRerank(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/rerank" {
|
||||
t.Errorf("path = %q", r.URL.Path)
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
// Out of score order: the client must sort descending.
|
||||
_, _ = w.Write([]byte(`{"results":[
|
||||
{"index":0,"relevance_score":0.11},
|
||||
{"index":2,"relevance_score":0.93},
|
||||
{"index":1,"relevance_score":0.42}
|
||||
]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
rm, err := p.RerankModel("rerank-bge-v2-m3")
|
||||
if err != nil {
|
||||
t.Fatalf("RerankModel: %v", err)
|
||||
}
|
||||
res, err := rm.Rerank(context.Background(),
|
||||
embeddings.RerankRequest{Query: "what is a panda?", Documents: []string{"a", "b", "c"}},
|
||||
embeddings.WithTopN(3))
|
||||
if err != nil {
|
||||
t.Fatalf("Rerank: %v", err)
|
||||
}
|
||||
if gotBody["top_n"] != float64(3) || gotBody["query"] != "what is a panda?" {
|
||||
t.Errorf("top_n/query = %v/%v", gotBody["top_n"], gotBody["query"])
|
||||
}
|
||||
if len(res.Results) != 3 || res.Results[0].Index != 2 || res.Results[2].Index != 0 {
|
||||
t.Fatalf("results = %+v (descending sort broken?)", res.Results)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRerankRejectsOutOfRangeIndex(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"results":[{"index":7,"relevance_score":0.9}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
rm, _ := p.RerankModel("rerank-bge-v2-m3")
|
||||
_, err := rm.Rerank(context.Background(),
|
||||
embeddings.RerankRequest{Query: "q", Documents: []string{"a"}})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for out-of-range index", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstructedQuery(t *testing.T) {
|
||||
got := embeddings.InstructedQuery("", "how tall is everest")
|
||||
want := "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: how tall is everest"
|
||||
if got != want {
|
||||
t.Errorf("InstructedQuery = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMusicGenerateSurvivesTransientPollFailures(t *testing.T) {
|
||||
old := musicPollInterval
|
||||
musicPollInterval = 5 * time.Millisecond
|
||||
defer func() { musicPollInterval = old }()
|
||||
|
||||
mp3 := []byte("ID3fakeaudio")
|
||||
var polls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/upstream/musicgen-acestep/release_task":
|
||||
_, _ = w.Write([]byte(`{"data": {"task_id": "t-3"}}`))
|
||||
case "/upstream/musicgen-acestep/query_result":
|
||||
switch polls.Add(1) {
|
||||
case 1:
|
||||
w.WriteHeader(http.StatusBadGateway) // transient transport blip
|
||||
case 2:
|
||||
_, _ = w.Write([]byte(`{"data": []}`)) // task momentarily absent
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"data": [{"task_id": "t-3", "status": 1,
|
||||
"result": "{\"file\": \"/v1/audio?path=out.mp3\"}"}]}`))
|
||||
}
|
||||
case "/upstream/musicgen-acestep/v1/audio":
|
||||
w.Header().Set("Content-Type", "audio/mpeg")
|
||||
_, _ = w.Write(mp3)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
mm, _ := p.MusicModel("musicgen-acestep")
|
||||
res, err := mm.Generate(context.Background(), musicgen.Request{Prompt: "p"})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate should survive 2 transient failures: %v", err)
|
||||
}
|
||||
if len(res.Audio.Data) == 0 {
|
||||
t.Fatal("no audio")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMusicGenerateRejectsHostileFileURL(t *testing.T) {
|
||||
old := musicPollInterval
|
||||
musicPollInterval = 5 * time.Millisecond
|
||||
defer func() { musicPollInterval = old }()
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/upstream/musicgen-acestep/release_task":
|
||||
_, _ = w.Write([]byte(`{"data": {"task_id": "t-4"}}`))
|
||||
case "/upstream/musicgen-acestep/query_result":
|
||||
_, _ = w.Write([]byte(`{"data": [{"task_id": "t-4", "status": 1,
|
||||
"result": "{\"file\": \"/v1/audio?path=../../api/models/unload\"}"}]}`))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
mm, _ := p.MusicModel("musicgen-acestep")
|
||||
_, err := mm.Generate(context.Background(), musicgen.Request{Prompt: "p"})
|
||||
if err == nil {
|
||||
t.Fatal("dot-dot result file URL accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMusicResultLiveShapes(t *testing.T) {
|
||||
// Live ACE-Step (2026-07-14): result is an ARRAY and carries raw
|
||||
// control characters inside string values.
|
||||
arrayWithCtrl := "[{\"file\": \"/v1/audio?path=x.mp3\", \"prompt\": \"line1\nline2\"}]"
|
||||
res, ok := parseMusicResult(arrayWithCtrl)
|
||||
if !ok || res.File != "/v1/audio?path=x.mp3" {
|
||||
t.Fatalf("array+ctrl: ok=%v res=%+v", ok, res)
|
||||
}
|
||||
// Docs shape (bare object) still parses.
|
||||
res, ok = parseMusicResult(`{"file": "/v1/audio?path=y.mp3"}`)
|
||||
if !ok || res.File != "/v1/audio?path=y.mp3" {
|
||||
t.Fatalf("object: ok=%v res=%+v", ok, res)
|
||||
}
|
||||
if _, ok := parseMusicResult("not json"); ok {
|
||||
t.Fatal("garbage parsed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeshResultSniffsActualFormat(t *testing.T) {
|
||||
// Live Hunyuan3D always returns GLB regardless of the requested
|
||||
// format — the result must be labelled by its magic bytes.
|
||||
glb := append([]byte("glTF"), []byte("\x02\x00\x00\x00rest")...)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
_, _ = w.Write(glb)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
mm, _ := p.MeshModel("image3d-hunyuan21")
|
||||
res, err := mm.Generate(context.Background(),
|
||||
meshgen.Request{Image: meshgen.Image{Data: []byte{1}}, Format: "stl"})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if res.Mesh.Format != "glb" || res.Mesh.MIME != "model/gltf-binary" {
|
||||
t.Fatalf("mesh labelled %s/%s, want glb (sniffed)", res.Mesh.Format, res.Mesh.MIME)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeshConverter(t *testing.T) {
|
||||
stl := []byte("solid m\nendsolid m\n")
|
||||
var gotTarget, gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("multipart: %v", err)
|
||||
}
|
||||
gotTarget = r.FormValue("target")
|
||||
w.Header().Set("Content-Type", "model/stl")
|
||||
_, _ = w.Write(stl)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
mc, err := p.MeshConverter("mediautils")
|
||||
if err != nil {
|
||||
t.Fatalf("MeshConverter: %v", err)
|
||||
}
|
||||
res, err := mc.Convert(context.Background(),
|
||||
meshgen.Mesh{Data: []byte("glTFxxxx"), Format: "glb"}, "stl")
|
||||
if err != nil {
|
||||
t.Fatalf("Convert: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/mediautils/v1/convert_mesh" || gotTarget != "stl" {
|
||||
t.Errorf("path/target = %q/%q", gotPath, gotTarget)
|
||||
}
|
||||
if res.Mesh.Format != "stl" {
|
||||
t.Errorf("format = %q", res.Mesh.Format)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
// ocr.go implements ocr.Provider against a Surya-style shim reached through
|
||||
// llama-swap's /upstream passthrough (ADR-0023):
|
||||
//
|
||||
// POST /upstream/<id>/v1/ocr multipart file[,langs,max_pages]
|
||||
//
|
||||
// The document may be an image or a PDF (the shim rasterizes PDFs itself).
|
||||
// The response is per-page JSON: {pages:[{number,text,lines,layout}]}. The
|
||||
// decode is tolerant — when a page carries no aggregate `text`, its line
|
||||
// texts are joined instead — and the full payload survives in Result.Raw for
|
||||
// callers that want bboxes/confidence/layout.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/ocr"
|
||||
)
|
||||
|
||||
// OCRModel implements ocr.Provider. The id selects which upstream llama-swap
|
||||
// loads.
|
||||
func (p *Provider) OCRModel(id string, opts ...ocr.ModelOption) (ocr.Model, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = ocr.ApplyModelOptions(opts)
|
||||
return &ocrModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type ocrModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// ocrResponse is the Surya shim's /v1/ocr shape (the subset this package
|
||||
// relies on; per-line bbox/confidence and layout stay in Raw).
|
||||
type ocrResponse struct {
|
||||
Pages []struct {
|
||||
Number int `json:"number"`
|
||||
Text string `json:"text"`
|
||||
Lines []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"lines"`
|
||||
} `json:"pages"`
|
||||
}
|
||||
|
||||
// Recognize implements ocr.Model.
|
||||
func (m *ocrModel) Recognize(ctx context.Context, req ocr.Request, opts ...ocr.Option) (*ocr.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Document) == 0 {
|
||||
return nil, fmt.Errorf("%w: ocr requires document bytes", llm.ErrUnsupported)
|
||||
}
|
||||
if req.MaxPages < 0 {
|
||||
return nil, fmt.Errorf("%w: ocr max pages must be >= 0, got %d", llm.ErrUnsupported, req.MaxPages)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/ocr")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maxPages := ""
|
||||
if req.MaxPages != 0 {
|
||||
maxPages = strconv.Itoa(req.MaxPages)
|
||||
}
|
||||
body, contentType, err := buildMultipart("build ocr form",
|
||||
filePart{field: "file", filename: documentFilename(req.Filename, req.MIME), data: req.Document},
|
||||
[]formField{
|
||||
{"langs", strings.Join(req.Languages, ","), false},
|
||||
{"max_pages", maxPages, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out ocrResponse
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: decode ocr response: %w", err)
|
||||
}
|
||||
if len(out.Pages) == 0 {
|
||||
// A blank page still comes back as a page with empty text; zero pages
|
||||
// is API drift or a soft error, never "the document was empty".
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: "ocr response contained no pages: " + truncateForError(raw)}
|
||||
}
|
||||
res := &ocr.Result{Raw: json.RawMessage(raw)}
|
||||
texts := make([]string, 0, len(out.Pages))
|
||||
for i, pg := range out.Pages {
|
||||
text := pg.Text
|
||||
if text == "" && len(pg.Lines) > 0 {
|
||||
lines := make([]string, 0, len(pg.Lines))
|
||||
for _, ln := range pg.Lines {
|
||||
lines = append(lines, ln.Text)
|
||||
}
|
||||
text = strings.Join(lines, "\n")
|
||||
}
|
||||
number := pg.Number
|
||||
if number == 0 {
|
||||
number = i + 1
|
||||
}
|
||||
res.Pages = append(res.Pages, ocr.Page{Number: number, Text: text})
|
||||
texts = append(texts, text)
|
||||
}
|
||||
res.Text = strings.Join(texts, "\n\n")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// documentFilename picks the multipart filename hint for an OCR document: the
|
||||
// caller's (sanitized), else one derived from the MIME subtype
|
||||
// ("document.pdf"), else "document". MIME parameters are stripped before
|
||||
// matching, mirroring transcriptionFilename.
|
||||
func documentFilename(filename, mimeType string) string {
|
||||
if name := sanitizeFilename(filename); name != "" {
|
||||
return name
|
||||
}
|
||||
mt := strings.ToLower(strings.TrimSpace(mimeType))
|
||||
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
||||
mt = parsed
|
||||
}
|
||||
switch mt {
|
||||
case "application/pdf":
|
||||
return "document.pdf"
|
||||
case "image/png":
|
||||
return "document.png"
|
||||
case "image/jpeg", "image/jpg":
|
||||
return "document.jpg"
|
||||
case "image/webp":
|
||||
return "document.webp"
|
||||
default:
|
||||
return "document"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/ocr"
|
||||
)
|
||||
|
||||
func TestRecognize(t *testing.T) {
|
||||
var gotPath, gotLangs, gotMaxPages, gotFilename string
|
||||
var gotFile []byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotLangs = r.FormValue("langs")
|
||||
gotMaxPages = r.FormValue("max_pages")
|
||||
f, hdr, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
t.Fatalf("form file: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
gotFile, _ = io.ReadAll(f)
|
||||
gotFilename = hdr.Filename
|
||||
_, _ = w.Write([]byte(`{"pages":[
|
||||
{"number":1,"text":"page one","lines":[{"text":"page","bbox":[0,0,1,1],"confidence":0.9},{"text":"one"}],"layout":{}},
|
||||
{"number":2,"text":"page two"}
|
||||
]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
om, err := p.OCRModel("ocr-surya")
|
||||
if err != nil {
|
||||
t.Fatalf("OCRModel: %v", err)
|
||||
}
|
||||
res, err := om.Recognize(context.Background(),
|
||||
ocr.Request{Document: []byte("%PDF"), MIME: "application/pdf"},
|
||||
ocr.WithLanguages("en", "de"), ocr.WithMaxPages(5))
|
||||
if err != nil {
|
||||
t.Fatalf("Recognize: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/ocr-surya/v1/ocr" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotLangs != "en,de" || gotMaxPages != "5" {
|
||||
t.Errorf("langs/max_pages = %q/%q", gotLangs, gotMaxPages)
|
||||
}
|
||||
if string(gotFile) != "%PDF" || gotFilename != "document.pdf" {
|
||||
t.Errorf("file = %q name = %q", gotFile, gotFilename)
|
||||
}
|
||||
if len(res.Pages) != 2 || res.Pages[0].Number != 1 || res.Pages[0].Text != "page one" ||
|
||||
res.Pages[1].Number != 2 || res.Pages[1].Text != "page two" {
|
||||
t.Errorf("pages = %+v", res.Pages)
|
||||
}
|
||||
if res.Text != "page one\n\npage two" {
|
||||
t.Errorf("text = %q", res.Text)
|
||||
}
|
||||
if res.Raw == nil {
|
||||
t.Error("Raw = nil, want raw payload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecognizeJoinsLinesWhenPageTextAbsent(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"pages":[{"lines":[{"text":"first line"},{"text":"second line"}]}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
om, _ := p.OCRModel("ocr-surya")
|
||||
res, err := om.Recognize(context.Background(), ocr.Request{Document: []byte("img")})
|
||||
if err != nil {
|
||||
t.Fatalf("Recognize: %v", err)
|
||||
}
|
||||
if len(res.Pages) != 1 || res.Pages[0].Text != "first line\nsecond line" {
|
||||
t.Errorf("pages = %+v", res.Pages)
|
||||
}
|
||||
// Missing page number defaults to position.
|
||||
if res.Pages[0].Number != 1 {
|
||||
t.Errorf("number = %d, want 1", res.Pages[0].Number)
|
||||
}
|
||||
if res.Text != "first line\nsecond line" {
|
||||
t.Errorf("text = %q", res.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecognizeOmitsUnsetFields(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
for _, k := range []string{"langs", "max_pages"} {
|
||||
if v, ok := r.MultipartForm.Value[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
if _, hdr, err := r.FormFile("file"); err == nil {
|
||||
if hdr.Filename != "document" {
|
||||
t.Errorf("filename = %q, want document fallback", hdr.Filename)
|
||||
}
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"pages":[{"number":1,"text":"x"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
om, _ := p.OCRModel("ocr-surya")
|
||||
if _, err := om.Recognize(context.Background(), ocr.Request{Document: []byte("img")}); err != nil {
|
||||
t.Fatalf("Recognize: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecognizeRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
om, _ := p.OCRModel("ocr-surya")
|
||||
if _, err := om.Recognize(context.Background(), ocr.Request{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no document: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := om.Recognize(context.Background(),
|
||||
ocr.Request{Document: []byte{1}, MaxPages: -1}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("negative max pages: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecognizeRejectsZeroPages(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"pages":[]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
om, _ := p.OCRModel("ocr-surya")
|
||||
_, err := om.Recognize(context.Background(), ocr.Request{Document: []byte("img")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for zero pages", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecognizeRejectsNonJSONResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte("<html>proxy error page</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
om, _ := p.OCRModel("ocr-surya")
|
||||
if _, err := om.Recognize(context.Background(), ocr.Request{Document: []byte("img")}); err == nil ||
|
||||
!strings.Contains(err.Error(), "decode ocr response") {
|
||||
t.Fatalf("err = %v, want decode error for non-JSON body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecognizeSurfacesAPIError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"unsupported file type"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
om, _ := p.OCRModel("ocr-surya")
|
||||
_, err := om.Recognize(context.Background(), ocr.Request{Document: []byte("bad")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %T %v, want *llm.APIError", err, err)
|
||||
}
|
||||
if apiErr.Status != http.StatusBadRequest || apiErr.Message != "unsupported file type" || apiErr.Model != "ocr-surya" {
|
||||
t.Errorf("apiErr = %+v", apiErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentFilename(t *testing.T) {
|
||||
cases := []struct {
|
||||
filename, mime, want string
|
||||
}{
|
||||
{"scan.pdf", "", "scan.pdf"},
|
||||
{"evil\r\nname.pdf", "", "evilname.pdf"},
|
||||
{"", "application/pdf", "document.pdf"},
|
||||
{"", "image/png", "document.png"},
|
||||
{"", "image/jpeg", "document.jpg"},
|
||||
{"", "image/webp; charset=binary", "document.webp"},
|
||||
{"", "", "document"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := documentFilename(tc.filename, tc.mime); got != tc.want {
|
||||
t.Errorf("documentFilename(%q, %q) = %q, want %q", tc.filename, tc.mime, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
// restore.go implements the imagegen.ColorizeProvider and
|
||||
// imagegen.FaceRestoreProvider surfaces against the mediautils shim reached
|
||||
// through llama-swap's /upstream passthrough (ADR-0023):
|
||||
//
|
||||
// colorize POST /upstream/<id>/v1/colorize (DDColor)
|
||||
// restore_faces POST /upstream/<id>/v1/restore_faces (GFPGAN)
|
||||
//
|
||||
// Both are one-file multipart in, one PNG out, mirroring mediautil.go.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// --- colorize ---
|
||||
|
||||
// ColorizeModel implements imagegen.ColorizeProvider against the mediautils
|
||||
// shim's POST /v1/colorize. The id selects which upstream llama-swap loads.
|
||||
func (p *Provider) ColorizeModel(id string, opts ...imagegen.ColorizeModelOption) (imagegen.Colorizer, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = imagegen.ApplyColorizeModelOptions(opts)
|
||||
return &colorizeModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type colorizeModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Colorize implements imagegen.Colorizer.
|
||||
func (m *colorizeModel) Colorize(ctx context.Context, req imagegen.ColorizeRequest, opts ...imagegen.ColorizeOption) (*imagegen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Image.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: colorization requires an image", llm.ErrUnsupported)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/colorize")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, contentType, err := buildMultipart("build colorize form",
|
||||
filePart{field: "file", filename: "image.png", data: req.Image.Data},
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxImageResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleImageResult(m.p.name, m.id, "colorize", raw, respType)
|
||||
}
|
||||
|
||||
// --- face restoration ---
|
||||
|
||||
// FaceRestoreModel implements imagegen.FaceRestoreProvider against the
|
||||
// mediautils shim's POST /v1/restore_faces.
|
||||
func (p *Provider) FaceRestoreModel(id string, opts ...imagegen.FaceRestoreModelOption) (imagegen.FaceRestorer, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = imagegen.ApplyFaceRestoreModelOptions(opts)
|
||||
return &faceRestoreModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type faceRestoreModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// RestoreFaces implements imagegen.FaceRestorer.
|
||||
func (m *faceRestoreModel) RestoreFaces(ctx context.Context, req imagegen.FaceRestoreRequest, opts ...imagegen.FaceRestoreOption) (*imagegen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Image.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: face restoration requires an image", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Upscale != 0 && req.Upscale != 1 && req.Upscale != 2 {
|
||||
return nil, fmt.Errorf("%w: face-restore upscale must be 1 or 2, got %d", llm.ErrUnsupported, req.Upscale)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/restore_faces")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upscale := ""
|
||||
if req.Upscale != 0 {
|
||||
upscale = strconv.Itoa(req.Upscale)
|
||||
}
|
||||
body, contentType, err := buildMultipart("build restore-faces form",
|
||||
filePart{field: "file", filename: "image.png", data: req.Image.Data},
|
||||
[]formField{{"upscale", upscale, false}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxImageResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleImageResult(m.p.name, m.id, "face restoration", raw, respType)
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// segment.go implements imagegen.SegmentationProvider against a
|
||||
// GroundingDINO+SAM shim (segment-langsam) reached through llama-swap's
|
||||
// /upstream passthrough (ADR-0023):
|
||||
//
|
||||
// POST /upstream/<id>/v1/segment multipart file,prompt[,threshold],output=mask
|
||||
//
|
||||
// The response is a single grayscale mask PNG where WHITE marks the prompted
|
||||
// region — directly usable as imagegen.EditRequest.Mask (white = repaint).
|
||||
// The shim also offers output=cutout|boxes; this client always requests the
|
||||
// mask, because a cutout is derivable client-side from mask+original with no
|
||||
// second GPU call.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// SegmentationModel implements imagegen.SegmentationProvider. The id selects
|
||||
// which upstream llama-swap loads.
|
||||
func (p *Provider) SegmentationModel(id string, opts ...imagegen.SegmentationModelOption) (imagegen.Segmenter, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = imagegen.ApplySegmentationModelOptions(opts)
|
||||
return &segmentationModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type segmentationModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Segment implements imagegen.Segmenter.
|
||||
func (m *segmentationModel) Segment(ctx context.Context, req imagegen.SegmentationRequest, opts ...imagegen.SegmentationOption) (*imagegen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Image.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: segmentation requires an image", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Prompt == "" {
|
||||
return nil, fmt.Errorf("%w: segmentation requires a prompt", llm.ErrUnsupported)
|
||||
}
|
||||
// NaN fails every comparison, so it would sail through a bare range
|
||||
// check and reach the upstream as the literal string "NaN".
|
||||
if math.IsNaN(req.Threshold) || req.Threshold < 0 || req.Threshold > 1 {
|
||||
return nil, fmt.Errorf("%w: segmentation threshold must be in [0,1], got %g", llm.ErrUnsupported, req.Threshold)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/segment")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
threshold := ""
|
||||
if req.Threshold != 0 {
|
||||
threshold = strconv.FormatFloat(req.Threshold, 'g', -1, 64)
|
||||
}
|
||||
body, contentType, err := buildMultipart("build segment form",
|
||||
filePart{field: "file", filename: "image.png", data: req.Image.Data},
|
||||
[]formField{
|
||||
{"prompt", req.Prompt, true},
|
||||
{"threshold", threshold, false},
|
||||
// Always the mask: white = prompted region, EditRequest.Mask
|
||||
// polarity. Cutouts are derived client-side.
|
||||
{"output", "mask", true},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxImageResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleImageResult(m.p.name, m.id, "segmentation", raw, respType)
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/imagegen"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
func TestSegment(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
var gotPath, gotPrompt, gotThreshold, gotOutput, gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
gotPrompt = r.FormValue("prompt")
|
||||
gotThreshold = r.FormValue("threshold")
|
||||
gotOutput = r.FormValue("output")
|
||||
if f, hdr, err := r.FormFile("file"); err == nil {
|
||||
gotFilename = hdr.Filename
|
||||
f.Close()
|
||||
} else {
|
||||
t.Errorf("file part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(png)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sg, err := p.SegmentationModel("segment-langsam")
|
||||
if err != nil {
|
||||
t.Fatalf("SegmentationModel: %v", err)
|
||||
}
|
||||
res, err := sg.Segment(context.Background(),
|
||||
imagegen.SegmentationRequest{Image: imagegen.Image{MIME: "image/png", Data: png}, Prompt: "the red car"},
|
||||
imagegen.WithSegmentationThreshold(0.35))
|
||||
if err != nil {
|
||||
t.Fatalf("Segment: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/segment-langsam/v1/segment" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotPrompt != "the red car" || gotThreshold != "0.35" || gotOutput != "mask" || gotFilename != "image.png" {
|
||||
t.Errorf("prompt/threshold/output/filename = %q/%q/%q/%q", gotPrompt, gotThreshold, gotOutput, gotFilename)
|
||||
}
|
||||
if len(res.Images) != 1 || res.Images[0].MIME != "image/png" {
|
||||
t.Fatalf("images = %+v", res.Images)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentOmitsDefaultThreshold(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
if _, ok := r.MultipartForm.Value["threshold"]; ok {
|
||||
t.Error("threshold field sent for default; want omitted")
|
||||
}
|
||||
if got := r.FormValue("output"); got != "mask" {
|
||||
t.Errorf("output = %q, want mask (always sent)", got)
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(png)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sg, _ := p.SegmentationModel("segment-langsam")
|
||||
if _, err := sg.Segment(context.Background(),
|
||||
imagegen.SegmentationRequest{Image: imagegen.Image{Data: png}, Prompt: "dog"}); err != nil {
|
||||
t.Fatalf("Segment: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
sg, _ := p.SegmentationModel("segment-langsam")
|
||||
if _, err := sg.Segment(context.Background(),
|
||||
imagegen.SegmentationRequest{Prompt: "dog"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no image: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := sg.Segment(context.Background(),
|
||||
imagegen.SegmentationRequest{Image: imagegen.Image{Data: []byte{1}}}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no prompt: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := sg.Segment(context.Background(),
|
||||
imagegen.SegmentationRequest{Image: imagegen.Image{Data: []byte{1}}, Prompt: "dog", Threshold: 1.5}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("threshold 1.5: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
for name, bad := range map[string]float64{
|
||||
"NaN": math.NaN(),
|
||||
"+Inf": math.Inf(1),
|
||||
"-Inf": math.Inf(-1),
|
||||
} {
|
||||
if _, err := sg.Segment(context.Background(),
|
||||
imagegen.SegmentationRequest{Image: imagegen.Image{Data: []byte{1}}, Prompt: "dog", Threshold: bad}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("threshold %s: err = %v, want ErrUnsupported", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentRejectsNonImageResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte("<html>proxy error page</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sg, _ := p.SegmentationModel("segment-langsam")
|
||||
_, err := sg.Segment(context.Background(),
|
||||
imagegen.SegmentationRequest{Image: imagegen.Image{Data: pngFixture(t)}, Prompt: "dog"})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-image body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColorize(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
var gotPath, gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
if f, hdr, err := r.FormFile("file"); err == nil {
|
||||
gotFilename = hdr.Filename
|
||||
f.Close()
|
||||
} else {
|
||||
t.Errorf("file part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(png)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
cl, err := p.ColorizeModel("mediautils")
|
||||
if err != nil {
|
||||
t.Fatalf("ColorizeModel: %v", err)
|
||||
}
|
||||
res, err := cl.Colorize(context.Background(),
|
||||
imagegen.ColorizeRequest{Image: imagegen.Image{Data: png}})
|
||||
if err != nil {
|
||||
t.Fatalf("Colorize: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/mediautils/v1/colorize" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotFilename != "image.png" {
|
||||
t.Errorf("filename = %q", gotFilename)
|
||||
}
|
||||
if len(res.Images) != 1 || res.Images[0].MIME != "image/png" {
|
||||
t.Fatalf("images = %+v", res.Images)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColorizeRejectsEmptyImage(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
cl, _ := p.ColorizeModel("mediautils")
|
||||
if _, err := cl.Colorize(context.Background(), imagegen.ColorizeRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestColorizeSurfacesAPIError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"model is loading"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
cl, _ := p.ColorizeModel("mediautils")
|
||||
_, err := cl.Colorize(context.Background(),
|
||||
imagegen.ColorizeRequest{Image: imagegen.Image{Data: pngFixture(t)}})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %T %v, want *llm.APIError", err, err)
|
||||
}
|
||||
if apiErr.Status != http.StatusServiceUnavailable || apiErr.Message != "model is loading" || apiErr.Model != "mediautils" {
|
||||
t.Errorf("apiErr = %+v", apiErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreFaces(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
var gotPath, gotUpscale string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
gotUpscale = r.FormValue("upscale")
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(png)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
fr, err := p.FaceRestoreModel("mediautils")
|
||||
if err != nil {
|
||||
t.Fatalf("FaceRestoreModel: %v", err)
|
||||
}
|
||||
res, err := fr.RestoreFaces(context.Background(),
|
||||
imagegen.FaceRestoreRequest{Image: imagegen.Image{Data: png}},
|
||||
imagegen.WithFaceRestoreUpscale(2))
|
||||
if err != nil {
|
||||
t.Fatalf("RestoreFaces: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/mediautils/v1/restore_faces" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotUpscale != "2" {
|
||||
t.Errorf("upscale = %q", gotUpscale)
|
||||
}
|
||||
if len(res.Images) != 1 {
|
||||
t.Fatalf("images = %+v", res.Images)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreFacesOmitsDefaultUpscale(t *testing.T) {
|
||||
png := pngFixture(t)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
if _, ok := r.MultipartForm.Value["upscale"]; ok {
|
||||
t.Error("upscale field sent for default; want omitted")
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(png)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
fr, _ := p.FaceRestoreModel("mediautils")
|
||||
if _, err := fr.RestoreFaces(context.Background(),
|
||||
imagegen.FaceRestoreRequest{Image: imagegen.Image{Data: png}}); err != nil {
|
||||
t.Fatalf("RestoreFaces: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreFacesRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
fr, _ := p.FaceRestoreModel("mediautils")
|
||||
if _, err := fr.RestoreFaces(context.Background(), imagegen.FaceRestoreRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no image: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := fr.RestoreFaces(context.Background(),
|
||||
imagegen.FaceRestoreRequest{Image: imagegen.Image{Data: []byte{1}}, Upscale: 3}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("upscale 3: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreFacesRejectsNonImageResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header()["Content-Type"] = nil // NO Content-Type at all
|
||||
_, _ = w.Write([]byte("502 bad gateway"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
fr, _ := p.FaceRestoreModel("mediautils")
|
||||
_, err := fr.RestoreFaces(context.Background(),
|
||||
imagegen.FaceRestoreRequest{Image: imagegen.Image{Data: pngFixture(t)}})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for headerless non-image body", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// sfx.go implements a second musicgen.Model factory against a Stable Audio
|
||||
// Open shim (sfxgen) reached through llama-swap's /upstream passthrough
|
||||
// (ADR-0024):
|
||||
//
|
||||
// POST /upstream/<id>/v1/sfx JSON {prompt, seconds?, steps?, cfg_scale?, seed?}
|
||||
//
|
||||
// Unlike the ACE-Step music path (music.go), /v1/sfx is SYNCHRONOUS: the
|
||||
// response body is the finished WAV clip — no job queue, no polling. The
|
||||
// surface reuses the musicgen types (a sound effect is a short audio clip
|
||||
// from a text prompt); only the provider method differs.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
|
||||
)
|
||||
|
||||
// SFXModel returns a musicgen.Model bound to a sync sound-effect backend.
|
||||
// The id selects which upstream llama-swap loads.
|
||||
func (p *Provider) SFXModel(id string, opts ...musicgen.ModelOption) (musicgen.Model, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = musicgen.ApplyModelOptions(opts)
|
||||
return &sfxModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type sfxModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// sfxRequest is the sfxgen shim's /v1/sfx shape. Optional fields stay off
|
||||
// the wire so the model's own defaults apply.
|
||||
type sfxRequest struct {
|
||||
Prompt string `json:"prompt"`
|
||||
Seconds int `json:"seconds,omitempty"`
|
||||
Steps *int `json:"steps,omitempty"`
|
||||
CFGScale *float64 `json:"cfg_scale,omitempty"`
|
||||
Seed *int64 `json:"seed,omitempty"`
|
||||
}
|
||||
|
||||
// Generate implements musicgen.Model. The clip-length ceiling (~11s for
|
||||
// Stable Audio Open Small) is the backend's to enforce, not this client's.
|
||||
func (m *sfxModel) Generate(ctx context.Context, req musicgen.Request, opts ...musicgen.Option) (*musicgen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Prompt) == "" {
|
||||
return nil, fmt.Errorf("%w: sfx generation requires a prompt", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Lyrics != "" {
|
||||
return nil, fmt.Errorf("%w: sfx generation does not support lyrics", llm.ErrUnsupported)
|
||||
}
|
||||
if req.DurationSeconds < 0 {
|
||||
return nil, fmt.Errorf("%w: duration must be >= 0, got %d", llm.ErrUnsupported, req.DurationSeconds)
|
||||
}
|
||||
if req.Steps != nil && *req.Steps <= 0 {
|
||||
return nil, fmt.Errorf("%w: inference steps must be > 0, got %d", llm.ErrUnsupported, *req.Steps)
|
||||
}
|
||||
// The endpoint emits WAV only; a caller asking for another container
|
||||
// would silently get mislabelled bytes — reject instead.
|
||||
if f := strings.ToLower(strings.TrimSpace(req.Format)); f != "" && f != "wav" {
|
||||
return nil, fmt.Errorf("%w: sfx output is wav only, got format %q", llm.ErrUnsupported, req.Format)
|
||||
}
|
||||
|
||||
path, err := upstreamPath(m.id, "/v1/sfx")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wire := sfxRequest{
|
||||
Prompt: req.Prompt,
|
||||
Seconds: req.DurationSeconds,
|
||||
Steps: req.Steps,
|
||||
CFGScale: req.CFGScale,
|
||||
Seed: req.Seed,
|
||||
}
|
||||
encoded, err := json.Marshal(wire)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: encode sfx request: %w", err)
|
||||
}
|
||||
raw, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, "application/json", bytes.NewReader(encoded), maxAudioResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "sfx response contained no audio"}
|
||||
}
|
||||
mimeType := audioResultMIME(contentType, raw)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("sfx response is not audio (Content-Type %q): %s", contentType, truncateForError(raw))}
|
||||
}
|
||||
return &musicgen.Result{Audio: musicgen.Audio{Data: raw, MIME: mimeType}}, nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
// stems.go implements audio.StemSeparationProvider against a Demucs shim
|
||||
// (audioutils) reached through llama-swap's /upstream passthrough (ADR-0024):
|
||||
//
|
||||
// POST /upstream/<id>/v1/stems multipart file[,model,two_stems,format]
|
||||
//
|
||||
// The response is a ZIP of the separated stems — one entry per stem, entry
|
||||
// name = stem name, extension = container. Zip transport keeps a 4-stem WAV
|
||||
// result (hundreds of MB decoded) off the JSON-of-base64 path entirely.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// maxStemsResponseBytes caps the /v1/stems zip body: four WAV stems of a
|
||||
// long song legitimately pass the 64MB JSON cap. Bounded so a buggy
|
||||
// upstream can't allocate without limit.
|
||||
const maxStemsResponseBytes = 512 << 20
|
||||
|
||||
// maxStemEntryBytes caps ONE decompressed zip entry — the zip-bomb guard.
|
||||
// A single WAV stem of even a very long song stays far under this.
|
||||
const maxStemEntryBytes = 256 << 20
|
||||
|
||||
// maxStemEntries caps how many stem entries are unpacked: Demucs emits at
|
||||
// most six, so anything past a small multiple of that is a hostile or
|
||||
// broken archive, not a result.
|
||||
const maxStemEntries = 16
|
||||
|
||||
// maxStemsTotalBytes caps the AGGREGATE decompressed size across entries —
|
||||
// the per-entry bound alone would still let a many-entry bomb multiply up.
|
||||
const maxStemsTotalBytes = 1 << 30
|
||||
|
||||
// StemSeparatorModel implements audio.StemSeparationProvider. The id selects
|
||||
// which upstream llama-swap loads.
|
||||
func (p *Provider) StemSeparatorModel(id string, opts ...audio.StemSeparatorModelOption) (audio.StemSeparator, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = audio.ApplyStemSeparatorModelOptions(opts)
|
||||
return &stemSeparatorModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type stemSeparatorModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// SeparateStems implements audio.StemSeparator.
|
||||
func (m *stemSeparatorModel) SeparateStems(ctx context.Context, req audio.StemSeparationRequest, opts ...audio.StemSeparationOption) (*audio.StemSeparationResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Audio) == 0 {
|
||||
return nil, fmt.Errorf("%w: stem separation requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
mode := strings.ToLower(strings.TrimSpace(req.Mode))
|
||||
if mode != "" && mode != "two" && mode != "four" {
|
||||
return nil, fmt.Errorf("%w: stem mode must be \"two\" or \"four\", got %q", llm.ErrUnsupported, req.Mode)
|
||||
}
|
||||
format := strings.ToLower(strings.TrimSpace(req.Format))
|
||||
if format != "" && format != "mp3" && format != "wav" {
|
||||
return nil, fmt.Errorf("%w: stem format must be \"mp3\" or \"wav\", got %q", llm.ErrUnsupported, req.Format)
|
||||
}
|
||||
upPath, err := upstreamPath(m.id, "/v1/stems")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Demucs' two-stem mode is "isolate one source vs the rest"; vocals is
|
||||
// the split this surface promises ("two" = vocals + accompaniment).
|
||||
twoStems := ""
|
||||
if mode == "two" {
|
||||
twoStems = "vocals"
|
||||
}
|
||||
body, contentType, err := buildMultipart("build stems form",
|
||||
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
|
||||
[]formField{
|
||||
{"model", req.Model, false},
|
||||
{"two_stems", twoStems, false},
|
||||
{"format", format, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, upPath, m.id, contentType, body, maxStemsResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems response contained no data"}
|
||||
}
|
||||
zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
|
||||
if err != nil {
|
||||
// A non-zip 2xx body is a misconfigured upstream (an HTML error page
|
||||
// behind a proxy, a JSON soft error) — fail loud, quoting a slice.
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("stems response is not a zip (Content-Type %q): %s", respType, truncateForError(raw))}
|
||||
}
|
||||
res := &audio.StemSeparationResult{}
|
||||
var totalBytes int64
|
||||
for _, f := range zr.File {
|
||||
if f.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
if len(res.Stems) >= maxStemEntries {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("stems zip holds more than %d entries", maxStemEntries)}
|
||||
}
|
||||
data, err := readZipEntry(f)
|
||||
if err != nil {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("stems zip entry %q: %v", f.Name, err)}
|
||||
}
|
||||
totalBytes += int64(len(data))
|
||||
if totalBytes > maxStemsTotalBytes {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("stems zip decompresses past %d bytes", int64(maxStemsTotalBytes))}
|
||||
}
|
||||
// Entry name → stem name; extension → MIME. Entries may sit in a
|
||||
// per-model directory ("htdemucs/vocals.mp3"), so use the base name.
|
||||
base := path.Base(f.Name)
|
||||
ext := path.Ext(base)
|
||||
res.Stems = append(res.Stems, audio.Stem{
|
||||
Name: strings.TrimSuffix(base, ext),
|
||||
Audio: data,
|
||||
MIME: stemMIME(ext),
|
||||
})
|
||||
}
|
||||
if len(res.Stems) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems zip contained no stems"}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// readZipEntry decompresses one entry, bounded by maxStemEntryBytes so a
|
||||
// zip bomb can't allocate without limit.
|
||||
func readZipEntry(f *zip.File) ([]byte, error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rc.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(rc, maxStemEntryBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > maxStemEntryBytes {
|
||||
return nil, fmt.Errorf("decompressed entry exceeds %d bytes", int64(maxStemEntryBytes))
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// stemMIME maps a stem file extension to its MIME type, reusing speechMIME's
|
||||
// format table ("" and unknown extensions land on the mp3 default — Demucs'
|
||||
// own default container).
|
||||
func stemMIME(ext string) string {
|
||||
return speechMIME("", strings.TrimPrefix(strings.ToLower(ext), "."))
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
|
||||
)
|
||||
|
||||
// stemsZipFixture builds a Demucs-style stems zip: entries under a per-model
|
||||
// directory, entry name = stem name, extension = container.
|
||||
func stemsZipFixture(t *testing.T, entries map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, data := range entries {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatalf("zip create: %v", err)
|
||||
}
|
||||
if _, err := w.Write([]byte(data)); err != nil {
|
||||
t.Fatalf("zip write: %v", err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("zip close: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestSeparateStems(t *testing.T) {
|
||||
zipBody := stemsZipFixture(t, map[string]string{
|
||||
"htdemucs/vocals.mp3": "VOX",
|
||||
"htdemucs/no_vocals.mp3": "ACC",
|
||||
})
|
||||
var gotPath, gotTwoStems, gotModel, gotFormat, gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotTwoStems = r.FormValue("two_stems")
|
||||
gotModel = r.FormValue("model")
|
||||
gotFormat = r.FormValue("format")
|
||||
if _, hdr, err := r.FormFile("file"); err == nil {
|
||||
gotFilename = hdr.Filename
|
||||
} else {
|
||||
t.Errorf("file part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = w.Write(zipBody)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, err := p.StemSeparatorModel("audioutils")
|
||||
if err != nil {
|
||||
t.Fatalf("StemSeparatorModel: %v", err)
|
||||
}
|
||||
res, err := ss.SeparateStems(context.Background(),
|
||||
audio.StemSeparationRequest{Audio: []byte("SONG"), MIME: "audio/mpeg"},
|
||||
audio.WithStemMode("two"), audio.WithStemModel("htdemucs_ft"), audio.WithStemFormat("mp3"))
|
||||
if err != nil {
|
||||
t.Fatalf("SeparateStems: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/audioutils/v1/stems" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotTwoStems != "vocals" || gotModel != "htdemucs_ft" || gotFormat != "mp3" || gotFilename != "audio.mp3" {
|
||||
t.Errorf("two_stems/model/format/filename = %q/%q/%q/%q", gotTwoStems, gotModel, gotFormat, gotFilename)
|
||||
}
|
||||
if len(res.Stems) != 2 {
|
||||
t.Fatalf("stems = %+v", res.Stems)
|
||||
}
|
||||
byName := map[string]audio.Stem{}
|
||||
for _, s := range res.Stems {
|
||||
byName[s.Name] = s
|
||||
}
|
||||
if v := byName["vocals"]; string(v.Audio) != "VOX" || v.MIME != "audio/mpeg" {
|
||||
t.Errorf("vocals = %+v", v)
|
||||
}
|
||||
if a := byName["no_vocals"]; string(a.Audio) != "ACC" || a.MIME != "audio/mpeg" {
|
||||
t.Errorf("no_vocals = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsOmitsUnsetFields(t *testing.T) {
|
||||
zipBody := stemsZipFixture(t, map[string]string{"vocals.wav": "V"})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
for _, k := range []string{"two_stems", "model", "format"} {
|
||||
if v, ok := r.MultipartForm.Value[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
_, _ = w.Write(zipBody)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
res, err := ss.SeparateStems(context.Background(),
|
||||
audio.StemSeparationRequest{Audio: []byte("SONG")})
|
||||
if err != nil {
|
||||
t.Fatalf("SeparateStems: %v", err)
|
||||
}
|
||||
// Top-level entry, wav extension.
|
||||
if len(res.Stems) != 1 || res.Stems[0].Name != "vocals" || res.Stems[0].MIME != "audio/wav" {
|
||||
t.Errorf("stems = %+v", res.Stems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
if _, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no audio: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ss.SeparateStems(context.Background(),
|
||||
audio.StemSeparationRequest{Audio: []byte{1}, Mode: "three"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("mode three: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ss.SeparateStems(context.Background(),
|
||||
audio.StemSeparationRequest{Audio: []byte{1}, Format: "flac"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("format flac: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsRejectsNonZip(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte("<html>proxy error page</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-zip body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsRejectsEmptyZip(t *testing.T) {
|
||||
zipBody := stemsZipFixture(t, map[string]string{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = w.Write(zipBody)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for stem-less zip", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsRejectsTooManyEntries(t *testing.T) {
|
||||
entries := map[string]string{}
|
||||
for i := 0; i <= maxStemEntries; i++ {
|
||||
entries[fmt.Sprintf("stem%02d.wav", i)] = "X"
|
||||
}
|
||||
zipBody := stemsZipFixture(t, entries)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = w.Write(zipBody)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) || !strings.Contains(apiErr.Message, "entries") {
|
||||
t.Fatalf("err = %v, want APIError for over-long stems zip", err)
|
||||
}
|
||||
}
|
||||
|
||||
// wavFixture is a minimal RIFF/WAVE header so http.DetectContentType sniffs
|
||||
// audio/wave.
|
||||
func wavFixture() []byte {
|
||||
return []byte("RIFF\x24\x00\x00\x00WAVEfmt ")
|
||||
}
|
||||
|
||||
func TestSFXGenerate(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, err := p.SFXModel("sfxgen-stableaudio")
|
||||
if err != nil {
|
||||
t.Fatalf("SFXModel: %v", err)
|
||||
}
|
||||
res, err := sm.Generate(context.Background(),
|
||||
musicgen.Request{Prompt: "glass shattering", DurationSeconds: 8},
|
||||
musicgen.WithSteps(50), musicgen.WithCFGScale(7), musicgen.WithSeed(42))
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/sfxgen-stableaudio/v1/sfx" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
want := map[string]any{"prompt": "glass shattering", "seconds": 8.0, "steps": 50.0, "cfg_scale": 7.0, "seed": 42.0}
|
||||
for k, w := range want {
|
||||
if gotBody[k] != w {
|
||||
t.Errorf("%s = %v, want %v", k, gotBody[k], w)
|
||||
}
|
||||
}
|
||||
if res.Audio.MIME != "audio/wav" || len(res.Audio.Data) == 0 {
|
||||
t.Errorf("audio = %q/%d bytes", res.Audio.MIME, len(res.Audio.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSFXOmitsDefaults(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
// No Content-Type: the RIFF sniff must still label the clip.
|
||||
w.Header()["Content-Type"] = nil
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SFXModel("sfxgen-stableaudio")
|
||||
res, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
for _, k := range []string{"seconds", "steps", "cfg_scale", "seed"} {
|
||||
if v, ok := gotBody[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
if res.Audio.MIME != "audio/wav" {
|
||||
t.Errorf("MIME = %q, want sniffed audio/wav", res.Audio.MIME)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSFXRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
sm, _ := p.SFXModel("sfxgen-stableaudio")
|
||||
if _, err := sm.Generate(context.Background(), musicgen.Request{Prompt: " "}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("empty prompt: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := sm.Generate(context.Background(),
|
||||
musicgen.Request{Prompt: "boom", Lyrics: "la la"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("lyrics: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := sm.Generate(context.Background(),
|
||||
musicgen.Request{Prompt: "boom", Format: "mp3"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("format mp3: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSFXRejectsNonAudioResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"detail":"queue full"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SFXModel("sfxgen-stableaudio")
|
||||
_, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-audio body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhance(t *testing.T) {
|
||||
var gotPath, gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
if _, hdr, err := r.FormFile("file"); err == nil {
|
||||
gotFilename = hdr.Filename
|
||||
} else {
|
||||
t.Errorf("file part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
en, err := p.SpeechEnhancerModel("audioutils")
|
||||
if err != nil {
|
||||
t.Fatalf("SpeechEnhancerModel: %v", err)
|
||||
}
|
||||
res, err := en.Enhance(context.Background(),
|
||||
audio.EnhancementRequest{Audio: []byte("NOISY"), MIME: "audio/ogg"})
|
||||
if err != nil {
|
||||
t.Fatalf("Enhance: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/audioutils/v1/enhance" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotFilename != "audio.ogg" {
|
||||
t.Errorf("filename = %q", gotFilename)
|
||||
}
|
||||
if res.MIME != "audio/wav" || len(res.Audio) == 0 {
|
||||
t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhanceRejectsEmptyAudio(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
en, _ := p.SpeechEnhancerModel("audioutils")
|
||||
if _, err := en.Enhance(context.Background(), audio.EnhancementRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhanceRejectsNonAudioResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte("<html>oops</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
en, _ := p.SpeechEnhancerModel("audioutils")
|
||||
_, err := en.Enhance(context.Background(), audio.EnhancementRequest{Audio: []byte("NOISY")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-audio body", err)
|
||||
}
|
||||
}
|
||||
@@ -16,12 +16,14 @@ import (
|
||||
//
|
||||
// Why reject rather than escape: same rationale as Unload — model ids
|
||||
// legitimately contain ":" but never path-structure characters, and escaping
|
||||
// would mask a config error instead of surfacing it.
|
||||
// would mask a config error instead of surfacing it. '%' is rejected too:
|
||||
// ids never legitimately carry percent-escapes, and %2F/%2E%2E would decode
|
||||
// back into path structure on the server side.
|
||||
func upstreamPath(model, rest string) (string, error) {
|
||||
if strings.TrimSpace(model) == "" {
|
||||
return "", fmt.Errorf("llama-swap: upstream call requires a model id")
|
||||
}
|
||||
if strings.ContainsAny(model, "/?#") || strings.Contains(model, "..") {
|
||||
if strings.ContainsAny(model, "/?#%") || strings.Contains(model, "..") {
|
||||
return "", fmt.Errorf("llama-swap: invalid model id %q for upstream call (contains a path separator)", model)
|
||||
}
|
||||
if !strings.HasPrefix(rest, "/") {
|
||||
@@ -49,14 +51,27 @@ type filePart struct {
|
||||
// writeFormFields). wrap labels errors. Returns the body and its content
|
||||
// type.
|
||||
func buildMultipart(wrap string, file filePart, fields []formField) (*bytes.Buffer, string, error) {
|
||||
return buildMultipartFiles(wrap, []filePart{file}, fields)
|
||||
}
|
||||
|
||||
// buildMultipartFiles is buildMultipart for endpoints taking SEVERAL files
|
||||
// (face swap sends a target and a source). Files are written in the given
|
||||
// order, then the fields. One writer loop serves both so the two cannot drift
|
||||
// in how they escape names or terminate the body.
|
||||
func buildMultipartFiles(wrap string, files []filePart, fields []formField) (*bytes.Buffer, string, error) {
|
||||
if len(files) == 0 {
|
||||
return nil, "", fmt.Errorf("llama-swap: %s: no file parts", wrap)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
fw, err := w.CreateFormFile(file.field, file.filename)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err)
|
||||
}
|
||||
if _, err := fw.Write(file.data); err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err)
|
||||
for _, file := range files {
|
||||
fw, err := w.CreateFormFile(file.field, file.filename)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err)
|
||||
}
|
||||
if _, err := fw.Write(file.data); err != nil {
|
||||
return nil, "", fmt.Errorf("llama-swap: %s: %w", wrap, err)
|
||||
}
|
||||
}
|
||||
if err := writeFormFields(w, wrap, fields); err != nil {
|
||||
return nil, "", err
|
||||
|
||||
+59
-39
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -39,10 +38,13 @@ type videoModel struct {
|
||||
// bound the call with a context deadline.
|
||||
//
|
||||
// Parameter names follow vLLM-Omni's videos API (num_frames, fps,
|
||||
// num_inference_steps, guidance_scale); the conditioning frame is sent as an
|
||||
// `input_reference` file part, following OpenAI's videos API. Upstreams
|
||||
// num_inference_steps, guidance_scale); the leading conditioning frame is sent
|
||||
// as an `input_reference` file part, following OpenAI's videos API, and a
|
||||
// trailing keyframe (Request.LastImage) as `input_reference_last`. Upstreams
|
||||
// ignore fields they don't understand, and optional fields stay off the wire
|
||||
// entirely so the model's own defaults apply.
|
||||
// entirely so the model's own defaults apply — which is also why a backend
|
||||
// without first-last-frame support returns an ordinary clip here rather than
|
||||
// an error.
|
||||
func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ...videogen.Option) (*videogen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Prompt) == "" {
|
||||
@@ -57,6 +59,9 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ..
|
||||
if req.InitImage != nil && len(req.InitImage.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: video init image has no bytes", llm.ErrUnsupported)
|
||||
}
|
||||
if req.LastImage != nil && len(req.LastImage.Data) == 0 {
|
||||
return nil, fmt.Errorf("%w: video last image has no bytes", llm.ErrUnsupported)
|
||||
}
|
||||
width, height, err := parseSize(req.Size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err)
|
||||
@@ -84,12 +89,19 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ..
|
||||
return nil, err
|
||||
}
|
||||
if req.InitImage != nil {
|
||||
fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||
if err := writeImagePart(w, "input_reference", "frame", req.InitImage); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := fw.Write(req.InitImage.Data); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: build video form: %w", err)
|
||||
}
|
||||
// The trailing keyframe rides a SEPARATE part rather than a second
|
||||
// `input_reference`: multipart permits repeated names, but the receiving
|
||||
// end would then have to rely on part ORDER to tell first from last, and
|
||||
// an ordering contract that is invisible in the field name is one nobody
|
||||
// can see they have broken. A backend that does not know the name ignores
|
||||
// the part, which is the same degradation as any other unknown field.
|
||||
if req.LastImage != nil {
|
||||
if err := writeImagePart(w, "input_reference_last", "frame_last", req.LastImage); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
@@ -100,22 +112,7 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts ..
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(videoBytes) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video response contained no video"}
|
||||
}
|
||||
mimeType := videoMIME(contentType, videoBytes)
|
||||
if mimeType == "" {
|
||||
// A 2xx body that is neither declared nor sniffable as video is a
|
||||
// misconfigured upstream (a JSON job envelope, an HTML error page
|
||||
// behind a proxy) — fail loud rather than hand back garbage as a
|
||||
// playable clip.
|
||||
return nil, &llm.APIError{
|
||||
Provider: m.p.name,
|
||||
Model: m.id,
|
||||
Message: fmt.Sprintf("video response is not a video (content-type %q)", contentType),
|
||||
}
|
||||
}
|
||||
return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: mimeType}}, nil
|
||||
return singleVideoResult(m.p.name, m.id, "video", videoBytes, contentType)
|
||||
}
|
||||
|
||||
// videoMIME resolves the result MIME type: the response Content-Type when it
|
||||
@@ -132,22 +129,45 @@ func videoMIME(contentType string, data []byte) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// initImageFilename picks the multipart filename hint for the conditioning
|
||||
// frame from its MIME subtype. The name is provider-chosen (never
|
||||
// caller-supplied), so no sanitization is needed.
|
||||
func initImageFilename(mimeType string) string {
|
||||
mt := strings.ToLower(strings.TrimSpace(mimeType))
|
||||
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
||||
mt = parsed
|
||||
// singleVideoResult wraps one raw video body into a videogen.Result,
|
||||
// requiring positive evidence of video-ness (declared video/* Content-Type
|
||||
// or sniffed mp4/webm magic) so a 2xx body that is anything else — a JSON
|
||||
// job envelope, an HTML error page behind a proxy — fails loud instead of
|
||||
// coming back as "the clip". The video sibling of singleImageResult, shared
|
||||
// by every surface whose response body IS the encoded clip.
|
||||
func singleVideoResult(provider, model, verb string, raw []byte, contentType string) (*videogen.Result, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: provider, Model: model, Message: verb + " response contained no video"}
|
||||
}
|
||||
switch mt {
|
||||
case "image/jpeg", "image/jpg":
|
||||
return "frame.jpg"
|
||||
case "image/webp":
|
||||
return "frame.webp"
|
||||
default: // unknown MIME — PNG is the safe hint
|
||||
return "frame.png"
|
||||
mimeType := videoMIME(contentType, raw)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: provider, Model: model,
|
||||
Message: fmt.Sprintf("%s response is not a video (Content-Type %q)", verb, contentType)}
|
||||
}
|
||||
return &videogen.Result{Video: videogen.Video{Data: raw, MIME: mimeType}}, nil
|
||||
}
|
||||
|
||||
// writeImagePart attaches one conditioning frame under the given field name,
|
||||
// with a filename derived from nameStem. Shared by the first- and last-frame
|
||||
// parts so the two cannot drift in how they encode, which is the usual way a
|
||||
// second copy of a block goes wrong.
|
||||
//
|
||||
// The two frames MUST carry DISTINCT filenames, not merely distinct field
|
||||
// names. Backends commonly stage an uploaded frame under a name derived from
|
||||
// the filename — our own ComfyUI shim posts to /upload/image with
|
||||
// overwrite=true — so two parts sharing "frame.png" would have the second
|
||||
// clobber the first, and BOTH keyframe inputs would then resolve to the same
|
||||
// stored image. The clip would render clean, pinned at both ends to the same
|
||||
// frame, with nothing anywhere reporting a problem.
|
||||
func writeImagePart(w *multipart.Writer, field, nameStem string, img *videogen.Image) error {
|
||||
fw, err := w.CreateFormFile(field, imageFilename(img.MIME, nameStem))
|
||||
if err != nil {
|
||||
return fmt.Errorf("llama-swap: build video form: %w", err)
|
||||
}
|
||||
if _, err := fw.Write(img.Data); err != nil {
|
||||
return fmt.Errorf("llama-swap: build video form: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// formatInt renders an optional int pointer for a form field; nil = "" (omit).
|
||||
|
||||
@@ -223,3 +223,134 @@ func TestVideoGenerateNonVideoBodyErrors(t *testing.T) {
|
||||
t.Errorf("message = %q, want mention of non-video body", apiErr.Message)
|
||||
}
|
||||
}
|
||||
|
||||
// Both keyframes reach the wire, under DISTINCT field names.
|
||||
//
|
||||
// The distinct-name property is the actual contract with the backend shim: the
|
||||
// two frames could have shared one repeated `input_reference` name, and then
|
||||
// which is first and which is last would depend on multipart part ORDER — an
|
||||
// ordering contract invisible in the payload, that nothing would notice
|
||||
// breaking. Asserting the names is what pins it.
|
||||
func TestVideoGenerateSendsBothKeyframes(t *testing.T) {
|
||||
var gotFirst, gotLast []byte
|
||||
var firstName, lastName string
|
||||
var sawLastPart bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
t.Errorf("parse form: %v", err)
|
||||
return
|
||||
}
|
||||
if f, hdr, err := r.FormFile("input_reference"); err == nil {
|
||||
gotFirst, _ = io.ReadAll(f)
|
||||
firstName = hdr.Filename
|
||||
f.Close()
|
||||
}
|
||||
if f, hdr, err := r.FormFile("input_reference_last"); err == nil {
|
||||
sawLastPart = true
|
||||
gotLast, _ = io.ReadAll(f)
|
||||
lastName = hdr.Filename
|
||||
f.Close()
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("fake-mp4-bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vm, err := p.VideoModel("videogen-minimax-h3")
|
||||
if err != nil {
|
||||
t.Fatalf("VideoModel: %v", err)
|
||||
}
|
||||
|
||||
first, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
last := append(append([]byte{}, first...), 0x00) // distinguishable from first
|
||||
|
||||
if _, err := vm.Generate(context.Background(), videogen.Request{
|
||||
Prompt: "a cat surfing",
|
||||
InitImage: &videogen.Image{MIME: "image/png", Data: first},
|
||||
LastImage: &videogen.Image{MIME: "image/png", Data: last},
|
||||
}); err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
|
||||
if !sawLastPart {
|
||||
t.Fatal("input_reference_last was not sent — a pinned end frame would be silently dropped")
|
||||
}
|
||||
if string(gotFirst) != string(first) {
|
||||
t.Errorf("input_reference = %d bytes, want %d", len(gotFirst), len(first))
|
||||
}
|
||||
if string(gotLast) != string(last) {
|
||||
t.Errorf("input_reference_last = %d bytes, want %d", len(gotLast), len(last))
|
||||
}
|
||||
// The two must not be the same bytes, or a swap/aliasing bug reads as a pass.
|
||||
if string(gotFirst) == string(gotLast) {
|
||||
t.Error("both parts carry identical bytes — the frames are being aliased")
|
||||
}
|
||||
// DISTINCT FILENAMES, not just distinct field names. Backends stage an
|
||||
// uploaded frame under a name derived from the filename (our ComfyUI shim
|
||||
// posts to /upload/image with overwrite=true), so two parts sharing
|
||||
// "frame.png" would have the second clobber the first and BOTH keyframes
|
||||
// would resolve to the same stored image — a clip pinned at both ends to
|
||||
// the same frame, rendering cleanly with nothing reporting a fault.
|
||||
if firstName == "" || lastName == "" {
|
||||
t.Fatalf("filenames = %q / %q, want both set", firstName, lastName)
|
||||
}
|
||||
if firstName == lastName {
|
||||
t.Errorf("both parts use filename %q — the second upload would clobber the first", firstName)
|
||||
}
|
||||
}
|
||||
|
||||
// LastImage alone (no InitImage) is a legitimate request: pin the destination
|
||||
// and let the model invent the approach. It must not require a first frame.
|
||||
func TestVideoGenerateLastImageAloneIsAllowed(t *testing.T) {
|
||||
var sawFirst, sawLast bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
t.Errorf("parse form: %v", err)
|
||||
return
|
||||
}
|
||||
if f, _, err := r.FormFile("input_reference"); err == nil {
|
||||
sawFirst = true
|
||||
f.Close()
|
||||
}
|
||||
if f, _, err := r.FormFile("input_reference_last"); err == nil {
|
||||
sawLast = true
|
||||
f.Close()
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write([]byte("fake-mp4-bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
vm, _ := p.VideoModel("videogen-minimax-h3")
|
||||
frame, _ := base64.StdEncoding.DecodeString(onePixelPNG)
|
||||
|
||||
if _, err := vm.Generate(context.Background(),
|
||||
videogen.Request{Prompt: "arrive here"},
|
||||
videogen.WithLastImage(videogen.Image{MIME: "image/png", Data: frame}),
|
||||
); err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if sawFirst {
|
||||
t.Error("input_reference sent, want omitted")
|
||||
}
|
||||
if !sawLast {
|
||||
t.Error("input_reference_last omitted, want sent")
|
||||
}
|
||||
}
|
||||
|
||||
// An empty LastImage is rejected before the request is built, matching
|
||||
// InitImage's existing contract — a zero-byte frame reaching the backend is a
|
||||
// confusing upstream error instead of a clear local one.
|
||||
func TestVideoGenerateRejectsEmptyLastImage(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
vm, _ := p.VideoModel("videogen-minimax-h3")
|
||||
_, err := vm.Generate(context.Background(), videogen.Request{
|
||||
Prompt: "x",
|
||||
LastImage: &videogen.Image{MIME: "image/png"},
|
||||
})
|
||||
if !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Fatalf("err = %v, want llm.ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
// videochain.go implements videogen.ChainerProvider against the videoutils
|
||||
// chain orchestrator reached through llama-swap's /upstream passthrough
|
||||
// (ADR-0025):
|
||||
//
|
||||
// POST /upstream/<id>/v1/video/chain JSON submit -> {job_id}
|
||||
// GET /upstream/<id>/v1/jobs/{id} -> {status,segment,total,segments}
|
||||
// GET /upstream/<id>/v1/jobs/{id}/result -> encoded clip
|
||||
// GET /upstream/<id>/v1/jobs/{id}/segments/{n} -> encoded clip
|
||||
//
|
||||
// Unlike the ACE-Step music path this client does NOT hide the job queue
|
||||
// behind a blocking call: a chain runs through multiple GPU swaps for many
|
||||
// minutes, and the caller (mort's long-video tool) owns the poll loop so it
|
||||
// can deliver PARTIAL results — completed segments survive a mid-chain
|
||||
// failure and stay fetchable via ChainSegmentResult.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
// ChainerModel implements videogen.ChainerProvider. The id selects which
|
||||
// upstream llama-swap loads (the videoutils orchestrator, which in turn
|
||||
// drives the generation model through llama-swap itself).
|
||||
func (p *Provider) ChainerModel(id string, opts ...videogen.ChainerModelOption) (videogen.Chainer, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = videogen.ApplyChainerModelOptions(opts)
|
||||
return &chainerModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type chainerModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// chainSubmitRequest is the videoutils POST /v1/video/chain shape. The init
|
||||
// image rides as base64 in the JSON body (`init_image_b64`) — the submit is
|
||||
// JSON, not multipart, per the pinned host contract.
|
||||
type chainSubmitRequest struct {
|
||||
Segments []chainSegmentWire `json:"segments"`
|
||||
InitImageB64 string `json:"init_image_b64,omitempty"`
|
||||
SmoothJoins bool `json:"smooth_joins,omitempty"`
|
||||
Size string `json:"size,omitempty"`
|
||||
}
|
||||
|
||||
type chainSegmentWire struct {
|
||||
Prompt string `json:"prompt"`
|
||||
Seconds float64 `json:"seconds,omitempty"`
|
||||
}
|
||||
|
||||
// SubmitChain implements videogen.Chainer.
|
||||
func (m *chainerModel) SubmitChain(ctx context.Context, req videogen.ChainRequest) (string, error) {
|
||||
if len(req.Segments) == 0 {
|
||||
return "", fmt.Errorf("%w: video chain requires at least one segment", llm.ErrUnsupported)
|
||||
}
|
||||
wire := chainSubmitRequest{
|
||||
SmoothJoins: req.SmoothJoins,
|
||||
Size: strings.TrimSpace(req.Size),
|
||||
}
|
||||
for i, seg := range req.Segments {
|
||||
if strings.TrimSpace(seg.Prompt) == "" {
|
||||
return "", fmt.Errorf("%w: video chain segment %d requires a prompt", llm.ErrUnsupported, i)
|
||||
}
|
||||
// NaN/±Inf would otherwise surface as an obscure json.Marshal error
|
||||
// (NaN fails every comparison; +Inf passes the >= 0 check).
|
||||
if seg.Seconds < 0 || math.IsNaN(seg.Seconds) || math.IsInf(seg.Seconds, 0) {
|
||||
return "", fmt.Errorf("%w: video chain segment %d seconds must be a finite value >= 0, got %g", llm.ErrUnsupported, i, seg.Seconds)
|
||||
}
|
||||
wire.Segments = append(wire.Segments, chainSegmentWire{Prompt: seg.Prompt, Seconds: seg.Seconds})
|
||||
}
|
||||
if len(req.InitImage) > 0 {
|
||||
wire.InitImageB64 = base64.StdEncoding.EncodeToString(req.InitImage)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/video/chain")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Tolerant envelope: {"job_id": ...} per the contract, with data-wrapped
|
||||
// and bare-id fallbacks (musicgen release_task precedent).
|
||||
var resp struct {
|
||||
JobID string `json:"job_id"`
|
||||
ID string `json:"id"`
|
||||
Data struct {
|
||||
JobID string `json:"job_id"`
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := m.p.doJSON(ctx, http.MethodPost, path, m.id, &wire, &resp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, id := range []string{resp.JobID, resp.Data.JobID, resp.ID, resp.Data.ID} {
|
||||
if id != "" {
|
||||
return id, nil
|
||||
}
|
||||
}
|
||||
return "", &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video chain submit returned no job_id"}
|
||||
}
|
||||
|
||||
// chainJobResponse is the GET /v1/jobs/{id} shape. `segments` entries are
|
||||
// tolerated as bare strings or objects keyed by id/segment_id.
|
||||
type chainJobResponse struct {
|
||||
Status string `json:"status"`
|
||||
Segment int `json:"segment"`
|
||||
Total int `json:"total"`
|
||||
Segments []json.RawMessage `json:"segments"`
|
||||
}
|
||||
|
||||
// ChainStatus implements videogen.Chainer.
|
||||
func (m *chainerModel) ChainStatus(ctx context.Context, jobID string) (*videogen.ChainJob, error) {
|
||||
path, err := m.jobPath(jobID, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var raw json.RawMessage
|
||||
if err := m.p.doJSON(ctx, http.MethodGet, path, m.id, nil, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out chainJobResponse
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: decode chain job status: %w", err)
|
||||
}
|
||||
job := &videogen.ChainJob{
|
||||
Status: out.Status,
|
||||
Segment: out.Segment,
|
||||
Total: out.Total,
|
||||
Raw: raw,
|
||||
}
|
||||
// Entries that carry no usable id — JSON null (which unmarshals into a
|
||||
// string as a no-op, leaving ""), an empty string, or an object with
|
||||
// neither key — are SKIPPED, never appended as "": SegmentIDs promises
|
||||
// fetchable artifacts, and the full payload stays in Raw for callers
|
||||
// that want the unfiltered list.
|
||||
for _, entry := range out.Segments {
|
||||
var s string
|
||||
if json.Unmarshal(entry, &s) == nil {
|
||||
if s != "" {
|
||||
job.SegmentIDs = append(job.SegmentIDs, s)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var obj struct {
|
||||
ID string `json:"id"`
|
||||
SegmentID string `json:"segment_id"`
|
||||
}
|
||||
if json.Unmarshal(entry, &obj) == nil {
|
||||
switch {
|
||||
case obj.ID != "":
|
||||
job.SegmentIDs = append(job.SegmentIDs, obj.ID)
|
||||
case obj.SegmentID != "":
|
||||
job.SegmentIDs = append(job.SegmentIDs, obj.SegmentID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if job.Status == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: "chain job status payload carried no status: " + truncateForError(raw)}
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// ChainResult implements videogen.Chainer.
|
||||
func (m *chainerModel) ChainResult(ctx context.Context, jobID string) (*videogen.Result, error) {
|
||||
path, err := m.jobPath(jobID, "/result")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodGet, path, m.id, "", nil, maxVideoResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleVideoResult(m.p.name, m.id, "video chain result", raw, respType)
|
||||
}
|
||||
|
||||
// ChainSegmentResult implements videogen.Chainer.
|
||||
func (m *chainerModel) ChainSegmentResult(ctx context.Context, jobID string, n int) (*videogen.Result, error) {
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("%w: chain segment index must be >= 0, got %d", llm.ErrUnsupported, n)
|
||||
}
|
||||
path, err := m.jobPath(jobID, "/segments/"+strconv.Itoa(n))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodGet, path, m.id, "", nil, maxVideoResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleVideoResult(m.p.name, m.id, "video chain segment", raw, respType)
|
||||
}
|
||||
|
||||
// jobPath builds /upstream/<model>/v1/jobs/<jobID><suffix>, refusing job ids
|
||||
// that carry path structure. The id is SERVER-SUPPLIED (echoed back from
|
||||
// SubmitChain), so like upstreamPath's rest-component checks this rejects
|
||||
// rather than escapes — a hostile/buggy upstream must not be able to steer
|
||||
// the follow-up request at another proxy endpoint.
|
||||
func (m *chainerModel) jobPath(jobID, suffix string) (string, error) {
|
||||
if strings.TrimSpace(jobID) == "" {
|
||||
return "", fmt.Errorf("llama-swap: chain job call requires a job id")
|
||||
}
|
||||
// '%' is rejected alongside the literal path characters: job ids never
|
||||
// legitimately carry percent-escapes, and %2F/%2E%2E would decode back
|
||||
// into path structure server-side.
|
||||
if strings.ContainsAny(jobID, "/?#%") || strings.Contains(jobID, "..") {
|
||||
return "", fmt.Errorf("llama-swap: invalid chain job id %q (contains path structure)", jobID)
|
||||
}
|
||||
return upstreamPath(m.id, "/v1/jobs/"+jobID+suffix)
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
func TestSubmitChain(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
_, _ = w.Write([]byte(`{"job_id":"job-123"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, err := p.ChainerModel("videoutils")
|
||||
if err != nil {
|
||||
t.Fatalf("ChainerModel: %v", err)
|
||||
}
|
||||
jobID, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
|
||||
Segments: []videogen.ChainSegment{
|
||||
{Prompt: "a cat walks in", Seconds: 5},
|
||||
{Prompt: "the cat sits down", Seconds: 5},
|
||||
},
|
||||
InitImage: []byte("IMG"),
|
||||
SmoothJoins: true,
|
||||
Size: "1280x704",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SubmitChain: %v", err)
|
||||
}
|
||||
if jobID != "job-123" {
|
||||
t.Errorf("jobID = %q", jobID)
|
||||
}
|
||||
if gotPath != "/upstream/videoutils/v1/video/chain" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
segments, _ := gotBody["segments"].([]any)
|
||||
if len(segments) != 2 {
|
||||
t.Fatalf("segments = %v", gotBody["segments"])
|
||||
}
|
||||
first, _ := segments[0].(map[string]any)
|
||||
if first["prompt"] != "a cat walks in" || first["seconds"] != 5.0 {
|
||||
t.Errorf("segment[0] = %v", first)
|
||||
}
|
||||
if gotBody["init_image_b64"] != base64.StdEncoding.EncodeToString([]byte("IMG")) {
|
||||
t.Errorf("init_image_b64 = %v", gotBody["init_image_b64"])
|
||||
}
|
||||
if gotBody["smooth_joins"] != true || gotBody["size"] != "1280x704" {
|
||||
t.Errorf("smooth_joins/size = %v/%v", gotBody["smooth_joins"], gotBody["size"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubmitChainOmitsUnsetFields(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
_, _ = w.Write([]byte(`{"data":{"job_id":"job-9"}}`)) // data-wrapped envelope
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
jobID, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
|
||||
Segments: []videogen.ChainSegment{{Prompt: "a dog"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SubmitChain: %v", err)
|
||||
}
|
||||
if jobID != "job-9" {
|
||||
t.Errorf("jobID = %q, want data-wrapped id", jobID)
|
||||
}
|
||||
for _, k := range []string{"init_image_b64", "smooth_joins", "size"} {
|
||||
if v, ok := gotBody[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
seg, _ := gotBody["segments"].([]any)
|
||||
if first, _ := seg[0].(map[string]any); first == nil {
|
||||
t.Fatalf("segments = %v", gotBody["segments"])
|
||||
} else if _, ok := first["seconds"]; ok {
|
||||
t.Error("zero seconds sent; want omitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubmitChainRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no segments: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
|
||||
Segments: []videogen.ChainSegment{{Prompt: " "}},
|
||||
}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("empty prompt: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
|
||||
Segments: []videogen.ChainSegment{{Prompt: "x", Seconds: -1}},
|
||||
}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("negative seconds: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
for name, bad := range map[string]float64{
|
||||
"NaN": math.NaN(),
|
||||
"+Inf": math.Inf(1),
|
||||
"-Inf": math.Inf(-1),
|
||||
} {
|
||||
if _, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
|
||||
Segments: []videogen.ChainSegment{{Prompt: "x", Seconds: bad}},
|
||||
}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("%s seconds: err = %v, want ErrUnsupported", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubmitChainRejectsMissingJobID(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
_, err := ch.SubmitChain(context.Background(), videogen.ChainRequest{
|
||||
Segments: []videogen.ChainSegment{{Prompt: "x"}},
|
||||
})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for missing job_id", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainStatus(t *testing.T) {
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_, _ = w.Write([]byte(`{"status":"running","segment":2,"total":3,"segments":["seg-0"]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
job, err := ch.ChainStatus(context.Background(), "job-123")
|
||||
if err != nil {
|
||||
t.Fatalf("ChainStatus: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/videoutils/v1/jobs/job-123" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if job.Status != "running" || job.Segment != 2 || job.Total != 3 {
|
||||
t.Errorf("job = %+v", job)
|
||||
}
|
||||
if !reflect.DeepEqual(job.SegmentIDs, []string{"seg-0"}) {
|
||||
t.Errorf("SegmentIDs = %v", job.SegmentIDs)
|
||||
}
|
||||
if job.Raw == nil {
|
||||
t.Error("Raw = nil, want raw payload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainStatusToleratesObjectSegments(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"status":"done","segment":2,"total":2,"segments":[{"id":"seg-0"},{"segment_id":"seg-1"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
job, err := ch.ChainStatus(context.Background(), "job-123")
|
||||
if err != nil {
|
||||
t.Fatalf("ChainStatus: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(job.SegmentIDs, []string{"seg-0", "seg-1"}) {
|
||||
t.Errorf("SegmentIDs = %v", job.SegmentIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainStatusSkipsIDLessSegments(t *testing.T) {
|
||||
// null, "", an id-less object, and a mistyped entry must all be
|
||||
// skipped — never appended as "" (SegmentIDs promises fetchable
|
||||
// artifacts; the unfiltered list stays in Raw).
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"status":"running","segment":3,"total":5,` +
|
||||
`"segments":[null,"seg-0","",{},{"id":"seg-1"},{"segment_id":""},42]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
job, err := ch.ChainStatus(context.Background(), "job-123")
|
||||
if err != nil {
|
||||
t.Fatalf("ChainStatus: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(job.SegmentIDs, []string{"seg-0", "seg-1"}) {
|
||||
t.Errorf("SegmentIDs = %v, want [seg-0 seg-1]", job.SegmentIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainStatusRejectsStatuslessPayload(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"detail":"no such job"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
_, err := ch.ChainStatus(context.Background(), "job-123")
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for statusless payload", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainJobPathRejectsHostileIDs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
for _, bad := range []string{"", "a/b", "a?b", "a#b", "..", "a..b", "a%2Fb", "%2e%2e", "a%b"} {
|
||||
if _, err := ch.ChainStatus(context.Background(), bad); err == nil {
|
||||
t.Errorf("ChainStatus(%q) succeeded; want error", bad)
|
||||
}
|
||||
if _, err := ch.ChainResult(context.Background(), bad); err == nil {
|
||||
t.Errorf("ChainResult(%q) succeeded; want error", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainResult(t *testing.T) {
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write(mp4Fixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
res, err := ch.ChainResult(context.Background(), "job-123")
|
||||
if err != nil {
|
||||
t.Fatalf("ChainResult: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/videoutils/v1/jobs/job-123/result" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if res.Video.MIME != "video/mp4" || len(res.Video.Data) == 0 {
|
||||
t.Fatalf("video = %q/%d bytes", res.Video.MIME, len(res.Video.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainSegmentResult(t *testing.T) {
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
_, _ = w.Write(mp4Fixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
res, err := ch.ChainSegmentResult(context.Background(), "job-123", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("ChainSegmentResult: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/videoutils/v1/jobs/job-123/segments/1" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if res.Video.MIME != "video/mp4" {
|
||||
t.Fatalf("video = %q", res.Video.MIME)
|
||||
}
|
||||
|
||||
if _, err := ch.ChainSegmentResult(context.Background(), "job-123", -1); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("negative segment: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainResultRejectsNonVideoResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"status":"running"}`)) // a status page, not the clip
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
_, err := ch.ChainResult(context.Background(), "job-123")
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-video body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainSurfacesAPIError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"job not found"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ch, _ := p.ChainerModel("videoutils")
|
||||
_, err := ch.ChainStatus(context.Background(), "job-void")
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %T %v, want *llm.APIError", err, err)
|
||||
}
|
||||
if apiErr.Status != http.StatusNotFound || apiErr.Message != "job not found" || apiErr.Model != "videoutils" {
|
||||
t.Errorf("apiErr = %+v", apiErr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// videoutil.go implements the videogen.VideoBackgroundRemovalProvider and
|
||||
// videogen.VideoUpscaleProvider surfaces against the mediautils shim reached
|
||||
// through llama-swap's /upstream passthrough (ADR-0025):
|
||||
//
|
||||
// matte POST /upstream/<id>/v1/video/matte (Robust Video Matting)
|
||||
// upscale POST /upstream/<id>/v1/video/upscale (per-frame Real-ESRGAN)
|
||||
//
|
||||
// Both are one-file multipart in, encoded clip out — the video siblings of
|
||||
// mediautil.go's still-image surfaces.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/videogen"
|
||||
)
|
||||
|
||||
// --- video background removal (matting) ---
|
||||
|
||||
// VideoBackgroundRemoverModel implements
|
||||
// videogen.VideoBackgroundRemovalProvider against the mediautils shim's
|
||||
// POST /v1/video/matte. The id selects which upstream llama-swap loads.
|
||||
func (p *Provider) VideoBackgroundRemoverModel(id string, opts ...videogen.VideoBackgroundRemoverModelOption) (videogen.VideoBackgroundRemover, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = videogen.ApplyVideoBackgroundRemoverModelOptions(opts)
|
||||
return &videoMatteModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type videoMatteModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// RemoveVideoBackground implements videogen.VideoBackgroundRemover.
|
||||
func (m *videoMatteModel) RemoveVideoBackground(ctx context.Context, req videogen.VideoBackgroundRemovalRequest, opts ...videogen.VideoBackgroundRemovalOption) (*videogen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Video) == 0 {
|
||||
return nil, fmt.Errorf("%w: video background removal requires a video", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Output != "" && req.Output != "greenscreen_mp4" && req.Output != "alpha_webm" {
|
||||
return nil, fmt.Errorf("%w: video matte output must be \"greenscreen_mp4\" or \"alpha_webm\", got %q", llm.ErrUnsupported, req.Output)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/video/matte")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, contentType, err := buildMultipart("build video-matte form",
|
||||
filePart{field: "file", filename: videoInputFilename(req.Filename, req.MIME), data: req.Video},
|
||||
[]formField{{"output", req.Output, false}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleVideoResult(m.p.name, m.id, "video matte", raw, respType)
|
||||
}
|
||||
|
||||
// --- video upscale ---
|
||||
|
||||
// VideoUpscalerModel implements videogen.VideoUpscaleProvider against the
|
||||
// mediautils shim's POST /v1/video/upscale.
|
||||
func (p *Provider) VideoUpscalerModel(id string, opts ...videogen.VideoUpscalerModelOption) (videogen.VideoUpscaler, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = videogen.ApplyVideoUpscalerModelOptions(opts)
|
||||
return &videoUpscaleModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type videoUpscaleModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// UpscaleVideo implements videogen.VideoUpscaler.
|
||||
func (m *videoUpscaleModel) UpscaleVideo(ctx context.Context, req videogen.VideoUpscaleRequest, opts ...videogen.VideoUpscaleOption) (*videogen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Video) == 0 {
|
||||
return nil, fmt.Errorf("%w: video upscale requires a video", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Scale != 0 && req.Scale != 2 && req.Scale != 4 {
|
||||
return nil, fmt.Errorf("%w: video upscale scale must be 2 or 4, got %d", llm.ErrUnsupported, req.Scale)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/video/upscale")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scale := ""
|
||||
if req.Scale != 0 {
|
||||
scale = strconv.Itoa(req.Scale)
|
||||
}
|
||||
body, contentType, err := buildMultipart("build video-upscale form",
|
||||
filePart{field: "file", filename: videoInputFilename(req.Filename, req.MIME), data: req.Video},
|
||||
[]formField{{"scale", scale, false}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxVideoResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return singleVideoResult(m.p.name, m.id, "video upscale", raw, respType)
|
||||
}
|
||||
|
||||
// videoInputFilename picks the multipart filename hint for a caller-supplied
|
||||
// video: the caller's (sanitized — upload metadata is untrusted), else one
|
||||
// derived from the MIME subtype ("video.mp4"), else "video". Mirrors
|
||||
// transcriptionFilename.
|
||||
func videoInputFilename(filename, mimeType string) string {
|
||||
if name := sanitizeFilename(filename); name != "" {
|
||||
return name
|
||||
}
|
||||
mt := strings.ToLower(strings.TrimSpace(mimeType))
|
||||
if parsed, _, err := mime.ParseMediaType(mt); err == nil {
|
||||
mt = parsed
|
||||
}
|
||||
switch mt {
|
||||
case "video/mp4":
|
||||
return "video.mp4"
|
||||
case "video/webm":
|
||||
return "video.webm"
|
||||
case "video/quicktime":
|
||||
return "video.mov"
|
||||
case "video/x-matroska":
|
||||
return "video.mkv"
|
||||
default:
|
||||
return "video"
|
||||
}
|
||||
}
|
||||
@@ -82,7 +82,7 @@ func (m *model) do(ctx context.Context, req llm.Request, stream bool) (*http.Res
|
||||
Model: m.id,
|
||||
Status: http.StatusUnauthorized,
|
||||
Code: "missing_api_key",
|
||||
Message: "no API key configured: set OPENAI_API_KEY or use WithAPIKey",
|
||||
Message: "no API key configured: set " + m.p.apiKeyName + " or use WithAPIKey",
|
||||
}
|
||||
}
|
||||
body, err := json.Marshal(m.buildRequest(req, stream))
|
||||
|
||||
@@ -34,6 +34,7 @@ const defaultBaseURL = "https://api.openai.com/v1"
|
||||
type Provider struct {
|
||||
name string
|
||||
apiKey string
|
||||
apiKeyName string
|
||||
baseURL string
|
||||
client *http.Client
|
||||
caps llm.Capabilities
|
||||
@@ -64,6 +65,14 @@ func WithHTTPClient(c *http.Client) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithAPIKeyName sets the environment-variable name shown in the missing-key
|
||||
// error (default "OPENAI_API_KEY"). Why: the same client serves compat
|
||||
// endpoints keyed by other env vars (e.g. KIMI_API_KEY), and the error should
|
||||
// name the one the operator actually needs to set.
|
||||
func WithAPIKeyName(name string) Option {
|
||||
return func(p *Provider) { p.apiKeyName = name }
|
||||
}
|
||||
|
||||
// WithName overrides the registry name ("openai" by default). Why: the same
|
||||
// client serves many OpenAI-compatible endpoints, and each needs a distinct
|
||||
// name in "provider/model" specs and error reporting.
|
||||
@@ -104,11 +113,12 @@ func defaultCapabilities() llm.Capabilities {
|
||||
// 401-style *llm.APIError at request time, not at construction.
|
||||
func New(opts ...Option) *Provider {
|
||||
p := &Provider{
|
||||
name: "openai",
|
||||
apiKey: os.Getenv("OPENAI_API_KEY"),
|
||||
baseURL: defaultBaseURL,
|
||||
client: http.DefaultClient,
|
||||
caps: defaultCapabilities(),
|
||||
name: "openai",
|
||||
apiKey: os.Getenv("OPENAI_API_KEY"),
|
||||
apiKeyName: "OPENAI_API_KEY",
|
||||
baseURL: defaultBaseURL,
|
||||
client: http.DefaultClient,
|
||||
caps: defaultCapabilities(),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(p)
|
||||
|
||||
+1
-1
@@ -263,7 +263,7 @@ func (r *Registry) providerFor(name string) (llm.Provider, error) {
|
||||
return nil, envErr
|
||||
}
|
||||
|
||||
envKey := "LLM_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
|
||||
envKey := envKeyForProvider(name)
|
||||
envVal := r.envLookup(envKey)
|
||||
if envVal == "" {
|
||||
return nil, fmt.Errorf("%w: %q (checked registry and %s env var)", ErrUnknownProvider, name, envKey)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package videogen
|
||||
|
||||
import "context"
|
||||
|
||||
// ChainSegment is one prompt in a multi-segment ("long video") chain.
|
||||
type ChainSegment struct {
|
||||
// Prompt describes this segment. Required.
|
||||
Prompt string
|
||||
|
||||
// Seconds is the segment's requested length; 0 = backend default.
|
||||
Seconds float64
|
||||
}
|
||||
|
||||
// ChainRequest asks a chain orchestrator to generate a long video as a
|
||||
// sequence of segments, each continuing from the previous segment's last
|
||||
// frame. Zero values mean "backend default" (ADR-0025).
|
||||
type ChainRequest struct {
|
||||
// Segments are the per-segment prompts in order. At least one required.
|
||||
Segments []ChainSegment
|
||||
|
||||
// InitImage optionally conditions the FIRST segment on a starting frame
|
||||
// (image-to-video); nil = pure text-to-video.
|
||||
InitImage []byte
|
||||
|
||||
// SmoothJoins asks the orchestrator to interpolate across segment
|
||||
// boundaries (RIFE-style) so cuts don't pop.
|
||||
SmoothJoins bool
|
||||
|
||||
// Size is the requested resolution, e.g. "1280x704"; "" = backend
|
||||
// default.
|
||||
Size string
|
||||
}
|
||||
|
||||
// ChainJob is a chain job's progress snapshot.
|
||||
type ChainJob struct {
|
||||
// Status is the backend's job state, passed through verbatim
|
||||
// (e.g. "queued", "running", "done", "failed").
|
||||
Status string
|
||||
|
||||
// Segment is the segment currently being generated (1-based); Total is
|
||||
// the segment count.
|
||||
Segment int
|
||||
Total int
|
||||
|
||||
// SegmentIDs name the COMPLETED per-segment artifacts, in order. A
|
||||
// mid-chain failure still leaves those segments retrievable via
|
||||
// ChainSegmentResult — note it takes the segment's index in the chain,
|
||||
// not an id string; this list tells you WHICH segments completed. So
|
||||
// multi-minute GPU output is never discarded. Entries the backend
|
||||
// reports without a usable id are skipped (the unfiltered list survives
|
||||
// in Raw).
|
||||
SegmentIDs []string
|
||||
|
||||
// Raw is the provider-native job payload. May be nil.
|
||||
Raw any
|
||||
}
|
||||
|
||||
// Chainer drives a multi-segment video-chain job. Unlike Model.Generate it
|
||||
// is deliberately ASYNC — a chain runs through multiple GPU loads for many
|
||||
// minutes, so callers submit, poll, and fetch instead of holding one
|
||||
// blocking call open.
|
||||
type Chainer interface {
|
||||
// SubmitChain starts a chain job and returns its job id.
|
||||
SubmitChain(ctx context.Context, req ChainRequest) (string, error)
|
||||
|
||||
// ChainStatus reports the job's progress. Polling also signals liveness
|
||||
// to backends that unload idle orchestrators.
|
||||
ChainStatus(ctx context.Context, jobID string) (*ChainJob, error)
|
||||
|
||||
// ChainResult fetches the finished, concatenated clip.
|
||||
ChainResult(ctx context.Context, jobID string) (*Result, error)
|
||||
|
||||
// ChainSegmentResult fetches one completed segment's clip (n indexes the
|
||||
// job's segment list) — the partial-delivery path when a chain dies
|
||||
// mid-run.
|
||||
ChainSegmentResult(ctx context.Context, jobID string, n int) (*Result, error)
|
||||
}
|
||||
|
||||
// ChainerModelOption configures a Chainer at construction time. Reserved for
|
||||
// future per-model settings.
|
||||
type ChainerModelOption func(*ChainerModelConfig)
|
||||
|
||||
// ChainerModelConfig carries per-model construction settings.
|
||||
type ChainerModelConfig struct{}
|
||||
|
||||
// ApplyChainerModelOptions folds options into a config.
|
||||
func ApplyChainerModelOptions(opts []ChainerModelOption) ChainerModelConfig {
|
||||
var cfg ChainerModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// ChainerProvider mints Chainers bound to one backend.
|
||||
type ChainerProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// ChainerModel returns a Chainer bound to the given id (passed through
|
||||
// to the backend verbatim; no catalog validation).
|
||||
ChainerModel(id string, opts ...ChainerModelOption) (Chainer, error)
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package videogen
|
||||
|
||||
import "context"
|
||||
|
||||
// LipsyncRequest asks a talking-head backend (SadTalker style) to animate a
|
||||
// still portrait so it speaks the given audio. Zero values mean "backend
|
||||
// default" (ADR-0025).
|
||||
type LipsyncRequest struct {
|
||||
// Image is the portrait to animate. Required.
|
||||
Image Image
|
||||
|
||||
// Audio is the encoded speech the head lip-syncs to. Required. Carried
|
||||
// as bytes (never a URL), mirroring audio.TranscriptionRequest.
|
||||
Audio []byte
|
||||
|
||||
// AudioMIME is the audio MIME type (e.g. "audio/wav"); "" = let the
|
||||
// backend sniff it.
|
||||
AudioMIME string
|
||||
|
||||
// AudioFilename is the multipart filename hint some backends key their
|
||||
// format detection on; "" derives one from AudioMIME or falls back to
|
||||
// "audio".
|
||||
AudioFilename string
|
||||
|
||||
// Still reduces head motion to blinks and lip movement (less uncanny on
|
||||
// formal portraits); false = backend default motion.
|
||||
Still bool
|
||||
|
||||
// Enhance runs the backend's face enhancer over the output frames.
|
||||
Enhance bool
|
||||
|
||||
// Preprocess selects how the backend frames the face: "crop" (animate
|
||||
// the face crop) or "full" (paste the animated face back into the whole
|
||||
// image); "" = backend default.
|
||||
Preprocess string
|
||||
}
|
||||
|
||||
// LipsyncOption mutates a LipsyncRequest before it is sent.
|
||||
type LipsyncOption func(*LipsyncRequest)
|
||||
|
||||
// WithLipsyncStill reduces head motion to blinks and lip movement.
|
||||
func WithLipsyncStill() LipsyncOption { return func(r *LipsyncRequest) { r.Still = true } }
|
||||
|
||||
// WithLipsyncEnhance runs the backend's face enhancer over the output.
|
||||
func WithLipsyncEnhance() LipsyncOption { return func(r *LipsyncRequest) { r.Enhance = true } }
|
||||
|
||||
// WithLipsyncPreprocess selects the face framing ("crop" or "full").
|
||||
func WithLipsyncPreprocess(p string) LipsyncOption {
|
||||
return func(r *LipsyncRequest) { r.Preprocess = p }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r LipsyncRequest) Apply(opts ...LipsyncOption) LipsyncRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Lipsyncer animates still portraits into talking-head clips. Its own small
|
||||
// interface rather than a method on Model: lip-syncers are not text-to-video
|
||||
// generators — they bind to a different backend id entirely.
|
||||
type Lipsyncer interface {
|
||||
// Lipsync returns the talking-head clip. Generation is slow (minutes);
|
||||
// bound the call with a context deadline.
|
||||
Lipsync(ctx context.Context, req LipsyncRequest, opts ...LipsyncOption) (*Result, error)
|
||||
}
|
||||
|
||||
// LipsyncModelOption configures a Lipsyncer at construction time. Reserved
|
||||
// for future per-model settings.
|
||||
type LipsyncModelOption func(*LipsyncModelConfig)
|
||||
|
||||
// LipsyncModelConfig carries per-model construction settings.
|
||||
type LipsyncModelConfig struct{}
|
||||
|
||||
// ApplyLipsyncModelOptions folds options into a config.
|
||||
func ApplyLipsyncModelOptions(opts []LipsyncModelOption) LipsyncModelConfig {
|
||||
var cfg LipsyncModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// LipsyncProvider mints Lipsyncers bound to one backend.
|
||||
type LipsyncProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// LipsyncModel returns a Lipsyncer bound to the given id (passed through
|
||||
// to the backend verbatim; no catalog validation).
|
||||
LipsyncModel(id string, opts ...LipsyncModelOption) (Lipsyncer, error)
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package videogen
|
||||
|
||||
import "context"
|
||||
|
||||
// VideoBackgroundRemovalRequest asks a video-matting backend (Robust Video
|
||||
// Matting style) to separate the foreground subject from the background of a
|
||||
// clip. Zero values mean "backend default" (ADR-0025).
|
||||
type VideoBackgroundRemovalRequest struct {
|
||||
// Video is the encoded clip to matte. Required. Carried as bytes (never
|
||||
// a URL).
|
||||
Video []byte
|
||||
|
||||
// MIME is the video MIME type (e.g. "video/mp4"); "" = let the backend
|
||||
// sniff it.
|
||||
MIME string
|
||||
|
||||
// Filename is the multipart filename hint some backends key their format
|
||||
// detection on; "" derives one from MIME ("video.mp4") or falls back to
|
||||
// "video".
|
||||
Filename string
|
||||
|
||||
// Output selects the delivery container: "greenscreen_mp4" (subject over
|
||||
// solid green, universally playable) or "alpha_webm" (true transparency,
|
||||
// VP9 alpha channel); "" = backend default.
|
||||
Output string
|
||||
}
|
||||
|
||||
// VideoBackgroundRemovalOption mutates a VideoBackgroundRemovalRequest
|
||||
// before it is sent.
|
||||
type VideoBackgroundRemovalOption func(*VideoBackgroundRemovalRequest)
|
||||
|
||||
// WithVideoBackgroundOutput selects the delivery container
|
||||
// ("greenscreen_mp4" or "alpha_webm").
|
||||
func WithVideoBackgroundOutput(o string) VideoBackgroundRemovalOption {
|
||||
return func(r *VideoBackgroundRemovalRequest) { r.Output = o }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r VideoBackgroundRemovalRequest) Apply(opts ...VideoBackgroundRemovalOption) VideoBackgroundRemovalRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// VideoBackgroundRemover mattes the subject out of video clips — the moving-
|
||||
// picture sibling of imagegen.BackgroundRemover.
|
||||
type VideoBackgroundRemover interface {
|
||||
// RemoveVideoBackground returns the matted clip. Matting is slow
|
||||
// (per-frame inference); bound the call with a context deadline.
|
||||
RemoveVideoBackground(ctx context.Context, req VideoBackgroundRemovalRequest, opts ...VideoBackgroundRemovalOption) (*Result, error)
|
||||
}
|
||||
|
||||
// VideoBackgroundRemoverModelOption configures a VideoBackgroundRemover at
|
||||
// construction time. Reserved for future per-model settings.
|
||||
type VideoBackgroundRemoverModelOption func(*VideoBackgroundRemoverModelConfig)
|
||||
|
||||
// VideoBackgroundRemoverModelConfig carries per-model construction settings.
|
||||
type VideoBackgroundRemoverModelConfig struct{}
|
||||
|
||||
// ApplyVideoBackgroundRemoverModelOptions folds options into a config.
|
||||
func ApplyVideoBackgroundRemoverModelOptions(opts []VideoBackgroundRemoverModelOption) VideoBackgroundRemoverModelConfig {
|
||||
var cfg VideoBackgroundRemoverModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// VideoBackgroundRemovalProvider mints VideoBackgroundRemovers bound to one
|
||||
// backend.
|
||||
type VideoBackgroundRemovalProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// VideoBackgroundRemoverModel returns a VideoBackgroundRemover bound to
|
||||
// the given id (passed through to the backend verbatim; no catalog
|
||||
// validation).
|
||||
VideoBackgroundRemoverModel(id string, opts ...VideoBackgroundRemoverModelOption) (VideoBackgroundRemover, error)
|
||||
}
|
||||
@@ -12,6 +12,8 @@
|
||||
// InitImage is a pure text prompt, a non-nil InitImage conditions generation
|
||||
// on that frame. Hybrid models (e.g. Wan 2.2 TI2V) serve both from the same
|
||||
// checkpoint, so unlike imagegen there is no separate Editor-style interface.
|
||||
// LastImage extends the same surface to the other end of the clip, so one
|
||||
// Request covers t2v, i2v, and first-last-frame-to-video without a mode flag.
|
||||
//
|
||||
// The first implementation is provider/llamaswap, which targets the blocking
|
||||
// OpenAI/vLLM-Omni-style POST /v1/videos/sync endpoint: the response body is
|
||||
@@ -51,6 +53,19 @@ type Request struct {
|
||||
// nil = pure text-to-video.
|
||||
InitImage *Image
|
||||
|
||||
// LastImage conditions generation on an ENDING frame. With InitImage it
|
||||
// pins both ends (first-last-frame-to-video); alone it pins only the
|
||||
// destination and lets the backend invent the approach.
|
||||
//
|
||||
// Support is per-model and NOT advertised anywhere in this contract: a
|
||||
// backend that does not understand a trailing keyframe ignores it and
|
||||
// returns an ordinary clip, which is indistinguishable from success.
|
||||
// There is no capability bit to consult, because the contract has no way
|
||||
// to learn one. A caller that needs to know whether the pin actually took
|
||||
// effect must establish that out of band — by configuration it controls,
|
||||
// not by inspecting the result.
|
||||
LastImage *Image
|
||||
|
||||
// Size is the requested resolution, e.g. "1280x704"; "" = backend default.
|
||||
Size string
|
||||
|
||||
@@ -92,6 +107,10 @@ type Option func(*Request)
|
||||
// WithInitImage conditions generation on a starting frame (image-to-video).
|
||||
func WithInitImage(img Image) Option { return func(r *Request) { r.InitImage = &img } }
|
||||
|
||||
// WithLastImage conditions generation on an ending frame. Combined with
|
||||
// WithInitImage this pins both ends of the clip.
|
||||
func WithLastImage(img Image) Option { return func(r *Request) { r.LastImage = &img } }
|
||||
|
||||
// WithSize sets the requested resolution (e.g. "1280x704").
|
||||
func WithSize(size string) Option { return func(r *Request) { r.Size = size } }
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package videogen
|
||||
|
||||
import "context"
|
||||
|
||||
// VideoUpscaleRequest asks a super-resolution backend (per-frame Real-ESRGAN
|
||||
// style) to enlarge a clip. Zero values mean "backend default" (ADR-0025).
|
||||
type VideoUpscaleRequest struct {
|
||||
// Video is the encoded clip to upscale. Required. Carried as bytes
|
||||
// (never a URL).
|
||||
Video []byte
|
||||
|
||||
// MIME is the video MIME type (e.g. "video/mp4"); "" = let the backend
|
||||
// sniff it.
|
||||
MIME string
|
||||
|
||||
// Filename is the multipart filename hint some backends key their format
|
||||
// detection on; "" derives one from MIME ("video.mp4") or falls back to
|
||||
// "video".
|
||||
Filename string
|
||||
|
||||
// Scale is the enlargement factor (2 or 4 on the reference backend);
|
||||
// 0 = backend default.
|
||||
Scale int
|
||||
}
|
||||
|
||||
// VideoUpscaleOption mutates a VideoUpscaleRequest before it is sent.
|
||||
type VideoUpscaleOption func(*VideoUpscaleRequest)
|
||||
|
||||
// WithVideoUpscaleScale sets the enlargement factor.
|
||||
func WithVideoUpscaleScale(s int) VideoUpscaleOption {
|
||||
return func(r *VideoUpscaleRequest) { r.Scale = s }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r VideoUpscaleRequest) Apply(opts ...VideoUpscaleOption) VideoUpscaleRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// VideoUpscaler enlarges video clips frame by frame — the moving-picture
|
||||
// sibling of imagegen.Upscaler.
|
||||
type VideoUpscaler interface {
|
||||
// UpscaleVideo returns the enlarged clip. Upscaling is slow (per-frame
|
||||
// inference); bound the call with a context deadline.
|
||||
UpscaleVideo(ctx context.Context, req VideoUpscaleRequest, opts ...VideoUpscaleOption) (*Result, error)
|
||||
}
|
||||
|
||||
// VideoUpscalerModelOption configures a VideoUpscaler at construction time.
|
||||
// Reserved for future per-model settings.
|
||||
type VideoUpscalerModelOption func(*VideoUpscalerModelConfig)
|
||||
|
||||
// VideoUpscalerModelConfig carries per-model construction settings.
|
||||
type VideoUpscalerModelConfig struct{}
|
||||
|
||||
// ApplyVideoUpscalerModelOptions folds options into a config.
|
||||
func ApplyVideoUpscalerModelOptions(opts []VideoUpscalerModelOption) VideoUpscalerModelConfig {
|
||||
var cfg VideoUpscalerModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// VideoUpscaleProvider mints VideoUpscalers bound to one backend.
|
||||
type VideoUpscaleProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// VideoUpscalerModel returns a VideoUpscaler bound to the given id
|
||||
// (passed through to the backend verbatim; no catalog validation).
|
||||
VideoUpscalerModel(id string, opts ...VideoUpscalerModelOption) (VideoUpscaler, error)
|
||||
}
|
||||
Reference in New Issue
Block a user