diff --git a/README.md b/README.md index e0e08a9..6ff4d7e 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,24 @@ tr, err := tm.Transcribe(ctx, audio.TranscriptionRequest{ voices, err := ls.ListVoices(ctx, "kokoro") // []string of voice ids ``` +## Video: text-to-video + image-to-video + +Video generation lives in the `videogen` package (ADR-0019), mirroring +imagegen/audio: one small `Model` contract, zero values mean backend +defaults, bytes in/out. Text-to-video and image-to-video are one surface — +a nil `InitImage` is a pure text prompt; setting it conditions generation +on that frame (hybrid checkpoints like Wan 2.2 TI2V serve both). First +backend: llama-swap (blocking `/v1/videos/sync`, vLLM-Omni style — the +response body is the encoded clip, so `Result` carries a single `Video`). +Generation runs for minutes; bound the call with a context deadline. + +```go +vm, _ := ls.VideoModel("videogen-wan22-5b") +res, err := vm.Generate(ctx, videogen.Request{Prompt: "a cat surfing"}, + videogen.WithSize("1280x704"), videogen.WithNumFrames(81)) +// res.Video.Data ([]byte) + res.Video.MIME ("video/mp4") +``` + ## Tool calls ```go @@ -398,7 +416,8 @@ response as a single delta plus final event. capabilities are present at the client level; whether a given call succeeds depends on the llama.cpp model llama-swap loads. llama-swap also provides **image generation + editing** (`imagegen`), **speech synthesis + -transcription** (`audio`) — separate axes, not shown above — plus a `Health` +transcription** (`audio`), **video generation** (`videogen`) — separate +axes, not shown above — plus a `Health` probe and management methods on `*llamaswap.Provider`. Notes: Ollama has no native tool_choice — `"none"` drops the tools; diff --git a/docs/adr/README.md b/docs/adr/README.md index 6818feb..0c641e3 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -22,3 +22,4 @@ One decision per file, append-only; supersede rather than rewrite. | [0016](0016-imagegen-interface.md) | imagegen — a canonical text-to-image interface | Accepted | | [0017](0017-audio-interfaces.md) | audio — canonical speech synthesis + transcription interfaces | Accepted | | [0018](0018-imagegen-editor.md) | imagegen.Editor — image-to-image as a separate optional interface | Accepted | +| [0019](0019-videogen-interface.md) | videogen — canonical video-generation surface | Accepted | diff --git a/provider/llamaswap/audio.go b/provider/llamaswap/audio.go index 77ffc6a..22ba06f 100644 --- a/provider/llamaswap/audio.go +++ b/provider/llamaswap/audio.go @@ -77,7 +77,7 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts . // is a concrete audio type, else a mapping from the requested format, else // audio/mpeg (the OpenAI endpoint's default container is mp3). func speechMIME(contentType, format string) string { - if mt, _, err := mime.ParseMediaType(contentType); err == nil && strings.HasPrefix(mt, "audio/") { + if mt := mimeFromContentType(contentType, "audio/"); mt != "" { return mt } switch strings.ToLower(strings.TrimSpace(format)) { @@ -128,24 +128,13 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.Transcrip if _, err := fw.Write(req.Audio); err != nil { return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) } - // Required fields always go on the wire (an empty model id should fail - // loudly upstream, not silently vanish); optional ones only when set. - fields := []struct { - key, value string - required bool - }{ + if err := writeFormFields(w, "build transcription form", []formField{ {"model", m.id, true}, {"response_format", "json", true}, {"language", req.Language, false}, {"prompt", req.Prompt, false}, - } - for _, f := range fields { - if !f.required && f.value == "" { - continue - } - if err := w.WriteField(f.key, f.value); err != nil { - return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) - } + }); err != nil { + return nil, err } if err := w.Close(); err != nil { return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) diff --git a/provider/llamaswap/llamaswap.go b/provider/llamaswap/llamaswap.go index c175a44..8dedece 100644 --- a/provider/llamaswap/llamaswap.go +++ b/provider/llamaswap/llamaswap.go @@ -30,6 +30,8 @@ import ( "encoding/json" "fmt" "io" + "mime" + "mime/multipart" "net/http" "strings" @@ -280,3 +282,34 @@ func (p *Provider) apiError(resp *http.Response, model string) error { e.Message = strings.TrimSpace(string(body)) return e } + +// formField is one multipart form entry for the media endpoints. Required +// fields always go on the wire (an empty model id should fail loudly +// upstream, not silently vanish); optional ones only when set. +type formField struct { + key, value string + required bool +} + +// writeFormFields appends fields to a multipart writer, skipping unset +// optional entries. wrap labels errors ("build transcription form", ...). +func writeFormFields(w *multipart.Writer, wrap string, fields []formField) error { + for _, f := range fields { + if !f.required && f.value == "" { + continue + } + if err := w.WriteField(f.key, f.value); err != nil { + return fmt.Errorf("llama-swap: %s: %w", wrap, err) + } + } + return nil +} + +// mimeFromContentType returns the parsed media type when it matches prefix +// ("audio/", "video/"), else "". +func mimeFromContentType(contentType, prefix string) string { + if mt, _, err := mime.ParseMediaType(contentType); err == nil && strings.HasPrefix(mt, prefix) { + return mt + } + return "" +} diff --git a/provider/llamaswap/video.go b/provider/llamaswap/video.go index 1823304..b0e893b 100644 --- a/provider/llamaswap/video.go +++ b/provider/llamaswap/video.go @@ -64,30 +64,24 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts .. var buf bytes.Buffer w := multipart.NewWriter(&buf) - // Required fields always go on the wire (an empty model id should fail - // loudly upstream, not silently vanish); optional ones only when set. - fields := []struct { - key, value string - required bool - }{ + if err := writeFormFields(w, "build video form", []formField{ {"model", m.id, true}, {"prompt", req.Prompt, true}, {"negative_prompt", req.NegativePrompt, false}, + // Resolution rides the wire twice: width/height (vLLM-Omni's + // names) AND the equivalent OpenAI-style size string, since + // upstreams silently ignore fields they don't understand and the + // values can never disagree. {"width", formatInt(width), false}, {"height", formatInt(height), false}, + {"size", strings.TrimSpace(req.Size), false}, {"num_frames", formatNonZero(req.NumFrames), false}, {"fps", formatNonZero(req.FPS), false}, {"num_inference_steps", formatInt(req.Steps), false}, {"guidance_scale", formatFloat(req.GuidanceScale), false}, {"seed", formatInt64(req.Seed), false}, - } - for _, f := range fields { - if !f.required && f.value == "" { - continue - } - if err := w.WriteField(f.key, f.value); err != nil { - return nil, fmt.Errorf("llama-swap: build video form: %w", err) - } + }); err != nil { + return nil, err } if req.InitImage != nil { fw, err := w.CreateFormFile("input_reference", initImageFilename(req.InitImage.MIME)) @@ -109,20 +103,33 @@ func (m *videoModel) Generate(ctx context.Context, req videogen.Request, opts .. if len(videoBytes) == 0 { return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "video response contained no video"} } - return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: videoMIME(contentType, videoBytes)}}, nil + mimeType := videoMIME(contentType, videoBytes) + if mimeType == "" { + // A 2xx body that is neither declared nor sniffable as video is a + // misconfigured upstream (a JSON job envelope, an HTML error page + // behind a proxy) — fail loud rather than hand back garbage as a + // playable clip. + return nil, &llm.APIError{ + Provider: m.p.name, + Model: m.id, + Message: fmt.Sprintf("video response is not a video (content-type %q)", contentType), + } + } + return &videogen.Result{Video: videogen.Video{Data: videoBytes, MIME: mimeType}}, nil } // videoMIME resolves the result MIME type: the response Content-Type when it -// is a concrete video type, else content sniffing, else video/mp4 (the -// endpoint's default container). +// is a concrete video type, else content sniffing (mp4/webm magic bytes), +// else "" — the caller treats undetectable as an upstream error, unlike the +// audio path where the request's format param implies the container. func videoMIME(contentType string, data []byte) string { - if mt, _, err := mime.ParseMediaType(contentType); err == nil && strings.HasPrefix(mt, "video/") { + if mt := mimeFromContentType(contentType, "video/"); mt != "" { return mt } if mt := http.DetectContentType(data); strings.HasPrefix(mt, "video/") { return mt } - return "video/mp4" + return "" } // initImageFilename picks the multipart filename hint for the conditioning diff --git a/provider/llamaswap/video_test.go b/provider/llamaswap/video_test.go index f4462e6..2684279 100644 --- a/provider/llamaswap/video_test.go +++ b/provider/llamaswap/video_test.go @@ -179,6 +179,9 @@ func TestVideoModelRequiresBaseURL(t *testing.T) { } func TestVideoMIME(t *testing.T) { + // A minimal ISO-BMFF prefix http.DetectContentType sniffs as video/mp4 + // (the "mp4" brand prefix must appear inside the declared ftyp box). + mp4Magic := append([]byte{0, 0, 0, 20}, []byte("ftypmp42\x00\x00\x00\x00mp42")...) cases := []struct { contentType string data []byte @@ -186,12 +189,36 @@ func TestVideoMIME(t *testing.T) { }{ {"video/webm", []byte("x"), "video/webm"}, {"video/mp4; charset=binary", []byte("x"), "video/mp4"}, - {"application/octet-stream", []byte("x"), "video/mp4"}, - {"", []byte("x"), "video/mp4"}, + {"application/octet-stream", mp4Magic, "video/mp4"}, + // Neither declared nor sniffable as video → "" (Generate errors). + {"application/octet-stream", []byte(`{"id":"job-1"}`), ""}, + {"", []byte("x"), ""}, } for _, tc := range cases { if got := videoMIME(tc.contentType, tc.data); got != tc.want { - t.Errorf("videoMIME(%q) = %q, want %q", tc.contentType, got, tc.want) + t.Errorf("videoMIME(%q, %q) = %q, want %q", tc.contentType, tc.data, got, tc.want) } } } + +func TestVideoGenerateNonVideoBodyErrors(t *testing.T) { + // A stock async /v1/videos handler mounted at the sync path (or an HTML + // error page behind a proxy) answers 200 with a non-video body — that + // must be an error, never a "successful" garbage clip. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"job-1","status":"queued"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + vm, _ := p.VideoModel("videogen-wan") + _, err := vm.Generate(context.Background(), videogen.Request{Prompt: "x"}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want *llm.APIError for non-video 2xx body", err) + } + if !strings.Contains(apiErr.Message, "not a video") { + t.Errorf("message = %q, want mention of non-video body", apiErr.Message) + } +}