diff --git a/internal/config/model_config.go b/internal/config/model_config.go index 7e354779..b85d7ce1 100644 --- a/internal/config/model_config.go +++ b/internal/config/model_config.go @@ -14,6 +14,7 @@ var validModalities = map[string]struct{}{ "text": {}, "audio": {}, "image": {}, + "video": {}, } // ModelCapConfig defines what modalities and features a model supports. @@ -37,12 +38,12 @@ func (c ModelCapConfig) Empty() bool { func (c ModelCapConfig) Validate() error { for _, m := range c.In { if _, ok := validModalities[m]; !ok { - return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image", m) + return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image, video", m) } } for _, m := range c.Out { if _, ok := validModalities[m]; !ok { - return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image", m) + return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image, video", m) } } if c.Context < 0 { diff --git a/internal/config/model_config_test.go b/internal/config/model_config_test.go index 622dab4e..eb72b4fd 100644 --- a/internal/config/model_config_test.go +++ b/internal/config/model_config_test.go @@ -296,20 +296,25 @@ func TestConfig_ModelCapabilities_Validate(t *testing.T) { assert.NoError(t, caps.Validate()) }) + t.Run("valid_video_modality", func(t *testing.T) { + caps := ModelCapConfig{In: []string{"text", "image"}, Out: []string{"video"}} + assert.NoError(t, caps.Validate()) + }) + t.Run("invalid_in_modality", func(t *testing.T) { - caps := ModelCapConfig{In: []string{"video"}} + caps := ModelCapConfig{In: []string{"smell"}} err := caps.Validate() assert.Error(t, err) assert.Contains(t, err.Error(), "capabilities.in") - assert.Contains(t, err.Error(), "video") + assert.Contains(t, err.Error(), "smell") }) t.Run("invalid_out_modality", func(t *testing.T) { - caps := ModelCapConfig{Out: []string{"video"}} + caps := ModelCapConfig{Out: []string{"smell"}} err := caps.Validate() assert.Error(t, err) assert.Contains(t, err.Error(), "capabilities.out") - assert.Contains(t, err.Error(), "video") + assert.Contains(t, err.Error(), "smell") }) t.Run("negative_context", func(t *testing.T) { @@ -327,10 +332,10 @@ models: capabilities: in: - text - - video + - smell ` _, err := LoadConfigFromReader(strings.NewReader(content)) assert.Error(t, err) - assert.Contains(t, err.Error(), "video") + assert.Contains(t, err.Error(), "smell") }) } diff --git a/internal/server/api.go b/internal/server/api.go index 0da4a781..b1b4f107 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -88,6 +88,12 @@ func renderCapabilities(caps config.ModelCapConfig) (arch map[string]any, capsMa if contains(caps.In, "image") && contains(caps.Out, "image") { capsMap["image_to_image"] = true } + if contains(caps.In, "text") && contains(caps.Out, "video") { + capsMap["video_generation"] = true + } + if contains(caps.In, "image") && contains(caps.Out, "video") { + capsMap["image_to_video"] = true + } } if caps.Tools { diff --git a/ui-svelte/src/components/playground/VideoInterface.svelte b/ui-svelte/src/components/playground/VideoInterface.svelte new file mode 100644 index 00000000..1181bd8a --- /dev/null +++ b/ui-svelte/src/components/playground/VideoInterface.svelte @@ -0,0 +1,359 @@ + + +
+ +
+ + + + + +
+ + + {#if showSettings} +
+
+ + + + + +
+ + +
+ {/if} + + + {#if !hasModels} +
+

No models configured. Add models to your configuration to generate videos.

+
+ {:else} + +
+ {#if isGenerating} +
+
+

Generating video… {formatElapsed(elapsedSeconds)}

+

+ Video generation takes minutes — a cold model load takes longer still. +

+
+ {:else if error} +
+

Error

+

{error}

+
+ {:else if videoUrl} +
+ + + +
+ {:else} +
+

Enter a prompt below to generate a short video clip

+

Attach an image to animate it (image-to-video)

+
+ {/if} +
+ + + {#if initImagePreview} +
+ Conditioning frame + Animating this image (image-to-video) + +
+ {/if} + + +
+ +
+ {#if isGenerating} + + {:else} + + + + {/if} +
+
+ {/if} +
diff --git a/ui-svelte/src/lib/types.ts b/ui-svelte/src/lib/types.ts index 907272ef..8d4238db 100644 --- a/ui-svelte/src/lib/types.ts +++ b/ui-svelte/src/lib/types.ts @@ -8,6 +8,8 @@ export interface ModelCapabilities { audio_speech?: boolean; image_generation?: boolean; image_to_image?: boolean; + video_generation?: boolean; + image_to_video?: boolean; function_calling?: boolean; reranker?: boolean; } diff --git a/ui-svelte/src/lib/videoApi.ts b/ui-svelte/src/lib/videoApi.ts new file mode 100644 index 00000000..971c674e --- /dev/null +++ b/ui-svelte/src/lib/videoApi.ts @@ -0,0 +1,57 @@ +export interface VideoGenerationOptions { + negativePrompt?: string; + size?: string; // "WxH" + numFrames?: number; + fps?: number; + steps?: number; + guidanceScale?: number; + seed?: number; // -1 = random (omitted) + initImage?: File | null; // image-to-video conditioning frame +} + +// POST /v1/videos/sync (multipart; llama-swap routes by the `model` field). +// The response body IS the encoded clip — returns an object URL for