Merge pull request 'feat(ui): Video playground tab + video capability plumbing' (#2) from feat/video-ui into main
Build CUDA image (fork) / build (push) Successful in 2m57s

Reviewed-on: #2
This commit was merged in pull request #2.
This commit is contained in:
2026-07-12 14:56:35 +00:00
8 changed files with 448 additions and 11 deletions
+3 -2
View File
@@ -14,6 +14,7 @@ var validModalities = map[string]struct{}{
"text": {},
"audio": {},
"image": {},
"video": {},
}
// ModelCapConfig defines what modalities and features a model supports.
@@ -37,12 +38,12 @@ func (c ModelCapConfig) Empty() bool {
func (c ModelCapConfig) Validate() error {
for _, m := range c.In {
if _, ok := validModalities[m]; !ok {
return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image", m)
return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image, video", m)
}
}
for _, m := range c.Out {
if _, ok := validModalities[m]; !ok {
return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image", m)
return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image, video", m)
}
}
if c.Context < 0 {
+11 -6
View File
@@ -296,20 +296,25 @@ func TestConfig_ModelCapabilities_Validate(t *testing.T) {
assert.NoError(t, caps.Validate())
})
t.Run("valid_video_modality", func(t *testing.T) {
caps := ModelCapConfig{In: []string{"text", "image"}, Out: []string{"video"}}
assert.NoError(t, caps.Validate())
})
t.Run("invalid_in_modality", func(t *testing.T) {
caps := ModelCapConfig{In: []string{"video"}}
caps := ModelCapConfig{In: []string{"smell"}}
err := caps.Validate()
assert.Error(t, err)
assert.Contains(t, err.Error(), "capabilities.in")
assert.Contains(t, err.Error(), "video")
assert.Contains(t, err.Error(), "smell")
})
t.Run("invalid_out_modality", func(t *testing.T) {
caps := ModelCapConfig{Out: []string{"video"}}
caps := ModelCapConfig{Out: []string{"smell"}}
err := caps.Validate()
assert.Error(t, err)
assert.Contains(t, err.Error(), "capabilities.out")
assert.Contains(t, err.Error(), "video")
assert.Contains(t, err.Error(), "smell")
})
t.Run("negative_context", func(t *testing.T) {
@@ -327,10 +332,10 @@ models:
capabilities:
in:
- text
- video
- smell
`
_, err := LoadConfigFromReader(strings.NewReader(content))
assert.Error(t, err)
assert.Contains(t, err.Error(), "video")
assert.Contains(t, err.Error(), "smell")
})
}
+6
View File
@@ -88,6 +88,12 @@ func renderCapabilities(caps config.ModelCapConfig) (arch map[string]any, capsMa
if contains(caps.In, "image") && contains(caps.Out, "image") {
capsMap["image_to_image"] = true
}
if contains(caps.In, "text") && contains(caps.Out, "video") {
capsMap["video_generation"] = true
}
if contains(caps.In, "image") && contains(caps.Out, "video") {
capsMap["image_to_video"] = true
}
}
if caps.Tools {
@@ -0,0 +1,359 @@
<script lang="ts">
import { models } from "../../stores/api";
import { persistentStore } from "../../stores/persistent";
import { generateVideoSync } from "../../lib/videoApi";
import { playgroundStores } from "../../stores/playgroundActivity";
import ModelSelector from "./ModelSelector.svelte";
import ExpandableTextarea from "./ExpandableTextarea.svelte";
const selectedModelStore = persistentStore<string>("playground-video-model", "");
const selectedSizeStore = persistentStore<string>("playground-video-size", "");
const negativePromptStore = persistentStore<string>("playground-video-negative-prompt", "");
const numFramesStore = persistentStore<number>("playground-video-num-frames", 0);
const fpsStore = persistentStore<number>("playground-video-fps", 0);
const stepsStore = persistentStore<number>("playground-video-steps", 0);
const guidanceStore = persistentStore<number>("playground-video-guidance", 0);
const seedStore = persistentStore<number>("playground-video-seed", -1);
let prompt = $state("");
let isGenerating = $state(false);
let videoUrl = $state<string | null>(null);
let videoMime = $state("video/mp4");
let videoBytes = $state(0);
let elapsedSeconds = $state(0);
let error = $state<string | null>(null);
let abortController = $state<AbortController | null>(null);
let showSettings = $state(false);
let initImage = $state<File | null>(null);
let initImagePreview = $state<string | null>(null);
let fileInput = $state<HTMLInputElement | null>(null);
let timer: ReturnType<typeof setInterval> | null = null;
let hasModels = $derived($models.some((m) => !m.unlisted));
$effect(() => {
playgroundStores.videoGenerating.set(isGenerating);
});
function onInitImageChange(event: Event) {
const input = event.target as HTMLInputElement;
const file = input.files?.[0] ?? null;
setInitImage(file);
}
function setInitImage(file: File | null) {
if (initImagePreview) URL.revokeObjectURL(initImagePreview);
initImage = file;
initImagePreview = file ? URL.createObjectURL(file) : null;
}
function clearInitImage() {
setInitImage(null);
if (fileInput) fileInput.value = "";
}
async function generate() {
const trimmedPrompt = prompt.trim();
if (!trimmedPrompt || !$selectedModelStore || isGenerating) return;
isGenerating = true;
error = null;
elapsedSeconds = 0;
abortController = new AbortController();
timer = setInterval(() => (elapsedSeconds += 1), 1000);
try {
const result = await generateVideoSync(
$selectedModelStore,
trimmedPrompt,
{
negativePrompt: $negativePromptStore || undefined,
size: $selectedSizeStore || undefined,
numFrames: $numFramesStore,
fps: $fpsStore,
steps: $stepsStore,
guidanceScale: $guidanceStore,
seed: $seedStore,
initImage,
},
abortController.signal
);
if (videoUrl) URL.revokeObjectURL(videoUrl);
videoUrl = result.url;
videoMime = result.mime;
videoBytes = result.sizeBytes;
} catch (err) {
if (err instanceof Error && err.name === "AbortError") {
// User cancelled
} else {
error = err instanceof Error ? err.message : "An error occurred";
}
} finally {
isGenerating = false;
abortController = null;
if (timer) clearInterval(timer);
timer = null;
}
}
function cancelGeneration() {
abortController?.abort();
}
function clearVideo() {
if (videoUrl) URL.revokeObjectURL(videoUrl);
videoUrl = null;
error = null;
prompt = "";
}
function downloadVideo() {
if (!videoUrl) return;
const ext = videoMime.includes("webm") ? "webm" : "mp4";
const link = document.createElement("a");
link.href = videoUrl;
link.download = `generated-video-${Date.now()}.${ext}`;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}
function formatBytes(n: number): string {
if (n > 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`;
if (n > 1024) return `${(n / 1024).toFixed(0)} KB`;
return `${n} B`;
}
function formatElapsed(s: number): string {
const m = Math.floor(s / 60);
return m > 0 ? `${m}m ${s % 60}s` : `${s}s`;
}
function handleKeyDown(event: KeyboardEvent) {
if (event.key === "Enter" && !event.shiftKey) {
event.preventDefault();
generate();
}
}
</script>
<div class="flex flex-col h-full">
<!-- Model selector and options -->
<div class="shrink-0 flex flex-wrap gap-2 mb-4">
<ModelSelector
bind:value={$selectedModelStore}
placeholder="Select a video model..."
disabled={isGenerating}
capabilities={["video_generation", "image_to_video"]}
matchAny={true}
/>
<select
class="px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
bind:value={$selectedSizeStore}
disabled={isGenerating}
>
<option value="">Model default size</option>
<optgroup label="Landscape">
<option value="1280x704">1280x704</option>
<option value="832x480">832x480</option>
</optgroup>
<optgroup label="Portrait">
<option value="704x1280">704x1280</option>
<option value="480x832">480x832</option>
</optgroup>
<optgroup label="Square">
<option value="704x704">704x704</option>
</optgroup>
</select>
<button
class="px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface hover:bg-secondary-hover transition-colors"
onclick={() => (showSettings = !showSettings)}
>
{showSettings ? "Hide Settings" : "Settings"}
</button>
</div>
<!-- Settings panel -->
{#if showSettings}
<div class="shrink-0 mb-4 p-4 rounded border border-gray-200 dark:border-white/10 bg-surface">
<div class="grid grid-cols-2 md:grid-cols-5 gap-3 mb-3">
<label class="flex flex-col gap-1">
<span class="text-xs text-txtsecondary">Frames (0 = default)</span>
<input
type="number"
class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
bind:value={$numFramesStore}
min="0"
max="1000"
/>
</label>
<label class="flex flex-col gap-1">
<span class="text-xs text-txtsecondary">FPS (0 = default)</span>
<input
type="number"
class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
bind:value={$fpsStore}
min="0"
max="60"
/>
</label>
<label class="flex flex-col gap-1">
<span class="text-xs text-txtsecondary">Steps (0 = default)</span>
<input
type="number"
class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
bind:value={$stepsStore}
min="0"
max="150"
/>
</label>
<label class="flex flex-col gap-1">
<span class="text-xs text-txtsecondary">Guidance (0 = default)</span>
<input
type="number"
class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
bind:value={$guidanceStore}
min="0"
max="30"
step="0.5"
/>
</label>
<label class="flex flex-col gap-1">
<span class="text-xs text-txtsecondary">Seed (-1 = random)</span>
<input
type="number"
class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
bind:value={$seedStore}
min="-1"
/>
</label>
</div>
<label class="flex flex-col gap-1">
<span class="text-xs text-txtsecondary">Negative Prompt</span>
<textarea
class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary resize-y text-sm"
bind:value={$negativePromptStore}
rows="2"
placeholder="Elements to avoid..."
></textarea>
</label>
</div>
{/if}
<!-- Empty state for no models configured -->
{#if !hasModels}
<div class="flex-1 flex items-center justify-center text-txtsecondary">
<p>No models configured. Add models to your configuration to generate videos.</p>
</div>
{:else}
<!-- Video display area -->
<div
class="flex-1 overflow-auto mb-4 flex items-center justify-center bg-surface border border-gray-200 dark:border-white/10 rounded"
>
{#if isGenerating}
<div class="text-center text-txtsecondary">
<div
class="inline-block w-8 h-8 border-4 border-primary border-t-transparent rounded-full animate-spin mb-2"
></div>
<p>Generating video… {formatElapsed(elapsedSeconds)}</p>
<p class="text-xs mt-1">
Video generation takes minutes — a cold model load takes longer still.
</p>
</div>
{:else if error}
<div class="text-center text-red-500 p-4">
<p class="font-medium">Error</p>
<p class="text-sm mt-1 break-all">{error}</p>
</div>
{:else if videoUrl}
<div class="relative max-w-full max-h-full flex items-center justify-center">
<!-- svelte-ignore a11y_media_has_caption -->
<video src={videoUrl} class="max-w-full max-h-full object-contain" controls autoplay loop></video>
<button
class="absolute bottom-2 right-2 p-2 bg-black/60 hover:bg-black/80 text-white rounded-full transition-colors"
onclick={downloadVideo}
aria-label="Download video"
title="Download ({formatBytes(videoBytes)})"
>
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4"
></path>
</svg>
</button>
</div>
{:else}
<div class="text-center text-txtsecondary">
<p>Enter a prompt below to generate a short video clip</p>
<p class="text-xs mt-1">Attach an image to animate it (image-to-video)</p>
</div>
{/if}
</div>
<!-- Init image chip -->
{#if initImagePreview}
<div class="shrink-0 flex items-center gap-2 mb-2">
<img src={initImagePreview} alt="Conditioning frame" class="h-12 rounded border border-gray-200 dark:border-white/10" />
<span class="text-xs text-txtsecondary">Animating this image (image-to-video)</span>
<button
class="px-1.5 py-0.5 text-xs rounded border border-gray-200 dark:border-white/10 hover:bg-red-500 hover:text-white hover:border-red-500 transition-colors"
onclick={clearInitImage}
disabled={isGenerating}
>
Remove
</button>
</div>
{/if}
<!-- Prompt input area -->
<div class="shrink-0 flex flex-col md:flex-row gap-2">
<ExpandableTextarea
bind:value={prompt}
placeholder="Describe the video you want to generate..."
rows={3}
onkeydown={handleKeyDown}
disabled={isGenerating || !$selectedModelStore}
/>
<div class="flex flex-row md:flex-col gap-2">
{#if isGenerating}
<button class="btn bg-red-500 hover:bg-red-600 text-white flex-1 md:flex-none" onclick={cancelGeneration}>
Cancel
</button>
{:else}
<button
class="btn bg-primary text-btn-primary-text hover:opacity-90 flex-1 md:flex-none"
onclick={generate}
disabled={!prompt.trim() || !$selectedModelStore}
>
Generate
</button>
<label
class="btn flex-1 md:flex-none text-center cursor-pointer {isGenerating ? 'opacity-50 pointer-events-none' : ''}"
>
Image…
<input
type="file"
accept="image/*"
class="hidden"
bind:this={fileInput}
onchange={onInitImageChange}
/>
</label>
<button
class="btn flex-1 md:flex-none"
onclick={clearVideo}
disabled={!videoUrl && !error && !prompt.trim()}
>
Clear
</button>
{/if}
</div>
</div>
{/if}
</div>
+2
View File
@@ -8,6 +8,8 @@ export interface ModelCapabilities {
audio_speech?: boolean;
image_generation?: boolean;
image_to_image?: boolean;
video_generation?: boolean;
image_to_video?: boolean;
function_calling?: boolean;
reranker?: boolean;
}
+57
View File
@@ -0,0 +1,57 @@
export interface VideoGenerationOptions {
negativePrompt?: string;
size?: string; // "WxH"
numFrames?: number;
fps?: number;
steps?: number;
guidanceScale?: number;
seed?: number; // -1 = random (omitted)
initImage?: File | null; // image-to-video conditioning frame
}
// POST /v1/videos/sync (multipart; llama-swap routes by the `model` field).
// The response body IS the encoded clip — returns an object URL for <video>.
// Callers must URL.revokeObjectURL() the result when done with it.
export async function generateVideoSync(
model: string,
prompt: string,
options: VideoGenerationOptions = {},
signal?: AbortSignal
): Promise<{ url: string; mime: string; sizeBytes: number }> {
const form = new FormData();
form.set("model", model);
form.set("prompt", prompt);
if (options.negativePrompt) form.set("negative_prompt", options.negativePrompt);
if (options.size) {
const [w, h] = options.size.split("x").map(Number);
if (w > 0 && h > 0) {
// Both conventions: vLLM-Omni reads width/height, OpenAI-shaped
// upstreams read size; they can never disagree.
form.set("width", String(w));
form.set("height", String(h));
form.set("size", options.size);
}
}
if (options.numFrames && options.numFrames > 0) form.set("num_frames", String(options.numFrames));
if (options.fps && options.fps > 0) form.set("fps", String(options.fps));
if (options.steps && options.steps > 0) form.set("num_inference_steps", String(options.steps));
if (options.guidanceScale && options.guidanceScale > 0)
form.set("guidance_scale", String(options.guidanceScale));
if (options.seed !== undefined && options.seed >= 0) form.set("seed", String(options.seed));
if (options.initImage) form.set("input_reference", options.initImage, options.initImage.name);
const response = await fetch("/v1/videos/sync", {
method: "POST",
body: form,
signal,
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`Video API error: ${response.status} - ${errorText}`);
}
const blob = await response.blob();
const mime = response.headers.get("Content-Type") || "video/mp4";
return { url: URL.createObjectURL(blob), mime, sizeBytes: blob.size };
}
+6 -1
View File
@@ -2,12 +2,13 @@
import { persistentStore } from "../stores/persistent";
import ChatInterface from "../components/playground/ChatInterface.svelte";
import ImageInterface from "../components/playground/ImageInterface.svelte";
import VideoInterface from "../components/playground/VideoInterface.svelte";
import AudioInterface from "../components/playground/AudioInterface.svelte";
import SpeechInterface from "../components/playground/SpeechInterface.svelte";
import RerankInterface from "../components/playground/RerankInterface.svelte";
import ConcurrencyInterface from "../components/playground/ConcurrencyInterface.svelte";
type Tab = "chat" | "images" | "speech" | "audio" | "rerank" | "concurrency";
type Tab = "chat" | "images" | "video" | "speech" | "audio" | "rerank" | "concurrency";
const selectedTabStore = persistentStore<Tab>("playground-selected-tab", "chat");
let mobileMenuOpen = $state(false);
@@ -15,6 +16,7 @@
const tabs: { id: Tab; label: string }[] = [
{ id: "chat", label: "Chat" },
{ id: "images", label: "Images" },
{ id: "video", label: "Video" },
{ id: "speech", label: "Speech" },
{ id: "audio", label: "Transcription" },
{ id: "rerank", label: "Rerank" },
@@ -92,6 +94,9 @@
<div class="h-full" class:tab-hidden={$selectedTabStore !== "images"}>
<ImageInterface />
</div>
<div class="h-full" class:tab-hidden={$selectedTabStore !== "video"}>
<VideoInterface />
</div>
<div class="h-full" class:tab-hidden={$selectedTabStore !== "speech"}>
<SpeechInterface />
</div>
+4 -2
View File
@@ -2,18 +2,20 @@ import { writable, derived } from "svelte/store";
const chatStreaming = writable(false);
const imageGenerating = writable(false);
const videoGenerating = writable(false);
const speechGenerating = writable(false);
const audioTranscribing = writable(false);
const rerankLoading = writable(false);
export const playgroundActivity = derived(
[chatStreaming, imageGenerating, speechGenerating, audioTranscribing, rerankLoading],
([$chat, $image, $speech, $audio, $rerank]) => $chat || $image || $speech || $audio || $rerank
[chatStreaming, imageGenerating, videoGenerating, speechGenerating, audioTranscribing, rerankLoading],
([$chat, $image, $video, $speech, $audio, $rerank]) => $chat || $image || $video || $speech || $audio || $rerank
);
export const playgroundStores = {
chatStreaming,
imageGenerating,
videoGenerating,
speechGenerating,
audioTranscribing,
rerankLoading,