Model capabilities 734 (#842)

internal/config,server: implement model capabilities

- define the capabilities of a model using a simple config block on the
model
- v1/models renders out capabilities to be compatible with openrouter,
huggingface chat, and mistral formats for broader compatibility
- add support for capabilities in UI

Fixes #734

ui-svelte/src/components/playground/AudioInterface.svelte

@@ -145,7 +145,7 @@
 <div class="flex flex-col h-full">
   <!-- Model selector -->
   <div class="shrink-0 flex flex-wrap gap-2 mb-4">
-    <ModelSelector bind:value={$selectedModelStore} placeholder="Select an audio model..." disabled={isTranscribing} />
+    <ModelSelector bind:value={$selectedModelStore} placeholder="Select an audio model..." disabled={isTranscribing} capabilities={["audio_transcriptions"]} />
   </div>
 
   <!-- Empty state for no models configured -->

ui-svelte/src/components/playground/ImageInterface.svelte

@@ -193,7 +193,7 @@
 <div class="flex flex-col h-full">
   <!-- Model selector and mode toggle -->
   <div class="shrink-0 flex flex-wrap gap-2 mb-4">
-    <ModelSelector bind:value={$selectedModelStore} placeholder="Select an image model..." disabled={isGenerating} />
+    <ModelSelector bind:value={$selectedModelStore} placeholder="Select an image model..." disabled={isGenerating} capabilities={["image_generation", "image_to_image"]} matchAny={true} />
 
     <select
       class="px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"

ui-svelte/src/components/playground/ModelSelector.svelte

@@ -6,12 +6,15 @@
     value: string;
     placeholder?: string;
     disabled?: boolean;
+    capabilities?: string[];
+    matchAny?: boolean;
   }
 
-  let { value = $bindable(), placeholder = "Select a model...", disabled = false }: Props = $props();
+  let { value = $bindable(), placeholder = "Select a model...", disabled = false, capabilities, matchAny = false }: Props = $props();
 
-  let grouped = $derived(groupModels($models));
-  let hasModels = $derived(grouped.local.length > 0 || Object.keys(grouped.peersByProvider).length > 0);
+  let grouped = $derived(groupModels($models, capabilities, matchAny));
+  let hasMatching = $derived(grouped.localMatching.length > 0);
+  let hasModels = $derived(hasMatching || grouped.local.length > 0 || Object.keys(grouped.peersByProvider).length > 0);
 </script>
 
 {#if hasModels}
@@ -21,6 +24,18 @@
     {disabled}
   >
     <option value="">{placeholder}</option>
+    {#if hasMatching}
+      <optgroup label="Matching Capabilities">
+        {#each grouped.localMatching as model (model.id)}
+          <option value={model.id}>{model.id}</option>
+          {#if model.aliases}
+            {#each model.aliases as alias (alias)}
+              <option value={alias}>  ↳ {alias}</option>
+            {/each}
+          {/if}
+        {/each}
+      </optgroup>
+    {/if}
     {#if grouped.local.length > 0}
       <optgroup label="Local">
         {#each grouped.local as model (model.id)}

ui-svelte/src/components/playground/RerankInterface.svelte

@@ -264,7 +264,7 @@
 <div class="flex flex-col h-full">
   <!-- Top bar: model selector + query input (table mode) + mode toggle -->
   <div class="shrink-0 flex flex-wrap gap-2 mb-4">
-    <ModelSelector bind:value={$selectedModelStore} placeholder="Select a rerank model..." disabled={isLoading} />
+    <ModelSelector bind:value={$selectedModelStore} placeholder="Select a rerank model..." disabled={isLoading} capabilities={["reranker"]} />
     {#if editorMode === "table"}
       <input
         type="text"

ui-svelte/src/components/playground/SpeechInterface.svelte

@@ -206,7 +206,7 @@
 <div class="flex flex-col h-full">
   <!-- Model and voice selectors -->
   <div class="shrink-0 flex gap-2 mb-4">
-    <ModelSelector bind:value={$selectedModelStore} placeholder="Select a speech model..." disabled={isGenerating} />
+    <ModelSelector bind:value={$selectedModelStore} placeholder="Select a speech model..." disabled={isGenerating} capabilities={["audio_speech"]} />
     <div class="flex gap-2">
       <select
         class="shrink-0 px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"

ui-svelte/src/lib/modelUtils.test.ts

@@ -0,0 +1,113 @@
+import { describe, it, expect } from "vitest";
+import { matchesCapabilities, groupModels } from "./modelUtils";
+import type { Model } from "./types";
+
+function makeModel(overrides: Partial<Model> = {}): Model {
+  return {
+    id: "test-model",
+    state: "ready",
+    name: "Test Model",
+    description: "",
+    unlisted: false,
+    peerID: "",
+    ...overrides,
+  };
+}
+
+describe("matchesCapabilities", () => {
+  it("returns true when required is empty", () => {
+    const model = makeModel();
+    expect(matchesCapabilities(model, [])).toBe(true);
+  });
+
+  it("returns false when model has no capabilities", () => {
+    const model = makeModel();
+    expect(matchesCapabilities(model, ["vision"])).toBe(false);
+  });
+
+  it("returns false when model has empty capabilities object", () => {
+    const model = makeModel({ capabilities: {} });
+    expect(matchesCapabilities(model, ["vision"])).toBe(false);
+  });
+
+  it("returns true when model has the single required capability", () => {
+    const model = makeModel({ capabilities: { vision: true } });
+    expect(matchesCapabilities(model, ["vision"])).toBe(true);
+  });
+
+  it("returns false when model lacks the required capability", () => {
+    const model = makeModel({ capabilities: { vision: true } });
+    expect(matchesCapabilities(model, ["audio_transcriptions"])).toBe(false);
+  });
+
+  it("AND semantics: returns true only when all required are present", () => {
+    const model = makeModel({ capabilities: { vision: true, audio_transcriptions: true } });
+    expect(matchesCapabilities(model, ["vision", "audio_transcriptions"])).toBe(true);
+    expect(matchesCapabilities(model, ["vision", "reranker"])).toBe(false);
+  });
+
+  it("matchAny=true: returns true when at least one required is present", () => {
+    const model = makeModel({ capabilities: { vision: true } });
+    expect(matchesCapabilities(model, ["vision", "reranker"], true)).toBe(true);
+    expect(matchesCapabilities(model, ["audio_transcriptions", "reranker"], true)).toBe(false);
+  });
+
+  it("matchAny=true with empty required returns true", () => {
+    const model = makeModel();
+    expect(matchesCapabilities(model, [], true)).toBe(true);
+  });
+});
+
+describe("groupModels", () => {
+  const models: Model[] = [
+    makeModel({ id: "chat-model", capabilities: { vision: true } }),
+    makeModel({ id: "audio-model", capabilities: { audio_transcriptions: true } }),
+    makeModel({ id: "no-caps-model" }),
+    makeModel({ id: "peer-model", peerID: "peer1" }),
+    makeModel({ id: "unlisted-model", unlisted: true, capabilities: { vision: true } }),
+  ];
+
+  it("filters out unlisted models", () => {
+    const result = groupModels(models);
+    expect(result.localMatching.length + result.local.length).toBe(3);
+    expect([...result.localMatching, ...result.local].every((m) => !m.unlisted)).toBe(true);
+  });
+
+  it("separates peer models into peersByProvider", () => {
+    const result = groupModels(models);
+    expect(result.peersByProvider["peer1"]).toHaveLength(1);
+    expect(result.peersByProvider["peer1"][0].id).toBe("peer-model");
+  });
+
+  it("without capabilities, all local models go to local (non-matching)", () => {
+    const result = groupModels(models);
+    expect(result.localMatching).toHaveLength(0);
+    expect(result.local).toHaveLength(3);
+  });
+
+  it("with capabilities, matching models go to localMatching", () => {
+    const result = groupModels(models, ["vision"]);
+    expect(result.localMatching).toHaveLength(1);
+    expect(result.localMatching[0].id).toBe("chat-model");
+    expect(result.local).toHaveLength(2);
+  });
+
+  it("with capabilities, models without capabilities go to local", () => {
+    const result = groupModels(models, ["vision"]);
+    expect(result.local.find((m) => m.id === "no-caps-model")).toBeDefined();
+  });
+
+  it("with matchAny, matches models with any listed capability", () => {
+    const result = groupModels(models, ["vision", "audio_transcriptions"], true);
+    expect(result.localMatching).toHaveLength(2);
+    expect(result.localMatching.map((m) => m.id)).toContain("chat-model");
+    expect(result.localMatching.map((m) => m.id)).toContain("audio-model");
+    expect(result.local).toHaveLength(1);
+  });
+
+  it("with empty capabilities array, all local go to local (non-matching)", () => {
+    const result = groupModels(models, []);
+    expect(result.localMatching).toHaveLength(0);
+    expect(result.local).toHaveLength(3);
+  });
+});

ui-svelte/src/lib/modelUtils.ts

@@ -2,14 +2,40 @@ import type { Model } from "./types";
 
 export interface GroupedModels {
   local: Model[];
+  localMatching: Model[];
   peersByProvider: Record<string, Model[]>;
 }
 
-export function groupModels(models: Model[]): GroupedModels {
+export function matchesCapabilities(model: Model, required: string[], matchAny = false): boolean {
+  if (!required.length) return true;
+  if (!model.capabilities) return false;
+  const caps = model.capabilities as Record<string, boolean>;
+  if (matchAny) {
+    return required.some((cap) => caps[cap] === true);
+  }
+  return required.every((cap) => caps[cap] === true);
+}
+
+export function groupModels(models: Model[], capabilities?: string[], matchAny = false): GroupedModels {
   const available = models.filter((m) => !m.unlisted);
   const local = available.filter((m) => !m.peerID);
   const peerModels = available.filter((m) => m.peerID);
 
+  let localMatching: Model[] = [];
+  let localRest: Model[] = [];
+
+  if (capabilities && capabilities.length > 0) {
+    for (const model of local) {
+      if (matchesCapabilities(model, capabilities, matchAny)) {
+        localMatching.push(model);
+      } else {
+        localRest.push(model);
+      }
+    }
+  } else {
+    localRest = local;
+  }
+
   const peersByProvider = peerModels.reduce(
     (acc, model) => {
       const peerId = model.peerID || "unknown";
@@ -20,5 +46,5 @@ export function groupModels(models: Model[]): GroupedModels {
     {} as Record<string, Model[]>
   );
 
-  return { local, peersByProvider };
+  return { local: localRest, localMatching, peersByProvider };
 }

ui-svelte/src/lib/types.ts

@@ -2,6 +2,16 @@ export type ConnectionState = "connected" | "connecting" | "disconnected";
 
 export type ModelStatus = "ready" | "starting" | "stopping" | "stopped" | "shutdown" | "unknown";
 
+export interface ModelCapabilities {
+  vision?: boolean;
+  audio_transcriptions?: boolean;
+  audio_speech?: boolean;
+  image_generation?: boolean;
+  image_to_image?: boolean;
+  function_calling?: boolean;
+  reranker?: boolean;
+}
+
 export interface Model {
   id: string;
   state: ModelStatus;
@@ -10,6 +20,7 @@ export interface Model {
   unlisted: boolean;
   peerID: string;
   aliases?: string[];
+  capabilities?: ModelCapabilities;
 }
 
 export interface TokenMetrics {