docs: update configuration.md for matrix

Add a new swap matrix to supersede groups for running concurrent models.
The matrix uses a solver that picks the lowest cost evictions to make a
requested model available. This simple approach along with a very basic
DSL grammar can enable very complex swapping scenarios.

- add DSL parser for set expressions with & (AND), | (OR), (), +ref
- add MatrixConfig structs, validation, and topological sort for +ref
- add MatrixSolver with cost-minimizing swap decisions
- add Matrix runtime integrating solver with Process lifecycle
- integrate matrix into ProxyManager with if-branches at all endpoints
- update config.example.yaml and config-schema.json with matrix schema
- config enforces groups XOR matrix (cannot use both)

fixes #643

.github/workflows/config-schema.yml

@@ -4,11 +4,15 @@ on:
   pull_request:
     paths:
       - "config-schema.json"
+      - "config.example.yaml"
+      - ".github/workflows/config-schema.yml"
   push:
     branches:
       - main
     paths:
       - "config-schema.json"
+      - "config.example.yaml"
+      - ".github/workflows/config-schema.yml"
 
   workflow_dispatch:
 
@@ -39,3 +43,14 @@ jobs:
           fi
 
           echo "✓ config-schema.json is valid"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Install check-jsonschema
+        run: pip install check-jsonschema
+
+      - name: Validate config.example.yaml against schema
+        run: check-jsonschema --schemafile config-schema.json config.example.yaml

.github/workflows/containers.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        platform: [intel, cuda, vulkan, cpu, musa, rocm]
+        platform: [intel, cuda, cuda13, vulkan, cpu, musa, rocm]
       fail-fast: false
     steps:
       - name: Checkout code

.github/workflows/go-ci.yml

@@ -2,69 +2,68 @@ name: Linux CI
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
     # only run when backend source changes
     # cmd/ is excluded because it contains utilities without tests
     paths:
-      - '**/*.go'
-      - '!cmd/**'
-      - 'go.mod'
-      - 'go.sum'
-      - 'Makefile'
-      - '.github/workflows/go-ci.yml'
+      - "**/*.go"
+      - "!cmd/**"
+      - "go.mod"
+      - "go.sum"
+      - "Makefile"
+      - ".github/workflows/go-ci.yml"
 
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
     paths:
-      - '**/*.go'
-      - '!cmd/**'
-      - 'go.mod'
-      - 'go.sum'
-      - 'Makefile'
-      - '.github/workflows/go-ci.yml'
+      - "**/*.go"
+      - "!cmd/**"
+      - "go.mod"
+      - "go.sum"
+      - "Makefile"
+      - ".github/workflows/go-ci.yml"
 
   # Allows manual triggering of the workflow
   workflow_dispatch:
 
 jobs:
-
   run-tests:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-    - name: Set up Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.23'
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
 
-    # Only run in this linux based runner
-    - name: Check Formatting
-      run: |
-        if [ "$(gofmt -l . | grep -v 'event/.*_test.go' | wc -l)" -gt 0 ]; then
-          gofmt -l . | grep -v 'event/.*_test.go'
-          exit 1
-        fi
-    # cache simple-responder to save the build time
-    - name: Restore Simple Responder
-      id: restore-simple-responder
-      uses: actions/cache/restore@v4
-      with:
-        path: ./build
-        key: ${{ runner.os }}-simple-responder-${{ hashFiles('misc/simple-responder/simple-responder.go') }}
+      # Only run in this linux based runner
+      - name: Check Formatting
+        run: |
+          if [ "$(gofmt -l . | grep -v 'event/.*_test.go' | wc -l)" -gt 0 ]; then
+            gofmt -l . | grep -v 'event/.*_test.go'
+            exit 1
+          fi
+      # cache simple-responder to save the build time
+      - name: Restore Simple Responder
+        id: restore-simple-responder
+        uses: actions/cache/restore@v4
+        with:
+          path: ./build
+          key: ${{ runner.os }}-simple-responder-${{ hashFiles('cmd/simple-responder/simple-responder.go') }}
 
-    # necessary for testing proxy/Process swapping
-    - name: Create simple-responder
-      run: make simple-responder
+      # necessary for testing proxy/Process swapping
+      - name: Create simple-responder
+        run: make simple-responder
 
-    - name: Save Simple Responder
-      # nothing new to save ... skip this step
-      if: steps.restore-simple-responder.outputs.cache-hit != 'true'
-      id: save-simple-responder
-      uses: actions/cache/save@v4
-      with:
-        path: ./build
-        key: ${{ runner.os }}-simple-responder-${{ hashFiles('misc/simple-responder/simple-responder.go') }}
+      - name: Save Simple Responder
+        # nothing new to save ... skip this step
+        if: steps.restore-simple-responder.outputs.cache-hit != 'true'
+        id: save-simple-responder
+        uses: actions/cache/save@v4
+        with:
+          path: ./build
+          key: ${{ runner.os }}-simple-responder-${{ hashFiles('cmd/simple-responder/simple-responder.go') }}
 
-    - name: Test all
-      run: make test-all
+      - name: Test all
+        run: make test-all

.github/workflows/unified-docker.yml

@@ -0,0 +1,131 @@
+name: Build Unified Docker Image
+
+on:
+  schedule:
+    - cron: "37 5 * * *"
+
+  workflow_dispatch:
+    inputs:
+      llama_cpp_ref:
+        description: "llama.cpp commit hash, tag, or branch"
+        required: false
+        default: "master"
+      whisper_ref:
+        description: "whisper.cpp commit hash, tag, or branch"
+        required: false
+        default: "master"
+      sd_ref:
+        description: "stable-diffusion.cpp commit hash, tag, or branch"
+        required: false
+        default: "master"
+      ik_llama_ref:
+        description: "ik_llama.cpp commit hash, tag, or branch (CUDA only)"
+        required: false
+        default: "main"
+      llama_swap_version:
+        description: "llama-swap version (e.g. v198, latest, main)"
+        required: false
+        default: "main"
+      build_cuda:
+        description: "Build CUDA image"
+        type: boolean
+        required: false
+        default: true
+      build_vulkan:
+        description: "Build Vulkan image"
+        type: boolean
+        required: false
+        default: true
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - id: set-matrix
+        run: |
+          backends=()
+          # schedule uses defaults (build both); workflow_dispatch respects inputs
+          if [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_cuda }}" == "true" ]]; then
+            backends+=("cuda")
+          fi
+          if [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_vulkan }}" == "true" ]]; then
+            backends+=("vulkan")
+          fi
+          matrix=$(printf '%s\n' "${backends[@]}" | jq -R . | jq -sc .)
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
+  build:
+    needs: setup
+    if: ${{ needs.setup.outputs.matrix != '[]' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: ${{ fromJSON(needs.setup.outputs.matrix) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Free up disk space
+        run: |
+          echo "Before cleanup:"
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker system prune -af
+          echo "After cleanup:"
+          df -h
+
+      # On GitHub Actions runners, create a fresh builder.
+      # When running locally under act, skip this and reuse the existing
+      # llama-swap-builder (which has ccache warm) to avoid exhausting disk.
+      - name: Set up Docker Buildx
+        if: ${{ !env.ACT }}
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        if: ${{ !env.ACT }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build unified Docker image (${{ matrix.backend }})
+        env:
+          LLAMA_REF: ${{ inputs.llama_cpp_ref || 'master' }}
+          WHISPER_REF: ${{ inputs.whisper_ref || 'master' }}
+          SD_REF: ${{ inputs.sd_ref || 'master' }}
+          IK_LLAMA_REF: ${{ inputs.ik_llama_ref || 'main' }}
+          LS_VERSION: ${{ inputs.llama_swap_version || 'main' }}
+          DOCKER_IMAGE_TAG: ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}
+          # When running under act, use the local builder that has warm ccache.
+          # On GitHub Actions, BUILDX_BUILDER is unset so docker uses the builder
+          # created by setup-buildx-action above.
+          BUILDX_BUILDER: ${{ env.ACT == 'true' && 'llama-swap-builder' || '' }}
+        run: |
+          chmod +x docker/unified/build-image.sh
+          docker/unified/build-image.sh --${{ matrix.backend }}
+
+      - name: Push to GitHub Container Registry
+        if: ${{ !env.ACT }}
+        run: |
+          BASE_TAG="ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}"
+          DATE_TAG=$(date -u +%Y-%m-%d)
+
+          docker push "${BASE_TAG}"
+          docker tag "${BASE_TAG}" "${BASE_TAG}-${DATE_TAG}"
+          docker push "${BASE_TAG}-${DATE_TAG}"
+
+          ROOTLESS_TAG="${BASE_TAG}-rootless"
+          docker push "${ROOTLESS_TAG}"
+          docker tag "${ROOTLESS_TAG}" "${ROOTLESS_TAG}-${DATE_TAG}"
+          docker push "${ROOTLESS_TAG}-${DATE_TAG}"

AGENTS.md

@@ -21,6 +21,7 @@ llama-swap is a light weight, transparent proxy server that provides automatic m
 
 - Follow test naming conventions like `TestProxyManager_<test name>`, `TestProcessGroup_<test name>`, etc.
 - Use `go test -v -run <name pattern for new tests>` to run any new tests you've written.
+- Run `gofmt -l .` before committing to verify formatting. Fix any reported files with `gofmt -w <file>`.
 - Use `make test-dev` after running new tests for a quick over all test run. This runs `go test` and `staticcheck`. Fix any static checking errors. Use this only when changes are made to any code under the `proxy/` directory
 - Use `make test-all` before completing work. This includes long running concurrency tests.
 

Makefile

@@ -51,7 +51,7 @@ mac: ui
 linux: ui
 	@echo "Building Linux binary..."
 	GOOS=linux GOARCH=amd64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-amd64
-	GOOS=linux GOARCH=arm64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-arm64
+#GOOS=linux GOARCH=arm64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-arm64
 
 # Build Windows binary
 windows: ui

README.md

@@ -5,7 +5,7 @@
 
 # llama-swap
 
-Run multiple LLM models on your machine and hot-swap between them as needed. llama-swap works with any OpenAI API-compatible server, giving you the flexibility to switch models without restarting your applications.
+Run multiple generative AI models on your machine and hot-swap between them on demand. llama-swap works with any OpenAI and Anthropic API compatible server and is used by thousands of people to power their local AI workflows.
 
 Built in Go for performance and simplicity, llama-swap has zero dependencies and is incredibly easy to set up. Get started in minutes - just one binary and one configuration file.
 
@@ -32,6 +32,10 @@ Built in Go for performance and simplicity, llama-swap has zero dependencies and
   - `v1/rerank`, `v1/reranking`, `/rerank`
   - `/infill` - for code infilling
   - `/completion` - for completion endpoint
+- ✅ SDAPI via [stable-diffusion.cpp's server](https://github.com/leejet/stable-diffusion.cpp/tree/master/examples/server)
+  - `/sdapi/v1/txt2img`
+  - `/sdapi/v1/img2img`
+  - `/sdapi/v1/loras` - requires `model` in request body to fetch the correct loras
 - ✅ llama-swap API
   - `/ui` - web UI
   - `/upstream/:model_id` - direct access to upstream server ([demo](https://github.com/mostlygeek/llama-swap/pull/31))
@@ -41,20 +45,32 @@ Built in Go for performance and simplicity, llama-swap has zero dependencies and
   - `/health` - just returns "OK"
 - ✅ API Key support - define keys to restrict access to API endpoints
 - ✅ Customizable
-  - Run multiple models at once with `Groups` ([#107](https://github.com/mostlygeek/llama-swap/issues/107))
+  - Run concurrent models with a custom DSL swap matrix ([#643](https://github.com/mostlygeek/llama-swap/issues/643))
   - Automatic unloading of models after timeout by setting a `ttl`
   - Reliable Docker and Podman support using `cmd` and `cmdStop` together
   - Preload models on startup with `hooks` ([#235](https://github.com/mostlygeek/llama-swap/pull/235))
 
 ### Web UI
 
-llama-swap includes a real time web interface for monitoring logs and controlling models:
+llama-swap includes a real time web interface with a playground for testing out all sorts of local models:
 
-<img width="1164" height="745" alt="image" src="https://github.com/user-attachments/assets/bacf3f9d-819f-430b-9ed2-1bfaa8d54579" />
+<img width="1125" height="876" alt="image" src="https://github.com/user-attachments/assets/8ee41947-97af-463d-b0f0-8e9c478fac07" />
 
-The Activity Page shows recent requests:
+View detailed token metrics:
 
-<img width="1360" height="963" alt="image" src="https://github.com/user-attachments/assets/5f3edee6-d03a-4ae5-ae06-b20ac1f135bd" />
+<img width="1111" height="515" alt="image" src="https://github.com/user-attachments/assets/64bfb280-d7a3-4126-971a-a128fd40410c" />
+
+Inspect request and responses:
+
+<img width="1111" height="720" alt="image" src="https://github.com/user-attachments/assets/24fe4aca-1448-4d7c-b9e8-a967589bda6c" />
+
+Manually load and unload models:
+
+<img width="1109" height="719" alt="image" src="https://github.com/user-attachments/assets/02b1e1f2-abd0-4050-84ae-facd66ff01c4" />
+
+Real time log streaming:
+
+<img width="1107" height="559" alt="image" src="https://github.com/user-attachments/assets/39669a10-cff2-409e-836a-5bad8bd0140c" />
 
 ## Installation
 
@@ -164,7 +180,7 @@ That's all you need to get started:
 Almost all configuration settings are optional and can be added one step at a time:
 
 - Advanced features
-  - `groups` to run multiple models at once
+  - `matrix` to run concurrent models with a custom swap logic DSL
   - `hooks` to run things on startup
   - `macros` reusable snippets
 - Model customization
@@ -182,7 +198,7 @@ See the [configuration documentation](docs/configuration.md) for all options.
 
 When a request is made to an OpenAI compatible endpoint, llama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to handle the request correctly.
 
-In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `groups` feature allows multiple models to be loaded at the same time. You have complete control over how your system resources are used.
+In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, using a `matrix` allows multiple models to be loaded at the same time. You have complete control over how your system resources are used.
 
 ## Reverse Proxy Configuration (nginx)
 

cmd/simple-responder/simple-responder.go

@@ -274,6 +274,43 @@ func main() {
 		c.String(200, fmt.Sprintf("%s %s", c.Request.Method, c.Request.URL.Path))
 	})
 
+	// SD API endpoints
+	r.POST("/sdapi/v1/txt2img", func(c *gin.Context) {
+		body, err := io.ReadAll(c.Request.Body)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read request body"})
+			return
+		}
+		defer c.Request.Body.Close()
+
+		modelName := gjson.GetBytes(body, "model").String()
+		c.JSON(http.StatusOK, gin.H{
+			"model":  modelName,
+			"images": []string{},
+		})
+	})
+
+	r.POST("/sdapi/v1/img2img", func(c *gin.Context) {
+		body, err := io.ReadAll(c.Request.Body)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read request body"})
+			return
+		}
+		defer c.Request.Body.Close()
+
+		modelName := gjson.GetBytes(body, "model").String()
+		c.JSON(http.StatusOK, gin.H{
+			"model":  modelName,
+			"images": []string{},
+		})
+	})
+
+	r.GET("/sdapi/v1/loras", func(c *gin.Context) {
+		c.JSON(http.StatusOK, gin.H{
+			"loras": []string{},
+		})
+	})
+
 	address := "127.0.0.1:" + *port // Address with the specified port
 
 	srv := &http.Server{

config-schema.json

@@ -39,6 +39,49 @@
             },
             "default": {},
             "description": "A dictionary of string substitutions. Macros are reusable snippets used in model cmd, cmdStop, proxy, checkEndpoint, filters.stripParams. Macro names must be <64 chars, match ^[a-zA-Z0-9_-]+$, and not be PORT or MODEL_ID. Values can be string, number, or boolean. Macros can reference other macros defined before them."
+        },
+        "timeouts": {
+            "type": "object",
+            "properties": {
+                "connect": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 30,
+                    "description": "TCP connection timeout in seconds. Set to 0 to disable."
+                },
+                "keepalive": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 30,
+                    "description": "TCP keepalive timeout in seconds. Set to 0 to disable."
+                },
+                "responseHeader": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 0,
+                    "description": "Time to wait for response headers in seconds. Set to 0 to disable."
+                },
+                "tlsHandshake": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 10,
+                    "description": "TLS handshake timeout in seconds. Set to 0 to disable."
+                },
+                "expectContinue": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 1,
+                    "description": "Expect-Continue timeout in seconds. Set to 0 to disable."
+                },
+                "idleConn": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 90,
+                    "description": "Idle connection timeout in seconds. Set to 0 to disable."
+                }
+            },
+            "additionalProperties": false,
+            "description": "Timeout settings for proxy connections."
         }
     },
     "properties": {
@@ -48,6 +91,12 @@
             "default": 120,
             "description": "Number of seconds to wait for a model to be ready to serve requests."
         },
+        "globalTTL": {
+            "type": "integer",
+            "minimum": 0,
+            "default": 0,
+            "description": "Default TTL for all models in seconds, 0 means no TTL and models will never be automatically unloaded"
+        },
         "logLevel": {
             "type": "string",
             "enum": [
@@ -177,9 +226,9 @@
                     },
                     "ttl": {
                         "type": "integer",
-                        "minimum": 0,
-                        "default": 0,
-                        "description": "Automatically unload the model after ttl seconds. 0 disables unloading. Must be >0 to enable."
+                        "minimum": -1,
+                        "default": -1,
+                        "description": "Automatically unload the model after ttl seconds. -1 uses the global TTL value, 0 disables unloading. Must be >0 to enable."
                     },
                     "useModelName": {
                         "type": "string",
@@ -200,11 +249,20 @@
                                 "additionalProperties": true,
                                 "default": {},
                                 "description": "Dictionary of parameters to set/override in requests. Useful for enforcing specific parameter values. Protected params like 'model' cannot be overridden. Values can be strings, numbers, booleans, arrays, or objects."
+                            },
+                            "setParamsByID": {
+                                "type": "object",
+                                "additionalProperties": {
+                                    "type": "object",
+                                    "additionalProperties": true
+                                },
+                                "default": {},
+                                "description": "Dictionary mapping requested model IDs (or aliases) to parameters to set/override in requests. Applied after setParams and can override those values. Useful with aliases to vary behaviour depending on which alias the client used (e.g. different reasoning_effort per alias). Keys support ${MODEL_ID} macro substitution. Protected params like 'model' cannot be overridden."
                             }
                         },
                         "additionalProperties": false,
                         "default": {},
-                        "description": "Dictionary of filter settings. Supports stripParams and setParams."
+                        "description": "Dictionary of filter settings. Supports stripParams, setParams, and setParamsByID."
                     },
                     "metadata": {
                         "type": "object",
@@ -226,6 +284,9 @@
                         "type": "boolean",
                         "default": false,
                         "description": "If true the model will not show up in /v1/models responses. It can still be used as normal in API requests."
+                    },
+                    "timeouts": {
+                        "$ref": "#/definitions/timeouts"
                     }
                 }
             }
@@ -264,6 +325,44 @@
             },
             "description": "A dictionary of group settings. Provides advanced controls over model swapping behaviour. Model IDs must be defined in models. A model can only be a member of one group. Behaviour controlled via swap, exclusive, persistent."
         },
+        "matrix": {
+            "type": "object",
+            "description": "Solver-based alternative to groups. Declares valid combinations of concurrent models. The solver minimizes eviction cost when swapping. A config must use either groups or matrix, not both.",
+            "required": [
+                "vars",
+                "sets"
+            ],
+            "properties": {
+                "vars": {
+                    "type": "object",
+                    "description": "Short names for models. Keys must be alphanumeric, 1-8 characters. All sets and evict_costs must use these IDs.",
+                    "minProperties": 1,
+                    "additionalProperties": {
+                        "type": "string"
+                    },
+                    "propertyNames": {
+                        "pattern": "^[a-zA-Z0-9]{1,8}$"
+                    }
+                },
+                "evict_costs": {
+                    "type": "object",
+                    "description": "Relative cost of evicting a running model. Models not listed default to 1. Values must be positive integers.",
+                    "additionalProperties": {
+                        "type": "integer",
+                        "minimum": 1
+                    }
+                },
+                "sets": {
+                    "type": "object",
+                    "description": "Named sets of concurrent model combinations. Values are DSL strings using & (AND), | (OR), () (grouping), and +ref (inline another set). Definition order is used for tie-breaking.",
+                    "minProperties": 1,
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                }
+            },
+            "additionalProperties": false
+        },
         "hooks": {
             "type": "object",
             "properties": {
@@ -352,11 +451,70 @@
                         "additionalProperties": false,
                         "default": {},
                         "description": "Dictionary of filter settings for peer requests. Supports stripParams and setParams."
+                    },
+                    "timeouts": {
+                        "type": "object",
+                        "properties": {
+                            "connect": {
+                                "type": "integer",
+                                "minimum": 0,
+                                "default": 30,
+                                "description": "TCP connection timeout in seconds."
+                            },
+                            "keepalive": {
+                                "type": "integer",
+                                "minimum": 0,
+                                "default": 30,
+                                "description": "TCP keepalive connection timeout in seconds."
+                            },
+                            "responseHeader": {
+                                "type": "integer",
+                                "minimum": 0,
+                                "default": 0,
+                                "description": "Time to wait for response headers in seconds."
+                            },
+                            "tlsHandshake": {
+                                "type": "integer",
+                                "minimum": 0,
+                                "default": 10,
+                                "description": "TLS handshake timeout in seconds."
+                            },
+                            "idleConn": {
+                                "type": "integer",
+                                "minimum": 0,
+                                "default": 90,
+                                "description": "Idle connection timeout in seconds."
+                            }
+                        },
+                        "additionalProperties": false,
+                        "description": "Timeout settings for proxy connections to this peer."
                     }
                 }
             },
             "default": {},
             "description": "A dictionary of remote peers and models they provide. Peers can be another llama-swap or any server that provides the /v1/ generative API endpoints supported by llama-swap."
         }
-    }
-}
+    },
+    "allOf": [
+        {
+            "if": {
+                "required": ["groups"]
+            },
+            "then": {
+                "not": {
+                    "required": ["matrix"]
+                }
+            }
+        },
+        {
+            "if": {
+                "required": ["matrix"]
+            },
+            "then": {
+                "not": {
+                    "required": ["groups"]
+                }
+            }
+        }
+    ]
+}

config.example.yaml

@@ -75,6 +75,11 @@ sendLoadingState: true
 #   all fields except for Id so chat UIs can use the alias equivalent to the original.
 includeAliasesInList: false
 
+# globalTTL: the default TTL in seconds before unloading a model
+# - optional, default: 0 (never automatically unload)
+# - must be >= 0
+globalTTL: 0
+
 # macros: a dictionary of string substitutions
 # - optional, default: empty dictionary
 # - macros are reusable snippets
@@ -126,7 +131,7 @@ apiKeys:
 # - below are examples of the all the settings a model can have
 models:
   # keys are the model names used in API requests
-  "llama":
+  "gpt-oss-120b":
     # macros: a dictionary of string substitutions specific to this model
     # - optional, default: empty dictionary
     # - macros defined here override macros defined in the global macros section
@@ -143,7 +148,7 @@ models:
     cmd: |
       # ${latest-llama} is a macro that is defined above
       ${latest-llama}
-      --model path/to/llama-8B-Q4_K_M.gguf
+      --model path/to/gpt-oss-120B.gguf
       --ctx-size ${default_ctx}
       --temperature ${temp}
 
@@ -151,13 +156,13 @@ models:
     # - optional, default: empty string
     # - if set, it will be used in the v1/models API response
     # - if not set, it will be omitted in the JSON model record
-    name: "llama 3.1 8B"
+    name: "gpt-oss 120B"
 
     # description: a description for the model
     # - optional, default: empty string
     # - if set, it will be used in the v1/models API response
     # - if not set, it will be omitted in the JSON model record
-    description: "A small but capable model used for quick testing"
+    description: "A thinking model from OpenAI"
 
     # env: define an array of environment variables to inject into cmd's environment
     # - optional, default: empty array
@@ -172,14 +177,6 @@ models:
     # - if you use a custom port in cmd this *must* be set
     proxy: http://127.0.0.1:8999
 
-    # aliases: alternative model names that this model configuration is used for
-    # - optional, default: empty array
-    # - aliases must be unique globally
-    # - useful for impersonating a specific model
-    aliases:
-      - "gpt-4o-mini"
-      - "gpt-3.5-turbo"
-
     # checkEndpoint: URL path to check if the server is ready
     # - optional, default: /health
     # - endpoint is expected to return an HTTP 200 response
@@ -188,8 +185,10 @@ models:
     checkEndpoint: /custom-endpoint
 
     # ttl: automatically unload the model after ttl seconds
-    # - optional, default: 0
-    # - ttl values must be a value greater than 0
+    # - optional, default: -1 (use global default)
+    # - ttl values must be a value greater than or equal to 0
+    # - a ttl of -1 will use the global TTL value as the default
+    # - a ttl of 0 will mean never unload
     # - a value of 0 disables automatic unloading of the model
     ttl: 60
 
@@ -197,7 +196,7 @@ models:
     # - optional, default: ""
     # - useful for when the upstream server expects a specific model name that
     #   is different from the model's ID
-    useModelName: "qwen:qwq"
+    useModelName: "openai/gpt-oss-120B"
 
     # filters: a dictionary of filter settings
     # - optional, default: empty dictionary
@@ -216,11 +215,38 @@ models:
       # - useful for enforcing specific parameter values
       # - protected params like "model" cannot be overridden
       # - values can be strings, numbers, booleans, arrays, or objects
+      # - always runs for the model
       setParams:
         # Example: enforce specific sampling parameters
         temperature: 0.7
         top_p: 0.9
 
+      # setParamsByID: a dictionary of parameters to set based the model ID
+      # - optional, default: empty dictionary
+      # - combine with aliases to create variant behaviour without reloading the model
+      # - parameters are set in the request body JSON
+      # - run after setParams so it will override any settings
+      # - protected params like "model" cannot be overridden
+      # - values can be strings, numbers, booleans, arrays, or objects
+      # - model aliases will be automatically created for each key
+      setParamsByID:
+        "${MODEL_ID}":
+          chat_template_kwargs:
+            reasoning_effort: medium
+        "${MODEL_ID}:high":
+          chat_template_kwargs:
+            reasoning_effort: high
+        "${MODEL_ID}:low":
+          chat_template_kwargs:
+            reasoning_effort: low
+
+    # aliases: alternative model names that this model configuration is used for
+    # - optional, default: empty array
+    # - aliases must be unique globally
+    # - useful for impersonating a specific model
+    aliases:
+      - "gpt-4o-mini"
+
     # metadata: a dictionary of arbitrary values that are included in /v1/models
     # - optional, default: empty dictionary
     # - while metadata can contains complex types it is recommended to keep it simple
@@ -258,6 +284,22 @@ models:
     # - optional, default: undefined (use global setting)
     sendLoadingState: false
 
+    # timeouts: configure proxy connection timeouts for this model
+    # - optional, defaults shown below
+    # - useful for models running on slower hardware that need longer timeouts
+    # - connect: TCP dial connection timeout in seconds, default: 30 seconds
+    # - keepalive: TCP connection keepalive timeout, default: 30 seconds
+    # - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
+    # - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
+    # - idleConn: idle connection timeout in seconds, default: 90 seconds
+    # - set any value to 0 to disable that timeout (not recommended)
+    timeouts:
+      connect: 30
+      keepalive: 0
+      responseHeader: 60
+      tlsHandshake: 10
+      idleConn: 90
+
   # Unlisted model example:
   "qwen-unlisted":
     # unlisted: boolean, true or false
@@ -289,68 +331,83 @@ models:
     # - processes have 5 seconds to shutdown until forceful termination is attempted
     cmdStop: docker stop ${MODEL_ID}
 
-# groups: a dictionary of group settings
-# - optional, default: empty dictionary
-# - provides advanced controls over model swapping behaviour
-# - using groups some models can be kept loaded indefinitely, while others are swapped out
-# - model IDs must be defined in the Models section
-# - a model can only be a member of one group
-# - group behaviour is controlled via the `swap`, `exclusive` and `persistent` fields
-# - see issue #109 for details
+# =============================================================================
+# matrix: run concurrent models with a solver-based swap DSL
+# =============================================================================
 #
-# NOTE: the example below uses model names that are not defined above for demonstration purposes
-groups:
-  # group1 works the same as the default behaviour of llama-swap where only one model is allowed
-  # to run a time across the whole llama-swap instance
-  "group1":
-    # swap: controls the model swapping behaviour in within the group
-    # - optional, default: true
-    # - true : only one model is allowed to run at a time
-    # - false: all models can run together, no swapping
-    swap: true
+# Note:
+# A config must use either a matrix or legacy groups, not both. A configuration error
+# will occur if both are defined. Configuration examples for legacy Groups can be found:
+# https://github.com/mostlygeek/llama-swap/blob/40e39f7/config.example.yaml#L334-L396
+#
+# The matrix declares valid combinations of models that can run concurrently.
+# When a model is requested, the solver finds the cheapest way to make it
+# available by evicting as few (and least costly) running models as possible.
+#
+# Solver behavior:
+#   1. Request arrives for model X
+#   2. If X is already running, forward immediately. Done.
+#   3. Find all sets containing X
+#   4. For each candidate set, compute cost: sum of evict_costs for
+#      every running model NOT in that set
+#   5. Pick lowest cost candidate. Ties broken by definition order.
+#   6. Evict what needs to stop. Start X. Forward request.
+#
+# Subset semantics: a set [a, b, c] means any subset is valid.
+# Only the requested model is started — others are not preloaded.
+#
+# A model not appearing in any set can only run alone.
+#
+matrix:
+  # vars: short names for models (alphanumeric, 1-8 chars)
+  # - required for sets and evict_costs settings
+  # - each entry is a short name to a real model ID. Do not use an alias
+  # - used to keep set DSL logic short and easier to read
+  # - sets and evict_costs only use identifiers defined in vars
+  vars:
+    g: gemma-model
+    q: qwen-model
+    m: mistral-model
+    v: voxtral-model
+    e: reranker-model
+    L: llama-70B
+    sd: stable-diffusion
 
-    # exclusive: controls how the group affects other groups
-    # - optional, default: true
-    # - true: causes all other groups to unload when this group runs a model
-    # - false: does not affect other groups
-    exclusive: true
+  # evict_costs: relative cost of losing a running model (default: 1)
+  evict_costs:
+    v: 50 # vllm backend, slow cold start
+    L: 30 # 70B weights, slow to load
 
-    # members references the models defined above
-    # required
-    members:
-      - "llama"
-      - "qwen-unlisted"
+  # sets: named sets of concurrent model combinations
+  # Values are DSL strings with operators:
+  #   &     AND (models run together)
+  #   |     OR  (alternatives)
+  #   ()    grouping
+  #   +ref  inline another set's expression
+  #
+  # Expansion examples:
+  #   "L"                  → [L]
+  #   "a & b"              → [a, b]
+  #   "a | b"              → [a], [b]
+  #   "(a | b) & c"        → [a, c], [b, c]
+  #   "(a | b) & (c | d)"  → [a,c], [a,d], [b,c], [b,d]
+  #   "+llms & v"          → expands llms inline, then applies & v
+  sets:
+    # LLM + TTS: switching between g/q/m won't evict v
+    # expands to: [g,v], [q,v], [m,v]
+    standard: "(g | q | m) & v"
 
-  # Example:
-  # - in group2 all models can run at the same time
-  # - when a different group is loaded it causes all running models in this group to unload
-  "group2":
-    swap: false
+    # LLM + TTS + reranker
+    # expands to: [g,v,e], [q,v,e]
+    with_rerank: "(g | q) & v & e"
 
-    # exclusive: false does not unload other groups when a model in group2 is requested
-    # - the models in group2 will be loaded but will not unload any other groups
-    exclusive: false
-    members:
-      - "docker-llama"
-      - "modelA"
-      - "modelB"
+    # LLM + image generation, no TTS
+    # expands to: [g,sd], [q,sd]
+    creative: "(g | q) & sd"
 
-  # Example:
-  # - a persistent group, prevents other groups from unloading it
-  "forever":
-    # persistent: prevents over groups from unloading the models in this group
-    # - optional, default: false
-    # - does not affect individual model behaviour
-    persistent: true
-
-    # set swap/exclusive to false to prevent swapping inside the group
-    # and the unloading of other groups
-    swap: false
-    exclusive: false
-    members:
-      - "forever-modelA"
-      - "forever-modelB"
-      - "forever-modelc"
+    # 70B model uses all GPUs, can only run alone
+    # expands to: [L]
+    full: "L"
 
 # hooks: a dictionary of event triggers and actions
 # - optional, default: empty dictionary
@@ -400,6 +457,17 @@ peers:
       - z-ai/glm-4.7
       - moonshotai/kimi-k2-0905
       - minimax/minimax-m2.1
+    # timeouts: configure proxy connection timeouts for this peer
+    # - optional, defaults shown below
+    # - useful when the peer runs on slower hardware
+    # - set any value to 0 to disable that timeout (not recommended)
+    timeouts:
+      connect: 30
+      keepalive: 30
+      responseHeader: 60
+      tlsHandshake: 10
+      idleConn: 90
+
     # filters: a dictionary of filter settings for peer requests
     # - optional, default: empty dictionary
     # - same capabilities as model filters (stripParams, setParams)

docker/build-container.sh

@@ -27,7 +27,7 @@ ARCH=$1
 PUSH_IMAGES=${2:-false}
 
 # List of allowed architectures
-ALLOWED_ARCHS=("intel" "vulkan" "musa" "cuda" "cpu" "rocm")
+ALLOWED_ARCHS=("intel" "vulkan" "musa" "cuda" "cuda13" "cpu" "rocm")
 
 # Check if ARCH is in the allowed list
 if [[ ! " ${ALLOWED_ARCHS[@]} " =~ " ${ARCH} " ]]; then

docker/build-image.sh

@@ -0,0 +1,305 @@
+#!/bin/bash
+#
+# Build script for llama-swap-docker with commit hash pinning
+#
+# Usage:
+#   ./build-image.sh --cuda                    # Build CUDA image
+#   ./build-image.sh --vulkan                  # Build Vulkan image
+#   ./build-image.sh --cuda --no-cache         # Build CUDA image without cache
+#   LLAMA_COMMIT_HASH=abc123 ./build-image.sh --cuda      # Override llama.cpp commit
+#   LLAMA_COMMIT_HASH=b8429 ./build-image.sh --vulkan    # Override llama.cpp release tag (vulkan uses prebuilt binaries)
+#   WHISPER_COMMIT_HASH=def456 ./build-image.sh --vulkan  # Override whisper.cpp commit
+#   SD_COMMIT_HASH=ghi789 ./build-image.sh --cuda        # Override stable-diffusion.cpp commit
+#
+# Features:
+#   - Auto-detects latest commit hashes from git repos
+#   - Builds llama-swap from local source code
+#   - Allows environment variable overrides for reproducible builds
+#   - Cache-friendly: changing commit hash busts cache appropriately
+#   - Supports both CUDA and Vulkan backends (requires explicit flag)
+#
+
+set -euo pipefail
+
+# Parse command line arguments
+BACKEND=""
+NO_CACHE=false
+
+if [[ $# -eq 0 ]]; then
+    echo "Error: No backend specified. Please use --cuda or --vulkan."
+    echo ""
+    echo "Usage: ./build-image.sh --cuda|--vulkan [--no-cache]"
+    echo ""
+    echo "Options:"
+    echo "  --cuda      Build CUDA image (NVIDIA GPUs)"
+    echo "  --vulkan    Build Vulkan image (AMD GPUs and compatible hardware)"
+    echo "  --no-cache  Force rebuild without using Docker cache"
+    echo "  --help, -h  Show this help message"
+    echo ""
+    echo "Environment variables:"
+    echo "  DOCKER_IMAGE_TAG     Set custom image tag (default: llama-swap:cuda or llama-swap:vulkan)"
+    echo "  LLAMA_COMMIT_HASH    Override llama.cpp commit hash"
+    echo "  WHISPER_COMMIT_HASH  Override whisper.cpp commit hash"
+    echo "  SD_COMMIT_HASH       Override stable-diffusion.cpp commit hash"
+    exit 1
+fi
+
+for arg in "$@"; do
+    case $arg in
+        --cuda)
+            BACKEND="cuda"
+            ;;
+        --vulkan)
+            BACKEND="vulkan"
+            ;;
+        --no-cache)
+            NO_CACHE=true
+            ;;
+        --help|-h)
+            echo "Usage: ./build-image.sh --cuda|--vulkan [--no-cache]"
+            echo ""
+            echo "Options:"
+            echo "  --cuda      Build CUDA image (NVIDIA GPUs)"
+            echo "  --vulkan    Build Vulkan image (AMD GPUs and compatible hardware)"
+            echo "  --no-cache  Force rebuild without using Docker cache"
+            echo "  --help, -h  Show this help message"
+            echo ""
+            echo "Environment variables:"
+            echo "  DOCKER_IMAGE_TAG     Set custom image tag (default: llama-swap:cuda or llama-swap:vulkan)"
+            echo "  LLAMA_COMMIT_HASH    Override llama.cpp commit hash"
+            echo "  WHISPER_COMMIT_HASH  Override whisper.cpp commit hash"
+            echo "  SD_COMMIT_HASH       Override stable-diffusion.cpp commit hash"
+            exit 0
+            ;;
+    esac
+done
+
+# Validate backend selection
+if [[ -z "$BACKEND" ]]; then
+    echo "Error: No backend specified. Please use --cuda or --vulkan."
+    exit 1
+fi
+
+# Configuration
+if [[ -n "${DOCKER_IMAGE_TAG:-}" ]]; then
+    # User provided a custom tag, use it as-is
+    :
+elif [[ "$BACKEND" == "vulkan" ]]; then
+    DOCKER_IMAGE_TAG="llama-swap:vulkan"
+else
+    DOCKER_IMAGE_TAG="llama-swap:cuda"
+fi
+DOCKER_BUILDKIT="${DOCKER_BUILDKIT:-1}"
+
+# Single unified Dockerfile, backend selected via build arg
+DOCKERFILE="Dockerfile"
+if [[ "$BACKEND" == "vulkan" ]]; then
+    echo "Building for: Vulkan (AMD GPUs and compatible hardware)"
+else
+    echo "Building for: CUDA (NVIDIA GPUs)"
+fi
+
+# Git repository URLs
+LLAMA_REPO="https://github.com/ggml-org/llama.cpp.git"
+WHISPER_REPO="https://github.com/ggml-org/whisper.cpp.git"
+SD_REPO="https://github.com/leejet/stable-diffusion.cpp.git"
+
+# Function to get the latest commit hash from a git repo's default branch
+get_latest_commit() {
+    local repo_url="$1"
+    local branch="${2:-master}"
+
+    # Try to get the latest commit hash for the specified branch
+    git ls-remote --heads "${repo_url}" "${branch}" 2>/dev/null | head -1 | cut -f1
+}
+
+# Function to get the default branch name (master or main)
+get_default_branch() {
+    local repo_url="$1"
+
+    # Check for master first
+    if git ls-remote --heads "${repo_url}" master &>/dev/null; then
+        echo "master"
+    elif git ls-remote --heads "${repo_url}" main &>/dev/null; then
+        echo "main"
+    else
+        echo "master"  # fallback
+    fi
+}
+
+# Function to get the latest release tag from a GitHub repo
+get_latest_release_tag() {
+    local owner_repo="$1"
+    curl -fsSL "https://api.github.com/repos/${owner_repo}/releases/latest" \
+        | grep '"tag_name"' | head -1 | cut -d'"' -f4
+}
+
+echo "=========================================="
+echo "llama-swap-docker Build Script"
+echo "=========================================="
+echo ""
+
+# Determine commit hashes / release tags - use env vars or auto-detect
+# For vulkan builds, llama and sd use GitHub release tags (prebuilt binaries).
+# For cuda builds (or whisper on any backend), use git commit hashes.
+if [[ -n "${LLAMA_COMMIT_HASH:-}" ]]; then
+    LLAMA_HASH="${LLAMA_COMMIT_HASH}"
+    echo "llama.cpp: Using provided version: ${LLAMA_HASH}"
+elif [[ "$BACKEND" == "vulkan" ]]; then
+    LLAMA_HASH=$(get_latest_release_tag "ggml-org/llama.cpp")
+    if [[ -z "${LLAMA_HASH}" ]]; then
+        echo "ERROR: Could not determine latest release tag for llama.cpp" >&2
+        exit 1
+    fi
+    echo "llama.cpp: Auto-detected latest release tag: ${LLAMA_HASH}"
+else
+    LLAMA_BRANCH=$(get_default_branch "${LLAMA_REPO}")
+    LLAMA_HASH=$(get_latest_commit "${LLAMA_REPO}" "${LLAMA_BRANCH}")
+    if [[ -z "${LLAMA_HASH}" ]]; then
+        echo "ERROR: Could not determine latest commit for llama.cpp" >&2
+        exit 1
+    fi
+    echo "llama.cpp: Auto-detected latest commit (${LLAMA_BRANCH}): ${LLAMA_HASH}"
+fi
+
+if [[ -n "${WHISPER_COMMIT_HASH:-}" ]]; then
+    WHISPER_HASH="${WHISPER_COMMIT_HASH}"
+    echo "whisper.cpp: Using provided commit hash: ${WHISPER_HASH}"
+else
+    WHISPER_BRANCH=$(get_default_branch "${WHISPER_REPO}")
+    WHISPER_HASH=$(get_latest_commit "${WHISPER_REPO}" "${WHISPER_BRANCH}")
+    if [[ -z "${WHISPER_HASH}" ]]; then
+        echo "ERROR: Could not determine latest commit for whisper.cpp" >&2
+        exit 1
+    fi
+    echo "whisper.cpp: Auto-detected latest commit (${WHISPER_BRANCH}): ${WHISPER_HASH}"
+fi
+
+if [[ -n "${SD_COMMIT_HASH:-}" ]]; then
+    SD_HASH="${SD_COMMIT_HASH}"
+    echo "stable-diffusion.cpp: Using provided version: ${SD_HASH}"
+elif [[ "$BACKEND" == "vulkan" ]]; then
+    SD_HASH=$(get_latest_release_tag "leejet/stable-diffusion.cpp")
+    if [[ -z "${SD_HASH}" ]]; then
+        echo "ERROR: Could not determine latest release tag for stable-diffusion.cpp" >&2
+        exit 1
+    fi
+    echo "stable-diffusion.cpp: Auto-detected latest release tag: ${SD_HASH}"
+else
+    SD_BRANCH=$(get_default_branch "${SD_REPO}")
+    SD_HASH=$(get_latest_commit "${SD_REPO}" "${SD_BRANCH}")
+    if [[ -z "${SD_HASH}" ]]; then
+        echo "ERROR: Could not determine latest commit for stable-diffusion.cpp" >&2
+        exit 1
+    fi
+    echo "stable-diffusion.cpp: Auto-detected latest commit (${SD_BRANCH}): ${SD_HASH}"
+fi
+
+echo ""
+echo "=========================================="
+echo "Starting Docker build..."
+echo "=========================================="
+echo ""
+
+# Build the Docker image with commit hashes as build args
+# Build context is the repository root (..) so the Dockerfile can access Go source
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+BUILD_ARGS=(
+    --build-arg "BACKEND=${BACKEND}"
+    --build-arg "LLAMA_COMMIT_HASH=${LLAMA_HASH}"
+    --build-arg "WHISPER_COMMIT_HASH=${WHISPER_HASH}"
+    --build-arg "SD_COMMIT_HASH=${SD_HASH}"
+    -t "${DOCKER_IMAGE_TAG}"
+    -f "${SCRIPT_DIR}/${DOCKERFILE}"
+)
+
+if [[ "$NO_CACHE" == true ]]; then
+    BUILD_ARGS+=(--no-cache)
+    echo "Note: Building without cache"
+fi
+
+# Use docker buildx with a custom builder for parallelism control
+# The legacy DOCKER_BUILDKIT=1 docker build doesn't respect BUILDKIT_MAX_PARALLELISM env var
+# We need to use a custom builder with a buildkitd.toml config file
+BUILDER_NAME="llama-swap-builder"
+
+# Check if our custom builder exists with the right config, create/update if needed
+if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
+    echo "Creating custom buildx builder with max-parallelism=1..."
+    
+    # Create buildkitd.toml config file
+    cat > buildkitd.toml << 'BUILDKIT_EOF'
+[worker.oci]
+  max-parallelism = 1
+BUILDKIT_EOF
+    
+    # Create the builder with the config
+    docker buildx create --name "$BUILDER_NAME" \
+        --driver docker-container \
+        --buildkitd-config buildkitd.toml \
+        --use
+else
+    # Switch to our builder
+    docker buildx use "$BUILDER_NAME"
+fi
+
+echo "Building with sequential stages (one at a time), each using all CPU cores..."
+echo "Using builder: $BUILDER_NAME"
+
+# Use docker buildx build with --load to load the image into Docker
+# The --builder flag ensures we use our custom builder with max-parallelism=1
+# Build context is the repository root so we can access Go source files
+docker buildx build --builder "$BUILDER_NAME" --load "${BUILD_ARGS[@]}" "${REPO_ROOT}"
+
+echo ""
+echo "=========================================="
+echo "Verifying build artifacts..."
+echo "=========================================="
+echo ""
+
+# Verify all expected binaries exist in the image
+MISSING_BINARIES=()
+
+for binary in llama-server llama-cli whisper-server whisper-cli sd-server sd-cli llama-swap; do
+    if ! docker run --rm "${DOCKER_IMAGE_TAG}" which "${binary}" >/dev/null 2>&1; then
+        MISSING_BINARIES+=("${binary}")
+    fi
+done
+
+if [[ ${#MISSING_BINARIES[@]} -gt 0 ]]; then
+    echo "ERROR: Build succeeded but the following binaries are missing from the image:"
+    for binary in "${MISSING_BINARIES[@]}"; do
+        echo "  - ${binary}"
+    done
+    echo ""
+    echo "This usually indicates a build stage failure. Try running with --no-cache flag:"
+    echo "  ./build-image.sh --vulkan --no-cache"
+    exit 1
+fi
+
+echo "All expected binaries verified: llama-server, llama-cli, whisper-server, whisper-cli, sd-server, sd-cli, llama-swap"
+
+echo ""
+echo "=========================================="
+echo "Build complete!"
+echo "=========================================="
+echo ""
+echo "Image tag: ${DOCKER_IMAGE_TAG}"
+echo ""
+echo "Built with:"
+echo "  llama.cpp:           ${LLAMA_HASH}"
+echo "  whisper.cpp:         ${WHISPER_HASH}"
+echo "  stable-diffusion.cpp: ${SD_HASH}"
+echo "  llama-swap:          $(docker run --rm "${DOCKER_IMAGE_TAG}" cat /versions.txt | grep llama-swap | cut -d' ' -f2-)"
+echo ""
+if [[ "$BACKEND" == "vulkan" ]]; then
+    echo "Run with:"
+    echo "  docker run -it --rm --device /dev/dri:/dev/dri ${DOCKER_IMAGE_TAG}"
+    echo ""
+    echo "Note: For AMD GPUs, you may also need to mount render devices:"
+    echo "  docker run -it --rm --device /dev/dri:/dev/dri --group-add video ${DOCKER_IMAGE_TAG}"
+else
+    echo "Run with:"
+    echo "  docker run -it --rm --gpus all ${DOCKER_IMAGE_TAG}"
+fi

docker/unified/Dockerfile

@@ -0,0 +1,203 @@
+# Unified multi-stage Dockerfile for AI inference tools
+# Supports CUDA and Vulkan backends via BACKEND build arg
+#
+# Usage:
+#   docker buildx build --build-arg BACKEND=cuda -t llama-swap:unified-cuda .
+#   docker buildx build --build-arg BACKEND=vulkan -t llama-swap:unified-vulkan .
+#   docker buildx build --build-arg BACKEND=cuda --build-arg CMAKE_CUDA_ARCHITECTURES="86;89" -t llama-swap:unified-cuda .
+#
+# Each project has its own install script that handles cloning, building,
+# and installing binaries. Build stages are independent for cache efficiency.
+
+ARG BACKEND=cuda
+
+# ── Builder bases ──────────────────────────────────────────────────────
+
+FROM nvidia/cuda:12.9.1-devel-ubuntu24.04 AS builder-base-cuda
+
+ARG CMAKE_CUDA_ARCHITECTURES="60;61;75;86;89"
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}
+ENV CCACHE_DIR=/ccache
+ENV CCACHE_MAXSIZE=2G
+ENV PATH="/usr/lib/ccache:${PATH}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential cmake git python3 python3-pip libssl-dev \
+    curl ca-certificates ccache make wget \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+
+# ──
+
+FROM ubuntu:24.04 AS builder-base-vulkan
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CCACHE_DIR=/ccache
+ENV CCACHE_MAXSIZE=2G
+ENV PATH="/usr/lib/ccache:${PATH}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential cmake git python3 python3-pip libssl-dev \
+    curl ca-certificates ccache make wget software-properties-common \
+    libvulkan-dev glslang-tools spirv-tools vulkan-validationlayers glslc \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+
+# ── Select builder base by BACKEND ────────────────────────────────────
+
+FROM builder-base-${BACKEND} AS builder-base
+
+# ── Build whisper.cpp (fastest build, run first) ──────────────────────
+
+FROM builder-base AS whisper-build
+ARG BACKEND=cuda
+ARG WHISPER_COMMIT_HASH=master
+COPY install-whisper.sh /build/
+RUN --mount=type=cache,id=ccache-${BACKEND},target=/ccache \
+    --mount=type=cache,id=whisper-${BACKEND},target=/src/whisper.cpp/build \
+    BACKEND=${BACKEND} bash /build/install-whisper.sh "${WHISPER_COMMIT_HASH}"
+
+# ── Build stable-diffusion.cpp ────────────────────────────────────────
+
+FROM builder-base AS sd-build
+ARG BACKEND=cuda
+ARG SD_COMMIT_HASH=master
+COPY install-sd.sh /build/
+RUN --mount=type=cache,id=ccache-${BACKEND},target=/ccache \
+    --mount=type=cache,id=sd-${BACKEND},target=/src/stable-diffusion.cpp/build \
+    BACKEND=${BACKEND} bash /build/install-sd.sh "${SD_COMMIT_HASH}"
+
+# ── Build llama.cpp (slowest build, run last) ─────────────────────────
+
+FROM builder-base AS llama-build
+ARG BACKEND=cuda
+ARG LLAMA_COMMIT_HASH=master
+COPY install-llama.sh /build/
+RUN --mount=type=cache,id=ccache-${BACKEND},target=/ccache \
+    --mount=type=cache,id=llama-${BACKEND},target=/src/llama.cpp/build \
+    BACKEND=${BACKEND} bash /build/install-llama.sh "${LLAMA_COMMIT_HASH}"
+
+# ── Build ik_llama.cpp (CUDA only) ────────────────────────────────────
+#
+# Two named stages allow ARG BACKEND to select at build time:
+#   - ik-llama-cuda  : real build (from builder-base-cuda)
+#   - ik-llama-vulkan: no-op (empty /install/bin, skips CUDA pull entirely)
+# BuildKit only evaluates the selected branch, so vulkan builds never
+# pull nvidia/cuda:*-devel or compile ik_llama.cpp.
+
+FROM builder-base-vulkan AS ik-llama-vulkan
+RUN mkdir -p /install/bin
+
+FROM builder-base-cuda AS ik-llama-cuda
+ARG IK_LLAMA_COMMIT_HASH=main
+COPY install-ik-llama.sh /build/
+RUN --mount=type=cache,id=ccache-cuda,target=/ccache \
+    --mount=type=cache,id=ik-llama-cuda,target=/src/ik_llama.cpp/build \
+    bash /build/install-ik-llama.sh "${IK_LLAMA_COMMIT_HASH}"
+
+ARG BACKEND=cuda
+FROM ik-llama-${BACKEND} AS ik-llama-build
+
+# ── Download llama-swap release binary ────────────────────────────────
+
+FROM builder-base AS llama-swap-download
+ARG LS_VERSION=latest
+COPY install-llama-swap.sh /build/
+RUN bash /build/install-llama-swap.sh "${LS_VERSION}"
+
+# ── Runtime bases ─────────────────────────────────────────────────────
+
+FROM nvidia/cuda:12.9.1-runtime-ubuntu24.04 AS runtime-cuda
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}"
+ENV PATH="/usr/local/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgomp1 python3 curl ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# CUDA stub drivers for container compatibility
+COPY --from=builder-base-cuda /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so
+COPY --from=builder-base-cuda /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
+
+# ──
+
+FROM ubuntu:24.04 AS runtime-vulkan
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PATH="/usr/local/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgomp1 libvulkan1 mesa-vulkan-drivers \
+    python3 curl ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# ── Select runtime base by BACKEND ────────────────────────────────────
+
+FROM runtime-${BACKEND} AS runtime
+
+ARG BACKEND=cuda
+ARG LLAMA_COMMIT_HASH=unknown
+ARG WHISPER_COMMIT_HASH=unknown
+ARG SD_COMMIT_HASH=unknown
+ARG IK_LLAMA_COMMIT_HASH=unknown
+ARG RUN_UID=0
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3-numpy python3-sentencepiece \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user when RUN_UID != 0
+RUN if [ "$RUN_UID" != "0" ]; then \
+      groupadd --system --gid $RUN_UID llama-swap && \
+      useradd --system --uid $RUN_UID --gid $RUN_UID \
+        --home /app --shell /sbin/nologin llama-swap; \
+    fi && \
+    mkdir -p /etc/llama-swap/config && \
+    chown -R ${RUN_UID}:${RUN_UID} /etc/llama-swap
+
+WORKDIR /app
+
+# Copy whisper.cpp binaries and libraries
+COPY --from=whisper-build /install/bin/whisper-server /usr/local/bin/
+COPY --from=whisper-build /install/bin/whisper-cli /usr/local/bin/
+COPY --from=whisper-build /install/lib/ /usr/local/lib/
+
+# Copy stable-diffusion.cpp binaries and libraries
+COPY --from=sd-build /install/bin/sd-server /usr/local/bin/
+COPY --from=sd-build /install/bin/sd-cli /usr/local/bin/
+COPY --from=sd-build /install/lib/ /usr/local/lib/
+
+# Copy llama.cpp binaries (statically linked)
+COPY --from=llama-build /install/bin/llama-server /usr/local/bin/
+COPY --from=llama-build /install/bin/llama-cli /usr/local/bin/
+
+# Copy ik-llama-server (CUDA only; empty copy for vulkan)
+COPY --from=ik-llama-build /install/bin/ /usr/local/bin/
+
+# Copy llama-swap binary
+COPY --from=llama-swap-download /install/bin/llama-swap /usr/local/bin/
+COPY --from=llama-swap-download /install/llama-swap-version /tmp/
+
+RUN ldconfig
+
+COPY config.example.yaml /etc/llama-swap/config/config.yaml
+
+# Version tracking
+RUN echo "llama.cpp: ${LLAMA_COMMIT_HASH}" > /versions.txt && \
+    echo "whisper.cpp: ${WHISPER_COMMIT_HASH}" >> /versions.txt && \
+    echo "stable-diffusion.cpp: ${SD_COMMIT_HASH}" >> /versions.txt && \
+    echo "ik_llama.cpp: ${IK_LLAMA_COMMIT_HASH}" >> /versions.txt && \
+    echo "llama-swap: $(cat /tmp/llama-swap-version)" >> /versions.txt && \
+    echo "backend: ${BACKEND}" >> /versions.txt && \
+    echo "build_timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /versions.txt
+
+RUN mkdir -p /models && chown ${RUN_UID}:${RUN_UID} /models
+WORKDIR /models
+USER ${RUN_UID}
+ENTRYPOINT ["llama-swap"]
+CMD ["-config", "/etc/llama-swap/config/config.yaml", "-listen", "0.0.0.0:8080"]

docker/unified/README.md

@@ -0,0 +1,8 @@
+# Unified Docker Container
+
+These scripts create a custom llama-swap container that contains:
+
+- llama-server for LLMs, rerank and embedding model support
+- sd-server (stable-diffusion.cpp) for image generation
+- whisper.cpp for ASR
+

docker/unified/build-image.sh

@@ -0,0 +1,303 @@
+#!/bin/bash
+#
+# Build script for unified container with version pinning
+#
+# Usage:
+#   ./build-image.sh --cuda                              # Build CUDA image
+#   ./build-image.sh --vulkan                            # Build Vulkan image
+#   ./build-image.sh --cuda --no-cache                   # Build without cache
+#   LLAMA_REF=b1234 ./build-image.sh --vulkan            # Pin llama.cpp to a commit hash
+#   LLAMA_REF=v1.2.3 ./build-image.sh --cuda             # Pin llama.cpp to a tag
+#   WHISPER_REF=v1.0.0 ./build-image.sh --vulkan         # Pin whisper.cpp to a tag
+#   SD_REF=master ./build-image.sh --cuda                # Pin stable-diffusion.cpp to a branch
+#   LS_VERSION=170 ./build-image.sh --cuda               # Override llama-swap version
+#   IK_LLAMA_REF=main ./build-image.sh --cuda            # Pin ik_llama.cpp to main branch (CUDA only)
+#
+
+set -euo pipefail
+
+BACKEND=""
+NO_CACHE=false
+
+for arg in "$@"; do
+    case $arg in
+        --cuda)
+            BACKEND="cuda"
+            ;;
+        --vulkan)
+            BACKEND="vulkan"
+            ;;
+        --no-cache)
+            NO_CACHE=true
+            ;;
+        --help|-h)
+            echo "Usage: ./build-image.sh --cuda|--vulkan [--no-cache]"
+            echo ""
+            echo "Options:"
+            echo "  --cuda      Build CUDA image (NVIDIA GPUs)"
+            echo "  --vulkan    Build Vulkan image (AMD GPUs and compatible hardware)"
+            echo "  --no-cache  Force rebuild without using Docker cache"
+            echo "  --help, -h  Show this help message"
+            echo ""
+            echo "Environment variables:"
+            echo "  DOCKER_IMAGE_TAG     Set custom image tag (default: llama-swap:unified-cuda or llama-swap:unified-vulkan)"
+            echo "  LLAMA_REF            Pin llama.cpp to a commit, tag, or branch"
+            echo "  WHISPER_REF          Pin whisper.cpp to a commit, tag, or branch"
+            echo "  SD_REF               Pin stable-diffusion.cpp to a commit, tag, or branch"
+            echo "  IK_LLAMA_REF         Pin ik_llama.cpp to a commit, tag, or branch (CUDA only)"
+            echo "  LS_VERSION           Override llama-swap version (e.g., '170' or 'latest')"
+            exit 0
+            ;;
+    esac
+done
+
+if [[ -z "$BACKEND" ]]; then
+    echo "Error: No backend specified. Please use --cuda or --vulkan."
+    echo ""
+    echo "Usage: ./build-image.sh --cuda|--vulkan [--no-cache]"
+    exit 1
+fi
+
+DOCKER_IMAGE_TAG="${DOCKER_IMAGE_TAG:-llama-swap:unified-${BACKEND}}"
+
+# Git repository URLs
+LLAMA_REPO="https://github.com/ggml-org/llama.cpp.git"
+WHISPER_REPO="https://github.com/ggml-org/whisper.cpp.git"
+SD_REPO="https://github.com/leejet/stable-diffusion.cpp.git"
+LLAMA_SWAP_REPO="https://github.com/mostlygeek/llama-swap.git"
+IK_LLAMA_REPO="https://github.com/ikawrakow/ik_llama.cpp.git"
+
+# Resolve a git ref (commit hash, tag, or branch) to a full commit hash.
+# Requires only: git, network access to the remote.
+resolve_ref() {
+    local repo_url="$1"
+    local ref="$2"
+
+    # Full 40-char SHA — use as-is
+    if [[ "${ref}" =~ ^[0-9a-f]{40}$ ]]; then
+        echo "${ref}"
+        return
+    fi
+
+    # Try tag then branch (exact match)
+    local hash
+    hash=$(git ls-remote "${repo_url}" "refs/tags/${ref}" "refs/heads/${ref}" 2>/dev/null | head -1 | cut -f1)
+    if [[ -n "${hash}" ]]; then
+        echo "${hash}"
+        return
+    fi
+
+    # Short hash (7+ chars): scan all refs for a SHA with this prefix
+    if [[ "${ref}" =~ ^[0-9a-f]{7,}$ ]]; then
+        hash=$(git ls-remote "${repo_url}" 2>/dev/null | grep "^${ref}" | head -1 | cut -f1)
+        if [[ -n "${hash}" ]]; then
+            echo "${hash}"
+            return
+        fi
+    fi
+
+    echo "ERROR: Could not resolve ref '${ref}' for ${repo_url}" >&2
+    if [[ "${ref}" =~ ^[0-9a-f]+$ && ${#ref} -lt 7 ]]; then
+        echo "  Short hashes must be at least 7 characters (got ${#ref})." >&2
+    else
+        echo "  Tried: tag, branch, git ls-remote prefix match" >&2
+    fi
+    echo "  Use a full 40-char SHA, a tag name, a branch name, or a 7+ char short hash." >&2
+    return 1
+}
+
+# Resolve HEAD of a repo without needing to know the default branch name.
+get_latest_hash() {
+    git ls-remote "${1}" HEAD 2>/dev/null | head -1 | cut -f1
+}
+
+echo "=========================================="
+echo "llama-swap Unified Build (${BACKEND})"
+echo "=========================================="
+echo ""
+
+# Resolve llama.cpp ref
+if [[ -n "${LLAMA_REF:-}" ]]; then
+    LLAMA_HASH=$(resolve_ref "${LLAMA_REPO}" "${LLAMA_REF}") || exit 1
+    echo "llama.cpp: ${LLAMA_REF} -> ${LLAMA_HASH}"
+else
+    LLAMA_HASH=$(get_latest_hash "${LLAMA_REPO}")
+    if [[ -z "${LLAMA_HASH}" ]]; then
+        echo "ERROR: Could not determine latest commit for llama.cpp" >&2
+        exit 1
+    fi
+    echo "llama.cpp: latest HEAD: ${LLAMA_HASH}"
+fi
+
+# Resolve whisper.cpp ref
+if [[ -n "${WHISPER_REF:-}" ]]; then
+    WHISPER_HASH=$(resolve_ref "${WHISPER_REPO}" "${WHISPER_REF}") || exit 1
+    echo "whisper.cpp: ${WHISPER_REF} -> ${WHISPER_HASH}"
+else
+    WHISPER_HASH=$(get_latest_hash "${WHISPER_REPO}")
+    if [[ -z "${WHISPER_HASH}" ]]; then
+        echo "ERROR: Could not determine latest commit for whisper.cpp" >&2
+        exit 1
+    fi
+    echo "whisper.cpp: latest HEAD: ${WHISPER_HASH}"
+fi
+
+# Resolve stable-diffusion.cpp ref
+if [[ -n "${SD_REF:-}" ]]; then
+    SD_HASH=$(resolve_ref "${SD_REPO}" "${SD_REF}") || exit 1
+    echo "stable-diffusion.cpp: ${SD_REF} -> ${SD_HASH}"
+else
+    SD_HASH=$(get_latest_hash "${SD_REPO}")
+    if [[ -z "${SD_HASH}" ]]; then
+        echo "ERROR: Could not determine latest commit for stable-diffusion.cpp" >&2
+        exit 1
+    fi
+    echo "stable-diffusion.cpp: latest HEAD: ${SD_HASH}"
+fi
+
+# Resolve ik_llama.cpp ref (CUDA only)
+if [[ "$BACKEND" == "cuda" ]]; then
+    if [[ -n "${IK_LLAMA_REF:-}" ]]; then
+        IK_LLAMA_HASH=$(resolve_ref "${IK_LLAMA_REPO}" "${IK_LLAMA_REF}") || exit 1
+        echo "ik_llama.cpp: ${IK_LLAMA_REF} -> ${IK_LLAMA_HASH}"
+    else
+        IK_LLAMA_HASH=$(get_latest_hash "${IK_LLAMA_REPO}")
+        if [[ -z "${IK_LLAMA_HASH}" ]]; then
+            echo "ERROR: Could not determine latest commit for ik_llama.cpp" >&2
+            exit 1
+        fi
+        echo "ik_llama.cpp: latest HEAD: ${IK_LLAMA_HASH}"
+    fi
+else
+    IK_LLAMA_HASH="n/a"
+    echo "ik_llama.cpp: skipped (vulkan build)"
+fi
+
+# Resolve llama-swap ref
+if [[ -n "${LS_VERSION:-}" ]]; then
+    LS_HASH=$(resolve_ref "${LLAMA_SWAP_REPO}" "${LS_VERSION}") || exit 1
+    echo "llama-swap: ${LS_VERSION} -> ${LS_HASH}"
+else
+    LS_HASH=$(get_latest_hash "${LLAMA_SWAP_REPO}")
+    if [[ -z "${LS_HASH}" ]]; then
+        echo "ERROR: Could not determine latest commit for llama-swap" >&2
+        exit 1
+    fi
+    echo "llama-swap: latest HEAD: ${LS_HASH}"
+fi
+
+echo ""
+echo "=========================================="
+echo "Starting Docker build..."
+echo "=========================================="
+echo ""
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+BUILD_ARGS=(
+    --build-arg "BACKEND=${BACKEND}"
+    --build-arg "LLAMA_COMMIT_HASH=${LLAMA_HASH}"
+    --build-arg "WHISPER_COMMIT_HASH=${WHISPER_HASH}"
+    --build-arg "SD_COMMIT_HASH=${SD_HASH}"
+    --build-arg "IK_LLAMA_COMMIT_HASH=${IK_LLAMA_HASH}"
+    --build-arg "LS_VERSION=${LS_HASH}"
+    -t "${DOCKER_IMAGE_TAG}"
+    -f "${SCRIPT_DIR}/Dockerfile"
+)
+
+if [[ "$NO_CACHE" == true ]]; then
+    BUILD_ARGS+=(--no-cache)
+    echo "Note: Building without cache"
+elif [[ "${GITHUB_ACTIONS:-}" == "true" && "${ACT:-}" != "true" ]]; then
+    CACHE_REF="ghcr.io/mostlygeek/llama-swap:unified-${BACKEND}-cache"
+    BUILD_ARGS+=(
+        --cache-from "type=registry,ref=${CACHE_REF}"
+        --cache-to "type=registry,ref=${CACHE_REF},mode=max"
+    )
+    echo "Note: Using registry cache (${CACHE_REF})"
+fi
+
+DOCKER_BUILDKIT=1 docker buildx build --load "${BUILD_ARGS[@]}" "${SCRIPT_DIR}"
+
+echo ""
+echo "=========================================="
+echo "Verifying build artifacts..."
+echo "=========================================="
+echo ""
+
+EXPECTED_BINARIES=(llama-server llama-cli whisper-server whisper-cli sd-server sd-cli llama-swap)
+if [[ "$BACKEND" == "cuda" ]]; then
+    EXPECTED_BINARIES+=(ik-llama-server)
+fi
+
+MISSING_BINARIES=()
+for binary in "${EXPECTED_BINARIES[@]}"; do
+    if ! docker run --rm --entrypoint which "${DOCKER_IMAGE_TAG}" "${binary}" >/dev/null 2>&1; then
+        MISSING_BINARIES+=("${binary}")
+    fi
+done
+
+if [[ ${#MISSING_BINARIES[@]} -gt 0 ]]; then
+    echo "ERROR: Build succeeded but the following binaries are missing:"
+    for binary in "${MISSING_BINARIES[@]}"; do
+        echo "  - ${binary}"
+    done
+    echo ""
+    echo "Try running with --no-cache flag:"
+    echo "  ./build-image.sh --${BACKEND} --no-cache"
+    exit 1
+fi
+
+VERIFIED_LIST="llama-server, llama-cli, whisper-server, whisper-cli, sd-server, sd-cli, llama-swap"
+if [[ "$BACKEND" == "cuda" ]]; then
+    VERIFIED_LIST="${VERIFIED_LIST}, ik-llama-server"
+fi
+echo "All expected binaries verified: ${VERIFIED_LIST}"
+
+echo ""
+echo "=========================================="
+echo "Building rootless image..."
+echo "=========================================="
+echo ""
+
+ROOTLESS_TAG="${DOCKER_IMAGE_TAG}-rootless"
+docker buildx build --load -t "${ROOTLESS_TAG}" - <<EOF
+FROM ${DOCKER_IMAGE_TAG}
+USER root
+RUN groupadd --system --gid 10001 llama-swap && \\
+    useradd --system --uid 10001 --gid 10001 \\
+      --home /app --shell /sbin/nologin llama-swap && \\
+    chown -R 10001:10001 /etc/llama-swap /models
+USER 10001
+EOF
+
+echo "Rootless image built: ${ROOTLESS_TAG}"
+
+echo ""
+echo "=========================================="
+echo "Build complete!"
+echo "=========================================="
+echo ""
+echo "Image tags:"
+echo "  ${DOCKER_IMAGE_TAG}"
+echo "  ${ROOTLESS_TAG}"
+echo ""
+echo "Built with:"
+echo "  llama.cpp:            ${LLAMA_HASH}"
+echo "  whisper.cpp:          ${WHISPER_HASH}"
+echo "  stable-diffusion.cpp: ${SD_HASH}"
+if [[ "$BACKEND" == "cuda" ]]; then
+    echo "  ik_llama.cpp:         ${IK_LLAMA_HASH}"
+fi
+echo "  llama-swap:           $(docker run --rm --entrypoint cat "${DOCKER_IMAGE_TAG}" /versions.txt | grep llama-swap | cut -d' ' -f2-)"
+echo ""
+if [[ "$BACKEND" == "vulkan" ]]; then
+    echo "Run with:"
+    echo "  docker run -it --rm --device /dev/dri:/dev/dri ${DOCKER_IMAGE_TAG}"
+    echo ""
+    echo "Note: For AMD GPUs, you may also need:"
+    echo "  docker run -it --rm --device /dev/dri:/dev/dri --group-add video ${DOCKER_IMAGE_TAG}"
+else
+    echo "Run with:"
+    echo "  docker run -it --rm --gpus all ${DOCKER_IMAGE_TAG}"
+fi

docker/unified/config.example.yaml

@@ -0,0 +1,33 @@
+# placeholder example configuration
+healthCheckTimeout: 300
+logRequests: true
+
+models:
+  "llama":
+    cmd: >
+      llama-server
+      -hf bartowski/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M
+      --port ${PORT}
+
+  "whisper":
+    checkEndpoint: /v1/audio/transcriptions/
+    cmd: >
+      whisper-server
+      --port ${PORT}
+      --m /models/whisper.bin
+      --flash-attn
+      --request-path /v1/audio/transcriptions --inference-path ""
+
+  "image":
+    checkEndpoint: /
+    cmd: |
+      /app/sd-server
+      --listen-port 9999
+      --diffusion-fa
+      --diffusion-model /models/z_image_turbo-Q8_0.gguf
+      --vae /models/ae.safetensors
+      --llm /models/qwen3-4b-instruct-2507-q8_0.gguf
+      --offload-to-cpu
+      --cfg-scale 1.0
+      --height 512 --width 512
+      --steps 8

docker/unified/install-ik-llama.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Install ik_llama.cpp - clone, build, and install binaries
+# Usage: ./install-ik-llama.sh <commit_hash>
+# Note: CUDA only; always built against builder-base-cuda
+set -e
+
+COMMIT_HASH="${1:-main}"
+
+mkdir -p /install/bin
+
+# Clone and checkout (init-based so cache-mounted build dir doesn't break clone)
+echo "=== Cloning ik_llama.cpp at ${COMMIT_HASH} ==="
+mkdir -p /src/ik_llama.cpp
+cd /src/ik_llama.cpp
+if [ ! -d .git ]; then
+    git init
+    git remote add origin https://github.com/ikawrakow/ik_llama.cpp.git
+fi
+git fetch --depth=1 origin "${COMMIT_HASH}"
+git checkout FETCH_HEAD
+
+CMAKE_FLAGS=(
+    -DGGML_NATIVE=OFF
+    -DBUILD_SHARED_LIBS=OFF
+    -DCMAKE_BUILD_TYPE=Release
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+    -DGGML_CUDA=ON
+    "-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:?CMAKE_CUDA_ARCHITECTURES must be set}"
+    "-DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler"
+    "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,/usr/local/cuda/lib64/stubs -lcuda -Wl,--allow-shlib-undefined"
+)
+
+rm -rf build/CMakeCache.txt build/CMakeFiles 2>/dev/null || true
+
+echo "=== Building ik_llama.cpp ==="
+cmake -B build "${CMAKE_FLAGS[@]}"
+cmake --build build --config Release -j"$(nproc)" --target llama-server
+
+if [ ! -f "build/bin/llama-server" ]; then
+    echo "FATAL: llama-server not found in build/bin/" >&2
+    exit 1
+fi
+
+# Install as ik-llama-server to avoid collision with llama.cpp's llama-server
+cp "build/bin/llama-server" "/install/bin/ik-llama-server"
+echo "=== ik_llama.cpp build complete ==="
+ls -la /install/bin/

docker/unified/install-llama-swap.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# Install llama-swap - download latest release binary from GitHub
+# Usage: ./install-llama-swap.sh [version]
+#   version: release version number (e.g., "170") or "latest" (default)
+set -e
+
+VERSION="${1:-latest}"
+REPO="mostlygeek/llama-swap"
+
+mkdir -p /install/bin
+
+# If a full commit hash is given, find the release tag that points to it
+if echo "${VERSION}" | grep -qE '^[0-9a-f]{40}$'; then
+    echo "=== Resolving commit ${VERSION:0:7} to release tag ==="
+    TAG=$(git ls-remote --tags "https://github.com/${REPO}.git" 2>/dev/null \
+        | grep "^${VERSION}" | sed 's|.*refs/tags/||' | grep -v '\^{}' | head -1)
+    if [ -n "${TAG}" ]; then
+        echo "Resolved to tag: ${TAG}"
+        VERSION="${TAG#v}"
+    else
+        echo "No release tag found for commit ${VERSION:0:7}, using latest"
+        VERSION="latest"
+    fi
+fi
+
+# Strip leading 'v' prefix so both "198" and "v198" work
+VERSION="${VERSION#v}"
+
+# Resolve "latest" to actual version number
+if [ "$VERSION" = "latest" ]; then
+    echo "=== Resolving latest llama-swap release ==="
+    VERSION=$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" \
+        | grep '"tag_name"' | head -1 | cut -d'"' -f4 | sed 's/^v//')
+    if [ -z "$VERSION" ]; then
+        echo "FATAL: Could not determine latest release version" >&2
+        exit 1
+    fi
+    echo "Latest version: ${VERSION}"
+fi
+
+# Download and extract
+URL="https://github.com/${REPO}/releases/download/v${VERSION}/llama-swap_${VERSION}_linux_amd64.tar.gz"
+echo "=== Downloading llama-swap v${VERSION} ==="
+echo "URL: $URL"
+curl -fSL -o /tmp/llama-swap.tar.gz "$URL"
+tar -xzf /tmp/llama-swap.tar.gz -C /install/bin/
+rm /tmp/llama-swap.tar.gz
+
+# Validate
+if [ ! -x "/install/bin/llama-swap" ]; then
+    echo "FATAL: llama-swap binary not found or not executable" >&2
+    ls -la /install/bin/ >&2
+    exit 1
+fi
+
+echo "$VERSION" > /install/llama-swap-version
+
+echo "=== llama-swap v${VERSION} installed ==="
+ls -la /install/bin/llama-swap

docker/unified/install-llama.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Install llama.cpp - clone, build, and install binaries
+# Usage: BACKEND=cuda|vulkan ./install-llama.sh <commit_hash>
+set -e
+
+COMMIT_HASH="${1:-master}"
+BACKEND="${BACKEND:-cuda}"
+
+mkdir -p /install/bin
+
+# Clone and checkout (init-based so cache-mounted /src/llama.cpp/build dir doesn't break clone)
+echo "=== Cloning llama.cpp at ${COMMIT_HASH} ==="
+mkdir -p /src/llama.cpp
+cd /src/llama.cpp
+if [ ! -d .git ]; then
+    git init
+    git remote add origin https://github.com/ggml-org/llama.cpp.git
+fi
+git fetch --depth=1 origin "${COMMIT_HASH}"
+git checkout FETCH_HEAD
+
+# Common cmake flags
+CMAKE_FLAGS=(
+    -DGGML_NATIVE=OFF
+    -DBUILD_SHARED_LIBS=OFF
+    -DCMAKE_BUILD_TYPE=Release
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+    -DLLAMA_BUILD_TESTS=OFF
+)
+
+if [ "$BACKEND" = "cuda" ]; then
+    CMAKE_FLAGS+=(
+        -DGGML_CUDA=ON
+        -DGGML_VULKAN=OFF
+        "-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:?CMAKE_CUDA_ARCHITECTURES must be set}"
+        "-DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler"
+        "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,/usr/local/cuda/lib64/stubs -lcuda"
+    )
+elif [ "$BACKEND" = "vulkan" ]; then
+    CMAKE_FLAGS+=(
+        -DGGML_CUDA=OFF
+        -DGGML_VULKAN=ON
+    )
+fi
+
+TARGETS=(llama-cli llama-server)
+
+rm -rf build/CMakeCache.txt build/CMakeFiles 2>/dev/null || true
+
+echo "=== Building llama.cpp for ${BACKEND} ==="
+cmake -B build "${CMAKE_FLAGS[@]}"
+cmake --build build --config Release -j"$(nproc)" --target "${TARGETS[@]}"
+
+for bin in "${TARGETS[@]}"; do
+    if [ ! -f "build/bin/$bin" ]; then
+        echo "FATAL: $bin not found in build/bin/" >&2
+        exit 1
+    fi
+    cp "build/bin/$bin" "/install/bin/"
+done
+echo "=== llama.cpp build complete ==="
+ls -la /install/bin/

docker/unified/install-sd.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Install stable-diffusion.cpp - clone, build, and install binaries and library
+# Usage: BACKEND=cuda|vulkan ./install-sd.sh <commit_hash>
+set -e
+
+COMMIT_HASH="${1:-master}"
+BACKEND="${BACKEND:-cuda}"
+
+mkdir -p /install/bin /install/lib
+
+# Clone and checkout (init-based so cache-mounted /src/stable-diffusion.cpp/build dir doesn't break clone)
+echo "=== Cloning stable-diffusion.cpp at ${COMMIT_HASH} ==="
+mkdir -p /src/stable-diffusion.cpp
+cd /src/stable-diffusion.cpp
+if [ ! -d .git ]; then
+    git init
+    git remote add origin https://github.com/leejet/stable-diffusion.cpp.git
+fi
+git fetch --depth=1 origin "${COMMIT_HASH}"
+git checkout FETCH_HEAD
+git submodule update --init --recursive --depth=1
+
+# Common cmake flags
+CMAKE_FLAGS=(
+    -DGGML_NATIVE=OFF
+    -DCMAKE_BUILD_TYPE=Release
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+    -DSD_BUILD_EXAMPLES=ON
+)
+
+if [ "$BACKEND" = "cuda" ]; then
+    CMAKE_FLAGS+=(
+        -DGGML_CUDA=ON
+        -DGGML_VULKAN=OFF
+        "-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:?CMAKE_CUDA_ARCHITECTURES must be set}"
+        "-DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler"
+        "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,/usr/local/cuda/lib64/stubs -lcuda"
+        "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath-link,/usr/local/cuda/lib64/stubs -lcuda"
+        -DSD_CUDA=ON
+    )
+elif [ "$BACKEND" = "vulkan" ]; then
+    CMAKE_FLAGS+=(
+        -DGGML_CUDA=OFF
+        -DGGML_VULKAN=ON
+        -DSD_VULKAN=ON
+    )
+fi
+
+TARGETS=(stable-diffusion sd-cli sd-server)
+
+rm -rf build/CMakeCache.txt build/CMakeFiles 2>/dev/null || true
+
+echo "=== Building stable-diffusion.cpp for ${BACKEND} ==="
+cmake -B build "${CMAKE_FLAGS[@]}"
+cmake --build build --config Release -j"$(nproc)" --target "${TARGETS[@]}"
+
+for bin in sd-cli sd-server; do
+    if [ ! -f "build/bin/$bin" ]; then
+        echo "FATAL: $bin not found in build/bin/" >&2
+        exit 1
+    fi
+    cp "build/bin/$bin" "/install/bin/"
+done
+find build -name "*.so*" -type f -exec cp {} /install/lib/ \;
+
+echo "=== stable-diffusion.cpp build complete ==="
+ls -la /install/bin/ /install/lib/

docker/unified/install-whisper.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Install whisper.cpp - clone, build, and install binaries
+# Usage: BACKEND=cuda|vulkan ./install-whisper.sh <commit_hash>
+set -e
+
+COMMIT_HASH="${1:-master}"
+BACKEND="${BACKEND:-cuda}"
+
+mkdir -p /install/bin /install/lib
+
+# Clone and checkout (init-based so cache-mounted /src/whisper.cpp/build dir doesn't break clone)
+echo "=== Cloning whisper.cpp at ${COMMIT_HASH} ==="
+mkdir -p /src/whisper.cpp
+cd /src/whisper.cpp
+if [ ! -d .git ]; then
+    git init
+    git remote add origin https://github.com/ggml-org/whisper.cpp.git
+fi
+git fetch --depth=1 origin "${COMMIT_HASH}"
+git checkout FETCH_HEAD
+
+# Common cmake flags
+CMAKE_FLAGS=(
+    -DGGML_NATIVE=OFF
+    -DCMAKE_BUILD_TYPE=Release
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+)
+
+if [ "$BACKEND" = "cuda" ]; then
+    CMAKE_FLAGS+=(
+        -DGGML_CUDA=ON
+        -DGGML_VULKAN=OFF
+        "-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:?CMAKE_CUDA_ARCHITECTURES must be set}"
+        "-DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler"
+        "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,/usr/local/cuda/lib64/stubs -lcuda"
+        "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath-link,/usr/local/cuda/lib64/stubs -lcuda"
+    )
+elif [ "$BACKEND" = "vulkan" ]; then
+    CMAKE_FLAGS+=(
+        -DGGML_CUDA=OFF
+        -DGGML_VULKAN=ON
+    )
+fi
+
+TARGETS=(whisper-cli whisper-server)
+
+rm -rf build/CMakeCache.txt build/CMakeFiles 2>/dev/null || true
+
+echo "=== Building whisper.cpp for ${BACKEND} ==="
+cmake -B build "${CMAKE_FLAGS[@]}"
+cmake --build build --config Release -j"$(nproc)" --target "${TARGETS[@]}"
+
+for bin in "${TARGETS[@]}"; do
+    if [ ! -f "build/bin/$bin" ]; then
+        echo "FATAL: $bin not found in build/bin/" >&2
+        exit 1
+    fi
+    cp "build/bin/$bin" "/install/bin/"
+done
+find build -name "*.so*" -type f -exec cp {} /install/lib/ \;
+
+echo "=== whisper.cpp build complete ==="
+ls -la /install/bin/

docs/configuration.md

@@ -22,7 +22,7 @@ models:
     cmd: llama-server --port ${PORT} -m /path/to/third_model.gguf
 ```
 
-With this configuration models will be hot swapped and loaded on demand. The special `${PORT}` macro provides a unique port per model. Useful if you want to run multiple models at the same time with the `groups` feature.
+With this configuration models will be hot swapped and loaded on demand. The special `${PORT}` macro provides a unique port per model which is useful if you want to run multiple models at the same time with the `matrix` feature.
 
 ## Advanced control with `cmd`
 
@@ -76,7 +76,7 @@ llama-swap supports many more features to customize how you want to manage your
 | --------- | ---------------------------------------------- |
 | `ttl`     | automatic unloading of models after a timeout  |
 | `macros`  | reusable snippets to use in configurations     |
-| `groups`  | run multiple models at a time                  |
+| `matrix`  | run multiple models at a time                  |
 | `hooks`   | event driven functionality                     |
 | `env`     | define environment variables per model         |
 | `aliases` | serve a model with different names             |
@@ -141,6 +141,11 @@ logToStdout: "proxy"
 # - useful for limiting memory usage when processing large volumes of metrics
 metricsMaxInMemory: 1000
 
+# captureBuffer: how many MBs to allocate for storing request/response captures
+# - optional, default: 10
+# - set to 0 to disable
+captureBuffer: 15
+
 # startPort: sets the starting port number for the automatic ${PORT} macro.
 # - optional, default: 5800
 # - the ${PORT} macro can be used in model.cmd and model.proxy settings
@@ -161,15 +166,10 @@ sendLoadingState: true
 #   all fields except for Id so chat UIs can use the alias equivalent to the original.
 includeAliasesInList: false
 
-# apiKeys: require an API key when making requests to inference endpoints
-# - optional, default: []
-# - when empty (the default) authorization will not be checked as llama-swap is default-allow
-# - each key is a non-empty string
-apiKeys:
-  - "sk-hunter2"
-  # hint, one liner: printf "sk-%s\n" "$(head -c 48 /dev/urandom | base64 )"
-  - "sk-gyCPiKUcIfPlaM4OSMZekkprgijPx6+OsmQs8Rsg0xZ9qpy6gKWsIKqHOk+cgXVx"
-  - "sk-+QtIn0Zjj4UHjiaZYiZEnru4mrwKM9RzhmJeK5SobNXLl8QMFXxGz1/2lEuvQpkb"
+# globalTTL: the default TTL in seconds before unloading a model
+# - optional, default: 0 (never automatically unload)
+# - must be >= 0
+globalTTL: 0
 
 # macros: a dictionary of string substitutions
 # - optional, default: empty dictionary
@@ -181,6 +181,9 @@ apiKeys:
 # - macro names must not be a reserved name: PORT or MODEL_ID
 # - macro values can be numbers, bools, or strings
 # - macros can contain other macros, but they must be defined before they are used
+# - environment variables can be referenced with ${env.VAR_NAME} syntax
+#   - env macros are substituted first, before regular macros
+#   - if the env var is not set, config loading will fail with an error
 macros:
   # Example of a multi-line macro
   "latest-llama": >
@@ -193,6 +196,24 @@ macros:
   # but they must be previously declared.
   "default_args": "--ctx-size ${default_ctx}"
 
+  # Example of environment variable macros
+  # - ${env.VAR_NAME} pulls the value from the system environment
+  # - useful for paths, secrets, or machine-specific configuration
+  "models_dir": "${env.HOME}/models"
+
+# apiKeys: require an API key when making requests to inference endpoints
+# - optional, default: []
+# - when empty (the default) authorization will not be checked as llama-swap is default-allow
+# - each key is a non-empty string
+apiKeys:
+  - "sk-hunter2"
+  # tip, one liner: printf "sk-%s\n" "$(head -c 48 /dev/urandom | base64 )"
+  - "sk-gyCPiKUcIfPlaM4OSMZekkprgijPx6+OsmQs8Rsg0xZ9qpy6gKWsIKqHOk+cgXVx"
+
+  # use environment variable macros to keep secrets out of the config
+  - "${env.API_KEY_1}"
+  - "${env.API_KEY_2}"
+
 # models: a dictionary of model configurations
 # - required
 # - each key is the model's ID, used in API requests
@@ -201,7 +222,7 @@ macros:
 # - below are examples of the all the settings a model can have
 models:
   # keys are the model names used in API requests
-  "llama":
+  "gpt-oss-120b":
     # macros: a dictionary of string substitutions specific to this model
     # - optional, default: empty dictionary
     # - macros defined here override macros defined in the global macros section
@@ -218,7 +239,7 @@ models:
     cmd: |
       # ${latest-llama} is a macro that is defined above
       ${latest-llama}
-      --model path/to/llama-8B-Q4_K_M.gguf
+      --model path/to/gpt-oss-120B.gguf
       --ctx-size ${default_ctx}
       --temperature ${temp}
 
@@ -226,13 +247,13 @@ models:
     # - optional, default: empty string
     # - if set, it will be used in the v1/models API response
     # - if not set, it will be omitted in the JSON model record
-    name: "llama 3.1 8B"
+    name: "gpt-oss 120B"
 
     # description: a description for the model
     # - optional, default: empty string
     # - if set, it will be used in the v1/models API response
     # - if not set, it will be omitted in the JSON model record
-    description: "A small but capable model used for quick testing"
+    description: "A thinking model from OpenAI"
 
     # env: define an array of environment variables to inject into cmd's environment
     # - optional, default: empty array
@@ -247,14 +268,6 @@ models:
     # - if you use a custom port in cmd this *must* be set
     proxy: http://127.0.0.1:8999
 
-    # aliases: alternative model names that this model configuration is used for
-    # - optional, default: empty array
-    # - aliases must be unique globally
-    # - useful for impersonating a specific model
-    aliases:
-      - "gpt-4o-mini"
-      - "gpt-3.5-turbo"
-
     # checkEndpoint: URL path to check if the server is ready
     # - optional, default: /health
     # - endpoint is expected to return an HTTP 200 response
@@ -263,8 +276,10 @@ models:
     checkEndpoint: /custom-endpoint
 
     # ttl: automatically unload the model after ttl seconds
-    # - optional, default: 0
-    # - ttl values must be a value greater than 0
+    # - optional, default: -1 (use global default)
+    # - ttl values must be a value greater than or equal to 0
+    # - a ttl of -1 will use the global TTL value as the default
+    # - a ttl of 0 will mean never unload
     # - a value of 0 disables automatic unloading of the model
     ttl: 60
 
@@ -272,11 +287,11 @@ models:
     # - optional, default: ""
     # - useful for when the upstream server expects a specific model name that
     #   is different from the model's ID
-    useModelName: "qwen:qwq"
+    useModelName: "openai/gpt-oss-120B"
 
     # filters: a dictionary of filter settings
     # - optional, default: empty dictionary
-    # - only stripParams is currently supported
+    # - same capabilities as peer filters (stripParams, setParams)
     filters:
       # stripParams: a comma separated list of parameters to remove from the request
       # - optional, default: ""
@@ -286,6 +301,43 @@ models:
       # - recommended to stick to sampling parameters
       stripParams: "temperature, top_p, top_k"
 
+      # setParams: a dictionary of parameters to set/override in requests
+      # - optional, default: empty dictionary
+      # - useful for enforcing specific parameter values
+      # - protected params like "model" cannot be overridden
+      # - values can be strings, numbers, booleans, arrays, or objects
+      # - always runs for the model
+      setParams:
+        # Example: enforce specific sampling parameters
+        temperature: 0.7
+        top_p: 0.9
+
+      # setParamsByID: a dictionary of parameters to set based the model ID
+      # - optional, default: empty dictionary
+      # - combine with aliases to create variant behaviour without reloading the model
+      # - parameters are set in the request body JSON
+      # - run after setParams so it will override any settings
+      # - protected params like "model" cannot be overridden
+      # - values can be strings, numbers, booleans, arrays, or objects
+      # - model aliases will be automatically created for each key
+      setParamsByID:
+        "${MODEL_ID}":
+          chat_template_kwargs:
+            reasoning_effort: medium
+        "${MODEL_ID}:high":
+          chat_template_kwargs:
+            reasoning_effort: high
+        "${MODEL_ID}:low":
+          chat_template_kwargs:
+            reasoning_effort: low
+
+    # aliases: alternative model names that this model configuration is used for
+    # - optional, default: empty array
+    # - aliases must be unique globally
+    # - useful for impersonating a specific model
+    aliases:
+      - "gpt-4o-mini"
+
     # metadata: a dictionary of arbitrary values that are included in /v1/models
     # - optional, default: empty dictionary
     # - while metadata can contains complex types it is recommended to keep it simple
@@ -323,6 +375,22 @@ models:
     # - optional, default: undefined (use global setting)
     sendLoadingState: false
 
+    # timeouts: configure proxy connection timeouts for this model
+    # - optional, defaults shown below
+    # - useful for models running on slower hardware that need longer timeouts
+    # - connect: TCP dial connection timeout in seconds, default: 30 seconds
+    # - keepalive: TCP connection keepalive timeout, default: 30 seconds
+    # - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
+    # - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
+    # - idleConn: idle connection timeout in seconds, default: 90 seconds
+    # - set any value to 0 to disable that timeout (not recommended)
+    timeouts:
+      connect: 30
+      keepalive: 0
+      responseHeader: 60
+      tlsHandshake: 10
+      idleConn: 90
+
   # Unlisted model example:
   "qwen-unlisted":
     # unlisted: boolean, true or false
@@ -354,68 +422,83 @@ models:
     # - processes have 5 seconds to shutdown until forceful termination is attempted
     cmdStop: docker stop ${MODEL_ID}
 
-# groups: a dictionary of group settings
-# - optional, default: empty dictionary
-# - provides advanced controls over model swapping behaviour
-# - using groups some models can be kept loaded indefinitely, while others are swapped out
-# - model IDs must be defined in the Models section
-# - a model can only be a member of one group
-# - group behaviour is controlled via the `swap`, `exclusive` and `persistent` fields
-# - see issue #109 for details
+# =============================================================================
+# matrix: run concurrent models with a solver-based swap DSL
+# =============================================================================
 #
-# NOTE: the example below uses model names that are not defined above for demonstration purposes
-groups:
-  # group1 works the same as the default behaviour of llama-swap where only one model is allowed
-  # to run a time across the whole llama-swap instance
-  "group1":
-    # swap: controls the model swapping behaviour in within the group
-    # - optional, default: true
-    # - true : only one model is allowed to run at a time
-    # - false: all models can run together, no swapping
-    swap: true
+# Note:
+# A config must use either a matrix or legacy groups, not both. A configuration error
+# will occur if both are defined. Configuration examples for legacy Groups can be found:
+# https://github.com/mostlygeek/llama-swap/blob/40e39f7/config.example.yaml#L334-L396
+#
+# The matrix declares valid combinations of models that can run concurrently.
+# When a model is requested, the solver finds the cheapest way to make it
+# available by evicting as few (and least costly) running models as possible.
+#
+# Solver behavior:
+#   1. Request arrives for model X
+#   2. If X is already running, forward immediately. Done.
+#   3. Find all sets containing X
+#   4. For each candidate set, compute cost: sum of evict_costs for
+#      every running model NOT in that set
+#   5. Pick lowest cost candidate. Ties broken by definition order.
+#   6. Evict what needs to stop. Start X. Forward request.
+#
+# Subset semantics: a set [a, b, c] means any subset is valid.
+# Only the requested model is started — others are not preloaded.
+#
+# A model not appearing in any set can only run alone.
+#
+matrix:
+  # vars: short names for models (alphanumeric, 1-8 chars)
+  # - required for sets and evict_costs settings
+  # - each entry is a short name to a real model ID. Do not use an alias
+  # - used to keep set DSL logic short and easier to read
+  # - sets and evict_costs only use identifiers defined in vars
+  vars:
+    g: gemma-model
+    q: qwen-model
+    m: mistral-model
+    v: voxtral-model
+    e: reranker-model
+    L: llama-70B
+    sd: stable-diffusion
 
-    # exclusive: controls how the group affects other groups
-    # - optional, default: true
-    # - true: causes all other groups to unload when this group runs a model
-    # - false: does not affect other groups
-    exclusive: true
+  # evict_costs: relative cost of losing a running model (default: 1)
+  evict_costs:
+    v: 50 # vllm backend, slow cold start
+    L: 30 # 70B weights, slow to load
 
-    # members references the models defined above
-    # required
-    members:
-      - "llama"
-      - "qwen-unlisted"
+  # sets: named sets of concurrent model combinations
+  # Values are DSL strings with operators:
+  #   &     AND (models run together)
+  #   |     OR  (alternatives)
+  #   ()    grouping
+  #   +ref  inline another set's expression
+  #
+  # Expansion examples:
+  #   "L"                  → [L]
+  #   "a & b"              → [a, b]
+  #   "a | b"              → [a], [b]
+  #   "(a | b) & c"        → [a, c], [b, c]
+  #   "(a | b) & (c | d)"  → [a,c], [a,d], [b,c], [b,d]
+  #   "+llms & v"          → expands llms inline, then applies & v
+  sets:
+    # LLM + TTS: switching between g/q/m won't evict v
+    # expands to: [g,v], [q,v], [m,v]
+    standard: "(g | q | m) & v"
 
-  # Example:
-  # - in group2 all models can run at the same time
-  # - when a different group is loaded it causes all running models in this group to unload
-  "group2":
-    swap: false
+    # LLM + TTS + reranker
+    # expands to: [g,v,e], [q,v,e]
+    with_rerank: "(g | q) & v & e"
 
-    # exclusive: false does not unload other groups when a model in group2 is requested
-    # - the models in group2 will be loaded but will not unload any other groups
-    exclusive: false
-    members:
-      - "docker-llama"
-      - "modelA"
-      - "modelB"
+    # LLM + image generation, no TTS
+    # expands to: [g,sd], [q,sd]
+    creative: "(g | q) & sd"
 
-  # Example:
-  # - a persistent group, prevents other groups from unloading it
-  "forever":
-    # persistent: prevents over groups from unloading the models in this group
-    # - optional, default: false
-    # - does not affect individual model behaviour
-    persistent: true
-
-    # set swap/exclusive to false to prevent swapping inside the group
-    # and the unloading of other groups
-    swap: false
-    exclusive: false
-    members:
-      - "forever-modelA"
-      - "forever-modelB"
-      - "forever-modelc"
+    # 70B model uses all GPUs, can only run alone
+    # expands to: [L]
+    full: "L"
 
 # hooks: a dictionary of event triggers and actions
 # - optional, default: empty dictionary
@@ -456,7 +539,8 @@ peers:
     # - optional, default: ""
     # - if blank, no key will be added to the request
     # - key will be injected into headers: Authorization: Bearer <key> and x-api-key: <key>
-    apiKey: sk-your-openrouter-key
+    # - can be a string or a macro
+    apiKey: ${env.OPENROUTER_API_KEY}
     models:
       - meta-llama/llama-3.1-8b-instruct
       - qwen/qwen3-235b-a22b-2507
@@ -464,4 +548,35 @@ peers:
       - z-ai/glm-4.7
       - moonshotai/kimi-k2-0905
       - minimax/minimax-m2.1
+    # timeouts: configure proxy connection timeouts for this peer
+    # - optional, defaults shown below
+    # - useful when the peer runs on slower hardware
+    # - set any value to 0 to disable that timeout (not recommended)
+    timeouts:
+      connect: 30
+      keepalive: 30
+      responseHeader: 60
+      tlsHandshake: 10
+      idleConn: 90
+
+    # filters: a dictionary of filter settings for peer requests
+    # - optional, default: empty dictionary
+    # - same capabilities as model filters (stripParams, setParams)
+    filters:
+      # stripParams: a comma separated list of parameters to remove from the request
+      # - optional, default: ""
+      # - useful for removing parameters that the peer doesn't support
+      # - the `model` parameter can never be removed
+      stripParams: "temperature, top_p"
+
+      # setParams: a dictionary of parameters to set/override in requests to this peer
+      # - optional, default: empty dictionary
+      # - useful for injecting provider-specific settings like data retention policies
+      # - protected params like "model" cannot be overridden
+      # - values can be strings, numbers, booleans, arrays, or objects
+      setParams:
+        # Example: enforce zero-data-retention for OpenRouter
+        provider:
+          data_collection: "deny"
+          zdr: true
 ```

go.mod

@@ -1,6 +1,6 @@
 module github.com/mostlygeek/llama-swap
 
-go 1.25.4
+go 1.26.1
 
 require (
 	github.com/billziss-gh/golib v0.2.0

proxy/config/config.go

@@ -124,10 +124,17 @@ type Config struct {
 	LogToStdout        string                 `yaml:"logToStdout"`
 	MetricsMaxInMemory int                    `yaml:"metricsMaxInMemory"`
 	CaptureBuffer      int                    `yaml:"captureBuffer"`
+	GlobalTTL          int                    `yaml:"globalTTL"`
 	Models             map[string]ModelConfig `yaml:"models"` /* key is model ID */
 	Profiles           map[string][]string    `yaml:"profiles"`
 	Groups             map[string]GroupConfig `yaml:"groups"` /* key is group ID */
 
+	// swap matrix: solver-based alternative to groups
+	Matrix *MatrixConfig `yaml:"matrix"`
+
+	// populated during validation when matrix is configured
+	ExpandedSets []ExpandedSet `yaml:"-"`
+
 	// for key/value replacements in model's cmd, cmdStop, proxy, checkEndPoint
 	Macros MacroList `yaml:"macros"`
 
@@ -203,6 +210,7 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 		LogToStdout:        LogToStdoutProxy,
 		MetricsMaxInMemory: 1000,
 		CaptureBuffer:      5,
+		GlobalTTL:          0,
 	}
 	if err = yaml.Unmarshal([]byte(yamlStr), &config); err != nil {
 		return Config{}, err
@@ -216,6 +224,10 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 		return Config{}, fmt.Errorf("startPort must be greater than 1")
 	}
 
+	if config.GlobalTTL < 0 {
+		return Config{}, fmt.Errorf("globalTTL must be >= 0")
+	}
+
 	switch config.LogToStdout {
 	case LogToStdoutProxy, LogToStdoutUpstream, LogToStdoutBoth, LogToStdoutNone:
 	default:
@@ -255,6 +267,15 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 		modelConfig.Cmd = StripComments(modelConfig.Cmd)
 		modelConfig.CmdStop = StripComments(modelConfig.CmdStop)
 
+		// set model TTL to globalTTL it is the default value
+		if modelConfig.UnloadAfter == MODEL_CONFIG_DEFAULT_TTL {
+			modelConfig.UnloadAfter = config.GlobalTTL
+		}
+
+		if modelConfig.UnloadAfter < 0 {
+			return Config{}, fmt.Errorf("model %s: invalid TTL value %d", modelId, modelConfig.UnloadAfter)
+		}
+
 		// Validate model macros
 		for _, macro := range modelConfig.Macros {
 			if err = validateMacro(macro.Name, macro.Value); err != nil {
@@ -293,6 +314,26 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 			modelConfig.Proxy = strings.ReplaceAll(modelConfig.Proxy, macroSlug, macroStr)
 			modelConfig.CheckEndpoint = strings.ReplaceAll(modelConfig.CheckEndpoint, macroSlug, macroStr)
 			modelConfig.Filters.StripParams = strings.ReplaceAll(modelConfig.Filters.StripParams, macroSlug, macroStr)
+			modelConfig.Name = strings.ReplaceAll(modelConfig.Name, macroSlug, macroStr)
+			modelConfig.Description = strings.ReplaceAll(modelConfig.Description, macroSlug, macroStr)
+
+			// Substitute macros in SetParamsByID keys and values
+			if len(modelConfig.Filters.SetParamsByID) > 0 {
+				newSetParamsByID := make(map[string]map[string]any, len(modelConfig.Filters.SetParamsByID))
+				for key, paramMap := range modelConfig.Filters.SetParamsByID {
+					newKey := strings.ReplaceAll(key, macroSlug, macroStr)
+					newValAny, err := substituteMacroInValue(any(paramMap), entry.Name, entry.Value)
+					if err != nil {
+						return Config{}, fmt.Errorf("model %s filters.setParamsByID: %s", modelId, err.Error())
+					}
+					newParamMap, ok := newValAny.(map[string]any)
+					if !ok {
+						return Config{}, fmt.Errorf("model %s filters.setParamsByID: unexpected type after macro substitution", modelId)
+					}
+					newSetParamsByID[newKey] = newParamMap
+				}
+				modelConfig.Filters.SetParamsByID = newSetParamsByID
+			}
 
 			// Substitute in metadata (type-preserving)
 			if len(modelConfig.Metadata) > 0 {
@@ -318,6 +359,8 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 			modelConfig.Cmd = strings.ReplaceAll(modelConfig.Cmd, macroSlug, macroStr)
 			modelConfig.CmdStop = strings.ReplaceAll(modelConfig.CmdStop, macroSlug, macroStr)
 			modelConfig.Proxy = strings.ReplaceAll(modelConfig.Proxy, macroSlug, macroStr)
+			modelConfig.Name = strings.ReplaceAll(modelConfig.Name, macroSlug, macroStr)
+			modelConfig.Description = strings.ReplaceAll(modelConfig.Description, macroSlug, macroStr)
 
 			if len(modelConfig.Metadata) > 0 {
 				result, err := substituteMacroInValue(modelConfig.Metadata, "PORT", nextPort)
@@ -337,6 +380,8 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 			"proxy":               modelConfig.Proxy,
 			"checkEndpoint":       modelConfig.CheckEndpoint,
 			"filters.stripParams": modelConfig.Filters.StripParams,
+			"name":                modelConfig.Name,
+			"description":         modelConfig.Description,
 		}
 
 		for fieldName, fieldValue := range fieldMap {
@@ -359,6 +404,34 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 			}
 		}
 
+		// Validate SetParamsByID keys and values
+		for key, paramMap := range modelConfig.Filters.SetParamsByID {
+			if matches := macroPatternRegex.FindAllStringSubmatch(key, -1); len(matches) > 0 {
+				return Config{}, fmt.Errorf("unknown macro '${%s}' found in model %s filters.setParamsByID key", matches[0][1], modelId)
+			}
+			if err := validateNestedForUnknownMacros(any(paramMap), fmt.Sprintf("model %s filters.setParamsByID[%s]", modelId, key)); err != nil {
+				return Config{}, err
+			}
+		}
+
+		// Auto-register setParamsByID keys as aliases (skip the model's own ID)
+		for key := range modelConfig.Filters.SetParamsByID {
+			if key == modelId {
+				continue
+			}
+			if _, exists := config.Models[key]; exists {
+				return Config{}, fmt.Errorf("model %s filters.setParamsByID: key '%s' conflicts with an existing model ID", modelId, key)
+			}
+			if existingModel, exists := config.aliases[key]; exists {
+				if existingModel != modelId {
+					return Config{}, fmt.Errorf("duplicate alias '%s' in model %s filters.setParamsByID, already used by model %s", key, modelId, existingModel)
+				}
+				continue // already registered as explicit alias for this model
+			}
+			config.aliases[key] = modelId
+			modelConfig.Aliases = append(modelConfig.Aliases, key)
+		}
+
 		if _, err := url.Parse(modelConfig.Proxy); err != nil {
 			return Config{}, fmt.Errorf("model %s: invalid proxy URL: %w", modelId, err)
 		}
@@ -371,22 +444,35 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 		config.Models[modelId] = modelConfig
 	}
 
-	config = AddDefaultGroupToConfig(config)
+	// groups XOR matrix
+	if config.Matrix != nil && len(config.Groups) > 0 {
+		return Config{}, fmt.Errorf("config cannot use both 'groups' and 'matrix'")
+	}
 
-	// Validate group members
-	memberUsage := make(map[string]string)
-	for groupID, groupConfig := range config.Groups {
-		prevSet := make(map[string]bool)
-		for _, member := range groupConfig.Members {
-			if _, found := prevSet[member]; found {
-				return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID)
-			}
-			prevSet[member] = true
+	if config.Matrix != nil {
+		expandedSets, err := ValidateMatrix(*config.Matrix, config.Models)
+		if err != nil {
+			return Config{}, fmt.Errorf("matrix: %w", err)
+		}
+		config.ExpandedSets = expandedSets
+	} else {
+		config = AddDefaultGroupToConfig(config)
 
-			if existingGroup, exists := memberUsage[member]; exists {
-				return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID)
+		// Validate group members
+		memberUsage := make(map[string]string)
+		for groupID, groupConfig := range config.Groups {
+			prevSet := make(map[string]bool)
+			for _, member := range groupConfig.Members {
+				if _, found := prevSet[member]; found {
+					return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID)
+				}
+				prevSet[member] = true
+
+				if existingGroup, exists := memberUsage[member]; exists {
+					return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID)
+				}
+				memberUsage[member] = groupID
 			}
-			memberUsage[member] = groupID
 		}
 	}
 

proxy/config/config_posix_test.go

@@ -163,6 +163,15 @@ groups:
 
 	modelLoadingState := false
 
+	defaultTimeout := TimeoutsConfig{
+		Connect:        30,
+		KeepAlive:      30,
+		ResponseHeader: 0,
+		TLSHandshake:   10,
+		ExpectContinue: 1,
+		IdleConn:       90,
+	}
+
 	expected := Config{
 		LogLevel:      "info",
 		LogTimeFormat: "",
@@ -187,6 +196,7 @@ groups:
 				Name:             "Model 1",
 				Description:      "This is model 1",
 				SendLoadingState: &modelLoadingState,
+				Timeouts:         defaultTimeout,
 			},
 			"model2": {
 				Cmd:              "path/to/server --arg1 one",
@@ -195,6 +205,7 @@ groups:
 				Env:              []string{},
 				CheckEndpoint:    "/",
 				SendLoadingState: &modelLoadingState,
+				Timeouts:         defaultTimeout,
 			},
 			"model3": {
 				Cmd:              "path/to/cmd --arg1 one",
@@ -203,6 +214,7 @@ groups:
 				Env:              []string{},
 				CheckEndpoint:    "/",
 				SendLoadingState: &modelLoadingState,
+				Timeouts:         defaultTimeout,
 			},
 			"model4": {
 				Cmd:              "path/to/cmd --arg1 one",
@@ -211,6 +223,7 @@ groups:
 				Aliases:          []string{},
 				Env:              []string{},
 				SendLoadingState: &modelLoadingState,
+				Timeouts:         defaultTimeout,
 			},
 		},
 		HealthCheckTimeout: 15,

proxy/config/config_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestConfig_GroupMemberIsUnique(t *testing.T) {
@@ -848,6 +849,71 @@ func TestConfig_APIKeys_EnvMacros(t *testing.T) {
 	})
 }
 
+func TestConfig_GlobalTTL(t *testing.T) {
+	t.Run("globalTTL sets default for models", func(t *testing.T) {
+		content := `
+globalTTL: 300
+models:
+  model1:
+    cmd: server --port ${PORT}
+`
+		config, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.NoError(t, err)
+		assert.Equal(t, 300, config.GlobalTTL)
+		assert.Equal(t, 300, config.Models["model1"].UnloadAfter)
+	})
+
+	t.Run("model ttl=0 overrides globalTTL", func(t *testing.T) {
+		content := `
+globalTTL: 300
+models:
+  model1:
+    cmd: server --port ${PORT}
+    ttl: 0
+`
+		config, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.NoError(t, err)
+		assert.Equal(t, 0, config.Models["model1"].UnloadAfter)
+	})
+
+	t.Run("model explicit ttl overrides globalTTL", func(t *testing.T) {
+		content := `
+globalTTL: 300
+models:
+  model1:
+    cmd: server --port ${PORT}
+    ttl: 600
+`
+		config, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.NoError(t, err)
+		assert.Equal(t, 600, config.Models["model1"].UnloadAfter)
+	})
+
+	t.Run("globalTTL defaults to 0", func(t *testing.T) {
+		content := `
+models:
+  model1:
+    cmd: server --port ${PORT}
+`
+		config, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.NoError(t, err)
+		assert.Equal(t, 0, config.GlobalTTL)
+		assert.Equal(t, 0, config.Models["model1"].UnloadAfter)
+	})
+
+	t.Run("negative globalTTL rejected", func(t *testing.T) {
+		content := `
+globalTTL: -1
+models:
+  model1:
+    cmd: server --port ${PORT}
+`
+		_, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "globalTTL must be >= 0")
+	})
+}
+
 func TestConfig_EnvMacros(t *testing.T) {
 	t.Run("basic env substitution in cmd", func(t *testing.T) {
 		t.Setenv("TEST_MODEL_PATH", "/opt/models")
@@ -1373,3 +1439,108 @@ models:
 	})
 
 }
+
+func TestConfig_TimeoutsParsing(t *testing.T) {
+	configYaml := `
+models:
+  model1:
+    cmd: test-server --port ${PORT}
+    timeouts:
+      connect: 45
+      responseHeader: 120
+`
+
+	config, err := LoadConfigFromReader(strings.NewReader(configYaml))
+	require.NoError(t, err)
+
+	modelConfig, found := config.Models["model1"]
+	require.True(t, found, "model1 should exist in config")
+
+	assert.Equal(t, 45, modelConfig.Timeouts.Connect)
+	assert.Equal(t, 120, modelConfig.Timeouts.ResponseHeader)
+}
+
+func TestConfig_TimeoutsDefaults(t *testing.T) {
+	configYaml := `
+models:
+  model1:
+    cmd: test-server --port ${PORT}
+`
+
+	config, err := LoadConfigFromReader(strings.NewReader(configYaml))
+	require.NoError(t, err)
+
+	modelConfig, found := config.Models["model1"]
+	require.True(t, found, "model1 should exist in config")
+
+	// Default values should be set during unmarshaling
+	assert.Equal(t, 30, modelConfig.Timeouts.Connect)
+	assert.Equal(t, 0, modelConfig.Timeouts.ResponseHeader)
+	assert.Equal(t, 10, modelConfig.Timeouts.TLSHandshake)
+	assert.Equal(t, 1, modelConfig.Timeouts.ExpectContinue)
+	assert.Equal(t, 90, modelConfig.Timeouts.IdleConn)
+}
+
+func TestConfig_TimeoutsZeroAllowed(t *testing.T) {
+	configYaml := `
+models:
+  model1:
+    cmd: test-server --port ${PORT}
+    timeouts:
+      connect: 0
+      responseHeader: 0
+`
+
+	config, err := LoadConfigFromReader(strings.NewReader(configYaml))
+	require.NoError(t, err)
+
+	modelConfig, found := config.Models["model1"]
+	require.True(t, found, "model1 should exist in config")
+
+	// Explicit 0 should be preserved (disables timeout)
+	assert.Equal(t, 0, modelConfig.Timeouts.Connect)
+	assert.Equal(t, 0, modelConfig.Timeouts.ResponseHeader)
+}
+
+func TestConfig_PeerTimeoutsParsing(t *testing.T) {
+	configYaml := `
+peers:
+  peer1:
+    proxy: http://example.com
+    models: [model1]
+    timeouts:
+      connect: 45
+      responseHeader: 120
+`
+
+	config, err := LoadConfigFromReader(strings.NewReader(configYaml))
+	require.NoError(t, err)
+
+	peerConfig, found := config.Peers["peer1"]
+	require.True(t, found, "peer1 should exist in config")
+
+	assert.Equal(t, 45, peerConfig.Timeouts.Connect)
+	assert.Equal(t, 120, peerConfig.Timeouts.ResponseHeader)
+}
+
+func TestConfig_PeerTimeoutsDefaults(t *testing.T) {
+	configYaml := `
+peers:
+  peer1:
+    proxy: http://example.com
+    models: [model1]
+`
+
+	config, err := LoadConfigFromReader(strings.NewReader(configYaml))
+	require.NoError(t, err)
+
+	peerConfig, found := config.Peers["peer1"]
+	require.True(t, found, "peer1 should exist in config")
+
+	// Default values should be set during unmarshaling
+	assert.Equal(t, 30, peerConfig.Timeouts.Connect)
+	assert.Equal(t, 60, peerConfig.Timeouts.ResponseHeader)
+	assert.Equal(t, 10, peerConfig.Timeouts.TLSHandshake)
+	assert.Equal(t, 1, peerConfig.Timeouts.ExpectContinue)
+	assert.Equal(t, 90, peerConfig.Timeouts.IdleConn)
+}

proxy/config/config_windows_test.go

@@ -155,6 +155,15 @@ groups:
 
 	modelLoadingState := false
 
+	defaultTimeout := TimeoutsConfig{
+		Connect:        30,
+		KeepAlive:      30,
+		ResponseHeader: 0,
+		TLSHandshake:   10,
+		ExpectContinue: 1,
+		IdleConn:       90,
+	}
+
 	expected := Config{
 		LogLevel:      "info",
 		LogTimeFormat: "",
@@ -173,6 +182,7 @@ groups:
 				Env:              []string{"VAR1=value1", "VAR2=value2"},
 				CheckEndpoint:    "/health",
 				SendLoadingState: &modelLoadingState,
+				Timeouts:         defaultTimeout,
 			},
 			"model2": {
 				Cmd:              "path/to/server --arg1 one",
@@ -182,6 +192,7 @@ groups:
 				Env:              []string{},
 				CheckEndpoint:    "/",
 				SendLoadingState: &modelLoadingState,
+				Timeouts:         defaultTimeout,
 			},
 			"model3": {
 				Cmd:              "path/to/cmd --arg1 one",
@@ -191,6 +202,7 @@ groups:
 				Env:              []string{},
 				CheckEndpoint:    "/",
 				SendLoadingState: &modelLoadingState,
+				Timeouts:         defaultTimeout,
 			},
 			"model4": {
 				Cmd:              "path/to/cmd --arg1 one",
@@ -200,6 +212,7 @@ groups:
 				Aliases:          []string{},
 				Env:              []string{},
 				SendLoadingState: &modelLoadingState,
+				Timeouts:         defaultTimeout,
 			},
 		},
 		HealthCheckTimeout: 15,

proxy/config/filters.go

@@ -20,6 +20,12 @@ type Filters struct {
 	// SetParams is a dictionary of parameters to set/override in requests
 	// Protected params (like "model") cannot be set
 	SetParams map[string]any `yaml:"setParams"`
+
+	// SetParamsByID maps requested model IDs to parameters to set/override in requests.
+	// Useful with aliases: a single loaded model can behave differently depending on
+	// which alias the client used. Applied after SetParams, so it can override those values.
+	// Protected params (like "model") cannot be set.
+	SetParamsByID map[string]map[string]any `yaml:"setParamsByID"`
 }
 
 // SanitizedStripParams returns a sorted list of parameters to strip,
@@ -51,6 +57,33 @@ func (f Filters) SanitizedStripParams() []string {
 	return cleaned
 }
 
+// SanitizedSetParamsByID returns the params to set for the given requestedModelID,
+// with protected params removed and keys sorted for consistent iteration order.
+// Returns nil if the ID has no entry or all its params are protected.
+func (f Filters) SanitizedSetParamsByID(requestedModelID string) (map[string]any, []string) {
+	if len(f.SetParamsByID) == 0 {
+		return nil, nil
+	}
+	params, found := f.SetParamsByID[requestedModelID]
+	if !found || len(params) == 0 {
+		return nil, nil
+	}
+	result := make(map[string]any, len(params))
+	keys := make([]string, 0, len(params))
+	for key, value := range params {
+		if slices.Contains(ProtectedParams, key) {
+			continue
+		}
+		result[key] = value
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	if len(result) == 0 {
+		return nil, nil
+	}
+	return result, keys
+}
+
 // SanitizedSetParams returns a copy of SetParams with protected params removed
 // and keys sorted for consistent iteration order
 func (f Filters) SanitizedSetParams() (map[string]any, []string) {

proxy/config/filters_test.go

@@ -162,6 +162,123 @@ func TestFilters_SanitizedSetParams(t *testing.T) {
 	}
 }
 
+func TestFilters_SanitizedSetParamsByID(t *testing.T) {
+	tests := []struct {
+		name             string
+		setParamsByID    map[string]map[string]any
+		requestedModelID string
+		wantParams       map[string]any
+		wantKeys         []string
+	}{
+		{
+			name:             "empty SetParamsByID returns nil",
+			setParamsByID:    nil,
+			requestedModelID: "model1",
+			wantParams:       nil,
+			wantKeys:         nil,
+		},
+		{
+			name:             "empty map returns nil",
+			setParamsByID:    map[string]map[string]any{},
+			requestedModelID: "model1",
+			wantParams:       nil,
+			wantKeys:         nil,
+		},
+		{
+			name: "non-matching model ID returns nil",
+			setParamsByID: map[string]map[string]any{
+				"model2": {"temperature": 0.9},
+			},
+			requestedModelID: "model1",
+			wantParams:       nil,
+			wantKeys:         nil,
+		},
+		{
+			name: "matching model ID returns correct params",
+			setParamsByID: map[string]map[string]any{
+				"model1": {"temperature": 0.7, "top_p": 0.9},
+				"model2": {"temperature": 0.5},
+			},
+			requestedModelID: "model1",
+			wantParams: map[string]any{
+				"temperature": 0.7,
+				"top_p":       0.9,
+			},
+			wantKeys: []string{"temperature", "top_p"},
+		},
+		{
+			name: "protected param model is filtered out",
+			setParamsByID: map[string]map[string]any{
+				"model1": {
+					"model":       "should-be-filtered",
+					"temperature": 0.7,
+				},
+			},
+			requestedModelID: "model1",
+			wantParams: map[string]any{
+				"temperature": 0.7,
+			},
+			wantKeys: []string{"temperature"},
+		},
+		{
+			name: "only protected param returns nil",
+			setParamsByID: map[string]map[string]any{
+				"model1": {
+					"model": "should-be-filtered",
+				},
+			},
+			requestedModelID: "model1",
+			wantParams:       nil,
+			wantKeys:         nil,
+		},
+		{
+			name: "keys are sorted",
+			setParamsByID: map[string]map[string]any{
+				"model1": {
+					"z_param": "z",
+					"a_param": "a",
+					"m_param": "m",
+				},
+			},
+			requestedModelID: "model1",
+			wantParams: map[string]any{
+				"z_param": "z",
+				"a_param": "a",
+				"m_param": "m",
+			},
+			wantKeys: []string{"a_param", "m_param", "z_param"},
+		},
+		{
+			name: "alias style key lookup",
+			setParamsByID: map[string]map[string]any{
+				"model1:high": {"reasoning_effort": "high"},
+				"model1:low":  {"reasoning_effort": "low"},
+			},
+			requestedModelID: "model1:high",
+			wantParams: map[string]any{
+				"reasoning_effort": "high",
+			},
+			wantKeys: []string{"reasoning_effort"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := Filters{SetParamsByID: tt.setParamsByID}
+			gotParams, gotKeys := f.SanitizedSetParamsByID(tt.requestedModelID)
+
+			if tt.wantParams == nil {
+				assert.Nil(t, gotParams)
+				assert.Nil(t, gotKeys)
+				return
+			}
+
+			assert.Equal(t, tt.wantKeys, gotKeys)
+			assert.Equal(t, tt.wantParams, gotParams)
+		})
+	}
+}
+
 func TestProtectedParams(t *testing.T) {
 	// Verify that "model" is protected
 	assert.Contains(t, ProtectedParams, "model")

proxy/config/macro_in_macro_test.go

@@ -104,6 +104,62 @@ models:
 	assert.Contains(t, err.Error(), "self-reference")
 }
 
+// Test macro substitution in name and description fields
+func TestConfig_MacroInNameAndDescription(t *testing.T) {
+	content := `
+startPort: 10000
+macros:
+  "VARIANT": "Q4_K_M"
+  "FAMILY": "llama"
+
+models:
+  my-model:
+    cmd: echo ok
+    proxy: http://localhost:8080
+    name: "${FAMILY} ${VARIANT}"
+    description: "A ${FAMILY} model in ${VARIANT} format"
+`
+
+	config, err := LoadConfigFromReader(strings.NewReader(content))
+	assert.NoError(t, err)
+	assert.Equal(t, "llama Q4_K_M", config.Models["my-model"].Name)
+	assert.Equal(t, "A llama model in Q4_K_M format", config.Models["my-model"].Description)
+}
+
+// Test MODEL_ID macro in name and description fields
+func TestConfig_ModelIDInNameAndDescription(t *testing.T) {
+	content := `
+startPort: 10000
+models:
+  llama-3b:
+    cmd: echo ok
+    proxy: http://localhost:8080
+    name: "Model: ${MODEL_ID}"
+    description: "Running ${MODEL_ID}"
+`
+
+	config, err := LoadConfigFromReader(strings.NewReader(content))
+	assert.NoError(t, err)
+	assert.Equal(t, "Model: llama-3b", config.Models["llama-3b"].Name)
+	assert.Equal(t, "Running llama-3b", config.Models["llama-3b"].Description)
+}
+
+// Test unknown macro in name or description returns an error
+func TestConfig_UnknownMacroInNameDescription(t *testing.T) {
+	content := `
+startPort: 10000
+models:
+  test:
+    cmd: echo ok
+    proxy: http://localhost:8080
+    name: "Model ${UNDEFINED}"
+`
+
+	_, err := LoadConfigFromReader(strings.NewReader(content))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "UNDEFINED")
+}
+
 // Test undefined macro reference error
 func TestConfig_UndefinedMacroReference(t *testing.T) {
 	content := `

proxy/config/matrix.go

@@ -0,0 +1,226 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+
+	"gopkg.in/yaml.v3"
+)
+
+var varKeyPattern = regexp.MustCompile(`^[a-zA-Z0-9]{1,8}$`)
+
+// MatrixConfig represents the swap matrix configuration block.
+type MatrixConfig struct {
+	Var        map[string]string `yaml:"vars"`
+	EvictCosts map[string]int    `yaml:"evict_costs"`
+	Sets       OrderedSets       `yaml:"sets"`
+}
+
+// SetEntry is a single named set with its DSL expression.
+type SetEntry struct {
+	Name string
+	DSL  string
+}
+
+// OrderedSets preserves YAML definition order of sets (used for tie-breaking).
+type OrderedSets []SetEntry
+
+func (os *OrderedSets) UnmarshalYAML(value *yaml.Node) error {
+	if value.Kind != yaml.MappingNode {
+		return fmt.Errorf("sets must be a mapping")
+	}
+
+	entries := make([]SetEntry, 0, len(value.Content)/2)
+	for i := 0; i < len(value.Content); i += 2 {
+		keyNode := value.Content[i]
+		valueNode := value.Content[i+1]
+
+		var name string
+		if err := keyNode.Decode(&name); err != nil {
+			return fmt.Errorf("failed to decode set name: %w", err)
+		}
+
+		var dsl string
+		if err := valueNode.Decode(&dsl); err != nil {
+			return fmt.Errorf("failed to decode DSL for set %q: %w", name, err)
+		}
+
+		entries = append(entries, SetEntry{Name: name, DSL: dsl})
+	}
+
+	*os = entries
+	return nil
+}
+
+// ExpandedSet is one valid combination of concurrent models (real model names).
+type ExpandedSet struct {
+	SetName string
+	DSL     string
+	Models  []string // real model names, sorted
+}
+
+// ValidateMatrix validates the matrix config and returns all expanded sets.
+func ValidateMatrix(matrix MatrixConfig, models map[string]ModelConfig) ([]ExpandedSet, error) {
+	if len(matrix.Sets) == 0 {
+		return nil, fmt.Errorf("matrix must define at least one set")
+	}
+
+	if len(matrix.Var) == 0 {
+		return nil, fmt.Errorf("matrix must define at least one var")
+	}
+
+	// Validate var entries
+	if matrix.Var != nil {
+		for id, modelName := range matrix.Var {
+			if !varKeyPattern.MatchString(id) {
+				return nil, fmt.Errorf("var key %q must be alphanumeric and 1-8 characters", id)
+			}
+			if _, exists := models[modelName]; !exists {
+				return nil, fmt.Errorf("var key %q references unknown model %q", id, modelName)
+			}
+		}
+	}
+
+	// Validate evict_costs
+	if matrix.EvictCosts != nil {
+		for key, cost := range matrix.EvictCosts {
+			if cost <= 0 {
+				return nil, fmt.Errorf("evict_cost for %q must be a positive integer, got %d", key, cost)
+			}
+			if _, ok := matrix.Var[key]; !ok {
+				return nil, fmt.Errorf("evict_costs: unknown var ID %q", key)
+			}
+		}
+	}
+
+	// Build dependency graph for +ref topological sort
+	setNames := make(map[string]bool)
+	for _, entry := range matrix.Sets {
+		setNames[entry.Name] = true
+	}
+
+	deps := make(map[string][]string) // setName -> set names it depends on
+	for _, entry := range matrix.Sets {
+		refs, err := extractRefs(entry.DSL)
+		if err != nil {
+			return nil, fmt.Errorf("set %q: %w", entry.Name, err)
+		}
+		for _, ref := range refs {
+			if !setNames[ref] {
+				return nil, fmt.Errorf("set %q references undefined set %q", entry.Name, ref)
+			}
+		}
+		deps[entry.Name] = refs
+	}
+
+	// Topological sort with cycle detection
+	order, err := topologicalSort(matrix.Sets, deps)
+	if err != nil {
+		return nil, err
+	}
+
+	// Expand sets in topological order
+	resolvedRefs := make(map[string][][]string) // set name -> expanded alias-level combos
+	var allExpanded []ExpandedSet
+	totalCombinations := 0
+
+	// Build ordered map for efficient lookup
+	setDSL := make(map[string]string)
+	for _, entry := range matrix.Sets {
+		setDSL[entry.Name] = entry.DSL
+	}
+
+	for _, name := range order {
+		dsl := setDSL[name]
+		combos, err := ParseAndExpandDSL(dsl, resolvedRefs)
+		if err != nil {
+			return nil, fmt.Errorf("set %q: %w", name, err)
+		}
+
+		resolvedRefs[name] = combos
+
+		// Resolve var IDs to real model names
+		for _, combo := range combos {
+			resolved := make([]string, len(combo))
+			for i, ident := range combo {
+				realName, ok := matrix.Var[ident]
+				if !ok {
+					return nil, fmt.Errorf("set %q: unknown var ID %q", name, ident)
+				}
+				resolved[i] = realName
+			}
+			sort.Strings(resolved)
+			allExpanded = append(allExpanded, ExpandedSet{
+				SetName: name,
+				DSL:     dsl,
+				Models:  resolved,
+			})
+		}
+
+		totalCombinations += len(combos)
+		if totalCombinations > maxDSLExpansions {
+			return nil, fmt.Errorf("total expanded combinations (%d) exceed limit of %d", totalCombinations, maxDSLExpansions)
+		}
+	}
+
+	return allExpanded, nil
+}
+
+// topologicalSort returns set names in dependency order.
+// Returns an error if a cycle is detected.
+func topologicalSort(sets OrderedSets, deps map[string][]string) ([]string, error) {
+	// States: 0 = unvisited, 1 = visiting, 2 = visited
+	state := make(map[string]int)
+	var order []string
+
+	var visit func(name string) error
+	visit = func(name string) error {
+		switch state[name] {
+		case 1:
+			return fmt.Errorf("circular reference detected involving set %q", name)
+		case 2:
+			return nil
+		}
+		state[name] = 1
+
+		for _, dep := range deps[name] {
+			if err := visit(dep); err != nil {
+				return err
+			}
+		}
+
+		state[name] = 2
+		order = append(order, name)
+		return nil
+	}
+
+	// Visit in definition order for deterministic output
+	for _, entry := range sets {
+		if state[entry.Name] == 0 {
+			if err := visit(entry.Name); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return order, nil
+}
+
+// ResolvedEvictCosts returns a map of real model name -> evict cost,
+// resolving var IDs. Models not listed default to 1.
+func (m *MatrixConfig) ResolvedEvictCosts() map[string]int {
+	costs := make(map[string]int)
+	if m.EvictCosts == nil {
+		return costs
+	}
+	for key, cost := range m.EvictCosts {
+		// Resolve var ID if present
+		if realName, ok := m.Var[key]; ok {
+			costs[realName] = cost
+		} else {
+			costs[key] = cost
+		}
+	}
+	return costs
+}

proxy/config/matrix_dsl.go

@@ -0,0 +1,376 @@
+package config
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"unicode"
+)
+
+const maxDSLExpansions = 1000
+
+// Token types for the DSL lexer
+type tokenType int
+
+const (
+	tokIdent  tokenType = iota // model alias or name
+	tokAnd                     // &
+	tokOr                      // |
+	tokLParen                  // (
+	tokRParen                  // )
+	tokRef                     // +setName
+	tokEOF
+)
+
+type token struct {
+	typ tokenType
+	val string
+}
+
+// tokenize splits a DSL string into tokens.
+func tokenize(input string) ([]token, error) {
+	var tokens []token
+	i := 0
+	runes := []rune(input)
+
+	for i < len(runes) {
+		ch := runes[i]
+
+		// skip whitespace
+		if unicode.IsSpace(ch) {
+			i++
+			continue
+		}
+
+		switch ch {
+		case '&':
+			tokens = append(tokens, token{tokAnd, "&"})
+			i++
+		case '|':
+			tokens = append(tokens, token{tokOr, "|"})
+			i++
+		case '(':
+			tokens = append(tokens, token{tokLParen, "("})
+			i++
+		case ')':
+			tokens = append(tokens, token{tokRParen, ")"})
+			i++
+		case '+':
+			// +ref: read the identifier that follows
+			i++
+			start := i
+			for i < len(runes) && isIdentChar(runes[i]) {
+				i++
+			}
+			if i == start {
+				return nil, fmt.Errorf("expected set name after '+' at position %d", start)
+			}
+			tokens = append(tokens, token{tokRef, string(runes[start:i])})
+		default:
+			if isIdentChar(ch) {
+				start := i
+				for i < len(runes) && isIdentChar(runes[i]) {
+					i++
+				}
+				tokens = append(tokens, token{tokIdent, string(runes[start:i])})
+			} else {
+				return nil, fmt.Errorf("unexpected character %q at position %d", ch, i)
+			}
+		}
+	}
+
+	tokens = append(tokens, token{tokEOF, ""})
+	return tokens, nil
+}
+
+func isIdentChar(ch rune) bool {
+	return unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '_' || ch == '-' || ch == '.'
+}
+
+// AST node types
+type dslNode interface {
+	dslNode()
+}
+
+type andNode struct {
+	children []dslNode
+}
+
+type orNode struct {
+	children []dslNode
+}
+
+type leafNode struct {
+	name string
+}
+
+type refNode struct {
+	setName string
+}
+
+func (andNode) dslNode()  {}
+func (orNode) dslNode()   {}
+func (leafNode) dslNode() {}
+func (refNode) dslNode()  {}
+
+// parser holds state for recursive-descent parsing.
+type parser struct {
+	tokens []token
+	pos    int
+}
+
+func (p *parser) peek() token {
+	if p.pos < len(p.tokens) {
+		return p.tokens[p.pos]
+	}
+	return token{tokEOF, ""}
+}
+
+func (p *parser) next() token {
+	t := p.peek()
+	if t.typ != tokEOF {
+		p.pos++
+	}
+	return t
+}
+
+func (p *parser) expect(typ tokenType) (token, error) {
+	t := p.next()
+	if t.typ != typ {
+		return t, fmt.Errorf("expected token type %d, got %q", typ, t.val)
+	}
+	return t, nil
+}
+
+// Grammar:
+//
+//	expr    = andExpr
+//	andExpr = orExpr ('&' orExpr)*
+//	orExpr  = atom ('|' atom)*
+//	atom    = ident | '+' ident | '(' expr ')'
+//
+// & binds tighter than |, so "a | b & c" means "a | (b & c)"
+func parse(tokens []token) (dslNode, error) {
+	p := &parser{tokens: tokens}
+	node, err := p.parseExpr()
+	if err != nil {
+		return nil, err
+	}
+	if p.peek().typ != tokEOF {
+		return nil, fmt.Errorf("unexpected token %q after expression", p.peek().val)
+	}
+	return node, nil
+}
+
+func (p *parser) parseExpr() (dslNode, error) {
+	return p.parseOrExpr()
+}
+
+func (p *parser) parseOrExpr() (dslNode, error) {
+	left, err := p.parseAndExpr()
+	if err != nil {
+		return nil, err
+	}
+
+	if p.peek().typ == tokOr {
+		children := []dslNode{left}
+		for p.peek().typ == tokOr {
+			p.next() // consume |
+			right, err := p.parseAndExpr()
+			if err != nil {
+				return nil, err
+			}
+			children = append(children, right)
+		}
+		return orNode{children: children}, nil
+	}
+
+	return left, nil
+}
+
+func (p *parser) parseAndExpr() (dslNode, error) {
+	left, err := p.parseAtom()
+	if err != nil {
+		return nil, err
+	}
+
+	if p.peek().typ == tokAnd {
+		children := []dslNode{left}
+		for p.peek().typ == tokAnd {
+			p.next() // consume &
+			right, err := p.parseAtom()
+			if err != nil {
+				return nil, err
+			}
+			children = append(children, right)
+		}
+		return andNode{children: children}, nil
+	}
+
+	return left, nil
+}
+
+func (p *parser) parseAtom() (dslNode, error) {
+	t := p.peek()
+
+	switch t.typ {
+	case tokIdent:
+		p.next()
+		return leafNode{name: t.val}, nil
+
+	case tokRef:
+		p.next()
+		return refNode{setName: t.val}, nil
+
+	case tokLParen:
+		p.next() // consume (
+		node, err := p.parseExpr()
+		if err != nil {
+			return nil, err
+		}
+		if _, err := p.expect(tokRParen); err != nil {
+			return nil, fmt.Errorf("missing closing parenthesis")
+		}
+		return node, nil
+
+	default:
+		return nil, fmt.Errorf("unexpected token %q", t.val)
+	}
+}
+
+// expand walks the AST and produces all combinations.
+// resolvedRefs contains previously expanded sets for +ref resolution.
+func expand(node dslNode, resolvedRefs map[string][][]string) ([][]string, error) {
+	switch n := node.(type) {
+	case leafNode:
+		return [][]string{{n.name}}, nil
+
+	case refNode:
+		expanded, ok := resolvedRefs[n.setName]
+		if !ok {
+			return nil, fmt.Errorf("unknown set reference +%s", n.setName)
+		}
+		// Return a copy
+		result := make([][]string, len(expanded))
+		for i, combo := range expanded {
+			result[i] = make([]string, len(combo))
+			copy(result[i], combo)
+		}
+		return result, nil
+
+	case orNode:
+		// Union of all children's expansions
+		var result [][]string
+		for _, child := range n.children {
+			childResult, err := expand(child, resolvedRefs)
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, childResult...)
+			if len(result) > maxDSLExpansions {
+				return nil, fmt.Errorf("DSL expansion exceeded %d combinations", maxDSLExpansions)
+			}
+		}
+		return result, nil
+
+	case andNode:
+		// Cartesian product across children
+		result := [][]string{{}} // start with one empty combo
+		for _, child := range n.children {
+			childResult, err := expand(child, resolvedRefs)
+			if err != nil {
+				return nil, err
+			}
+			result, err = cartesianProduct(result, childResult, maxDSLExpansions)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return result, nil
+
+	default:
+		return nil, fmt.Errorf("unknown node type %T", node)
+	}
+}
+
+// cartesianProduct computes the cartesian product of two sets of combinations.
+// It returns an error if the product would exceed cap.
+func cartesianProduct(left, right [][]string, cap int) ([][]string, error) {
+	if int64(len(left))*int64(len(right)) > int64(cap) {
+		return nil, fmt.Errorf("DSL expansion exceeded %d combinations", cap)
+	}
+	result := make([][]string, 0, len(left)*len(right))
+	for _, l := range left {
+		for _, r := range right {
+			combo := make([]string, 0, len(l)+len(r))
+			combo = append(combo, l...)
+			combo = append(combo, r...)
+			result = append(result, combo)
+		}
+	}
+	return result, nil
+}
+
+// ParseAndExpandDSL tokenizes, parses, and expands a DSL string.
+// resolvedRefs contains previously expanded sets for +ref inlining.
+func ParseAndExpandDSL(dsl string, resolvedRefs map[string][][]string) ([][]string, error) {
+	dsl = strings.TrimSpace(dsl)
+	if dsl == "" {
+		return nil, fmt.Errorf("empty DSL expression")
+	}
+
+	tokens, err := tokenize(dsl)
+	if err != nil {
+		return nil, fmt.Errorf("tokenize: %w", err)
+	}
+
+	tree, err := parse(tokens)
+	if err != nil {
+		return nil, fmt.Errorf("parse: %w", err)
+	}
+
+	result, err := expand(tree, resolvedRefs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Deduplicate models within each combination and sort for consistency
+	for i, combo := range result {
+		result[i] = dedupAndSort(combo)
+	}
+
+	return result, nil
+}
+
+// dedupAndSort removes duplicate entries and sorts alphabetically.
+func dedupAndSort(items []string) []string {
+	seen := make(map[string]bool, len(items))
+	var unique []string
+	for _, item := range items {
+		if !seen[item] {
+			seen[item] = true
+			unique = append(unique, item)
+		}
+	}
+	sort.Strings(unique)
+	return unique
+}
+
+// extractRefs scans a DSL string for +ref tokens without full parsing.
+// Used for building the dependency graph for topological sorting.
+func extractRefs(dsl string) ([]string, error) {
+	tokens, err := tokenize(dsl)
+	if err != nil {
+		return nil, err
+	}
+
+	var refs []string
+	seen := make(map[string]bool)
+	for _, t := range tokens {
+		if t.typ == tokRef && !seen[t.val] {
+			seen[t.val] = true
+			refs = append(refs, t.val)
+		}
+	}
+	return refs, nil
+}

proxy/config/matrix_dsl_test.go

@@ -0,0 +1,300 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDSL_Tokenize(t *testing.T) {
+	tests := []struct {
+		name   string
+		input  string
+		expect []token
+		errMsg string
+	}{
+		{
+			name:  "single identifier",
+			input: "abc",
+			expect: []token{
+				{tokIdent, "abc"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "identifier with hyphens and dots",
+			input: "model-name.v2",
+			expect: []token{
+				{tokIdent, "model-name.v2"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "and expression",
+			input: "a & b",
+			expect: []token{
+				{tokIdent, "a"},
+				{tokAnd, "&"},
+				{tokIdent, "b"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "or expression",
+			input: "a | b",
+			expect: []token{
+				{tokIdent, "a"},
+				{tokOr, "|"},
+				{tokIdent, "b"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "parentheses",
+			input: "(a | b) & c",
+			expect: []token{
+				{tokLParen, "("},
+				{tokIdent, "a"},
+				{tokOr, "|"},
+				{tokIdent, "b"},
+				{tokRParen, ")"},
+				{tokAnd, "&"},
+				{tokIdent, "c"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "ref token",
+			input: "+llms & v",
+			expect: []token{
+				{tokRef, "llms"},
+				{tokAnd, "&"},
+				{tokIdent, "v"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "no whitespace",
+			input: "(a|b)&c",
+			expect: []token{
+				{tokLParen, "("},
+				{tokIdent, "a"},
+				{tokOr, "|"},
+				{tokIdent, "b"},
+				{tokRParen, ")"},
+				{tokAnd, "&"},
+				{tokIdent, "c"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:   "empty ref",
+			input:  "+",
+			errMsg: "expected set name after '+'",
+		},
+		{
+			name:   "invalid character",
+			input:  "a @ b",
+			errMsg: "unexpected character",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tokens, err := tokenize(tt.input)
+			if tt.errMsg != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.errMsg)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tt.expect, tokens)
+			}
+		})
+	}
+}
+
+func TestDSL_ParseAndExpand(t *testing.T) {
+	tests := []struct {
+		name   string
+		dsl    string
+		refs   map[string][][]string
+		expect [][]string
+		errMsg string
+	}{
+		{
+			name:   "single model",
+			dsl:    "L",
+			expect: [][]string{{"L"}},
+		},
+		{
+			name:   "two models with AND",
+			dsl:    "a & b",
+			expect: [][]string{{"a", "b"}},
+		},
+		{
+			name:   "two models with OR",
+			dsl:    "a | b",
+			expect: [][]string{{"a"}, {"b"}},
+		},
+		{
+			name:   "three models with OR",
+			dsl:    "a | b | c",
+			expect: [][]string{{"a"}, {"b"}, {"c"}},
+		},
+		{
+			name: "cartesian product (a|b) & (c|d)",
+			dsl:  "(a | b) & (c | d)",
+			expect: [][]string{
+				{"a", "c"},
+				{"a", "d"},
+				{"b", "c"},
+				{"b", "d"},
+			},
+		},
+		{
+			name: "three-way AND",
+			dsl:  "a & b & c",
+			expect: [][]string{
+				{"a", "b", "c"},
+			},
+		},
+		{
+			name: "(g | q | m) & v",
+			dsl:  "(g | q | m) & v",
+			expect: [][]string{
+				{"g", "v"},
+				{"q", "v"},
+				{"m", "v"},
+			},
+		},
+		{
+			name: "(g | q) & v & e",
+			dsl:  "(g | q) & v & e",
+			expect: [][]string{
+				{"e", "g", "v"},
+				{"e", "q", "v"},
+			},
+		},
+		{
+			name: "precedence: a | b & c means a | (b & c)",
+			dsl:  "a | b & c",
+			expect: [][]string{
+				{"a"},
+				{"b", "c"},
+			},
+		},
+		{
+			name: "+ref inlining",
+			dsl:  "+llms & v",
+			refs: map[string][][]string{
+				"llms": {{"g"}, {"q"}, {"m"}},
+			},
+			expect: [][]string{
+				{"g", "v"},
+				{"q", "v"},
+				{"m", "v"},
+			},
+		},
+		{
+			name: "+ref chained",
+			dsl:  "+with_tts & e",
+			refs: map[string][][]string{
+				"with_tts": {{"g", "v"}, {"q", "v"}, {"m", "v"}},
+			},
+			expect: [][]string{
+				{"e", "g", "v"},
+				{"e", "q", "v"},
+				{"e", "m", "v"},
+			},
+		},
+		{
+			name: "dedup within combination",
+			dsl:  "a & a",
+			expect: [][]string{
+				{"a"},
+			},
+		},
+		{
+			name:   "empty expression",
+			dsl:    "",
+			errMsg: "empty DSL expression",
+		},
+		{
+			name:   "unmatched open paren",
+			dsl:    "(a | b",
+			errMsg: "missing closing parenthesis",
+		},
+		{
+			name:   "unmatched close paren",
+			dsl:    "a | b)",
+			errMsg: "unexpected token",
+		},
+		{
+			name:   "unknown ref",
+			dsl:    "+unknown",
+			errMsg: "unknown set reference +unknown",
+		},
+		{
+			name:   "empty parens",
+			dsl:    "()",
+			errMsg: "unexpected token",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			refs := tt.refs
+			if refs == nil {
+				refs = map[string][][]string{}
+			}
+			result, err := ParseAndExpandDSL(tt.dsl, refs)
+			if tt.errMsg != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.errMsg)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tt.expect, result)
+			}
+		})
+	}
+}
+
+func TestDSL_ExpansionCap(t *testing.T) {
+	// Build an expression that would exceed 1000 combinations:
+	// (a1|a2|...|a32) & (b1|b2|...|b32) = 1024 combos
+	var aItems, bItems []string
+	for i := 0; i < 32; i++ {
+		aItems = append(aItems, fmt.Sprintf("a%d", i))
+		bItems = append(bItems, fmt.Sprintf("b%d", i))
+	}
+	dsl := fmt.Sprintf("(%s) & (%s)",
+		join(aItems, " | "),
+		join(bItems, " | "),
+	)
+	_, err := ParseAndExpandDSL(dsl, map[string][][]string{})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "exceeded")
+}
+
+func TestDSL_ExtractRefs(t *testing.T) {
+	refs, err := extractRefs("+llms & v & +other")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"llms", "other"}, refs)
+
+	refs, err = extractRefs("a & b")
+	require.NoError(t, err)
+	assert.Empty(t, refs)
+}
+
+func join(items []string, sep string) string {
+	result := ""
+	for i, item := range items {
+		if i > 0 {
+			result += sep
+		}
+		result += item
+	}
+	return result
+}

proxy/config/matrix_test.go

@@ -0,0 +1,305 @@
+package config
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func makeModels(names ...string) map[string]ModelConfig {
+	m := make(map[string]ModelConfig)
+	for _, name := range names {
+		m[name] = ModelConfig{Cmd: "echo " + name}
+	}
+	return m
+}
+
+func TestValidateMatrix_Basic(t *testing.T) {
+	models := makeModels("gemma", "qwen", "mistral", "voxtral", "llama70B")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{
+			"g": "gemma",
+			"q": "qwen",
+			"m": "mistral",
+			"v": "voxtral",
+			"L": "llama70B",
+		},
+		EvictCosts: map[string]int{
+			"L": 30,
+			"v": 50,
+		},
+		Sets: OrderedSets{
+			{Name: "standard", DSL: "(g | q | m) & v"},
+			{Name: "full", DSL: "L"},
+		},
+	}
+
+	expanded, err := ValidateMatrix(matrix, models)
+	require.NoError(t, err)
+
+	// standard expands to [gemma,voxtral], [qwen,voxtral], [mistral,voxtral]
+	// full expands to [llama70B]
+	assert.Len(t, expanded, 4)
+
+	assert.Equal(t, "standard", expanded[0].SetName)
+	assert.Equal(t, []string{"gemma", "voxtral"}, expanded[0].Models)
+
+	assert.Equal(t, "standard", expanded[1].SetName)
+	assert.Equal(t, []string{"qwen", "voxtral"}, expanded[1].Models)
+
+	assert.Equal(t, "standard", expanded[2].SetName)
+	assert.Equal(t, []string{"mistral", "voxtral"}, expanded[2].Models)
+
+	assert.Equal(t, "full", expanded[3].SetName)
+	assert.Equal(t, []string{"llama70B"}, expanded[3].Models)
+}
+
+func TestValidateMatrix_WithRef(t *testing.T) {
+	models := makeModels("gemma", "qwen", "mistral", "voxtral", "reranker")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{
+			"g": "gemma",
+			"q": "qwen",
+			"m": "mistral",
+			"v": "voxtral",
+			"e": "reranker",
+		},
+		Sets: OrderedSets{
+			{Name: "llms", DSL: "g | q | m"},
+			{Name: "with_tts", DSL: "+llms & v"},
+			{Name: "mega", DSL: "+with_tts & e"},
+		},
+	}
+
+	expanded, err := ValidateMatrix(matrix, models)
+	require.NoError(t, err)
+
+	// llms: [gemma], [qwen], [mistral]
+	// with_tts: [gemma,voxtral], [qwen,voxtral], [mistral,voxtral]
+	// mega: [gemma,reranker,voxtral], [qwen,reranker,voxtral], [mistral,reranker,voxtral]
+	assert.Len(t, expanded, 9)
+
+	// Check mega entries
+	megaEntries := filterBySetName(expanded, "mega")
+	assert.Len(t, megaEntries, 3)
+	assert.Equal(t, []string{"gemma", "reranker", "voxtral"}, megaEntries[0].Models)
+}
+
+func TestValidateMatrix_MapIDRequired(t *testing.T) {
+	// DSL cannot use real model names directly — must use var IDs
+	models := makeModels("gemma", "voxtral")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{"g": "gemma"},
+		Sets: OrderedSets{
+			{Name: "combo", DSL: "g & voxtral"},
+		},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unknown var ID")
+}
+
+func TestValidateMatrix_InvalidAliasKey(t *testing.T) {
+	models := makeModels("gemma")
+
+	tests := []struct {
+		name   string
+		alias  string
+		errMsg string
+	}{
+		{"too long", "abcdefghi", "alphanumeric and 1-8 characters"},
+		{"has underscore", "a_b", "alphanumeric and 1-8 characters"},
+		{"has hyphen", "a-b", "alphanumeric and 1-8 characters"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			matrix := MatrixConfig{
+				Var:  map[string]string{tt.alias: "gemma"},
+				Sets: OrderedSets{{Name: "s", DSL: tt.alias}},
+			}
+			_, err := ValidateMatrix(matrix, models)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tt.errMsg)
+		})
+	}
+}
+
+func TestValidateMatrix_AliasReferencesUnknownModel(t *testing.T) {
+	models := makeModels("gemma")
+
+	matrix := MatrixConfig{
+		Var:  map[string]string{"x": "nonexistent"},
+		Sets: OrderedSets{{Name: "s", DSL: "x"}},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unknown model")
+}
+
+func TestValidateMatrix_EvictCostInvalid(t *testing.T) {
+	models := makeModels("gemma")
+
+	t.Run("zero cost", func(t *testing.T) {
+		matrix := MatrixConfig{
+			Var:        map[string]string{"g": "gemma"},
+			EvictCosts: map[string]int{"g": 0},
+			Sets:       OrderedSets{{Name: "s", DSL: "g"}},
+		}
+		_, err := ValidateMatrix(matrix, models)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "positive integer")
+	})
+
+	t.Run("negative cost", func(t *testing.T) {
+		matrix := MatrixConfig{
+			Var:        map[string]string{"g": "gemma"},
+			EvictCosts: map[string]int{"g": -1},
+			Sets:       OrderedSets{{Name: "s", DSL: "g"}},
+		}
+		_, err := ValidateMatrix(matrix, models)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "positive integer")
+	})
+
+	t.Run("unknown var ID in evict_costs", func(t *testing.T) {
+		matrix := MatrixConfig{
+			Var:        map[string]string{"g": "gemma"},
+			EvictCosts: map[string]int{"unknown": 5},
+			Sets:       OrderedSets{{Name: "s", DSL: "g"}},
+		}
+		_, err := ValidateMatrix(matrix, models)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "unknown var ID")
+	})
+}
+
+func TestValidateMatrix_CycleDetection(t *testing.T) {
+	models := makeModels("gemma")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{"g": "gemma"},
+		Sets: OrderedSets{
+			{Name: "a", DSL: "+b"},
+			{Name: "b", DSL: "+a"},
+		},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "circular reference")
+}
+
+func TestValidateMatrix_UndefinedRefTarget(t *testing.T) {
+	models := makeModels("gemma")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{"g": "gemma"},
+		Sets: OrderedSets{
+			{Name: "a", DSL: "+nonexistent"},
+		},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "references undefined set")
+}
+
+func TestValidateMatrix_NoSets(t *testing.T) {
+	_, err := ValidateMatrix(MatrixConfig{}, makeModels("gemma"))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "at least one set")
+}
+
+func TestValidateMatrix_UnknownMapIDInDSL(t *testing.T) {
+	models := makeModels("gemma")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{"g": "gemma"},
+		Sets: OrderedSets{
+			{Name: "s", DSL: "g & nonexistent"},
+		},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unknown var ID")
+}
+
+func TestValidateMatrix_ResolvedEvictCosts(t *testing.T) {
+	mc := &MatrixConfig{
+		Var: map[string]string{
+			"g": "gemma",
+			"L": "llama70B",
+		},
+		EvictCosts: map[string]int{
+			"L": 30,
+			"g": 5,
+		},
+	}
+
+	costs := mc.ResolvedEvictCosts()
+	assert.Equal(t, 30, costs["llama70B"])
+	assert.Equal(t, 5, costs["gemma"])
+}
+
+func TestValidateMatrix_ConfigXOR(t *testing.T) {
+	// groups and matrix both defined
+	yaml := `
+models:
+  model1:
+    cmd: echo model1
+    proxy: http://localhost:8080
+groups:
+  group1:
+    members:
+      - model1
+matrix:
+  sets:
+    s: "model1"
+`
+	_, err := LoadConfigFromReader(strings.NewReader(yaml))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "cannot use both")
+}
+
+func TestValidateMatrix_ConfigMatrixOnly(t *testing.T) {
+	yaml := `
+models:
+  gemma:
+    cmd: echo gemma
+    proxy: http://localhost:8080
+  qwen:
+    cmd: echo qwen
+    proxy: http://localhost:8081
+matrix:
+  vars:
+    g: gemma
+    q: qwen
+  sets:
+    combo: "g | q"
+`
+	cfg, err := LoadConfigFromReader(strings.NewReader(yaml))
+	require.NoError(t, err)
+	assert.NotNil(t, cfg.Matrix)
+	assert.Len(t, cfg.ExpandedSets, 2)
+	// Groups should be empty when matrix is used
+	assert.Empty(t, cfg.Groups)
+}
+
+func filterBySetName(sets []ExpandedSet, name string) []ExpandedSet {
+	var result []ExpandedSet
+	for _, s := range sets {
+		if s.SetName == name {
+			result = append(result, s)
+		}
+	}
+	return result
+}

proxy/config/model_config.go

@@ -5,6 +5,21 @@ import (
 	"runtime"
 )
 
+const (
+	MODEL_CONFIG_DEFAULT_TTL = -1
+)
+
+// TimeoutsConfig holds timeout settings for proxy connections
+// 0 = no timeout
+type TimeoutsConfig struct {
+	Connect        int `yaml:"connect"`
+	KeepAlive      int `yaml:"keepalive"`
+	ResponseHeader int `yaml:"responseHeader"`
+	TLSHandshake   int `yaml:"tlsHandshake"`
+	ExpectContinue int `yaml:"expectContinue"`
+	IdleConn       int `yaml:"idleConn"`
+}
+
 type ModelConfig struct {
 	Cmd           string   `yaml:"cmd"`
 	CmdStop       string   `yaml:"cmdStop"`
@@ -36,6 +51,9 @@ type ModelConfig struct {
 
 	// override global setting
 	SendLoadingState *bool `yaml:"sendLoadingState"`
+
+	// Timeout settings for proxy connections
+	Timeouts TimeoutsConfig `yaml:"timeouts"`
 }
 
 func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -47,12 +65,22 @@ func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		Aliases:          []string{},
 		Env:              []string{},
 		CheckEndpoint:    "/health",
-		UnloadAfter:      0,
+		UnloadAfter:      MODEL_CONFIG_DEFAULT_TTL, // use GlobalTTL
 		Unlisted:         false,
 		UseModelName:     "",
 		ConcurrencyLimit: 0,
 		Name:             "",
 		Description:      "",
+
+		// matches http.DefaultTransport
+		Timeouts: TimeoutsConfig{
+			Connect:        30,
+			KeepAlive:      30,
+			ResponseHeader: 0,
+			TLSHandshake:   10,
+			ExpectContinue: 1,
+			IdleConn:       90,
+		},
 	}
 
 	// the default cmdStop to taskkill /f /t /pid ${PID}

proxy/config/model_config_test.go

@@ -73,6 +73,72 @@ models:
 	}
 }
 
+func TestConfig_SetParamsByIDAutoAlias(t *testing.T) {
+	content := `
+models:
+  model1:
+    cmd: path/to/cmd --port ${PORT}
+    filters:
+      setParamsByID:
+        "${MODEL_ID}:high":
+          reasoning_effort: high
+        "${MODEL_ID}:low":
+          reasoning_effort: low
+`
+	cfg, err := LoadConfigFromReader(strings.NewReader(content))
+	assert.NoError(t, err)
+
+	// Keys (other than the model's own ID) should be registered as aliases
+	realName, found := cfg.RealModelName("model1:high")
+	assert.True(t, found, "model1:high should be an auto-registered alias")
+	assert.Equal(t, "model1", realName)
+
+	realName, found = cfg.RealModelName("model1:low")
+	assert.True(t, found, "model1:low should be an auto-registered alias")
+	assert.Equal(t, "model1", realName)
+
+	// Auto-aliases should also appear in modelConfig.Aliases
+	aliases := cfg.Models["model1"].Aliases
+	assert.Contains(t, aliases, "model1:high")
+	assert.Contains(t, aliases, "model1:low")
+}
+
+func TestConfig_SetParamsByIDAutoAliasConflictWithModelID(t *testing.T) {
+	content := `
+models:
+  model1:
+    cmd: path/to/cmd --port ${PORT}
+    filters:
+      setParamsByID:
+        model2:
+          reasoning_effort: high
+  model2:
+    cmd: path/to/cmd --port ${PORT}
+`
+	_, err := LoadConfigFromReader(strings.NewReader(content))
+	assert.ErrorContains(t, err, "conflicts with an existing model ID")
+}
+
+func TestConfig_SetParamsByIDAutoAliasConflictWithOtherModel(t *testing.T) {
+	content := `
+models:
+  model1:
+    cmd: path/to/cmd --port ${PORT}
+    filters:
+      setParamsByID:
+        "shared-alias":
+          reasoning_effort: high
+  model2:
+    cmd: path/to/cmd --port ${PORT}
+    filters:
+      setParamsByID:
+        "shared-alias":
+          reasoning_effort: low
+`
+	_, err := LoadConfigFromReader(strings.NewReader(content))
+	assert.ErrorContains(t, err, "duplicate alias")
+}
+
 func TestConfig_ModelFiltersWithSetParams(t *testing.T) {
 	content := `
 models:

proxy/config/peer.go

@@ -12,6 +12,9 @@ type PeerConfig struct {
 	ApiKey   string   `yaml:"apiKey"`
 	Models   []string `yaml:"models"`
 	Filters  Filters  `yaml:"filters"`
+
+	// Timeout settings for proxy connections
+	Timeouts TimeoutsConfig `yaml:"timeouts"`
 }
 
 func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -21,6 +24,17 @@ func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		ApiKey:  "",
 		Models:  []string{},
 		Filters: Filters{},
+
+		// mostly matches http.DefaultTransport but with a 60s ResponseHeader timeout
+		// to match the pre PR #619 functionality
+		Timeouts: TimeoutsConfig{
+			Connect:        30,
+			KeepAlive:      30,
+			ResponseHeader: 60,
+			TLSHandshake:   10,
+			ExpectContinue: 1,
+			IdleConn:       90,
+		},
 	}
 
 	if err := unmarshal(&defaults); err != nil {

proxy/events.go

@@ -8,6 +8,7 @@ const ConfigFileChangedEventID = 0x03
 const LogDataEventID = 0x04
 const TokenMetricsEventID = 0x05
 const ModelPreloadedEventID = 0x06
+const InFlightRequestsEventID = 0x07
 
 type ProcessStateChangeEvent struct {
 	ProcessName string
@@ -58,3 +59,11 @@ type ModelPreloadedEvent struct {
 func (e ModelPreloadedEvent) Type() uint32 {
 	return ModelPreloadedEventID
 }
+
+type InFlightRequestsEvent struct {
+	Total int
+}
+
+func (e InFlightRequestsEvent) Type() uint32 {
+	return InFlightRequestsEventID
+}

proxy/matrix.go

@@ -0,0 +1,298 @@
+package proxy
+
+import (
+	"fmt"
+	"net/http"
+	"slices"
+	"sort"
+	"sync"
+
+	"github.com/mostlygeek/llama-swap/proxy/config"
+)
+
+// MatrixSolver contains pure swap-decision logic with no Process dependencies.
+// It is safe for concurrent reads after construction.
+type MatrixSolver struct {
+	expandedSets []config.ExpandedSet // all valid model combinations
+	evictCosts   map[string]int       // real model name -> eviction cost (default 1)
+	modelToSets  map[string][]int     // model name -> indices into expandedSets
+}
+
+// NewMatrixSolver builds a solver from expanded sets and eviction costs.
+func NewMatrixSolver(expandedSets []config.ExpandedSet, evictCosts map[string]int) *MatrixSolver {
+	modelToSets := make(map[string][]int)
+	for i, es := range expandedSets {
+		for _, model := range es.Models {
+			modelToSets[model] = append(modelToSets[model], i)
+		}
+	}
+
+	return &MatrixSolver{
+		expandedSets: expandedSets,
+		evictCosts:   evictCosts,
+		modelToSets:  modelToSets,
+	}
+}
+
+// SolveResult describes what the solver decided.
+type SolveResult struct {
+	Evict     []string // running models that must be stopped
+	TargetSet []string // the chosen set of models (for informational purposes)
+	SetName   string   // name of the chosen set
+	DSL       string   // original DSL expression for the chosen set
+	TotalCost int      // total eviction cost
+}
+
+// Solve determines which models to evict when a model is requested.
+//
+// Algorithm:
+//  1. If requestedModel is already running, no eviction needed.
+//  2. Find all sets containing requestedModel.
+//  3. If no sets found, the model runs alone; evict all running models.
+//  4. For each candidate set, compute cost = sum of evict_costs for running
+//     models NOT in that set.
+//  5. Pick lowest cost. Ties broken by definition order (index in expandedSets).
+//  6. Return models to evict and the chosen set.
+func (s *MatrixSolver) Solve(requestedModel string, runningModels []string) (SolveResult, error) {
+	// If already running, nothing to do (but fill in set info for logging)
+	if slices.Contains(runningModels, requestedModel) {
+		setName, dsl := s.findMatchingSet(requestedModel, runningModels)
+		return SolveResult{
+			TargetSet: runningModels,
+			SetName:   setName,
+			DSL:       dsl,
+		}, nil
+	}
+
+	candidateIndices := s.modelToSets[requestedModel]
+
+	// Model not in any set: runs alone, evict everything
+	if len(candidateIndices) == 0 {
+		evict := make([]string, len(runningModels))
+		copy(evict, runningModels)
+		return SolveResult{
+			Evict:     evict,
+			TargetSet: []string{requestedModel},
+		}, nil
+	}
+
+	// Find the cheapest candidate set
+	bestCost := -1
+	bestIdx := -1
+
+	for _, idx := range candidateIndices {
+		setModels := s.expandedSets[idx].Models
+		cost := 0
+		for _, running := range runningModels {
+			if !slices.Contains(setModels, running) {
+				cost += s.evictCost(running)
+			}
+		}
+
+		if bestCost < 0 || cost < bestCost || (cost == bestCost && idx < bestIdx) {
+			bestCost = cost
+			bestIdx = idx
+		}
+	}
+
+	// Determine which running models to evict
+	chosen := s.expandedSets[bestIdx]
+	var evict []string
+	for _, running := range runningModels {
+		if !slices.Contains(chosen.Models, running) {
+			evict = append(evict, running)
+		}
+	}
+
+	return SolveResult{
+		Evict:     evict,
+		TargetSet: chosen.Models,
+		SetName:   chosen.SetName,
+		DSL:       chosen.DSL,
+		TotalCost: bestCost,
+	}, nil
+}
+
+// findMatchingSet finds the expanded set that contains all running models.
+// Returns the set name and DSL, or empty strings if no match.
+func (s *MatrixSolver) findMatchingSet(requestedModel string, runningModels []string) (string, string) {
+	for _, idx := range s.modelToSets[requestedModel] {
+		set := s.expandedSets[idx]
+		allInSet := true
+		for _, m := range runningModels {
+			if !slices.Contains(set.Models, m) {
+				allInSet = false
+				break
+			}
+		}
+		if allInSet {
+			return set.SetName, set.DSL
+		}
+	}
+	return "", ""
+}
+
+func (s *MatrixSolver) evictCost(model string) int {
+	if cost, ok := s.evictCosts[model]; ok {
+		return cost
+	}
+	return 1
+}
+
+// Matrix manages processes using solver-based swap logic.
+type Matrix struct {
+	sync.Mutex
+	solver         *MatrixSolver
+	processes      map[string]*Process // all processes keyed by real model name
+	config         config.Config
+	proxyLogger    *LogMonitor
+	upstreamLogger *LogMonitor
+}
+
+// NewMatrix creates a Matrix from config. It creates a Process for every
+// model defined in the config (any model can run alone even if not in a set).
+func NewMatrix(cfg config.Config, proxyLogger, upstreamLogger *LogMonitor) *Matrix {
+	processes := make(map[string]*Process)
+	for modelID, modelConfig := range cfg.Models {
+		processLogger := NewLogMonitorWriter(upstreamLogger)
+		process := NewProcess(modelID, cfg.HealthCheckTimeout, modelConfig, processLogger, proxyLogger)
+		processes[modelID] = process
+	}
+
+	evictCosts := cfg.Matrix.ResolvedEvictCosts()
+
+	return &Matrix{
+		solver:         NewMatrixSolver(cfg.ExpandedSets, evictCosts),
+		processes:      processes,
+		config:         cfg,
+		proxyLogger:    proxyLogger,
+		upstreamLogger: upstreamLogger,
+	}
+}
+
+// ProxyRequest handles the swap logic and proxies the request to the model.
+func (m *Matrix) ProxyRequest(modelID string, w http.ResponseWriter, r *http.Request) error {
+	process, ok := m.processes[modelID]
+	if !ok {
+		return fmt.Errorf("model %s not found in matrix", modelID)
+	}
+
+	m.Lock()
+	running := m.runningModels()
+	result, err := m.solver.Solve(modelID, running)
+	if err != nil {
+		m.Unlock()
+		return fmt.Errorf("matrix solver error: %w", err)
+	}
+
+	// Log solver decision
+	if len(result.Evict) > 0 {
+		m.proxyLogger.Infof("Matrix: model=%s set=%s dsl=%q evict=%v target=%v cost=%d",
+			modelID, result.SetName, result.DSL, result.Evict, result.TargetSet, result.TotalCost)
+	} else if len(running) == 0 {
+		m.proxyLogger.Infof("Matrix: model=%s starting (no models running)", modelID)
+	} else {
+		m.proxyLogger.Debugf("Matrix: model=%s already running in set=%s dsl=%q", modelID, result.SetName, result.DSL)
+	}
+
+	// Evict models that need to be stopped
+	if len(result.Evict) > 0 {
+		var wg sync.WaitGroup
+		for _, evictModel := range result.Evict {
+			if p, exists := m.processes[evictModel]; exists {
+				wg.Add(1)
+				go func(p *Process) {
+					defer wg.Done()
+					p.Stop()
+				}(p)
+			}
+		}
+		wg.Wait()
+	}
+	m.Unlock()
+
+	// Proxy the request (Process handles on-demand start)
+	process.ProxyRequest(w, r)
+	return nil
+}
+
+// StopProcesses stops all running processes.
+func (m *Matrix) StopProcesses(strategy StopStrategy) {
+	m.Lock()
+	defer m.Unlock()
+
+	var wg sync.WaitGroup
+	for _, process := range m.processes {
+		wg.Add(1)
+		go func(p *Process) {
+			defer wg.Done()
+			switch strategy {
+			case StopImmediately:
+				p.StopImmediately()
+			default:
+				p.Stop()
+			}
+		}(process)
+	}
+	wg.Wait()
+}
+
+// StopProcess stops a single process by model ID.
+func (m *Matrix) StopProcess(modelID string, strategy StopStrategy) error {
+	process, ok := m.processes[modelID]
+	if !ok {
+		return fmt.Errorf("process not found for %s", modelID)
+	}
+
+	switch strategy {
+	case StopImmediately:
+		process.StopImmediately()
+	default:
+		process.Stop()
+	}
+	return nil
+}
+
+// Shutdown shuts down all processes.
+func (m *Matrix) Shutdown() {
+	var wg sync.WaitGroup
+	for _, process := range m.processes {
+		wg.Add(1)
+		go func(p *Process) {
+			defer wg.Done()
+			p.Shutdown()
+		}(process)
+	}
+	wg.Wait()
+}
+
+// RunningModels returns model names currently in StateReady.
+func (m *Matrix) RunningModels() []string {
+	m.Lock()
+	defer m.Unlock()
+	return m.runningModels()
+}
+
+// runningModels returns running model names (caller must hold lock).
+func (m *Matrix) runningModels() []string {
+	var running []string
+	for id, process := range m.processes {
+		if process.CurrentState() == StateReady {
+			running = append(running, id)
+		}
+	}
+	sort.Strings(running)
+	return running
+}
+
+// GetProcess returns the Process for a model.
+func (m *Matrix) GetProcess(modelID string) (*Process, bool) {
+	p, ok := m.processes[modelID]
+	return p, ok
+}
+
+// HasModel returns true if the model is managed by this matrix.
+func (m *Matrix) HasModel(modelID string) bool {
+	_, ok := m.processes[modelID]
+	return ok
+}

proxy/matrix_test.go

@@ -0,0 +1,227 @@
+package proxy
+
+import (
+	"testing"
+
+	"github.com/mostlygeek/llama-swap/proxy/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Helper to build expanded sets for solver tests
+func makeExpandedSets(sets ...struct {
+	name   string
+	models []string
+}) []config.ExpandedSet {
+	var result []config.ExpandedSet
+	for _, s := range sets {
+		result = append(result, config.ExpandedSet{
+			SetName: s.name,
+			Models:  s.models,
+		})
+	}
+	return result
+}
+
+func es(name string, models ...string) struct {
+	name   string
+	models []string
+} {
+	return struct {
+		name   string
+		models []string
+	}{name, models}
+}
+
+func TestMatrixSolver_AlreadyRunning(t *testing.T) {
+	solver := NewMatrixSolver(
+		makeExpandedSets(es("s1", "a", "b")),
+		nil,
+	)
+
+	result, err := solver.Solve("a", []string{"a"})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"a"}, result.TargetSet)
+	assert.Equal(t, "s1", result.SetName)
+}
+
+func TestMatrixSolver_NotInAnySet_RunsAlone(t *testing.T) {
+	solver := NewMatrixSolver(
+		makeExpandedSets(es("s1", "a", "b")),
+		nil,
+	)
+
+	// Model "c" not in any set
+	result, err := solver.Solve("c", []string{"a", "b"})
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"a", "b"}, result.Evict)
+	assert.Equal(t, []string{"c"}, result.TargetSet)
+}
+
+func TestMatrixSolver_NotInAnySet_NothingRunning(t *testing.T) {
+	solver := NewMatrixSolver(
+		makeExpandedSets(es("s1", "a", "b")),
+		nil,
+	)
+
+	result, err := solver.Solve("c", []string{})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"c"}, result.TargetSet)
+}
+
+func TestMatrixSolver_SingleSet_EvictsNonMembers(t *testing.T) {
+	// Set: [a, b]. Request a when b and c are running.
+	solver := NewMatrixSolver(
+		makeExpandedSets(es("s1", "a", "b")),
+		nil,
+	)
+
+	result, err := solver.Solve("a", []string{"b", "c"})
+	require.NoError(t, err)
+	// c is not in the set, so it gets evicted. b is in the set, so it stays.
+	assert.Equal(t, []string{"c"}, result.Evict)
+	assert.Equal(t, []string{"a", "b"}, result.TargetSet)
+}
+
+func TestMatrixSolver_PicksLowestCost(t *testing.T) {
+	// Two sets containing model "a":
+	// s1: [a, v]  — if v is running, cost=0; if L is running, cost=30
+	// s2: [a, L]  — if L is running, cost=0; if v is running, cost=50
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("s1", "a", "v"),
+			es("s2", "a", "L"),
+		),
+		map[string]int{"v": 50, "L": 30},
+	)
+
+	// v is running. Switching to a:
+	// s1 cost: v is in s1, so 0
+	// s2 cost: v is NOT in s2, so 50
+	// => pick s1
+	result, err := solver.Solve("a", []string{"v"})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"a", "v"}, result.TargetSet)
+
+	// L is running. Switching to a:
+	// s1 cost: L is NOT in s1, so 30
+	// s2 cost: L is in s2, so 0
+	// => pick s2
+	result, err = solver.Solve("a", []string{"L"})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"a", "L"}, result.TargetSet)
+}
+
+func TestMatrixSolver_TieBreakingByDefinitionOrder(t *testing.T) {
+	// Two sets with identical cost. Definition order should win.
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("s1", "a", "x"),
+			es("s2", "a", "y"),
+		),
+		nil,
+	)
+
+	// Nothing running, both sets cost 0. s1 is first.
+	result, err := solver.Solve("a", []string{})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"a", "x"}, result.TargetSet)
+}
+
+func TestMatrixSolver_EvictCostPreservesExpensive(t *testing.T) {
+	// Model "v" costs 50 to evict, "m" costs 1 (default).
+	// Sets: [g,v], [g,m]
+	// Running: v, m. Request g.
+	// s1=[g,v]: evict m (cost 1), keep v
+	// s2=[g,m]: evict v (cost 50), keep m
+	// => pick s1
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("s1", "g", "v"),
+			es("s2", "g", "m"),
+		),
+		map[string]int{"v": 50},
+	)
+
+	result, err := solver.Solve("g", []string{"v", "m"})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"m"}, result.Evict)
+	assert.Equal(t, []string{"g", "v"}, result.TargetSet)
+}
+
+func TestMatrixSolver_NothingRunning(t *testing.T) {
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("s1", "g", "v"),
+			es("s2", "q", "v"),
+		),
+		nil,
+	)
+
+	result, err := solver.Solve("g", []string{})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"g", "v"}, result.TargetSet)
+}
+
+func TestMatrixSolver_FullScenario(t *testing.T) {
+	// Simulates the example config:
+	// standard: [g,v], [q,v], [m,v]
+	// with_rerank: [g,v,e], [q,v,e]
+	// creative: [g,sd], [q,sd]
+	// full: [L]
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("standard", "g", "v"),
+			es("standard", "q", "v"),
+			es("standard", "m", "v"),
+			es("with_rerank", "e", "g", "v"),
+			es("with_rerank", "e", "q", "v"),
+			es("creative", "g", "sd"),
+			es("creative", "q", "sd"),
+			es("full", "L"),
+		),
+		map[string]int{"v": 50, "L": 30, "whisper": 10},
+	)
+
+	// Running: g, v. Request q.
+	// standard[q,v]: evict g (cost 1), keep v. Total: 1.
+	// with_rerank[q,v,e]: evict g (cost 1), keep v. Total: 1.
+	// => tie, pick first by definition order = standard[q,v]
+	result, err := solver.Solve("q", []string{"g", "v"})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"g"}, result.Evict)
+	assert.Equal(t, []string{"q", "v"}, result.TargetSet)
+
+	// Running: g, v. Request L.
+	// full[L]: evict g (cost 1) + v (cost 50). Total: 51.
+	// Only one set contains L, so pick it.
+	result, err = solver.Solve("L", []string{"g", "v"})
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"g", "v"}, result.Evict)
+	assert.Equal(t, []string{"L"}, result.TargetSet)
+
+	// Running: g, v. Request sd.
+	// creative[g,sd]: evict v (cost 50). Total: 50.
+	// creative[q,sd]: evict g (cost 1) + v (cost 50). Total: 51.
+	// => pick creative[g,sd]
+	result, err = solver.Solve("sd", []string{"g", "v"})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"v"}, result.Evict)
+	assert.Equal(t, []string{"g", "sd"}, result.TargetSet)
+
+	// Running: q, v, e. Request g.
+	// standard[g,v]: evict q (1) + e (1). Total: 2.
+	// with_rerank[g,v,e]: evict q (1). Total: 1.
+	// creative[g,sd]: evict q (1) + v (50) + e (1). Total: 52.
+	// => pick with_rerank[g,v,e]
+	result, err = solver.Solve("g", []string{"e", "q", "v"})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"q"}, result.Evict)
+	assert.Equal(t, []string{"e", "g", "v"}, result.TargetSet)
+}

proxy/metrics_monitor.go

@@ -350,6 +350,11 @@ func processStreamingResponse(modelID string, start time.Time, body []byte) (Tok
 			usage := parsed.Get("usage")
 			timings := parsed.Get("timings")
 
+			// v1/responses format nests usage under response.usage
+			if !usage.Exists() {
+				usage = parsed.Get("response.usage")
+			}
+
 			if usage.Exists() || timings.Exists() {
 				return parseMetrics(modelID, start, usage, timings)
 			}
@@ -360,6 +365,8 @@ func processStreamingResponse(modelID string, start time.Time, body []byte) (Tok
 }
 
 func parseMetrics(modelID string, start time.Time, usage, timings gjson.Result) (TokenMetrics, error) {
+	wallDurationMs := int(time.Since(start).Milliseconds())
+
 	// default values
 	cachedTokens := -1 // unknown or missing data
 	outputTokens := 0
@@ -368,7 +375,7 @@ func parseMetrics(modelID string, start time.Time, usage, timings gjson.Result)
 	// timings data
 	tokensPerSecond := -1.0
 	promptPerSecond := -1.0
-	durationMs := int(time.Since(start).Milliseconds())
+	durationMs := wallDurationMs
 
 	if usage.Exists() {
 		if pt := usage.Get("prompt_tokens"); pt.Exists() {
@@ -397,7 +404,10 @@ func parseMetrics(modelID string, start time.Time, usage, timings gjson.Result)
 		outputTokens = int(timings.Get("predicted_n").Int())
 		promptPerSecond = timings.Get("prompt_per_second").Float()
 		tokensPerSecond = timings.Get("predicted_per_second").Float()
-		durationMs = int(timings.Get("prompt_ms").Float() + timings.Get("predicted_ms").Float())
+		timingsDurationMs := int(timings.Get("prompt_ms").Float() + timings.Get("predicted_ms").Float())
+		if timingsDurationMs > durationMs {
+			durationMs = timingsDurationMs
+		}
 
 		if cachedValue := timings.Get("cache_n"); cachedValue.Exists() {
 			cachedTokens = int(cachedValue.Int())
@@ -503,9 +513,9 @@ func filterAcceptEncoding(acceptEncoding string) string {
 	supported := map[string]bool{"gzip": true, "deflate": true}
 	var filtered []string
 
-	for _, part := range strings.Split(acceptEncoding, ",") {
+	for part := range strings.SplitSeq(acceptEncoding, ",") {
 		// Parse encoding and optional quality value (e.g., "gzip;q=1.0")
-		encoding := strings.TrimSpace(strings.Split(part, ";")[0])
+		encoding, _, _ := strings.Cut(strings.TrimSpace(part), ";")
 		if supported[strings.ToLower(encoding)] {
 			filtered = append(filtered, strings.TrimSpace(part))
 		}

proxy/metrics_monitor_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/mostlygeek/llama-swap/event"
 	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
 )
 
 func TestMetricsMonitor_AddMetrics(t *testing.T) {
@@ -570,6 +571,27 @@ func TestMetricsMonitor_Concurrent(t *testing.T) {
 }
 
 func TestMetricsMonitor_ParseMetrics(t *testing.T) {
+	t.Run("keeps wall clock duration when timings underreport request time", func(t *testing.T) {
+		start := time.Now().Add(-5 * time.Second)
+		usage := gjson.Parse(`{"prompt_tokens": 5, "completion_tokens": 1}`)
+		timings := gjson.Parse(`{
+			"prompt_n": 5,
+			"predicted_n": 1,
+			"prompt_per_second": 10.0,
+			"predicted_per_second": 2.0,
+			"prompt_ms": 5.0,
+			"predicted_ms": 15.0
+		}`)
+
+		metrics, err := parseMetrics("test-model", start, usage, timings)
+		assert.NoError(t, err)
+		assert.Equal(t, 5, metrics.InputTokens)
+		assert.Equal(t, 1, metrics.OutputTokens)
+		assert.Equal(t, 10.0, metrics.PromptPerSecond)
+		assert.Equal(t, 2.0, metrics.TokensPerSecond)
+		assert.GreaterOrEqual(t, metrics.DurationMs, 5000)
+	})
+
 	t.Run("prefers timings over usage data", func(t *testing.T) {
 		mm := newMetricsMonitor(testLogger, 10, 0)
 
@@ -709,6 +731,35 @@ data: [DONE]
 		assert.Equal(t, 0, metrics[0].OutputTokens)
 	})
 
+	t.Run("v1/responses format with nested response.usage", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10, 0)
+
+		// v1/responses SSE format: usage is nested under response.usage
+		responseBody := "event: response.completed\n" +
+			`data: {"type":"response.completed","response":{"id":"resp_abc","object":"response","created_at":1773416985,"status":"completed","model":"test-model","output":[],"usage":{"input_tokens":17,"output_tokens":23,"total_tokens":40}}}` +
+			"\n\n"
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "text/event-stream")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/v1/responses", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 1, len(metrics))
+		assert.Equal(t, "test-model", metrics[0].Model)
+		assert.Equal(t, 17, metrics[0].InputTokens)
+		assert.Equal(t, 23, metrics[0].OutputTokens)
+	})
+
 	t.Run("handles empty streaming response records minimal metrics", func(t *testing.T) {
 		mm := newMetricsMonitor(testLogger, 10, 0)
 

proxy/peerproxy.go

@@ -34,23 +34,25 @@ func NewPeerProxy(peers config.PeerDictionaryConfig, proxyLogger *LogMonitor) (*
 	}
 	sort.Strings(peerIDs)
 
-	// Create a shared transport with reasonable timeouts for peer connections
-	// these can be tuned with feedback later
-	peerTransport := &http.Transport{
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second, // Connection timeout
-			KeepAlive: 30 * time.Second,
-		}).DialContext,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ResponseHeaderTimeout: 60 * time.Second, // Time to wait for response headers
-		ExpectContinueTimeout: 1 * time.Second,
-		MaxIdleConns:          100,
-		MaxIdleConnsPerHost:   10,
-		IdleConnTimeout:       90 * time.Second,
-	}
-
 	for _, peerID := range peerIDs {
 		peer := peers[peerID]
+
+		// Create a transport with per-peer timeout configuration
+		peerTransport := &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   time.Duration(peer.Timeouts.Connect) * time.Second,
+				KeepAlive: time.Duration(peer.Timeouts.KeepAlive) * time.Second,
+			}).DialContext,
+			TLSHandshakeTimeout:   time.Duration(peer.Timeouts.TLSHandshake) * time.Second,
+			ResponseHeaderTimeout: time.Duration(peer.Timeouts.ResponseHeader) * time.Second,
+			ExpectContinueTimeout: time.Duration(peer.Timeouts.ExpectContinue) * time.Second,
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          100,
+			MaxIdleConnsPerHost:   10,
+			IdleConnTimeout:       time.Duration(peer.Timeouts.IdleConn) * time.Second,
+		}
+
 		// Create reverse proxy for this peer
 		reverseProxy := httputil.NewSingleHostReverseProxy(peer.ProxyURL)
 		reverseProxy.Transport = peerTransport

proxy/peerproxy_test.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/mostlygeek/llama-swap/proxy/config"
 	"github.com/stretchr/testify/assert"
@@ -266,3 +267,45 @@ func TestProxyRequest_SSEHeaderModification(t *testing.T) {
 	// The X-Accel-Buffering header should be set to "no" for SSE
 	assert.Equal(t, "no", w.Header().Get("X-Accel-Buffering"))
 }
+
+func TestNewPeerProxy_CustomTimeouts(t *testing.T) {
+	proxyURL, _ := url.Parse("http://localhost:8080")
+
+	peers := config.PeerDictionaryConfig{
+		"test-peer": config.PeerConfig{
+			Proxy:    "http://localhost:8080",
+			ProxyURL: proxyURL,
+			Models:   []string{"model1"},
+			Timeouts: config.TimeoutsConfig{
+				Connect:        45,
+				ResponseHeader: 300,
+				TLSHandshake:   15,
+				ExpectContinue: 2,
+				IdleConn:       120,
+			},
+		},
+	}
+
+	peerProxy, err := NewPeerProxy(peers, testLogger)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, peerProxy)
+	assert.True(t, peerProxy.HasPeerModel("model1"))
+
+	// Verify the timeout values are actually applied to the transport
+	member, found := peerProxy.proxyMap["model1"]
+	require.True(t, found, "model1 should exist in proxyMap")
+	assert.NotNil(t, member.reverseProxy)
+	assert.NotNil(t, member.reverseProxy.Transport)
+
+	transport, ok := member.reverseProxy.Transport.(*http.Transport)
+	require.True(t, ok, "Transport should be *http.Transport")
+
+	// Verify all timeout values are correctly applied
+	assert.Equal(t, 300*time.Second, transport.ResponseHeaderTimeout)
+	assert.Equal(t, 15*time.Second, transport.TLSHandshakeTimeout)
+	assert.Equal(t, 2*time.Second, transport.ExpectContinueTimeout)
+	assert.Equal(t, 120*time.Second, transport.IdleConnTimeout)
+	// ForceAttemptHTTP2 should be enabled
+	assert.True(t, transport.ForceAttemptHTTP2)
+}

proxy/process.go

@@ -96,6 +96,24 @@ func NewProcess(ID string, healthCheckTimeout int, config config.ModelConfig, pr
 	var reverseProxy *httputil.ReverseProxy
 	if proxyURL != nil {
 		reverseProxy = httputil.NewSingleHostReverseProxy(proxyURL)
+
+		// Create custom transport with configured timeouts
+		transport := &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   time.Duration(config.Timeouts.Connect) * time.Second,
+				KeepAlive: time.Duration(config.Timeouts.KeepAlive) * time.Second,
+			}).DialContext,
+			TLSHandshakeTimeout:   time.Duration(config.Timeouts.TLSHandshake) * time.Second,
+			ResponseHeaderTimeout: time.Duration(config.Timeouts.ResponseHeader) * time.Second,
+			ExpectContinueTimeout: time.Duration(config.Timeouts.ExpectContinue) * time.Second,
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          100,
+			MaxIdleConnsPerHost:   10,
+			IdleConnTimeout:       time.Duration(config.Timeouts.IdleConn) * time.Second,
+		}
+		reverseProxy.Transport = transport
+
 		reverseProxy.ModifyResponse = func(resp *http.Response) error {
 			// prevent nginx from buffering streaming responses (e.g., SSE)
 			if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {

proxy/process_test.go

@@ -2,6 +2,7 @@ package proxy
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -117,12 +118,12 @@ func TestProcess_UnloadAfterTTL(t *testing.T) {
 	}
 
 	expectedMessage := "I_sense_imminent_danger"
-	config := getTestSimpleResponderConfig(expectedMessage)
-	assert.Equal(t, 0, config.UnloadAfter)
-	config.UnloadAfter = 3 // seconds
-	assert.Equal(t, 3, config.UnloadAfter)
+	conf := getTestSimpleResponderConfig(expectedMessage)
+	assert.Equal(t, config.MODEL_CONFIG_DEFAULT_TTL, conf.UnloadAfter)
+	conf.UnloadAfter = 3 // seconds
+	assert.Equal(t, 3, conf.UnloadAfter)
 
-	process := NewProcess("ttl_test", 2, config, debugLogger, debugLogger)
+	process := NewProcess("ttl_test", 2, conf, debugLogger, debugLogger)
 	defer process.Stop()
 
 	// this should take 4 seconds
@@ -159,12 +160,12 @@ func TestProcess_LowTTLValue(t *testing.T) {
 		t.Skip("skipping test, edit process_test.go to run it ")
 	}
 
-	config := getTestSimpleResponderConfig("fast_ttl")
-	assert.Equal(t, 0, config.UnloadAfter)
-	config.UnloadAfter = 1 // second
-	assert.Equal(t, 1, config.UnloadAfter)
+	conf := getTestSimpleResponderConfig("fast_ttl")
+	assert.Equal(t, config.MODEL_CONFIG_DEFAULT_TTL, conf.UnloadAfter)
+	conf.UnloadAfter = 1 // second
+	assert.Equal(t, 1, conf.UnloadAfter)
 
-	process := NewProcess("ttl", 2, config, debugLogger, debugLogger)
+	process := NewProcess("ttl", 2, conf, debugLogger, debugLogger)
 	defer process.Stop()
 
 	for i := 0; i < 100; i++ {
@@ -569,3 +570,39 @@ func (w *panicOnWriteResponseWriter) Write(b []byte) (int, error) {
 	}
 	return w.ResponseRecorder.Write(b)
 }
+
+func TestProcess_CustomTimeouts(t *testing.T) {
+	modelConfig := config.ModelConfig{
+		Cmd:           "echo test",
+		Proxy:         "http://localhost:8080",
+		CheckEndpoint: "/health",
+		Timeouts: config.TimeoutsConfig{
+			Connect:        45,
+			ResponseHeader: 120,
+			TLSHandshake:   15,
+			ExpectContinue: 2,
+			IdleConn:       120,
+		},
+	}
+
+	debugLogger := NewLogMonitorWriter(io.Discard)
+	process := NewProcess("test-model", 30, modelConfig, debugLogger, debugLogger)
+
+	// Verify the process was created successfully
+	assert.NotNil(t, process)
+	assert.Equal(t, "test-model", process.ID)
+	assert.NotNil(t, process.reverseProxy)
+	assert.NotNil(t, process.reverseProxy.Transport)
+
+	// Verify it's using http.Transport (not some other type)
+	transport, ok := process.reverseProxy.Transport.(*http.Transport)
+	assert.True(t, ok, "Transport should be *http.Transport")
+	assert.NotNil(t, transport)
+
+	// Verify the timeouts are correctly applied
+	assert.Equal(t, 120*time.Second, transport.ResponseHeaderTimeout)
+	assert.Equal(t, 15*time.Second, transport.TLSHandshakeTimeout)
+	assert.Equal(t, 2*time.Second, transport.ExpectContinueTimeout)
+	assert.Equal(t, 120*time.Second, transport.IdleConnTimeout)
+	assert.True(t, transport.ForceAttemptHTTP2)
+}

proxy/proxymanager.go

@@ -28,6 +28,40 @@ const (
 
 type proxyCtxKey string
 
+type InflightCounter struct {
+	mu    sync.Mutex
+	total int
+}
+
+func newInflightCounter() *InflightCounter {
+	return &InflightCounter{}
+}
+
+func (ic *InflightCounter) Current() int {
+	ic.mu.Lock()
+	total := ic.total
+	ic.mu.Unlock()
+	return total
+}
+
+func (ic *InflightCounter) Increment() int {
+	ic.mu.Lock()
+	ic.total++
+	total := ic.total
+	ic.mu.Unlock()
+	return total
+}
+
+func (ic *InflightCounter) Decrement() int {
+	ic.mu.Lock()
+	if ic.total > 0 {
+		ic.total--
+	}
+	total := ic.total
+	ic.mu.Unlock()
+	return total
+}
+
 type ProxyManager struct {
 	sync.Mutex
 
@@ -43,6 +77,11 @@ type ProxyManager struct {
 
 	processGroups map[string]*ProcessGroup
 
+	// matrix-based swap (mutually exclusive with processGroups)
+	matrix *Matrix
+
+	inFlightCounter *InflightCounter
+
 	// shutdown signaling
 	shutdownCtx    context.Context
 	shutdownCancel context.CancelFunc
@@ -155,6 +194,8 @@ func New(proxyConfig config.Config) *ProxyManager {
 
 		processGroups: make(map[string]*ProcessGroup),
 
+		inFlightCounter: newInflightCounter(),
+
 		shutdownCtx:    shutdownCtx,
 		shutdownCancel: shutdownCancel,
 
@@ -165,10 +206,14 @@ func New(proxyConfig config.Config) *ProxyManager {
 		peerProxy: peerProxy,
 	}
 
-	// create the process groups
-	for groupID := range proxyConfig.Groups {
-		processGroup := NewProcessGroup(groupID, proxyConfig, proxyLogger, upstreamLogger)
-		pm.processGroups[groupID] = processGroup
+	// create either matrix or process groups (mutually exclusive)
+	if proxyConfig.Matrix != nil {
+		pm.matrix = NewMatrix(proxyConfig, proxyLogger, upstreamLogger)
+	} else {
+		for groupID := range proxyConfig.Groups {
+			processGroup := NewProcessGroup(groupID, proxyConfig, proxyLogger, upstreamLogger)
+			pm.processGroups[groupID] = processGroup
+		}
 	}
 
 	pm.setupGinEngine()
@@ -187,18 +232,29 @@ func New(proxyConfig config.Config) *ProxyManager {
 				}
 
 				proxyLogger.Infof("Preloading model: %s", modelID)
-				processGroup, err := pm.swapProcessGroup(modelID)
 
-				if err != nil {
+				var preloadErr error
+				req, _ := http.NewRequest("GET", "/", nil)
+
+				if pm.matrix != nil {
+					preloadErr = pm.matrix.ProxyRequest(modelID, discardWriter, req)
+				} else {
+					processGroup, err := pm.swapProcessGroup(modelID)
+					if err != nil {
+						preloadErr = err
+					} else {
+						preloadErr = processGroup.ProxyRequest(modelID, discardWriter, req)
+					}
+				}
+
+				if preloadErr != nil {
 					event.Emit(ModelPreloadedEvent{
 						ModelName: modelID,
 						Success:   false,
 					})
-					proxyLogger.Errorf("Failed to preload model %s: %v", modelID, err)
+					proxyLogger.Errorf("Failed to preload model %s: %v", modelID, preloadErr)
 					continue
 				} else {
-					req, _ := http.NewRequest("GET", "/", nil)
-					processGroup.ProxyRequest(modelID, discardWriter, req)
 					event.Emit(ModelPreloadedEvent{
 						ModelName: modelID,
 						Success:   true,
@@ -276,37 +332,42 @@ func (pm *ProxyManager) setupGinEngine() {
 
 	// Set up routes using the Gin engine
 	// Protected routes use pm.apiKeyAuth() middleware
-	pm.ginEngine.POST("/v1/chat/completions", pm.apiKeyAuth(), pm.proxyInferenceHandler)
-	pm.ginEngine.POST("/v1/responses", pm.apiKeyAuth(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/chat/completions", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/responses", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
 	// Support legacy /v1/completions api, see issue #12
-	pm.ginEngine.POST("/v1/completions", pm.apiKeyAuth(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/completions", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
 	// Support anthropic /v1/messages (added https://github.com/ggml-org/llama.cpp/pull/17570)
-	pm.ginEngine.POST("/v1/messages", pm.apiKeyAuth(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/messages", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
 	// Support anthropic count_tokens API (Also added in the above PR)
-	pm.ginEngine.POST("/v1/messages/count_tokens", pm.apiKeyAuth(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/messages/count_tokens", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
 
 	// Support embeddings and reranking
-	pm.ginEngine.POST("/v1/embeddings", pm.apiKeyAuth(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/embeddings", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
 
 	// llama-server's /reranking endpoint + aliases
-	pm.ginEngine.POST("/reranking", pm.apiKeyAuth(), pm.proxyInferenceHandler)
-	pm.ginEngine.POST("/rerank", pm.apiKeyAuth(), pm.proxyInferenceHandler)
-	pm.ginEngine.POST("/v1/rerank", pm.apiKeyAuth(), pm.proxyInferenceHandler)
-	pm.ginEngine.POST("/v1/reranking", pm.apiKeyAuth(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/reranking", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/rerank", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/rerank", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/reranking", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
 
 	// llama-server's /infill endpoint for code infilling
-	pm.ginEngine.POST("/infill", pm.apiKeyAuth(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/infill", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
 
 	// llama-server's /completion endpoint
-	pm.ginEngine.POST("/completion", pm.apiKeyAuth(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/completion", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
 
 	// Support audio/speech endpoint
-	pm.ginEngine.POST("/v1/audio/speech", pm.apiKeyAuth(), pm.proxyInferenceHandler)
-	pm.ginEngine.POST("/v1/audio/voices", pm.apiKeyAuth(), pm.proxyInferenceHandler)
-	pm.ginEngine.GET("/v1/audio/voices", pm.apiKeyAuth(), pm.proxyGETModelHandler)
-	pm.ginEngine.POST("/v1/audio/transcriptions", pm.apiKeyAuth(), pm.proxyOAIPostFormHandler)
-	pm.ginEngine.POST("/v1/images/generations", pm.apiKeyAuth(), pm.proxyInferenceHandler)
-	pm.ginEngine.POST("/v1/images/edits", pm.apiKeyAuth(), pm.proxyOAIPostFormHandler)
+	pm.ginEngine.POST("/v1/audio/speech", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/audio/voices", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.GET("/v1/audio/voices", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyGETModelHandler)
+	pm.ginEngine.POST("/v1/audio/transcriptions", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyOAIPostFormHandler)
+	pm.ginEngine.POST("/v1/images/generations", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/v1/images/edits", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyOAIPostFormHandler)
+
+	// sd.cpp /sdapi/v1 endpoints
+	pm.ginEngine.POST("/sdapi/v1/txt2img", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.POST("/sdapi/v1/img2img", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyInferenceHandler)
+	pm.ginEngine.GET("/sdapi/v1/loras", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyGETModelHandler)
 
 	pm.ginEngine.GET("/v1/models", pm.apiKeyAuth(), pm.listModelsHandler)
 
@@ -325,7 +386,7 @@ func (pm *ProxyManager) setupGinEngine() {
 	pm.ginEngine.GET("/upstream", func(c *gin.Context) {
 		c.Redirect(http.StatusFound, "/ui/models")
 	})
-	pm.ginEngine.Any("/upstream/*upstreamPath", pm.apiKeyAuth(), pm.proxyToUpstream)
+	pm.ginEngine.Any("/upstream/*upstreamPath", pm.apiKeyAuth(), pm.trackInflight(), pm.proxyToUpstream)
 	pm.ginEngine.GET("/unload", pm.apiKeyAuth(), pm.unloadAllModelsHandler)
 	pm.ginEngine.GET("/running", pm.apiKeyAuth(), pm.listRunningProcessesHandler)
 	pm.ginEngine.GET("/health", func(c *gin.Context) {
@@ -389,6 +450,14 @@ func (pm *ProxyManager) setupGinEngine() {
 	gin.DisableConsoleColor()
 }
 
+func (pm *ProxyManager) trackInflight() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		event.Emit(InFlightRequestsEvent{Total: pm.inFlightCounter.Increment()})
+		defer event.Emit(InFlightRequestsEvent{Total: pm.inFlightCounter.Decrement()})
+		c.Next()
+	}
+}
+
 // ServeHTTP implements http.Handler interface
 func (pm *ProxyManager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	pm.ginEngine.ServeHTTP(w, r)
@@ -402,6 +471,11 @@ func (pm *ProxyManager) StopProcesses(strategy StopStrategy) {
 	pm.Lock()
 	defer pm.Unlock()
 
+	if pm.matrix != nil {
+		pm.matrix.StopProcesses(strategy)
+		return
+	}
+
 	// stop Processes in parallel
 	var wg sync.WaitGroup
 	for _, processGroup := range pm.processGroups {
@@ -422,6 +496,12 @@ func (pm *ProxyManager) Shutdown() {
 
 	pm.proxyLogger.Debug("Shutdown() called in proxy manager")
 
+	if pm.matrix != nil {
+		pm.matrix.Shutdown()
+		pm.shutdownCancel()
+		return
+	}
+
 	var wg sync.WaitGroup
 	// Send shutdown signal to all process in groups
 	for _, processGroup := range pm.processGroups {
@@ -588,10 +668,16 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
 		return
 	}
 
-	processGroup, err := pm.swapProcessGroup(modelID)
-	if err != nil {
-		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
-		return
+	var handler func(string, http.ResponseWriter, *http.Request) error
+	if pm.matrix != nil {
+		handler = pm.matrix.ProxyRequest
+	} else {
+		processGroup, err := pm.swapProcessGroup(modelID)
+		if err != nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+			return
+		}
+		handler = processGroup.ProxyRequest
 	}
 
 	// rewrite the path
@@ -600,13 +686,13 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
 
 	// attempt to record metrics if it is a POST request
 	if pm.metricsMonitor != nil && c.Request.Method == "POST" {
-		if err := pm.metricsMonitor.wrapHandler(modelID, c.Writer, c.Request, processGroup.ProxyRequest); err != nil {
+		if err := pm.metricsMonitor.wrapHandler(modelID, c.Writer, c.Request, handler); err != nil {
 			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying metrics wrapped request: %s", err.Error()))
 			pm.proxyLogger.Errorf("Error proxying wrapped upstream request for model %s, path=%s", modelID, originalPath)
 			return
 		}
 	} else {
-		if err := processGroup.ProxyRequest(modelID, c.Writer, c.Request); err != nil {
+		if err := handler(modelID, c.Writer, c.Request); err != nil {
 			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error()))
 			pm.proxyLogger.Errorf("Error proxying upstream request for model %s, path=%s", modelID, originalPath)
 			return
@@ -632,10 +718,16 @@ func (pm *ProxyManager) proxyInferenceHandler(c *gin.Context) {
 
 	modelID, found := pm.config.RealModelName(requestedModel)
 	if found {
-		processGroup, err := pm.swapProcessGroup(modelID)
-		if err != nil {
-			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
-			return
+		var localHandler func(string, http.ResponseWriter, *http.Request) error
+		if pm.matrix != nil {
+			localHandler = pm.matrix.ProxyRequest
+		} else {
+			processGroup, err := pm.swapProcessGroup(modelID)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+				return
+			}
+			localHandler = processGroup.ProxyRequest
 		}
 
 		// issue #69 allow custom model names to be sent to upstream
@@ -674,8 +766,19 @@ func (pm *ProxyManager) proxyInferenceHandler(c *gin.Context) {
 			}
 		}
 
+		// setParamsByID: set params based on the requested model ID (runs after setParams, can override it)
+		setParamsByIDParams, setParamsByIDKeys := pm.config.Models[modelID].Filters.SanitizedSetParamsByID(requestedModel)
+		for _, key := range setParamsByIDKeys {
+			pm.proxyLogger.Debugf("<%s> setting param by id: %s", requestedModel, key)
+			bodyBytes, err = sjson.SetBytes(bodyBytes, key, setParamsByIDParams[key])
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error setting parameter %s in request", key))
+				return
+			}
+		}
+
 		pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
-		nextHandler = processGroup.ProxyRequest
+		nextHandler = localHandler
 	} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
 		pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
 		modelID = requestedModel
@@ -761,15 +864,19 @@ func (pm *ProxyManager) proxyOAIPostFormHandler(c *gin.Context) {
 
 	modelID, found := pm.config.RealModelName(requestedModel)
 	if found {
-		processGroup, err := pm.swapProcessGroup(modelID)
-		if err != nil {
-			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
-			return
+		if pm.matrix != nil {
+			nextHandler = pm.matrix.ProxyRequest
+		} else {
+			processGroup, err := pm.swapProcessGroup(modelID)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+				return
+			}
+			nextHandler = processGroup.ProxyRequest
 		}
 
 		useModelName = pm.config.Models[modelID].UseModelName
 		pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
-		nextHandler = processGroup.ProxyRequest
 	} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
 		pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
 		modelID = requestedModel
@@ -880,14 +987,18 @@ func (pm *ProxyManager) proxyGETModelHandler(c *gin.Context) {
 	var modelID string
 
 	if realModelID, found := pm.config.RealModelName(requestedModel); found {
-		processGroup, err := pm.swapProcessGroup(realModelID)
-		if err != nil {
-			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
-			return
-		}
 		modelID = realModelID
+		if pm.matrix != nil {
+			nextHandler = pm.matrix.ProxyRequest
+		} else {
+			processGroup, err := pm.swapProcessGroup(realModelID)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+				return
+			}
+			nextHandler = processGroup.ProxyRequest
+		}
 		pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
-		nextHandler = processGroup.ProxyRequest
 	} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
 		modelID = requestedModel
 		pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
@@ -986,9 +1097,9 @@ func (pm *ProxyManager) listRunningProcessesHandler(context *gin.Context) {
 	context.Header("Content-Type", "application/json")
 	runningProcesses := make([]gin.H, 0) // Default to an empty response.
 
-	for _, processGroup := range pm.processGroups {
-		for _, process := range processGroup.processes {
-			if process.CurrentState() == StateReady {
+	if pm.matrix != nil {
+		for _, modelID := range pm.matrix.RunningModels() {
+			if process, ok := pm.matrix.GetProcess(modelID); ok {
 				runningProcesses = append(runningProcesses, gin.H{
 					"model":       process.ID,
 					"state":       process.state,
@@ -1000,6 +1111,22 @@ func (pm *ProxyManager) listRunningProcessesHandler(context *gin.Context) {
 				})
 			}
 		}
+	} else {
+		for _, processGroup := range pm.processGroups {
+			for _, process := range processGroup.processes {
+				if process.CurrentState() == StateReady {
+					runningProcesses = append(runningProcesses, gin.H{
+						"model":       process.ID,
+						"state":       process.state,
+						"cmd":         process.config.Cmd,
+						"proxy":       process.config.Proxy,
+						"ttl":         process.config.UnloadAfter,
+						"name":        process.config.Name,
+						"description": process.config.Description,
+					})
+				}
+			}
+		}
 	}
 
 	// Put the results under the `running` key.

proxy/proxymanager_api.go

@@ -14,12 +14,13 @@ import (
 )
 
 type Model struct {
-	Id          string `json:"id"`
-	Name        string `json:"name"`
-	Description string `json:"description"`
-	State       string `json:"state"`
-	Unlisted    bool   `json:"unlisted"`
-	PeerID      string `json:"peerID"`
+	Id          string   `json:"id"`
+	Name        string   `json:"name"`
+	Description string   `json:"description"`
+	State       string   `json:"state"`
+	Unlisted    bool     `json:"unlisted"`
+	PeerID      string   `json:"peerID"`
+	Aliases     []string `json:"aliases,omitempty"`
 }
 
 func addApiHandlers(pm *ProxyManager) {
@@ -54,27 +55,28 @@ func (pm *ProxyManager) getModelStatus() []Model {
 	// Iterate over sorted keys
 	for _, modelID := range modelIDs {
 		// Get process state
-		processGroup := pm.findGroupByModelName(modelID)
 		state := "unknown"
-		if processGroup != nil {
-			process := processGroup.processes[modelID]
-			if process != nil {
-				var stateStr string
-				switch process.CurrentState() {
-				case StateReady:
-					stateStr = "ready"
-				case StateStarting:
-					stateStr = "starting"
-				case StateStopping:
-					stateStr = "stopping"
-				case StateShutdown:
-					stateStr = "shutdown"
-				case StateStopped:
-					stateStr = "stopped"
-				default:
-					stateStr = "unknown"
-				}
-				state = stateStr
+		var process *Process
+		if pm.matrix != nil {
+			process, _ = pm.matrix.GetProcess(modelID)
+		} else {
+			processGroup := pm.findGroupByModelName(modelID)
+			if processGroup != nil {
+				process = processGroup.processes[modelID]
+			}
+		}
+		if process != nil {
+			switch process.CurrentState() {
+			case StateReady:
+				state = "ready"
+			case StateStarting:
+				state = "starting"
+			case StateStopping:
+				state = "stopping"
+			case StateShutdown:
+				state = "shutdown"
+			case StateStopped:
+				state = "stopped"
 			}
 		}
 		models = append(models, Model{
@@ -83,6 +85,7 @@ func (pm *ProxyManager) getModelStatus() []Model {
 			Description: pm.config.Models[modelID].Description,
 			State:       state,
 			Unlisted:    pm.config.Models[modelID].Unlisted,
+			Aliases:     pm.config.Models[modelID].Aliases,
 		})
 	}
 
@@ -107,6 +110,7 @@ const (
 	msgTypeModelStatus messageType = "modelStatus"
 	msgTypeLogData     messageType = "logData"
 	msgTypeMetrics     messageType = "metrics"
+	msgTypeInFlight    messageType = "inflight"
 )
 
 type messageEnvelope struct {
@@ -166,6 +170,18 @@ func (pm *ProxyManager) apiSendEvents(c *gin.Context) {
 		}
 	}
 
+	sendInFlight := func(total int) {
+		jsonData, err := json.Marshal(gin.H{"total": total})
+		if err == nil {
+			select {
+			case sendBuffer <- messageEnvelope{Type: msgTypeInFlight, Data: string(jsonData)}:
+			case <-ctx.Done():
+				return
+			default:
+			}
+		}
+	}
+
 	/**
 	 * Send updated models list
 	 */
@@ -193,11 +209,19 @@ func (pm *ProxyManager) apiSendEvents(c *gin.Context) {
 		sendMetrics([]TokenMetrics{e.Metrics})
 	})()
 
+	/**
+	 * Send in-flight request stats related to token stats "Waiting: N" count.
+	 */
+	defer event.On(func(e InFlightRequestsEvent) {
+		sendInFlight(e.Total)
+	})()
+
 	// send initial batch of data
 	sendLogData("proxy", pm.proxyLogger.GetHistory())
 	sendLogData("upstream", pm.upstreamLogger.GetHistory())
 	sendModels()
 	sendMetrics(pm.metricsMonitor.getMetrics())
+	sendInFlight(pm.inFlightCounter.Current())
 
 	for {
 		select {
@@ -231,18 +255,23 @@ func (pm *ProxyManager) apiUnloadSingleModelHandler(c *gin.Context) {
 		return
 	}
 
-	processGroup := pm.findGroupByModelName(realModelName)
-	if processGroup == nil {
-		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("process group not found for model %s", requestedModel))
-		return
+	var stopErr error
+	if pm.matrix != nil {
+		stopErr = pm.matrix.StopProcess(realModelName, StopImmediately)
+	} else {
+		processGroup := pm.findGroupByModelName(realModelName)
+		if processGroup == nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("process group not found for model %s", requestedModel))
+			return
+		}
+		stopErr = processGroup.StopProcess(realModelName, StopImmediately)
 	}
 
-	if err := processGroup.StopProcess(realModelName, StopImmediately); err != nil {
-		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error stopping process: %s", err.Error()))
+	if stopErr != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error stopping process: %s", stopErr.Error()))
 		return
-	} else {
-		c.String(http.StatusOK, "OK")
 	}
+	c.String(http.StatusOK, "OK")
 }
 
 func (pm *ProxyManager) apiGetVersion(c *gin.Context) {

proxy/proxymanager_test.go

@@ -730,7 +730,7 @@ func TestProxyManager_RunningEndpoint(t *testing.T) {
 		// Verify extended fields are present
 		assert.NotEmpty(t, response.Running[0].Cmd, "cmd should be populated")
 		assert.NotEmpty(t, response.Running[0].Proxy, "proxy should be populated")
-		assert.Equal(t, 0, response.Running[0].TTL, "ttl should default to 0")
+		assert.Equal(t, -1, response.Running[0].TTL, "ttl should default to -1 (use globalTTL)")
 	})
 }
 
@@ -1046,6 +1046,61 @@ func TestProxyManager_FiltersStripParams(t *testing.T) {
 	// t.Logf("%v", response)
 }
 
+func TestProxyManager_FiltersSetParamsByID(t *testing.T) {
+	// no explicit aliases — setParamsByID keys are auto-registered as aliases
+	configStr := strings.Replace(`
+logLevel: error
+models:
+  model1:
+    cmd: 'SRPATH --port ${PORT} --silent --respond model1'
+    proxy: "http://127.0.0.1:${PORT}"
+    filters:
+      setParams:
+        reasoning_effort: medium
+      setParamsByID:
+        "${MODEL_ID}:high":
+          reasoning_effort: high
+        "${MODEL_ID}:low":
+          reasoning_effort: low
+`, "SRPATH", simpleResponderPath, -1)
+
+	cfg, err := config.LoadConfigFromReader(strings.NewReader(configStr))
+	if !assert.NoError(t, err, "invalid test configuration") {
+		return
+	}
+
+	proxy := New(cfg)
+	defer proxy.StopProcesses(StopWaitForInflightRequest)
+
+	tests := []struct {
+		requestedModel string
+		wantEffort     string
+	}{
+		// setParams applies, no setParamsByID match
+		{requestedModel: "model1", wantEffort: "medium"},
+		// setParamsByID overrides setParams
+		{requestedModel: "model1:high", wantEffort: "high"},
+		{requestedModel: "model1:low", wantEffort: "low"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.requestedModel, func(t *testing.T) {
+			reqBody := fmt.Sprintf(`{"model":%q}`, tt.requestedModel)
+			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+			w := CreateTestResponseRecorder()
+			proxy.ServeHTTP(w, req)
+			assert.Equal(t, http.StatusOK, w.Code)
+
+			var response map[string]interface{}
+			assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+
+			requestBody, _ := response["request_body"].(string)
+			gotEffort := gjson.Get(requestBody, "reasoning_effort").String()
+			assert.Equal(t, tt.wantEffort, gotEffort, "reasoning_effort mismatch for model %s", tt.requestedModel)
+		})
+	}
+}
+
 func TestProxyManager_HealthEndpoint(t *testing.T) {
 	config := config.AddDefaultGroupToConfig(config.Config{
 		HealthCheckTimeout: 15,
@@ -1604,3 +1659,82 @@ models:
 		assert.Equal(t, "no", w.Header().Get("X-Accel-Buffering"))
 	})
 }
+
+func TestProxyManager_SdApiTxt2ImgRouting(t *testing.T) {
+	conf := config.AddDefaultGroupToConfig(config.Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]config.ModelConfig{
+			"sd-model": getTestSimpleResponderConfig("sd-model"),
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(conf)
+	defer proxy.StopProcesses(StopWaitForInflightRequest)
+
+	t.Run("successful txt2img with model", func(t *testing.T) {
+		reqBody := `{"model":"sd-model","prompt":"a cat"}`
+		req := httptest.NewRequest("POST", "/sdapi/v1/txt2img", bytes.NewBufferString(reqBody))
+		w := CreateTestResponseRecorder()
+
+		proxy.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Contains(t, w.Body.String(), "sd-model")
+	})
+
+	t.Run("successful img2img with model", func(t *testing.T) {
+		reqBody := `{"model":"sd-model","prompt":"a cat","init_images":[]}`
+		req := httptest.NewRequest("POST", "/sdapi/v1/img2img", bytes.NewBufferString(reqBody))
+		w := CreateTestResponseRecorder()
+
+		proxy.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Contains(t, w.Body.String(), "sd-model")
+	})
+
+	t.Run("missing model returns 400", func(t *testing.T) {
+		reqBody := `{"prompt":"a cat"}`
+		req := httptest.NewRequest("POST", "/sdapi/v1/txt2img", bytes.NewBufferString(reqBody))
+		w := CreateTestResponseRecorder()
+
+		proxy.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusBadRequest, w.Code)
+		assert.Contains(t, w.Body.String(), "missing or invalid 'model' key")
+	})
+}
+
+func TestProxyManager_SdApiGetLoras(t *testing.T) {
+	conf := config.AddDefaultGroupToConfig(config.Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]config.ModelConfig{
+			"sd-model": getTestSimpleResponderConfig("sd-model"),
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(conf)
+	defer proxy.StopProcesses(StopWaitForInflightRequest)
+
+	t.Run("successful GET loras with model query param", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/sdapi/v1/loras?model=sd-model", nil)
+		w := CreateTestResponseRecorder()
+		proxy.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+	})
+
+	t.Run("missing model query param returns 400", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/sdapi/v1/loras", nil)
+		w := CreateTestResponseRecorder()
+		proxy.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusBadRequest, w.Code)
+		assert.Contains(t, w.Body.String(), "missing required 'model' query parameter")
+	})
+
+	t.Run("unknown model returns 400", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/sdapi/v1/loras?model=nonexistent", nil)
+		w := CreateTestResponseRecorder()
+		proxy.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusBadRequest, w.Code)
+		assert.Contains(t, w.Body.String(), "could not find suitable handler")
+	})
+}

ui-svelte/.npmrc

@@ -0,0 +1 @@
+legacy-peer-deps=true

ui-svelte/package.json

@@ -12,18 +12,18 @@
     "test:watch": "vitest"
   },
   "devDependencies": {
-    "@sveltejs/vite-plugin-svelte": "^5.0.3",
+    "@sveltejs/vite-plugin-svelte": "^7.0.0",
     "@tailwindcss/vite": "^4.1.8",
     "@tsconfig/svelte": "^5.0.4",
     "@types/hast": "^3.0.4",
     "@types/node": "^25.1.0",
-    "svelte": "^5.19.0",
+    "svelte": "^5.46.4",
     "svelte-check": "^4.1.4",
     "tailwindcss": "^4.1.8",
     "typescript": "~5.8.3",
-    "vite": "^6.3.5",
-    "vite-plugin-compression2": "^2.4.0",
-    "vitest": "^4.0.18"
+    "vite": "^8.0.0",
+    "vite-plugin-compression2": "^2.5.1",
+    "vitest": "^4.1.0"
   },
   "dependencies": {
     "highlight.js": "^11.11.1",

ui-svelte/src/components/LogPanel.svelte

@@ -65,10 +65,17 @@
   });
 
   let preElement: HTMLPreElement;
+  let userScrolledUp = $state(false);
 
-  // Auto scroll to bottom when logs change
+  function handleScroll() {
+    if (!preElement) return;
+    const { scrollTop, scrollHeight, clientHeight } = preElement;
+    userScrolledUp = scrollHeight - scrollTop - clientHeight > 40;
+  }
+
+  // Auto scroll to bottom when logs change, unless user has scrolled up
   $effect(() => {
-    if (preElement && filteredLogs) {
+    if (preElement && filteredLogs && !userScrolledUp) {
       preElement.scrollTop = preElement.scrollHeight;
     }
   });
@@ -82,7 +89,7 @@
       <div class="flex gap-2 items-center">
         <button class="btn border-0" onclick={toggleFontSize} title="Change font size">
           <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class="w-4 h-4">
-            <path fill-rule="evenodd" d="M10.5 3.75a6 6 0 0 0-5.98 6.496A5.25 5.25 0 0 0 6.75 20.25H18a4.5 4.5 0 0 0 2.206-8.423 3.75 3.75 0 0 0-4.133-4.303A6.001 6.001 0 0 0 10.5 3.75Zm2.25 6a.75.75 0 0 0-1.5 0v4.94l-1.72-1.72a.75.75 0 0 0-1.06 1.06l3 3a.75.75 0 0 0 1.06 0l3-3a.75.75 0 1 0-1.06-1.06l-1.72 1.72V9.75Z" clip-rule="evenodd" />
+            <path d="M2 4v3h5v12h3V7h5V4H2zm19 5h-9v3h3v7h3v-7h3V9z"/>
           </svg>
         </button>
         <button class="btn border-0" onclick={toggleWrapText} title="Toggle text wrap">
@@ -127,6 +134,6 @@
     {/if}
   </div>
   <div class="rounded-lg bg-background font-mono text-sm flex-1 overflow-hidden">
-    <pre bind:this={preElement} class="{textWrapClass} {fontSizeClass} h-full overflow-auto p-4">{filteredLogs}</pre>
+    <pre bind:this={preElement} onscroll={handleScroll} class="{textWrapClass} {fontSizeClass} h-full overflow-auto p-4">{filteredLogs}</pre>
   </div>
 </div>

ui-svelte/src/components/ModelsPanel.svelte

@@ -165,6 +165,9 @@
               {#if model.description}
                 <p class={model.unlisted ? "text-opacity-70" : ""}><em>{model.description}</em></p>
               {/if}
+              {#if model.aliases && model.aliases.length > 0}
+                <p class="text-xs text-txtsecondary">Aliases: {model.aliases.join(", ")}</p>
+              {/if}
             </td>
             <td class="w-12">
               {#if model.state === "stopped"}

ui-svelte/src/components/StatsPanel.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { metrics } from "../stores/api";
+  import { inFlightRequests, metrics } from "../stores/api";
   import TokenHistogram from "./TokenHistogram.svelte";
 
   interface HistogramData {
@@ -15,7 +15,14 @@
   let stats = $derived.by(() => {
     const totalRequests = $metrics.length;
     if (totalRequests === 0) {
-      return { totalRequests: 0, totalInputTokens: 0, totalOutputTokens: 0, tokenStats: { p99: "0", p95: "0", p50: "0" }, histogramData: null };
+      return {
+        totalRequests: 0,
+        totalInputTokens: 0,
+        totalOutputTokens: 0,
+        inFlightRequests: $inFlightRequests,
+        tokenStats: { p99: "0", p95: "0", p50: "0" },
+        histogramData: null,
+      };
     }
 
     const totalInputTokens = $metrics.reduce((sum, m) => sum + m.input_tokens, 0);
@@ -24,7 +31,14 @@
     // Calculate token statistics using output_tokens and duration_ms
     const validMetrics = $metrics.filter((m) => m.duration_ms > 0 && m.output_tokens > 0);
     if (validMetrics.length === 0) {
-      return { totalRequests, totalInputTokens, totalOutputTokens, tokenStats: { p99: "0", p95: "0", p50: "0" }, histogramData: null };
+      return {
+        totalRequests,
+        totalInputTokens,
+        totalOutputTokens,
+        inFlightRequests: $inFlightRequests,
+        tokenStats: { p99: "0", p95: "0", p50: "0" },
+        histogramData: null,
+      };
     }
 
     // Calculate tokens/second for each valid metric
@@ -63,6 +77,7 @@
       totalRequests,
       totalInputTokens,
       totalOutputTokens,
+      inFlightRequests: $inFlightRequests,
       tokenStats: {
         p99: p99.toFixed(2),
         p95: p95.toFixed(2),
@@ -95,7 +110,12 @@
 
       <tbody class="bg-surface divide-y divide-card-border-inner">
         <tr class="hover:bg-secondary">
-          <td class="px-4 py-4 text-sm font-semibold text-gray-900 dark:text-white">{stats.totalRequests}</td>
+          <td class="px-4 py-4 text-sm font-semibold text-gray-900 dark:text-white">
+            <div class="flex flex-col gap-1">
+              <span class="text-xs font-medium text-gray-500 dark:text-gray-400">Completed: {nf.format(stats.totalRequests)}</span>
+              <span class="text-xs font-medium text-gray-500 dark:text-gray-400">Waiting: {nf.format(stats.inFlightRequests)}</span>
+            </div>
+          </td>
 
           <td class="px-4 py-4 text-sm text-gray-700 dark:text-gray-300 border-l border-gray-200 dark:border-white/10">
             <div class="flex items-center gap-2">

ui-svelte/src/components/playground/AudioInterface.svelte

@@ -16,7 +16,7 @@
   let fileInput = $state<HTMLInputElement | null>(null);
   let copied = $state(false);
 
-  const ACCEPTED_FORMATS = ['.mp3', '.wav'];
+  const ACCEPTED_FORMATS = ['.mp3', '.wav', '.ogg'];
   const MAX_FILE_SIZE = 25 * 1024 * 1024; // 25MB
 
   let hasModels = $derived($models.some((m) => !m.unlisted));
@@ -31,7 +31,7 @@
     const ext = '.' + file.name.split('.').pop()?.toLowerCase();
 
     if (!ACCEPTED_FORMATS.includes(ext)) {
-      return { valid: false, error: 'Invalid file type. Accepted: MP3, WAV' };
+      return { valid: false, error: 'Invalid file type. Accepted: MP3, WAV, OGG' };
     }
 
     if (file.size > MAX_FILE_SIZE) {
@@ -208,7 +208,7 @@
           <div>
             <p class="mb-2">Drag and drop an audio file here</p>
             <p class="text-sm">or use the Browse button below</p>
-            <p class="text-xs mt-4">Accepted formats: MP3, WAV (max 25MB)</p>
+            <p class="text-xs mt-4">Accepted formats: MP3, WAV, OGG (max 25MB)</p>
           </div>
         </div>
       {/if}
@@ -218,7 +218,7 @@
     <div class="shrink-0 flex gap-2">
       <input
         type="file"
-        accept=".mp3,.wav"
+        accept=".mp3,.wav,.ogg"
         class="hidden"
         onchange={handleFileSelect}
         bind:this={fileInput}

ui-svelte/src/components/playground/ChatMessage.svelte

@@ -116,6 +116,47 @@
       cancelEdit();
     }
   }
+
+  const COPY_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="14" height="14" x="8" y="8" rx="2" ry="2"/><path d="M4 16c-1.1 0-2-.9-2-2V4c0-1.1.9-2 2-2h10c1.1 0 2 .9 2 2"/></svg>`;
+  const CHECK_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M20 6 9 17l-5-5"/></svg>`;
+
+  function codeBlockCopy(node: HTMLElement) {
+    function attachButtons() {
+      node.querySelectorAll<HTMLPreElement>('pre:not([data-copy-btn])').forEach(pre => {
+        pre.setAttribute('data-copy-btn', 'true');
+        const btn = document.createElement('button');
+        btn.className = 'code-copy-btn';
+        btn.title = 'Copy code';
+        btn.innerHTML = COPY_SVG;
+        btn.addEventListener('click', async () => {
+          const text = pre.querySelector('code')?.textContent ?? pre.textContent ?? '';
+          try {
+            if (navigator.clipboard && window.isSecureContext) {
+              await navigator.clipboard.writeText(text);
+            } else {
+              const ta = document.createElement('textarea');
+              ta.value = text;
+              ta.style.cssText = 'position:fixed;left:-9999px';
+              document.body.appendChild(ta);
+              ta.select();
+              document.execCommand('copy');
+              document.body.removeChild(ta);
+            }
+            btn.innerHTML = CHECK_SVG;
+            btn.classList.add('copied');
+            setTimeout(() => { btn.innerHTML = COPY_SVG; btn.classList.remove('copied'); }, 2000);
+          } catch (e) {
+            console.error('copy failed', e);
+          }
+        });
+        pre.appendChild(btn);
+      });
+    }
+    attachButtons();
+    const mo = new MutationObserver(attachButtons);
+    mo.observe(node, { childList: true, subtree: true });
+    return { destroy: () => mo.disconnect() };
+  }
 </script>
 
 <div class="flex {role === 'user' ? 'justify-end' : 'justify-start'} mb-4">
@@ -174,7 +215,7 @@
       {#if showRaw}
         <div class="whitespace-pre-wrap font-mono text-sm">{textContent}</div>
       {:else}
-        <div class="prose prose-sm dark:prose-invert max-w-none">
+        <div class="prose prose-sm dark:prose-invert max-w-none" use:codeBlockCopy>
           {#each renderedParts.blocks as block (block.id)}
             {@html block.html}
           {/each}
@@ -299,14 +340,42 @@
 
 <style>
   .prose :global(pre) {
+    position: relative;
     background-color: var(--color-surface);
     border: 1px solid var(--color-border, rgba(128, 128, 128, 0.2));
     border-radius: 0.375rem;
     padding: 0.75rem;
+    padding-right: 2.5rem;
     overflow-x: auto;
     margin: 0.5rem 0;
   }
 
+  .prose :global(.code-copy-btn) {
+    position: absolute;
+    top: 0.375rem;
+    right: 0.375rem;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    padding: 0.25rem;
+    border-radius: 0.25rem;
+    border: 1px solid var(--color-border);
+    background: var(--color-surface);
+    color: var(--color-txtsecondary);
+    cursor: pointer;
+    transition: background-color 0.15s;
+    line-height: 0;
+  }
+
+  .prose :global(.code-copy-btn:hover) {
+    background: var(--color-secondary);
+  }
+
+  .prose :global(.code-copy-btn.copied) {
+    color: var(--color-success);
+    opacity: 1;
+  }
+
   .prose :global(code) {
     font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
     font-size: 0.875em;

ui-svelte/src/components/playground/ImageInterface.svelte

@@ -2,26 +2,91 @@
   import { models } from "../../stores/api";
   import { persistentStore } from "../../stores/persistent";
   import { generateImage } from "../../lib/imageApi";
+  import { generateSdImage, fetchSdLoras } from "../../lib/sdApi";
   import { playgroundStores } from "../../stores/playgroundActivity";
   import ModelSelector from "./ModelSelector.svelte";
   import ExpandableTextarea from "./ExpandableTextarea.svelte";
+  import type { ImageApiMode, SdApiLora, SdApiLoraRef } from "../../lib/types";
 
   const selectedModelStore = persistentStore<string>("playground-image-model", "");
   const selectedSizeStore = persistentStore<string>("playground-image-size", "1024x1024");
+  const apiModeStore = persistentStore<ImageApiMode>("playground-image-api-mode", "openai");
+
+  // SDAPI persistent settings
+  const sdNegativePromptStore = persistentStore<string>("playground-sdapi-negative-prompt", "");
+  const sdStepsStore = persistentStore<number>("playground-sdapi-steps", 20);
+  const sdCfgScaleStore = persistentStore<number>("playground-sdapi-cfg-scale", 7);
+  const sdSeedStore = persistentStore<number>("playground-sdapi-seed", -1);
+  const sdSamplerStore = persistentStore<string>("playground-sdapi-sampler", "");
+  const sdSchedulerStore = persistentStore<string>("playground-sdapi-scheduler", "");
+  const sdBatchSizeStore = persistentStore<number>("playground-sdapi-batch-size", 1);
 
   let prompt = $state("");
   let isGenerating = $state(false);
-  let generatedImage = $state<string | null>(null);
+  let generatedImages = $state<string[]>([]);
   let error = $state<string | null>(null);
   let abortController = $state<AbortController | null>(null);
   let showFullscreen = $state(false);
+  let fullscreenIndex = $state(0);
+  let showSettings = $state(false);
+
+  // SDAPI lora state
+  let availableLoras = $state<SdApiLora[]>([]);
+  let selectedLoras = $state<SdApiLoraRef[]>([]);
+  let isLoadingLoras = $state(false);
+  let lorasLoaded = $state(false);
+  let loraError = $state<string | null>(null);
 
   let hasModels = $derived($models.some((m) => !m.unlisted));
+  let isSdapi = $derived($apiModeStore === "sdapi");
 
   $effect(() => {
     playgroundStores.imageGenerating.set(isGenerating);
   });
 
+  async function loadLoras() {
+    if (!$selectedModelStore || isLoadingLoras) return;
+    isLoadingLoras = true;
+    loraError = null;
+    try {
+      const loras = await fetchSdLoras($selectedModelStore);
+      availableLoras = loras;
+      lorasLoaded = true;
+    } catch (err) {
+      availableLoras = [];
+      loraError = err instanceof Error ? err.message : "Failed to load LoRAs";
+      lorasLoaded = false;
+    } finally {
+      isLoadingLoras = false;
+    }
+  }
+
+  function addLora(event: Event) {
+    const select = event.target as HTMLSelectElement;
+    const path = select.value;
+    if (!path) return;
+
+    const lora = availableLoras.find((l) => l.path === path);
+    if (lora && !selectedLoras.some((l) => l.path === path)) {
+      selectedLoras = [...selectedLoras, { path: lora.path, multiplier: 1.0 }];
+    }
+    select.value = "";
+  }
+
+  function removeLora(path: string) {
+    selectedLoras = selectedLoras.filter((l) => l.path !== path);
+  }
+
+  function updateLoraMultiplier(path: string, multiplier: number) {
+    selectedLoras = selectedLoras.map((l) =>
+      l.path === path ? { ...l, multiplier } : l
+    );
+  }
+
+  function getLoraName(path: string): string {
+    return availableLoras.find((l) => l.path === path)?.name ?? path;
+  }
+
   async function generate() {
     const trimmedPrompt = prompt.trim();
     if (!trimmedPrompt || !$selectedModelStore || isGenerating) return;
@@ -31,19 +96,44 @@
     abortController = new AbortController();
 
     try {
-      const response = await generateImage(
-        $selectedModelStore,
-        trimmedPrompt,
-        $selectedSizeStore,
-        abortController.signal
-      );
+      if (isSdapi) {
+        const [w, h] = $selectedSizeStore.split("x").map(Number);
+        const request = {
+          model: $selectedModelStore,
+          prompt: trimmedPrompt,
+          negative_prompt: $sdNegativePromptStore || undefined,
+          width: w,
+          height: h,
+          steps: $sdStepsStore,
+          cfg_scale: $sdCfgScaleStore,
+          seed: $sdSeedStore,
+          batch_size: $sdBatchSizeStore,
+          sampler_name: $sdSamplerStore || undefined,
+          scheduler: $sdSchedulerStore || undefined,
+          lora: selectedLoras.length > 0 ? selectedLoras : undefined,
+        };
 
-      if (response.data && response.data.length > 0) {
-        const imageData = response.data[0];
-        if (imageData.b64_json) {
-          generatedImage = `data:image/png;base64,${imageData.b64_json}`;
-        } else if (imageData.url) {
-          generatedImage = imageData.url;
+        const response = await generateSdImage(request, abortController.signal);
+        if (response.images && response.images.length > 0) {
+          generatedImages = response.images.map(
+            (img) => `data:image/png;base64,${img}`
+          );
+        }
+      } else {
+        const response = await generateImage(
+          $selectedModelStore,
+          trimmedPrompt,
+          $selectedSizeStore,
+          abortController.signal
+        );
+
+        if (response.data && response.data.length > 0) {
+          const imageData = response.data[0];
+          if (imageData.b64_json) {
+            generatedImages = [`data:image/png;base64,${imageData.b64_json}`];
+          } else if (imageData.url) {
+            generatedImages = [imageData.url];
+          }
         }
       }
     } catch (err) {
@@ -63,28 +153,29 @@
   }
 
   function clearImage() {
-    generatedImage = null;
+    generatedImages = [];
     error = null;
     prompt = "";
   }
 
-  function downloadImage() {
-    if (!generatedImage) return;
+  function downloadImage(index: number = 0) {
+    const img = generatedImages[index];
+    if (!img) return;
 
     const link = document.createElement("a");
-    link.href = generatedImage;
-    link.download = `generated-image-${Date.now()}.png`;
+    link.href = img;
+    link.download = `generated-image-${Date.now()}-${index}.png`;
     document.body.appendChild(link);
     link.click();
     document.body.removeChild(link);
   }
 
-  function openFullscreen() {
+  function openFullscreen(index: number = 0) {
+    fullscreenIndex = index;
     showFullscreen = true;
   }
 
   function closeFullscreen(event?: MouseEvent) {
-    // Only close if clicking the background, not the image
     if (event && event.target !== event.currentTarget) {
       return;
     }
@@ -100,9 +191,19 @@
 </script>
 
 <div class="flex flex-col h-full">
-  <!-- Model selector -->
+  <!-- Model selector and mode toggle -->
   <div class="shrink-0 flex flex-wrap gap-2 mb-4">
     <ModelSelector bind:value={$selectedModelStore} placeholder="Select an image model..." disabled={isGenerating} />
+
+    <select
+      class="px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+      bind:value={$apiModeStore}
+      disabled={isGenerating}
+    >
+      <option value="openai">OpenAI</option>
+      <option value="sdapi">SDAPI</option>
+    </select>
+
     <select
       class="px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
       bind:value={$selectedSizeStore}
@@ -123,8 +224,166 @@
         <option value="1024x1792">1024x1792 (SDXL)</option>
       </optgroup>
     </select>
+
+    {#if isSdapi}
+      <button
+        class="px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface hover:bg-secondary-hover transition-colors"
+        onclick={() => showSettings = !showSettings}
+      >
+        {showSettings ? "Hide Settings" : "Settings"}
+      </button>
+    {/if}
   </div>
 
+  <!-- SDAPI Settings Panel -->
+  {#if isSdapi && showSettings}
+    <div class="shrink-0 mb-4 p-4 rounded border border-gray-200 dark:border-white/10 bg-surface">
+      <div class="grid grid-cols-2 md:grid-cols-4 gap-3 mb-3">
+        <label class="flex flex-col gap-1">
+          <span class="text-xs text-txtsecondary">Steps</span>
+          <input
+            type="number"
+            class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+            bind:value={$sdStepsStore}
+            min="1"
+            max="150"
+          />
+        </label>
+        <label class="flex flex-col gap-1">
+          <span class="text-xs text-txtsecondary">CFG Scale</span>
+          <input
+            type="number"
+            class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+            bind:value={$sdCfgScaleStore}
+            min="1"
+            max="30"
+            step="0.5"
+          />
+        </label>
+        <label class="flex flex-col gap-1">
+          <span class="text-xs text-txtsecondary">Seed (-1 = random)</span>
+          <input
+            type="number"
+            class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+            bind:value={$sdSeedStore}
+            min="-1"
+          />
+        </label>
+        <label class="flex flex-col gap-1">
+          <span class="text-xs text-txtsecondary">Batch Size</span>
+          <input
+            type="number"
+            class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+            bind:value={$sdBatchSizeStore}
+            min="1"
+            max="8"
+          />
+        </label>
+        <label class="flex flex-col gap-1">
+          <span class="text-xs text-txtsecondary">Sampler</span>
+          <select
+            class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+            bind:value={$sdSamplerStore}
+          >
+            <option value="">Default</option>
+            <option value="euler_a">euler_a</option>
+            <option value="euler">euler</option>
+            <option value="heun">heun</option>
+            <option value="dpm2">dpm2</option>
+            <option value="dpmpp2s_a">dpmpp2s_a</option>
+            <option value="dpmpp2m">dpmpp2m</option>
+            <option value="dpmpp2mv2">dpmpp2mv2</option>
+            <option value="ipndm">ipndm</option>
+            <option value="ipndm_v">ipndm_v</option>
+            <option value="lcm">lcm</option>
+            <option value="ddim_trailing">ddim_trailing</option>
+            <option value="tcd">tcd</option>
+          </select>
+        </label>
+        <label class="flex flex-col gap-1">
+          <span class="text-xs text-txtsecondary">Scheduler</span>
+          <select
+            class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+            bind:value={$sdSchedulerStore}
+          >
+            <option value="">Auto for model</option>
+            <option value="discrete">discrete</option>
+            <option value="karras">karras</option>
+            <option value="exponential">exponential</option>
+            <option value="ays">ays</option>
+            <option value="gits">gits</option>
+          </select>
+        </label>
+      </div>
+
+      <label class="flex flex-col gap-1 mb-3">
+        <span class="text-xs text-txtsecondary">Negative Prompt</span>
+        <textarea
+          class="px-2 py-1 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary resize-y text-sm"
+          bind:value={$sdNegativePromptStore}
+          rows="2"
+          placeholder="Elements to avoid..."
+        ></textarea>
+      </label>
+
+      <!-- LoRA Selection -->
+      <div>
+        <span class="text-xs text-txtsecondary block mb-1">LoRAs</span>
+        <div class="flex items-center gap-2 mb-2">
+          <button
+            class="px-3 py-1.5 text-sm rounded border border-gray-200 dark:border-white/10 bg-surface hover:bg-secondary-hover transition-colors disabled:opacity-50"
+            onclick={loadLoras}
+            disabled={!$selectedModelStore || isLoadingLoras}
+          >
+            {isLoadingLoras ? "Loading..." : lorasLoaded ? "Reload LoRAs" : "Load LoRAs"}
+          </button>
+          {#if lorasLoaded && availableLoras.length > 0}
+            <select
+              class="flex-1 px-2 py-1.5 text-sm rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+              onchange={addLora}
+            >
+              <option value="">Add a LoRA...</option>
+              {#each availableLoras.filter((l) => !selectedLoras.some((s) => s.path === l.path)) as lora}
+                <option value={lora.path}>{lora.name}</option>
+              {/each}
+            </select>
+          {/if}
+        </div>
+        {#if loraError}
+          <p class="text-xs text-red-500 mb-1">{loraError}</p>
+        {/if}
+        {#if lorasLoaded && availableLoras.length === 0}
+          <p class="text-xs text-txtsecondary">No LoRAs available</p>
+        {/if}
+        {#if selectedLoras.length > 0}
+          <div class="flex flex-col gap-1.5">
+            {#each selectedLoras as lora}
+              <div class="flex items-center gap-2 text-sm">
+                <span class="flex-1 truncate">{getLoraName(lora.path)}</span>
+                <input
+                  type="number"
+                  class="w-20 px-1.5 py-1 text-xs rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-1 focus:ring-primary"
+                  value={lora.multiplier}
+                  oninput={(e) => updateLoraMultiplier(lora.path, parseFloat((e.target as HTMLInputElement).value) || 1)}
+                  min="0"
+                  max="2"
+                  step="0.1"
+                />
+                <button
+                  class="px-1.5 py-0.5 text-xs rounded border border-gray-200 dark:border-white/10 hover:bg-red-500 hover:text-white hover:border-red-500 transition-colors"
+                  onclick={() => removeLora(lora.path)}
+                  aria-label="Remove LoRA"
+                >
+                  x
+                </button>
+              </div>
+            {/each}
+          </div>
+        {/if}
+      </div>
+    </div>
+  {/if}
+
   <!-- Empty state for no models configured -->
   {#if !hasModels}
     <div class="flex-1 flex items-center justify-center text-txtsecondary">
@@ -143,22 +402,50 @@
           <p class="font-medium">Error</p>
           <p class="text-sm mt-1">{error}</p>
         </div>
-      {:else if generatedImage}
+      {:else if generatedImages.length > 1}
+        <!-- Grid for multiple images (batch) -->
+        <div class="grid grid-cols-2 gap-2 p-2 w-full h-full overflow-auto">
+          {#each generatedImages as img, i}
+            <div class="relative flex items-center justify-center">
+              <button
+                class="p-0 border-0 bg-transparent cursor-pointer"
+                onclick={() => openFullscreen(i)}
+                aria-label="View fullscreen"
+              >
+                <img
+                  src={img}
+                  alt="AI generated content {i + 1}"
+                  class="max-w-full max-h-full object-contain hover:opacity-90 transition-opacity"
+                />
+              </button>
+              <button
+                class="absolute bottom-2 right-2 p-1.5 bg-black/60 hover:bg-black/80 text-white rounded-full transition-colors"
+                onclick={(e) => { e.stopPropagation(); downloadImage(i); }}
+                aria-label="Download image"
+              >
+                <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                  <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4"></path>
+                </svg>
+              </button>
+            </div>
+          {/each}
+        </div>
+      {:else if generatedImages.length === 1}
         <div class="relative max-w-full max-h-full flex items-center justify-center">
           <button
             class="p-0 border-0 bg-transparent cursor-pointer"
-            onclick={openFullscreen}
+            onclick={() => openFullscreen(0)}
             aria-label="View fullscreen"
           >
             <img
-              src={generatedImage}
+              src={generatedImages[0]}
               alt="AI generated content"
               class="max-w-full max-h-full object-contain hover:opacity-90 transition-opacity"
             />
           </button>
           <button
             class="absolute bottom-2 right-2 p-2 bg-black/60 hover:bg-black/80 text-white rounded-full transition-colors"
-            onclick={(e) => { e.stopPropagation(); downloadImage(); }}
+            onclick={(e) => { e.stopPropagation(); downloadImage(0); }}
             aria-label="Download image"
           >
             <svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
@@ -198,7 +485,7 @@
           <button
             class="btn flex-1 md:flex-none"
             onclick={clearImage}
-            disabled={!generatedImage && !error && !prompt.trim()}
+            disabled={generatedImages.length === 0 && !error && !prompt.trim()}
           >
             Clear
           </button>
@@ -209,7 +496,7 @@
 </div>
 
 <!-- Fullscreen dialog -->
-{#if showFullscreen && generatedImage}
+{#if showFullscreen && generatedImages[fullscreenIndex]}
   <div
     class="fixed inset-0 bg-black/90 z-50 flex items-center justify-center p-4"
     onclick={(e) => closeFullscreen(e)}
@@ -226,7 +513,7 @@
       ×
     </button>
     <img
-      src={generatedImage}
+      src={generatedImages[fullscreenIndex]}
       alt="AI generated content"
       class="max-w-full max-h-full object-contain pointer-events-none"
     />

ui-svelte/src/components/playground/ModelSelector.svelte

@@ -25,6 +25,11 @@
       <optgroup label="Local">
         {#each grouped.local as model (model.id)}
           <option value={model.id}>{model.id}</option>
+          {#if model.aliases}
+            {#each model.aliases as alias (alias)}
+              <option value={alias}>  ↳ {alias}</option>
+            {/each}
+          {/if}
         {/each}
       </optgroup>
     {/if}

ui-svelte/src/components/playground/RerankInterface.svelte

@@ -0,0 +1,406 @@
+<script lang="ts">
+  import { models } from "../../stores/api";
+  import { persistentStore } from "../../stores/persistent";
+  import { rerank } from "../../lib/rerankApi";
+  import { playgroundStores } from "../../stores/playgroundActivity";
+  import ModelSelector from "./ModelSelector.svelte";
+
+  type RerankRow = { doc: string; score: number | null };
+  type SortOrder = "none" | "asc" | "desc";
+  type EditorMode = "table" | "json";
+
+  const selectedModelStore = persistentStore<string>("playground-rerank-model", "");
+
+  const defaultQuery = "How do LLM's work?";
+  const defaultDocs = [
+    "Large language models (LLMs) use transformer architectures to predict the next token in a sequence based on massive amounts of text data.",
+    "LLMs are trained on diverse internet text, learning statistical patterns of language that allow them to generate coherent responses.",
+    "During training, LLMs minimize a loss function that measures the difference between predicted and actual tokens across billions of examples.",
+    "Attention mechanisms in transformers enable LLMs to weigh the importance of different words when generating output.",
+    "Fine\u2011tuning allows a pre\u2011trained LLM to adapt to a specific downstream task with a smaller dataset.",
+    "Neural networks consist of layers of interconnected neurons that adjust their weights during back\u2011propagation.",
+    "The history of the Roman Empire spanned over a thousand years.",
+    "Soccer is the most popular sport in many countries around the world.",
+    "Quantum computing uses qubits to perform calculations that are intractable for classical computers.",
+  ];
+
+  let query = $state(defaultQuery);
+  let rows = $state<RerankRow[]>([
+    ...defaultDocs.map((doc) => ({ doc, score: null })),
+    { doc: "", score: null },
+  ]);
+  let isLoading = $state(false);
+  let error = $state<string | null>(null);
+  let usage = $state<{ prompt_tokens: number; total_tokens: number } | null>(null);
+  let abortController: AbortController | null = null;
+  let sortOrder = $state<SortOrder>("desc");
+  let editorMode = $state<EditorMode>("table");
+  let jsonText = $state("");
+  let jsonError = $state<string | null>(null);
+
+  let hasModels = $derived($models.some((m) => !m.unlisted));
+
+  let canSubmit = $derived((() => {
+    if (!$selectedModelStore || isLoading) return false;
+    if (editorMode === "json") {
+      try {
+        const parsed = JSON.parse(jsonText) as Record<string, unknown>;
+        return (
+          typeof parsed.query === "string" &&
+          parsed.query.trim() !== "" &&
+          Array.isArray(parsed.documents) &&
+          (parsed.documents as unknown[]).some(
+            (d) => typeof d === "string" && (d as string).trim() !== ""
+          )
+        );
+      } catch {
+        return false;
+      }
+    }
+    return query.trim() !== "" && rows.some((r) => r.doc.trim() !== "");
+  })());
+
+  // Display rows with sort applied (display-only transform, rows[] is never mutated by sorting)
+  let displayRows = $derived((() => {
+    const indexed = rows.map((row, i) => ({ row, i }));
+    if (sortOrder === "none") return indexed;
+    return [...indexed].sort((a, b) => {
+      if (a.row.score === null && b.row.score === null) return 0;
+      if (a.row.score === null) return 1;
+      if (b.row.score === null) return -1;
+      return sortOrder === "desc"
+        ? b.row.score - a.row.score
+        : a.row.score - b.row.score;
+    });
+  })());
+
+  // Auto-add a new empty row when the last row gets content (table mode only)
+  $effect(() => {
+    if (editorMode === "table" && rows[rows.length - 1]?.doc.trim() !== "") {
+      rows = [...rows, { doc: "", score: null }];
+    }
+  });
+
+  // Sync loading state to activity store
+  $effect(() => {
+    playgroundStores.rerankLoading.set(isLoading);
+  });
+
+  function switchToJson() {
+    if (editorMode === "json") return;
+    const docs = rows.filter((r) => r.doc.trim() !== "").map((r) => r.doc);
+    jsonText = JSON.stringify({ query, documents: docs }, null, 2);
+    jsonError = null;
+    editorMode = "json";
+  }
+
+  function switchToTable() {
+    if (editorMode === "table") return;
+    if (jsonText.trim() === "") {
+      query = "";
+      rows = [{ doc: "", score: null }];
+      jsonError = null;
+      editorMode = "table";
+      return;
+    }
+    try {
+      const parsed = JSON.parse(jsonText) as unknown;
+      if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+        throw new Error("Expected a JSON object");
+      }
+      const obj = parsed as Record<string, unknown>;
+      if (typeof obj.query !== "string") throw new Error('"query" must be a string');
+      if (!Array.isArray(obj.documents)) throw new Error('"documents" must be an array');
+      query = obj.query;
+      const newRows: RerankRow[] = (obj.documents as unknown[]).map((d) => ({
+        doc: typeof d === "string" ? d : String(d),
+        score: null,
+      }));
+      if (newRows.length === 0 || newRows[newRows.length - 1].doc.trim() !== "") {
+        newRows.push({ doc: "", score: null });
+      }
+      rows = newRows;
+      jsonError = null;
+      editorMode = "table";
+    } catch (err) {
+      jsonError = err instanceof Error ? err.message : "Invalid JSON";
+    }
+  }
+
+  function cycleSortOrder() {
+    sortOrder = sortOrder === "none" ? "desc" : sortOrder === "desc" ? "asc" : "none";
+  }
+
+  function sortIndicator(): string {
+    if (sortOrder === "desc") return " ↓";
+    if (sortOrder === "asc") return " ↑";
+    return "";
+  }
+
+  async function submit() {
+    if (!canSubmit) return;
+
+    let submitQuery: string;
+    let nonEmptyEntries: { originalIndex: number; doc: string }[];
+
+    if (editorMode === "json") {
+      // Parse JSON, sync state to table, then submit
+      try {
+        const parsed = JSON.parse(jsonText) as Record<string, unknown>;
+        submitQuery = parsed.query as string;
+        const docs = (parsed.documents as string[]).filter((d) => d.trim() !== "");
+        const newRows: RerankRow[] = docs.map((d) => ({ doc: d, score: null }));
+        newRows.push({ doc: "", score: null });
+        rows = newRows;
+        query = submitQuery;
+        editorMode = "table";
+      } catch {
+        error = "Invalid JSON — fix before submitting";
+        return;
+      }
+      nonEmptyEntries = rows
+        .map((r, i) => ({ originalIndex: i, doc: r.doc }))
+        .filter((e) => e.doc.trim() !== "");
+    } else {
+      submitQuery = query;
+      nonEmptyEntries = rows
+        .map((r, i) => ({ originalIndex: i, doc: r.doc }))
+        .filter((e) => e.doc.trim() !== "");
+    }
+
+    isLoading = true;
+    error = null;
+    usage = null;
+
+    // Clear previous scores
+    rows = rows.map((r) => ({ ...r, score: null }));
+
+    abortController = new AbortController();
+
+    try {
+      const response = await rerank(
+        $selectedModelStore,
+        submitQuery,
+        nonEmptyEntries.map((e) => e.doc),
+        abortController.signal
+      );
+
+      usage = response.usage;
+
+      // Map result.index (position in submitted docs array) back to original rows[] index
+      const updated = rows.map((r) => ({ ...r }));
+      for (const result of response.results) {
+        const entry = nonEmptyEntries[result.index];
+        if (entry !== undefined) {
+          updated[entry.originalIndex].score = result.relevance_score;
+        }
+      }
+      rows = updated;
+    } catch (err) {
+      if (err instanceof Error && err.name === "AbortError") {
+        // User cancelled
+      } else {
+        error = err instanceof Error ? err.message : "An error occurred";
+      }
+    } finally {
+      isLoading = false;
+      abortController = null;
+    }
+  }
+
+  function cancel() {
+    abortController?.abort();
+  }
+
+  function clear() {
+    query = defaultQuery;
+    rows = [...defaultDocs.map((doc) => ({ doc, score: null })), { doc: "", score: null }];
+    error = null;
+    usage = null;
+    sortOrder = "desc";
+    jsonText = "";
+    jsonError = null;
+  }
+
+  function deleteRow(originalIndex: number) {
+    if (rows.length <= 1) return;
+    rows = rows.filter((_, i) => i !== originalIndex);
+  }
+
+  function updateDoc(originalIndex: number, value: string) {
+    const updated = rows.map((r) => ({ ...r }));
+    updated[originalIndex].doc = value;
+    rows = updated;
+  }
+
+  function scoreColor(score: number | null): string {
+    if (score === null) return "text-txtsecondary";
+    if (score > 0) return "text-green-600 dark:text-green-400";
+    return "text-red-500 dark:text-red-400";
+  }
+
+  function formatScore(score: number | null): string {
+    if (score === null) return "—";
+    return score.toFixed(3);
+  }
+
+  function handleKeyDown(e: KeyboardEvent) {
+    if (e.key === "Enter" && !e.shiftKey) {
+      e.preventDefault();
+      submit();
+    }
+  }
+
+  let isCleared = $derived(
+    query === defaultQuery &&
+    rows.every((r, i) => r.score === null && r.doc === (defaultDocs[i] ?? "")) &&
+    rows.length === defaultDocs.length + 1 &&
+    !jsonText.trim() &&
+    !error &&
+    !usage
+  );
+</script>
+
+<div class="flex flex-col h-full">
+  <!-- Top bar: model selector + query input (table mode) + mode toggle -->
+  <div class="shrink-0 flex flex-wrap gap-2 mb-4">
+    <ModelSelector bind:value={$selectedModelStore} placeholder="Select a rerank model..." disabled={isLoading} />
+    {#if editorMode === "table"}
+      <input
+        type="text"
+        class="min-w-0 flex-1 basis-48 px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary"
+        placeholder="Query..."
+        bind:value={query}
+        disabled={isLoading}
+        onkeydown={handleKeyDown}
+      />
+    {/if}
+    <!-- Table / JSON toggle -->
+    <div class="flex rounded border border-gray-200 dark:border-white/10 overflow-hidden shrink-0">
+      <button
+        class="px-3 py-1.5 text-sm transition-colors {editorMode === 'table'
+          ? 'bg-primary text-btn-primary-text'
+          : 'bg-surface hover:bg-secondary-hover'}"
+        onclick={switchToTable}
+        disabled={isLoading}
+      >
+        Table
+      </button>
+      <button
+        class="px-3 py-1.5 text-sm border-l border-gray-200 dark:border-white/10 transition-colors {editorMode === 'json'
+          ? 'bg-primary text-btn-primary-text'
+          : 'bg-surface hover:bg-secondary-hover'}"
+        onclick={switchToJson}
+        disabled={isLoading}
+      >
+        JSON
+      </button>
+    </div>
+  </div>
+
+  {#if !hasModels}
+    <div class="flex-1 flex items-center justify-center text-txtsecondary">
+      <p>No models configured. Add models to your configuration to use reranking.</p>
+    </div>
+  {:else if editorMode === "json"}
+    <!-- JSON editor -->
+    <div class="flex-1 flex flex-col min-h-0 mb-4">
+      <textarea
+        class="flex-1 w-full font-mono text-sm px-3 py-2 rounded border border-gray-200 dark:border-white/10 bg-surface focus:outline-none focus:ring-2 focus:ring-primary resize-none"
+        bind:value={jsonText}
+        disabled={isLoading}
+        placeholder={'{\n  "query": "your search query",\n  "documents": [\n    "document one",\n    "document two"\n  ]\n}'}
+        spellcheck={false}
+      ></textarea>
+      {#if jsonError}
+        <p class="mt-1 text-sm text-red-500">{jsonError}</p>
+      {/if}
+    </div>
+  {:else}
+    <!-- Document table -->
+    <div class="flex-1 overflow-y-auto mb-4 border border-gray-200 dark:border-white/10 rounded">
+      <table class="w-full border-collapse table-fixed">
+        <colgroup>
+          <col class="w-auto" />
+          <col style="width: 120px" />
+          <col style="width: 40px" />
+        </colgroup>
+        <thead class="sticky top-0 bg-surface border-b border-gray-200 dark:border-white/10">
+          <tr>
+            <th class="px-3 py-2 text-left text-sm font-medium text-txtsecondary">Document</th>
+            <th
+              class="px-3 py-2 text-right text-sm font-medium text-txtsecondary cursor-pointer select-none hover:text-txtprimary transition-colors"
+              onclick={cycleSortOrder}
+            >
+              Score{sortIndicator()}
+            </th>
+            <th class="px-2 py-2"></th>
+          </tr>
+        </thead>
+        <tbody>
+          {#each displayRows as { row, i } (i)}
+            <tr class="border-b border-gray-100 dark:border-white/5 last:border-0">
+              <td class="px-3 py-1.5">
+                <input
+                  type="text"
+                  class="w-full bg-transparent focus:outline-none focus:ring-1 focus:ring-primary rounded px-1 py-0.5"
+                  placeholder={i === rows.length - 1 ? "Add document..." : "Document text..."}
+                  value={row.doc}
+                  oninput={(e) => updateDoc(i, (e.target as HTMLInputElement).value)}
+                  disabled={isLoading}
+                  onkeydown={handleKeyDown}
+                />
+              </td>
+              <td class="px-3 py-1.5 text-right font-mono text-sm {scoreColor(row.score)}">
+                {#if isLoading && row.score === null && row.doc.trim() !== ""}
+                  <span class="inline-block w-4 h-4 border-2 border-current border-t-transparent rounded-full animate-spin align-middle"></span>
+                {:else}
+                  {formatScore(row.score)}
+                {/if}
+              </td>
+              <td class="px-2 py-1.5 text-center">
+                <button
+                  class="w-7 h-7 flex items-center justify-center text-txtsecondary hover:text-red-500 transition-colors rounded disabled:opacity-30 disabled:cursor-not-allowed"
+                  onclick={() => deleteRow(i)}
+                  disabled={rows.length <= 1}
+                  tabindex="-1"
+                  aria-label="Remove row"
+                >
+                  ×
+                </button>
+              </td>
+            </tr>
+          {/each}
+        </tbody>
+      </table>
+    </div>
+  {/if}
+
+  <!-- Bottom toolbar -->
+  {#if hasModels}
+    <div class="shrink-0 flex flex-wrap items-center gap-2">
+      {#if isLoading}
+        <button class="btn bg-red-500 hover:bg-red-600 text-white" onclick={cancel}>
+          Cancel
+        </button>
+      {:else}
+        <button
+          class="btn bg-primary text-btn-primary-text hover:opacity-90"
+          onclick={submit}
+          disabled={!canSubmit}
+        >
+          Rerank
+        </button>
+        <button class="btn" onclick={clear} disabled={isCleared}>
+          Clear
+        </button>
+      {/if}
+
+      {#if error}
+        <span class="text-sm text-red-500 ml-2">{error}</span>
+      {:else if usage}
+        <span class="text-sm text-txtsecondary ml-2">{usage.total_tokens} tokens</span>
+      {/if}
+    </div>
+  {/if}
+</div>

ui-svelte/src/lib/rerankApi.ts

@@ -0,0 +1,27 @@
+export interface RerankResult {
+  index: number;
+  relevance_score: number;
+}
+
+export interface RerankResponse {
+  model: string;
+  object: string;
+  usage: { prompt_tokens: number; total_tokens: number };
+  results: RerankResult[];
+}
+
+export async function rerank(
+  model: string,
+  query: string,
+  documents: string[],
+  signal: AbortSignal
+): Promise<RerankResponse> {
+  const response = await fetch("/v1/rerank", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ model, query, documents }),
+    signal,
+  });
+  if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);
+  return response.json();
+}

ui-svelte/src/lib/sdApi.ts

@@ -0,0 +1,39 @@
+import type { SdApiTxt2ImgRequest, SdApiResponse, SdApiLora } from "./types";
+
+export async function generateSdImage(
+  request: SdApiTxt2ImgRequest,
+  signal?: AbortSignal
+): Promise<SdApiResponse> {
+  const response = await fetch("/sdapi/v1/txt2img", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(request),
+    signal,
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`SDAPI error: ${response.status} - ${errorText}`);
+  }
+
+  return response.json();
+}
+
+export async function fetchSdLoras(
+  model: string,
+  signal?: AbortSignal
+): Promise<SdApiLora[]> {
+  const response = await fetch(
+    `/sdapi/v1/loras?model=${encodeURIComponent(model)}`,
+    { signal }
+  );
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`SDAPI loras error: ${response.status} - ${errorText}`);
+  }
+
+  return response.json();
+}

ui-svelte/src/lib/types.ts

@@ -9,6 +9,7 @@ export interface Model {
   description: string;
   unlisted: boolean;
   peerID: string;
+  aliases?: string[];
 }
 
 export interface Metrics {
@@ -38,8 +39,12 @@ export interface LogData {
   data: string;
 }
 
+export interface InFlightStats {
+  total: number;
+}
+
 export interface APIEventEnvelope {
-  type: "modelStatus" | "logData" | "metrics";
+  type: "modelStatus" | "logData" | "metrics" | "inflight";
   data: string;
 }
 
@@ -110,6 +115,40 @@ export interface ImageGenerationResponse {
   }>;
 }
 
+// SDAPI types (stable-diffusion.cpp)
+export type ImageApiMode = "openai" | "sdapi";
+
+export interface SdApiLora {
+  name: string;
+  path: string;
+}
+
+export interface SdApiLoraRef {
+  path: string;
+  multiplier: number;
+}
+
+export interface SdApiTxt2ImgRequest {
+  model?: string;
+  prompt: string;
+  negative_prompt?: string;
+  width?: number;
+  height?: number;
+  steps?: number;
+  cfg_scale?: number;
+  seed?: number;
+  batch_size?: number;
+  sampler_name?: string;
+  scheduler?: string;
+  lora?: SdApiLoraRef[];
+}
+
+export interface SdApiResponse {
+  images: string[];
+  parameters: Record<string, unknown>;
+  info: string;
+}
+
 export interface AudioTranscriptionRequest {
   file: File;
   model: string;

ui-svelte/src/routes/Playground.svelte

@@ -4,8 +4,9 @@
   import ImageInterface from "../components/playground/ImageInterface.svelte";
   import AudioInterface from "../components/playground/AudioInterface.svelte";
   import SpeechInterface from "../components/playground/SpeechInterface.svelte";
+  import RerankInterface from "../components/playground/RerankInterface.svelte";
 
-  type Tab = "chat" | "images" | "speech" | "audio";
+  type Tab = "chat" | "images" | "speech" | "audio" | "rerank";
 
   const selectedTabStore = persistentStore<Tab>("playground-selected-tab", "chat");
   let mobileMenuOpen = $state(false);
@@ -15,6 +16,7 @@
     { id: "images", label: "Images" },
     { id: "speech", label: "Speech" },
     { id: "audio", label: "Transcription" },
+    { id: "rerank", label: "Rerank" },
   ];
 
   function selectTab(tab: Tab) {
@@ -89,6 +91,9 @@
     <div class="h-full" class:tab-hidden={$selectedTabStore !== "audio"}>
       <AudioInterface />
     </div>
+    <div class="h-full" class:tab-hidden={$selectedTabStore !== "rerank"}>
+      <RerankInterface />
+    </div>
   </div>
 </div>
 

ui-svelte/src/stores/api.ts

@@ -1,5 +1,5 @@
 import { writable } from "svelte/store";
-import type { Model, Metrics, VersionInfo, LogData, APIEventEnvelope, ReqRespCapture } from "../lib/types";
+import type { Model, Metrics, VersionInfo, LogData, APIEventEnvelope, ReqRespCapture, InFlightStats } from "../lib/types";
 import { connectionState } from "./theme";
 
 const LOG_LENGTH_LIMIT = 1024 * 100; /* 100KB of log data */
@@ -9,6 +9,7 @@ export const models = writable<Model[]>([]);
 export const proxyLogs = writable<string>("");
 export const upstreamLogs = writable<string>("");
 export const metrics = writable<Metrics[]>([]);
+export const inFlightRequests = writable<number>(0);
 export const versionInfo = writable<VersionInfo>({
   build_date: "unknown",
   commit: "unknown",
@@ -29,6 +30,7 @@ export function enableAPIEvents(enabled: boolean): void {
     apiEventSource?.close();
     apiEventSource = null;
     metrics.set([]);
+    inFlightRequests.set(0);
     return;
   }
 
@@ -46,6 +48,7 @@ export function enableAPIEvents(enabled: boolean): void {
       proxyLogs.set("");
       upstreamLogs.set("");
       metrics.set([]);
+      inFlightRequests.set(0);
       models.set([]);
       retryCount = 0;
       connectionState.set("connected");
@@ -59,7 +62,7 @@ export function enableAPIEvents(enabled: boolean): void {
             const newModels = JSON.parse(message.data) as Model[];
             // Sort models by name and id
             newModels.sort((a, b) => {
-              return (a.name + a.id).localeCompare(b.name + b.id);
+              return (a.name + a.id).localeCompare(b.name + b.id, undefined, { numeric : true} );
             });
             models.set(newModels);
             break;
@@ -83,6 +86,11 @@ export function enableAPIEvents(enabled: boolean): void {
             metrics.update((prevMetrics) => [...newMetrics, ...prevMetrics]);
             break;
           }
+          case "inflight": {
+            const stats = JSON.parse(message.data) as InFlightStats;
+            inFlightRequests.set(stats.total ?? 0);
+            break;
+          }
         }
       } catch (err) {
         console.error(e.data, err);

ui-svelte/src/stores/playgroundActivity.ts

@@ -4,10 +4,11 @@ const chatStreaming = writable(false);
 const imageGenerating = writable(false);
 const speechGenerating = writable(false);
 const audioTranscribing = writable(false);
+const rerankLoading = writable(false);
 
 export const playgroundActivity = derived(
-  [chatStreaming, imageGenerating, speechGenerating, audioTranscribing],
-  ([$chat, $image, $speech, $audio]) => $chat || $image || $speech || $audio
+  [chatStreaming, imageGenerating, speechGenerating, audioTranscribing, rerankLoading],
+  ([$chat, $image, $speech, $audio, $rerank]) => $chat || $image || $speech || $audio || $rerank
 );
 
 export const playgroundStores = {
@@ -15,4 +16,5 @@ export const playgroundStores = {
   imageGenerating,
   speechGenerating,
   audioTranscribing,
+  rerankLoading,
 };

ui-svelte/vite.config.ts

@@ -32,6 +32,7 @@ export default defineConfig({
       "/upstream": "http://localhost:8080",
       "/unload": "http://localhost:8080",
       "/v1": "http://localhost:8080",
+      "/sdapi": "http://localhost:8080",
     },
   },
 });