ui-svelte: fix histogram calculation (#695)

- Fix the histogram calculation to use server provided generation
tokens/second.
- Move histogram to Activities page where it can exist with the rest of
the token metrics

Fixes #681

.github/workflows/go-ci.yml

@@ -2,69 +2,68 @@ name: Linux CI
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
     # only run when backend source changes
     # cmd/ is excluded because it contains utilities without tests
     paths:
-      - '**/*.go'
-      - '!cmd/**'
-      - 'go.mod'
-      - 'go.sum'
-      - 'Makefile'
-      - '.github/workflows/go-ci.yml'
+      - "**/*.go"
+      - "!cmd/**"
+      - "go.mod"
+      - "go.sum"
+      - "Makefile"
+      - ".github/workflows/go-ci.yml"
 
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
     paths:
-      - '**/*.go'
-      - '!cmd/**'
-      - 'go.mod'
-      - 'go.sum'
-      - 'Makefile'
-      - '.github/workflows/go-ci.yml'
+      - "**/*.go"
+      - "!cmd/**"
+      - "go.mod"
+      - "go.sum"
+      - "Makefile"
+      - ".github/workflows/go-ci.yml"
 
   # Allows manual triggering of the workflow
   workflow_dispatch:
 
 jobs:
-
   run-tests:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-    - name: Set up Go
-      uses: actions/setup-go@v4
-      with:
-        go-version-file: go.mod
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
 
-    # Only run in this linux based runner
-    - name: Check Formatting
-      run: |
-        if [ "$(gofmt -l . | grep -v 'event/.*_test.go' | wc -l)" -gt 0 ]; then
-          gofmt -l . | grep -v 'event/.*_test.go'
-          exit 1
-        fi
-    # cache simple-responder to save the build time
-    - name: Restore Simple Responder
-      id: restore-simple-responder
-      uses: actions/cache/restore@v4
-      with:
-        path: ./build
-        key: ${{ runner.os }}-simple-responder-${{ hashFiles('cmd/simple-responder/simple-responder.go') }}
+      # Only run in this linux based runner
+      - name: Check Formatting
+        run: |
+          if [ "$(gofmt -l . | grep -v 'event/.*_test.go' | wc -l)" -gt 0 ]; then
+            gofmt -l . | grep -v 'event/.*_test.go'
+            exit 1
+          fi
+      # cache simple-responder to save the build time
+      - name: Restore Simple Responder
+        id: restore-simple-responder
+        uses: actions/cache/restore@v4
+        with:
+          path: ./build
+          key: ${{ runner.os }}-simple-responder-${{ hashFiles('cmd/simple-responder/simple-responder.go') }}
 
-    # necessary for testing proxy/Process swapping
-    - name: Create simple-responder
-      run: make simple-responder
+      # necessary for testing proxy/Process swapping
+      - name: Create simple-responder
+        run: make simple-responder
 
-    - name: Save Simple Responder
-      # nothing new to save ... skip this step
-      if: steps.restore-simple-responder.outputs.cache-hit != 'true'
-      id: save-simple-responder
-      uses: actions/cache/save@v4
-      with:
-        path: ./build
-        key: ${{ runner.os }}-simple-responder-${{ hashFiles('misc/simple-responder/simple-responder.go') }}
+      - name: Save Simple Responder
+        # nothing new to save ... skip this step
+        if: steps.restore-simple-responder.outputs.cache-hit != 'true'
+        id: save-simple-responder
+        uses: actions/cache/save@v4
+        with:
+          path: ./build
+          key: ${{ runner.os }}-simple-responder-${{ hashFiles('cmd/simple-responder/simple-responder.go') }}
 
-    - name: Test all
-      run: make test-all
+      - name: Test all
+        run: make test-all

.github/workflows/ui-tests.yml

@@ -19,9 +19,6 @@ jobs:
 
   run-tests:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ui-svelte
     steps:
     - uses: actions/checkout@v4
 
@@ -32,11 +29,5 @@ jobs:
         cache: 'npm'
         cache-dependency-path: ui-svelte/package-lock.json
 
-    - name: Install dependencies
-      run: npm ci
-
-    - name: Type check
-      run: npm run check
-
-    - name: Run tests
-      run: npm test
+    - name: Run UI tests
+      run: make test-ui

.github/workflows/unified-docker.yml

@@ -36,6 +36,11 @@ on:
         type: boolean
         required: false
         default: true
+      push_to_ghcr:
+        description: "Push images to ghcr.io"
+        type: boolean
+        required: false
+        default: true
 
 permissions:
   contents: read
@@ -68,13 +73,6 @@ jobs:
       fail-fast: false
       matrix:
         backend: ${{ fromJSON(needs.setup.outputs.matrix) }}
-        variant:
-          - name: root
-            uid: "0"
-            suffix: ""
-          - name: rootless
-            uid: "10001"
-            suffix: "-rootless"
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -106,15 +104,14 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build unified Docker image (${{ matrix.backend }}, ${{ matrix.variant.name }})
+      - name: Build unified Docker image (${{ matrix.backend }})
         env:
           LLAMA_REF: ${{ inputs.llama_cpp_ref || 'master' }}
           WHISPER_REF: ${{ inputs.whisper_ref || 'master' }}
           SD_REF: ${{ inputs.sd_ref || 'master' }}
           IK_LLAMA_REF: ${{ inputs.ik_llama_ref || 'main' }}
           LS_VERSION: ${{ inputs.llama_swap_version || 'main' }}
-          RUN_UID: ${{ matrix.variant.uid }}
-          DOCKER_IMAGE_TAG: ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}${{ matrix.variant.suffix }}
+          DOCKER_IMAGE_TAG: ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}
           # When running under act, use the local builder that has warm ccache.
           # On GitHub Actions, BUILDX_BUILDER is unset so docker uses the builder
           # created by setup-buildx-action above.
@@ -124,10 +121,16 @@ jobs:
           docker/unified/build-image.sh --${{ matrix.backend }}
 
       - name: Push to GitHub Container Registry
-        if: ${{ !env.ACT }}
+        if: ${{ !env.ACT && (github.event_name == 'schedule' || inputs.push_to_ghcr == true) }}
         run: |
-          TAG="ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}${{ matrix.variant.suffix }}"
-          docker push "${TAG}"
+          BASE_TAG="ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}"
           DATE_TAG=$(date -u +%Y-%m-%d)
-          docker tag "${TAG}" "${TAG}-${DATE_TAG}"
-          docker push "${TAG}-${DATE_TAG}"
+
+          docker push "${BASE_TAG}"
+          docker tag "${BASE_TAG}" "${BASE_TAG}-${DATE_TAG}"
+          docker push "${BASE_TAG}-${DATE_TAG}"
+
+          ROOTLESS_TAG="${BASE_TAG}-rootless"
+          docker push "${ROOTLESS_TAG}"
+          docker tag "${ROOTLESS_TAG}" "${ROOTLESS_TAG}-${DATE_TAG}"
+          docker push "${ROOTLESS_TAG}-${DATE_TAG}"

AGENTS.md

@@ -24,6 +24,7 @@ llama-swap is a light weight, transparent proxy server that provides automatic m
 - Run `gofmt -l .` before committing to verify formatting. Fix any reported files with `gofmt -w <file>`.
 - Use `make test-dev` after running new tests for a quick over all test run. This runs `go test` and `staticcheck`. Fix any static checking errors. Use this only when changes are made to any code under the `proxy/` directory
 - Use `make test-all` before completing work. This includes long running concurrency tests.
+- Use `make test-ui` after making changes to the UI in ui-svelte/
 
 ### Commit message example format:
 

Makefile

@@ -48,10 +48,15 @@ mac: ui
 	GOOS=darwin GOARCH=arm64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-darwin-arm64
 
 # Build Linux binary
-linux: ui
-	@echo "Building Linux binary..."
+linux: linux-arm64 linux-amd64
+
+linux-amd64: ui
+	@echo "Building Linux AMD64 binary..."
 	GOOS=linux GOARCH=amd64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-amd64
-#GOOS=linux GOARCH=arm64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-arm64
+
+linux-arm64: ui
+	@echo "Building Linux ARM64 binary..."
+	GOOS=linux GOARCH=arm64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-arm64
 
 # Build Windows binary
 windows: ui
@@ -92,5 +97,9 @@ wol-proxy: $(BUILD_DIR)
 	@echo "Building wol-proxy"
 	go build -o $(BUILD_DIR)/wol-proxy-$(GOOS)-$(GOARCH)-$(shell date +%Y-%m-%d) cmd/wol-proxy/wol-proxy.go
 
+test-ui:
+	cd ui-svelte && npm ci && npm run check && npm test
+
 # Phony targets
-.PHONY: all clean ui mac linux windows simple-responder simple-responder-windows test test-all test-dev wol-proxy
+.PHONY: all clean ui mac windows simple-responder simple-responder-windows test test-all test-dev test-ui wol-proxy
+.PHONE: linux linux-arm64 linux-amd64

README.md

@@ -5,7 +5,7 @@
 
 # llama-swap
 
-Run multiple generative AI models on your machine and hot-swap between them on demand. llama-swap works with any OpenAI and Anthropic API compatible server and is used by thousands of people to power their local AI workflows. 
+Run multiple generative AI models on your machine and hot-swap between them on demand. llama-swap works with any OpenAI and Anthropic API compatible server and is used by thousands of people to power their local AI workflows.
 
 Built in Go for performance and simplicity, llama-swap has zero dependencies and is incredibly easy to set up. Get started in minutes - just one binary and one configuration file.
 
@@ -45,14 +45,14 @@ Built in Go for performance and simplicity, llama-swap has zero dependencies and
   - `/health` - just returns "OK"
 - ✅ API Key support - define keys to restrict access to API endpoints
 - ✅ Customizable
-  - Run multiple models at once with `Groups` ([#107](https://github.com/mostlygeek/llama-swap/issues/107))
+  - Run concurrent models with a custom DSL swap matrix ([#643](https://github.com/mostlygeek/llama-swap/issues/643))
   - Automatic unloading of models after timeout by setting a `ttl`
   - Reliable Docker and Podman support using `cmd` and `cmdStop` together
   - Preload models on startup with `hooks` ([#235](https://github.com/mostlygeek/llama-swap/pull/235))
 
 ### Web UI
 
-llama-swap includes a real time web interface with a playground for testing out all sorts of local models: 
+llama-swap includes a real time web interface with a playground for testing out all sorts of local models:
 
 <img width="1125" height="876" alt="image" src="https://github.com/user-attachments/assets/8ee41947-97af-463d-b0f0-8e9c478fac07" />
 
@@ -64,16 +64,14 @@ Inspect request and responses:
 
 <img width="1111" height="720" alt="image" src="https://github.com/user-attachments/assets/24fe4aca-1448-4d7c-b9e8-a967589bda6c" />
 
-Manually load and unload models: 
+Manually load and unload models:
 
 <img width="1109" height="719" alt="image" src="https://github.com/user-attachments/assets/02b1e1f2-abd0-4050-84ae-facd66ff01c4" />
 
-
-Real time log streaming: 
+Real time log streaming:
 
 <img width="1107" height="559" alt="image" src="https://github.com/user-attachments/assets/39669a10-cff2-409e-836a-5bad8bd0140c" />
 
-
 ## Installation
 
 llama-swap can be installed in multiple ways
@@ -182,7 +180,7 @@ That's all you need to get started:
 Almost all configuration settings are optional and can be added one step at a time:
 
 - Advanced features
-  - `groups` to run multiple models at once
+  - `matrix` to run concurrent models with a custom swap logic DSL
   - `hooks` to run things on startup
   - `macros` reusable snippets
 - Model customization
@@ -200,7 +198,7 @@ See the [configuration documentation](docs/configuration.md) for all options.
 
 When a request is made to an OpenAI compatible endpoint, llama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to handle the request correctly.
 
-In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `groups` feature allows multiple models to be loaded at the same time. You have complete control over how your system resources are used.
+In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, using a `matrix` allows multiple models to be loaded at the same time. You have complete control over how your system resources are used.
 
 ## Reverse Proxy Configuration (nginx)
 

ai-plans/improve-tests-655.md

@@ -0,0 +1,183 @@
+# Improve Testability (#655)
+
+## Current Pain Points
+
+1. **Tests bypass config loading** - ~80% of tests build `config.Config` structs directly, skipping YAML parsing, env var substitution, macro expansion, and `${PORT}` assignment. Config bugs in those paths go untested.
+
+2. **simple-responder is everywhere** - Every proxy/routing test launches a real subprocess, waits for health checks (~healthCheckTimeout: 15), and manages process lifecycle just to test HTTP routing. Most of that overhead is wasted.
+
+3. **Port counter is fragile** - A global `nextTestPort` counter starting at 12000 with a mutex. Parallel tests or leftover processes can collide.
+
+## Stages
+
+### Stage 1: YAML-based test config helper
+
+**Goal:** Tests go through the real `LoadConfigFromReader` path instead of hand-building structs.
+
+**Effort:** Low | **Impact:** Config bugs caught earlier | **Risk:** None
+
+Create a test helper in `proxy/helpers_test.go`:
+
+```go
+// testConfigFromYAML substitutes simple-responder paths and loads through
+// the real config pipeline (env vars, macros, port assignment, etc.)
+func testConfigFromYAML(t *testing.T, yamlTmpl string) config.Config {
+    t.Helper()
+    yamlStr := strings.ReplaceAll(yamlTmpl, "{{RESPONDER}}", filepath.ToSlash(simpleResponderPath))
+    cfg, err := config.LoadConfigFromReader(strings.NewReader(yamlStr))
+    require.NoError(t, err)
+    return cfg
+}
+```
+
+Tests would then look like:
+
+```go
+func TestProxyManager_SwapProcessCorrectly(t *testing.T) {
+    config := testConfigFromYAML(t, `
+healthCheckTimeout: 15
+logLevel: error
+models:
+  model1:
+    cmd: {{RESPONDER}} --port ${PORT} -silent -respond model1
+  model2:
+    cmd: {{RESPONDER}} --port ${PORT} -silent -respond model2
+`)
+    proxy := New(config)
+    // ... same assertions
+}
+```
+
+**Why this stage first:** Zero production code changes. Pure test-side refactoring. Can be done incrementally - migrate tests one at a time. Each migrated test now validates the full config pipeline.
+
+**Scope:** ~20-30 tests in `proxymanager_test.go`, `processgroup_test.go`, `peerproxy_test.go`.
+
+### Stage 2: Injected test handler (eliminate simple-responder for routing tests)
+
+**Goal:** Replace simple-responder subprocess launches with an injected `http.Handler` for tests that don't specifically test process lifecycle.
+
+**Effort:** Medium | **Impact:** 10-100x faster routing tests | **Risk:** Low (additive, no existing code broken)
+
+Add a `testHandler http.Handler` field to `Process`. When set, `ProxyRequest` delegates directly to this handler instead of going through the reverse proxy. No subprocess, no health checks, no TCP roundtrip.
+
+**2a. Add testHandler to Process:**
+
+```go
+// In Process struct (process.go):
+testHandler http.Handler  // set only in tests; bypasses subprocess and reverse proxy
+```
+
+In `Process.Start()`, skip subprocess + health check when handler is set:
+
+```go
+func (p *Process) start() error {
+    if p.testHandler != nil {
+        p.setState(StateReady)
+        return nil
+    }
+    // existing subprocess logic...
+}
+```
+
+In `Process.ProxyRequest()`, delegate directly to the handler:
+
+```go
+// Before the reverseProxy.ServeHTTP call:
+if p.testHandler != nil {
+    p.testHandler.ServeHTTP(w, r)
+    return
+}
+```
+
+**2b. Test helper to create the handler:**
+
+```go
+// newTestHandler returns an http.Handler that mimics llama.cpp's API
+// (same endpoints as simple-responder).
+func newTestHandler(respond string) http.Handler {
+    mux := http.NewServeMux()
+    mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { ... })
+    mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { ... })
+    // ... other endpoints
+    return mux
+}
+```
+
+Tests for routing/auth/CORS/streaming then become:
+
+```go
+func TestProxyManager_AuthRequired(t *testing.T) {
+    handler := newTestHandler("model1")
+
+    config := testConfigFromYAML(t, `
+healthCheckTimeout: 15
+logLevel: error
+requiredAPIKeys: [test-key]
+models:
+  model1:
+    cmd: {{RESPONDER}} --port ${PORT} -silent -respond model1
+`)
+    pm := NewProxyManager(config)
+    // inject handler — skips subprocess, health check, port allocation
+    pm.processGroups["model1"].process.testHandler = handler
+}
+```
+
+**Why this matters:** The handler is called directly in-process. No subprocess spawn, no health check timeout, no port allocation, no TCP roundtrip, no reverse proxy overhead. Routing tests go from ~100ms each (process startup + health check) to ~1ms. Unlike an `httptest.Server` approach, there are zero network hops.
+
+**Why not blank-cmd + proxy URL:** A blank `cmd` with a `proxy` field pointing at `httptest.Server` still requires a real TCP roundtrip through the reverse proxy and introduces "external process" semantics to the config schema. Injecting the handler directly keeps it purely a test concern with no config changes.
+
+**Scope:** Most tests in `proxymanager_test.go` (auth, CORS, model listing, streaming, peer proxy), `peerproxy_test.go`, `metrics_monitor_test.go`.
+
+### Stage 3: Migrate tests incrementally
+
+**Goal:** Convert existing tests to use the Stage 1 + Stage 2 helpers.
+
+**Effort:** Medium | **Impact:** Cleaner, more reliable tests | **Risk:** None
+
+Priority order:
+1. `proxymanager_test.go` routing tests (highest count, most repetition)
+2. `peerproxy_test.go` (straightforward, all HTTP routing)
+3. `metrics_monitor_test.go` (capture logic doesn't need real processes)
+4. `processgroup_test.go` swap tests (keep simple-responder for actual swap lifecycle tests)
+
+Tests that **must keep simple-responder:**
+- Process lifecycle: start/stop, SIGKILL, SIGTERM, TTL expiry, health check failures, failed start counting
+- ProcessGroup swap concurrency (the port-collision test in `TestProcessGroup_ProxyRequestSwapIsTrueParallel`)
+
+**Scope:** ~60-70% of tests can drop simple-responder.
+
+### Stage 4 (optional): Process interface for ProcessGroup
+
+**Goal:** Enable pure unit tests of ProcessGroup's swap/exclusive/concurrency logic without any HTTP server at all.
+
+**Effort:** High | **Impact:** Pure unit tests possible | **Risk:** Medium (refactor core code)
+
+```go
+type ProcessController interface {
+    Start() error
+    Stop(StopStrategy)
+    ProxyRequest(http.ResponseWriter, *http.Request) error
+    CurrentState() ProcessState
+    ID() string
+    SetState(ProcessState)  // for test setup
+}
+```
+
+This requires:
+- Extracting the interface
+- A `MockProcess` implementation
+- Refactoring `ProcessGroup` to use the interface instead of `*Process`
+
+**Recommendation:** Only do this if ProcessGroup grows significantly more complex. Stages 1-3 give 80% of the benefit for 20% of the effort.
+
+## Effort/Impact Summary
+
+| Stage | Effort | Impact | Risk |
+|-------|--------|--------|------|
+| 1. YAML config helper | Low | Config bugs caught earlier | None |
+| 2. Injected test handler | Medium | 10-100x faster routing tests | Low |
+| 3. Migrate tests | Medium | Cleaner, more reliable tests | None |
+| 4. Process interface | High | Pure unit tests possible | Medium |
+
+**Recommended approach:** Do stages 1-3 in order. Each stage is independently valuable and can ship on its own. Stage 4 is deferred unless there's a specific need.

config-schema.json

@@ -47,31 +47,37 @@
                     "type": "integer",
                     "minimum": 0,
                     "default": 30,
-                    "description": "TCP connection timeout in seconds. Set to 0 to disable (not recommended)."
+                    "description": "TCP connection timeout in seconds. Set to 0 to disable."
+                },
+                "keepalive": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 30,
+                    "description": "TCP keepalive timeout in seconds. Set to 0 to disable."
                 },
                 "responseHeader": {
                     "type": "integer",
                     "minimum": 0,
-                    "default": 60,
-                    "description": "Time to wait for response headers in seconds. Set to 0 to disable (not recommended)."
+                    "default": 0,
+                    "description": "Time to wait for response headers in seconds. Set to 0 to disable."
                 },
                 "tlsHandshake": {
                     "type": "integer",
                     "minimum": 0,
                     "default": 10,
-                    "description": "TLS handshake timeout in seconds. Set to 0 to disable (not recommended)."
+                    "description": "TLS handshake timeout in seconds. Set to 0 to disable."
                 },
                 "expectContinue": {
                     "type": "integer",
                     "minimum": 0,
                     "default": 1,
-                    "description": "Expect-Continue timeout in seconds. Set to 0 to disable (not recommended)."
+                    "description": "Expect-Continue timeout in seconds. Set to 0 to disable."
                 },
                 "idleConn": {
                     "type": "integer",
                     "minimum": 0,
                     "default": 90,
-                    "description": "Idle connection timeout in seconds. Set to 0 to disable (not recommended)."
+                    "description": "Idle connection timeout in seconds. Set to 0 to disable."
                 }
             },
             "additionalProperties": false,
@@ -319,6 +325,44 @@
             },
             "description": "A dictionary of group settings. Provides advanced controls over model swapping behaviour. Model IDs must be defined in models. A model can only be a member of one group. Behaviour controlled via swap, exclusive, persistent."
         },
+        "matrix": {
+            "type": "object",
+            "description": "Solver-based alternative to groups. Declares valid combinations of concurrent models. The solver minimizes eviction cost when swapping. A config must use either groups or matrix, not both.",
+            "required": [
+                "vars",
+                "sets"
+            ],
+            "properties": {
+                "vars": {
+                    "type": "object",
+                    "description": "Short names for models. Keys must be alphanumeric, 1-8 characters. All sets and evict_costs must use these IDs.",
+                    "minProperties": 1,
+                    "additionalProperties": {
+                        "type": "string"
+                    },
+                    "propertyNames": {
+                        "pattern": "^[a-zA-Z0-9]{1,8}$"
+                    }
+                },
+                "evict_costs": {
+                    "type": "object",
+                    "description": "Relative cost of evicting a running model. Models not listed default to 1. Values must be positive integers.",
+                    "additionalProperties": {
+                        "type": "integer",
+                        "minimum": 1
+                    }
+                },
+                "sets": {
+                    "type": "object",
+                    "description": "Named sets of concurrent model combinations. Values are DSL strings using & (AND), | (OR), () (grouping), and +ref (inline another set). Definition order is used for tie-breaking.",
+                    "minProperties": 1,
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                }
+            },
+            "additionalProperties": false
+        },
         "hooks": {
             "type": "object",
             "properties": {
@@ -413,25 +457,31 @@
                         "properties": {
                             "connect": {
                                 "type": "integer",
-                                "minimum": 1,
+                                "minimum": 0,
                                 "default": 30,
                                 "description": "TCP connection timeout in seconds."
                             },
+                            "keepalive": {
+                                "type": "integer",
+                                "minimum": 0,
+                                "default": 30,
+                                "description": "TCP keepalive connection timeout in seconds."
+                            },
                             "responseHeader": {
                                 "type": "integer",
-                                "minimum": 1,
-                                "default": 60,
+                                "minimum": 0,
+                                "default": 0,
                                 "description": "Time to wait for response headers in seconds."
                             },
                             "tlsHandshake": {
                                 "type": "integer",
-                                "minimum": 1,
+                                "minimum": 0,
                                 "default": 10,
                                 "description": "TLS handshake timeout in seconds."
                             },
                             "idleConn": {
                                 "type": "integer",
-                                "minimum": 1,
+                                "minimum": 0,
                                 "default": 90,
                                 "description": "Idle connection timeout in seconds."
                             }
@@ -444,5 +494,27 @@
             "default": {},
             "description": "A dictionary of remote peers and models they provide. Peers can be another llama-swap or any server that provides the /v1/ generative API endpoints supported by llama-swap."
         }
-    }
+    },
+    "allOf": [
+        {
+            "if": {
+                "required": ["groups"]
+            },
+            "then": {
+                "not": {
+                    "required": ["matrix"]
+                }
+            }
+        },
+        {
+            "if": {
+                "required": ["matrix"]
+            },
+            "then": {
+                "not": {
+                    "required": ["groups"]
+                }
+            }
+        }
+    ]
 }

config.example.yaml

@@ -287,14 +287,15 @@ models:
     # timeouts: configure proxy connection timeouts for this model
     # - optional, defaults shown below
     # - useful for models running on slower hardware that need longer timeouts
-    # - connect: TCP connection timeout in seconds
-    # - responseHeader: time to wait for response headers in seconds
-    #   (increasing this helps avoid 502 errors on slow hardware)
-    # - tlsHandshake: TLS handshake timeout in seconds
-    # - idleConn: idle connection timeout in seconds
+    # - connect: TCP dial connection timeout in seconds, default: 30 seconds
+    # - keepalive: TCP connection keepalive timeout, default: 30 seconds
+    # - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
+    # - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
+    # - idleConn: idle connection timeout in seconds, default: 90 seconds
     # - set any value to 0 to disable that timeout (not recommended)
     timeouts:
       connect: 30
+      keepalive: 0
       responseHeader: 60
       tlsHandshake: 10
       idleConn: 90
@@ -330,68 +331,83 @@ models:
     # - processes have 5 seconds to shutdown until forceful termination is attempted
     cmdStop: docker stop ${MODEL_ID}
 
-# groups: a dictionary of group settings
-# - optional, default: empty dictionary
-# - provides advanced controls over model swapping behaviour
-# - using groups some models can be kept loaded indefinitely, while others are swapped out
-# - model IDs must be defined in the Models section
-# - a model can only be a member of one group
-# - group behaviour is controlled via the `swap`, `exclusive` and `persistent` fields
-# - see issue #109 for details
+# =============================================================================
+# matrix: run concurrent models with a solver-based swap DSL
+# =============================================================================
 #
-# NOTE: the example below uses model names that are not defined above for demonstration purposes
-groups:
-  # group1 works the same as the default behaviour of llama-swap where only one model is allowed
-  # to run a time across the whole llama-swap instance
-  "group1":
-    # swap: controls the model swapping behaviour in within the group
-    # - optional, default: true
-    # - true : only one model is allowed to run at a time
-    # - false: all models can run together, no swapping
-    swap: true
+# Note:
+# A config must use either a matrix or legacy groups, not both. A configuration error
+# will occur if both are defined. Configuration examples for legacy Groups can be found:
+# https://github.com/mostlygeek/llama-swap/blob/40e39f7/config.example.yaml#L334-L396
+#
+# The matrix declares valid combinations of models that can run concurrently.
+# When a model is requested, the solver finds the cheapest way to make it
+# available by evicting as few (and least costly) running models as possible.
+#
+# Solver behavior:
+#   1. Request arrives for model X
+#   2. If X is already running, forward immediately. Done.
+#   3. Find all sets containing X
+#   4. For each candidate set, compute cost: sum of evict_costs for
+#      every running model NOT in that set
+#   5. Pick lowest cost candidate. Ties broken by definition order.
+#   6. Evict what needs to stop. Start X. Forward request.
+#
+# Subset semantics: a set [a, b, c] means any subset is valid.
+# Only the requested model is started — others are not preloaded.
+#
+# A model not appearing in any set can only run alone.
+#
+matrix:
+  # vars: short names for models (alphanumeric, 1-8 chars)
+  # - required for sets and evict_costs settings
+  # - each entry is a short name to a real model ID. Do not use an alias
+  # - used to keep set DSL logic short and easier to read
+  # - sets and evict_costs only use identifiers defined in vars
+  vars:
+    g: gemma-model
+    q: qwen-model
+    m: mistral-model
+    v: voxtral-model
+    e: reranker-model
+    L: llama-70B
+    sd: stable-diffusion
 
-    # exclusive: controls how the group affects other groups
-    # - optional, default: true
-    # - true: causes all other groups to unload when this group runs a model
-    # - false: does not affect other groups
-    exclusive: true
+  # evict_costs: relative cost of losing a running model (default: 1)
+  evict_costs:
+    v: 50 # vllm backend, slow cold start
+    L: 30 # 70B weights, slow to load
 
-    # members references the models defined above
-    # required
-    members:
-      - "llama"
-      - "qwen-unlisted"
+  # sets: named sets of concurrent model combinations
+  # Values are DSL strings with operators:
+  #   &     AND (models run together)
+  #   |     OR  (alternatives)
+  #   ()    grouping
+  #   +ref  inline another set's expression
+  #
+  # Expansion examples:
+  #   "L"                  → [L]
+  #   "a & b"              → [a, b]
+  #   "a | b"              → [a], [b]
+  #   "(a | b) & c"        → [a, c], [b, c]
+  #   "(a | b) & (c | d)"  → [a,c], [a,d], [b,c], [b,d]
+  #   "+llms & v"          → expands llms inline, then applies & v
+  sets:
+    # LLM + TTS: switching between g/q/m won't evict v
+    # expands to: [g,v], [q,v], [m,v]
+    standard: "(g | q | m) & v"
 
-  # Example:
-  # - in group2 all models can run at the same time
-  # - when a different group is loaded it causes all running models in this group to unload
-  "group2":
-    swap: false
+    # LLM + TTS + reranker
+    # expands to: [g,v,e], [q,v,e]
+    with_rerank: "(g | q) & v & e"
 
-    # exclusive: false does not unload other groups when a model in group2 is requested
-    # - the models in group2 will be loaded but will not unload any other groups
-    exclusive: false
-    members:
-      - "docker-llama"
-      - "modelA"
-      - "modelB"
+    # LLM + image generation, no TTS
+    # expands to: [g,sd], [q,sd]
+    creative: "(g | q) & sd"
 
-  # Example:
-  # - a persistent group, prevents other groups from unloading it
-  "forever":
-    # persistent: prevents over groups from unloading the models in this group
-    # - optional, default: false
-    # - does not affect individual model behaviour
-    persistent: true
-
-    # set swap/exclusive to false to prevent swapping inside the group
-    # and the unloading of other groups
-    swap: false
-    exclusive: false
-    members:
-      - "forever-modelA"
-      - "forever-modelB"
-      - "forever-modelc"
+    # 70B model uses all GPUs, can only run alone
+    # expands to: [L]
+    full: "L"
 
 # hooks: a dictionary of event triggers and actions
 # - optional, default: empty dictionary
@@ -447,6 +463,7 @@ peers:
     # - set any value to 0 to disable that timeout (not recommended)
     timeouts:
       connect: 30
+      keepalive: 30
       responseHeader: 60
       tlsHandshake: 10
       idleConn: 90

docker/unified/Dockerfile

@@ -42,6 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential cmake git python3 python3-pip libssl-dev \
     curl ca-certificates ccache make wget software-properties-common \
     libvulkan-dev glslang-tools spirv-tools vulkan-validationlayers glslc \
+    spirv-headers \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /build
@@ -148,7 +149,7 @@ ARG IK_LLAMA_COMMIT_HASH=unknown
 ARG RUN_UID=0
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    python3-numpy python3-sentencepiece \
+    python3-numpy python3-sentencepiece python3-pip \
     && rm -rf /var/lib/apt/lists/*
 
 # Create non-root user when RUN_UID != 0
@@ -179,6 +180,9 @@ COPY --from=llama-build /install/bin/llama-cli /usr/local/bin/
 # Copy ik-llama-server (CUDA only; empty copy for vulkan)
 COPY --from=ik-llama-build /install/bin/ /usr/local/bin/
 
+# Install uv
+RUN pip install uv --break-system-packages
+
 # Copy llama-swap binary
 COPY --from=llama-swap-download /install/bin/llama-swap /usr/local/bin/
 COPY --from=llama-swap-download /install/llama-swap-version /tmp/

docker/unified/build-image.sh

@@ -201,7 +201,6 @@ BUILD_ARGS=(
     --build-arg "SD_COMMIT_HASH=${SD_HASH}"
     --build-arg "IK_LLAMA_COMMIT_HASH=${IK_LLAMA_HASH}"
     --build-arg "LS_VERSION=${LS_HASH}"
-    --build-arg "RUN_UID=${RUN_UID:-0}"
     -t "${DOCKER_IMAGE_TAG}"
     -f "${SCRIPT_DIR}/Dockerfile"
 )
@@ -255,12 +254,33 @@ if [[ "$BACKEND" == "cuda" ]]; then
 fi
 echo "All expected binaries verified: ${VERIFIED_LIST}"
 
+echo ""
+echo "=========================================="
+echo "Building rootless image..."
+echo "=========================================="
+echo ""
+
+ROOTLESS_TAG="${DOCKER_IMAGE_TAG}-rootless"
+docker buildx build --load -t "${ROOTLESS_TAG}" - <<EOF
+FROM ${DOCKER_IMAGE_TAG}
+USER root
+RUN groupadd --system --gid 10001 llama-swap && \\
+    useradd --system --uid 10001 --gid 10001 \\
+      --home /app --shell /sbin/nologin llama-swap && \\
+    chown -R 10001:10001 /etc/llama-swap /models
+USER 10001
+EOF
+
+echo "Rootless image built: ${ROOTLESS_TAG}"
+
 echo ""
 echo "=========================================="
 echo "Build complete!"
 echo "=========================================="
 echo ""
-echo "Image tag: ${DOCKER_IMAGE_TAG}"
+echo "Image tags:"
+echo "  ${DOCKER_IMAGE_TAG}"
+echo "  ${ROOTLESS_TAG}"
 echo ""
 echo "Built with:"
 echo "  llama.cpp:            ${LLAMA_HASH}"

docs/configuration.md

@@ -22,7 +22,7 @@ models:
     cmd: llama-server --port ${PORT} -m /path/to/third_model.gguf
 ```
 
-With this configuration models will be hot swapped and loaded on demand. The special `${PORT}` macro provides a unique port per model. Useful if you want to run multiple models at the same time with the `groups` feature.
+With this configuration models will be hot swapped and loaded on demand. The special `${PORT}` macro provides a unique port per model which is useful if you want to run multiple models at the same time with the `matrix` feature.
 
 ## Advanced control with `cmd`
 
@@ -76,7 +76,7 @@ llama-swap supports many more features to customize how you want to manage your
 | --------- | ---------------------------------------------- |
 | `ttl`     | automatic unloading of models after a timeout  |
 | `macros`  | reusable snippets to use in configurations     |
-| `groups`  | run multiple models at a time                  |
+| `matrix`  | run multiple models at a time                  |
 | `hooks`   | event driven functionality                     |
 | `env`     | define environment variables per model         |
 | `aliases` | serve a model with different names             |
@@ -141,6 +141,11 @@ logToStdout: "proxy"
 # - useful for limiting memory usage when processing large volumes of metrics
 metricsMaxInMemory: 1000
 
+# captureBuffer: how many MBs to allocate for storing request/response captures
+# - optional, default: 10
+# - set to 0 to disable
+captureBuffer: 15
+
 # startPort: sets the starting port number for the automatic ${PORT} macro.
 # - optional, default: 5800
 # - the ${PORT} macro can be used in model.cmd and model.proxy settings
@@ -161,15 +166,10 @@ sendLoadingState: true
 #   all fields except for Id so chat UIs can use the alias equivalent to the original.
 includeAliasesInList: false
 
-# apiKeys: require an API key when making requests to inference endpoints
-# - optional, default: []
-# - when empty (the default) authorization will not be checked as llama-swap is default-allow
-# - each key is a non-empty string
-apiKeys:
-  - "sk-hunter2"
-  # hint, one liner: printf "sk-%s\n" "$(head -c 48 /dev/urandom | base64 )"
-  - "sk-gyCPiKUcIfPlaM4OSMZekkprgijPx6+OsmQs8Rsg0xZ9qpy6gKWsIKqHOk+cgXVx"
-  - "sk-+QtIn0Zjj4UHjiaZYiZEnru4mrwKM9RzhmJeK5SobNXLl8QMFXxGz1/2lEuvQpkb"
+# globalTTL: the default TTL in seconds before unloading a model
+# - optional, default: 0 (never automatically unload)
+# - must be >= 0
+globalTTL: 0
 
 # macros: a dictionary of string substitutions
 # - optional, default: empty dictionary
@@ -181,6 +181,9 @@ apiKeys:
 # - macro names must not be a reserved name: PORT or MODEL_ID
 # - macro values can be numbers, bools, or strings
 # - macros can contain other macros, but they must be defined before they are used
+# - environment variables can be referenced with ${env.VAR_NAME} syntax
+#   - env macros are substituted first, before regular macros
+#   - if the env var is not set, config loading will fail with an error
 macros:
   # Example of a multi-line macro
   "latest-llama": >
@@ -193,6 +196,24 @@ macros:
   # but they must be previously declared.
   "default_args": "--ctx-size ${default_ctx}"
 
+  # Example of environment variable macros
+  # - ${env.VAR_NAME} pulls the value from the system environment
+  # - useful for paths, secrets, or machine-specific configuration
+  "models_dir": "${env.HOME}/models"
+
+# apiKeys: require an API key when making requests to inference endpoints
+# - optional, default: []
+# - when empty (the default) authorization will not be checked as llama-swap is default-allow
+# - each key is a non-empty string
+apiKeys:
+  - "sk-hunter2"
+  # tip, one liner: printf "sk-%s\n" "$(head -c 48 /dev/urandom | base64 )"
+  - "sk-gyCPiKUcIfPlaM4OSMZekkprgijPx6+OsmQs8Rsg0xZ9qpy6gKWsIKqHOk+cgXVx"
+
+  # use environment variable macros to keep secrets out of the config
+  - "${env.API_KEY_1}"
+  - "${env.API_KEY_2}"
+
 # models: a dictionary of model configurations
 # - required
 # - each key is the model's ID, used in API requests
@@ -201,7 +222,7 @@ macros:
 # - below are examples of the all the settings a model can have
 models:
   # keys are the model names used in API requests
-  "llama":
+  "gpt-oss-120b":
     # macros: a dictionary of string substitutions specific to this model
     # - optional, default: empty dictionary
     # - macros defined here override macros defined in the global macros section
@@ -218,7 +239,7 @@ models:
     cmd: |
       # ${latest-llama} is a macro that is defined above
       ${latest-llama}
-      --model path/to/llama-8B-Q4_K_M.gguf
+      --model path/to/gpt-oss-120B.gguf
       --ctx-size ${default_ctx}
       --temperature ${temp}
 
@@ -226,13 +247,13 @@ models:
     # - optional, default: empty string
     # - if set, it will be used in the v1/models API response
     # - if not set, it will be omitted in the JSON model record
-    name: "llama 3.1 8B"
+    name: "gpt-oss 120B"
 
     # description: a description for the model
     # - optional, default: empty string
     # - if set, it will be used in the v1/models API response
     # - if not set, it will be omitted in the JSON model record
-    description: "A small but capable model used for quick testing"
+    description: "A thinking model from OpenAI"
 
     # env: define an array of environment variables to inject into cmd's environment
     # - optional, default: empty array
@@ -247,14 +268,6 @@ models:
     # - if you use a custom port in cmd this *must* be set
     proxy: http://127.0.0.1:8999
 
-    # aliases: alternative model names that this model configuration is used for
-    # - optional, default: empty array
-    # - aliases must be unique globally
-    # - useful for impersonating a specific model
-    aliases:
-      - "gpt-4o-mini"
-      - "gpt-3.5-turbo"
-
     # checkEndpoint: URL path to check if the server is ready
     # - optional, default: /health
     # - endpoint is expected to return an HTTP 200 response
@@ -263,8 +276,10 @@ models:
     checkEndpoint: /custom-endpoint
 
     # ttl: automatically unload the model after ttl seconds
-    # - optional, default: 0
-    # - ttl values must be a value greater than 0
+    # - optional, default: -1 (use global default)
+    # - ttl values must be a value greater than or equal to 0
+    # - a ttl of -1 will use the global TTL value as the default
+    # - a ttl of 0 will mean never unload
     # - a value of 0 disables automatic unloading of the model
     ttl: 60
 
@@ -272,11 +287,11 @@ models:
     # - optional, default: ""
     # - useful for when the upstream server expects a specific model name that
     #   is different from the model's ID
-    useModelName: "qwen:qwq"
+    useModelName: "openai/gpt-oss-120B"
 
     # filters: a dictionary of filter settings
     # - optional, default: empty dictionary
-    # - only stripParams is currently supported
+    # - same capabilities as peer filters (stripParams, setParams)
     filters:
       # stripParams: a comma separated list of parameters to remove from the request
       # - optional, default: ""
@@ -286,6 +301,43 @@ models:
       # - recommended to stick to sampling parameters
       stripParams: "temperature, top_p, top_k"
 
+      # setParams: a dictionary of parameters to set/override in requests
+      # - optional, default: empty dictionary
+      # - useful for enforcing specific parameter values
+      # - protected params like "model" cannot be overridden
+      # - values can be strings, numbers, booleans, arrays, or objects
+      # - always runs for the model
+      setParams:
+        # Example: enforce specific sampling parameters
+        temperature: 0.7
+        top_p: 0.9
+
+      # setParamsByID: a dictionary of parameters to set based the model ID
+      # - optional, default: empty dictionary
+      # - combine with aliases to create variant behaviour without reloading the model
+      # - parameters are set in the request body JSON
+      # - run after setParams so it will override any settings
+      # - protected params like "model" cannot be overridden
+      # - values can be strings, numbers, booleans, arrays, or objects
+      # - model aliases will be automatically created for each key
+      setParamsByID:
+        "${MODEL_ID}":
+          chat_template_kwargs:
+            reasoning_effort: medium
+        "${MODEL_ID}:high":
+          chat_template_kwargs:
+            reasoning_effort: high
+        "${MODEL_ID}:low":
+          chat_template_kwargs:
+            reasoning_effort: low
+
+    # aliases: alternative model names that this model configuration is used for
+    # - optional, default: empty array
+    # - aliases must be unique globally
+    # - useful for impersonating a specific model
+    aliases:
+      - "gpt-4o-mini"
+
     # metadata: a dictionary of arbitrary values that are included in /v1/models
     # - optional, default: empty dictionary
     # - while metadata can contains complex types it is recommended to keep it simple
@@ -319,33 +371,26 @@ models:
     # - recommended to be omitted and the default used
     concurrencyLimit: 0
 
-    # timeouts: configure proxy connection timeouts for this model
-    # - optional, defaults shown below
-    # - useful for models on slower hardware that need longer timeouts
-    # - increase responseHeader to avoid "timeout awaiting response headers" errors
-    # - set any value to 0 to disable that timeout (not recommended)
-    timeouts:
-      # connect: TCP connection timeout in seconds
-      # - default: 30
-      connect: 30
-
-      # responseHeader: time to wait for response headers in seconds
-      # - default: 60
-      # - for slow image generation or large models, consider increasing to 300+ seconds
-      responseHeader: 60
-
-      # tlsHandshake: TLS handshake timeout in seconds
-      # - default: 10
-      tlsHandshake: 10
-
-      # idleConn: idle connection timeout in seconds
-      # - default: 90
-      idleConn: 90
-
     # sendLoadingState: overrides the global sendLoadingState setting for this model
     # - optional, default: undefined (use global setting)
     sendLoadingState: false
 
+    # timeouts: configure proxy connection timeouts for this model
+    # - optional, defaults shown below
+    # - useful for models running on slower hardware that need longer timeouts
+    # - connect: TCP dial connection timeout in seconds, default: 30 seconds
+    # - keepalive: TCP connection keepalive timeout, default: 30 seconds
+    # - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
+    # - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
+    # - idleConn: idle connection timeout in seconds, default: 90 seconds
+    # - set any value to 0 to disable that timeout (not recommended)
+    timeouts:
+      connect: 30
+      keepalive: 0
+      responseHeader: 60
+      tlsHandshake: 10
+      idleConn: 90
+
   # Unlisted model example:
   "qwen-unlisted":
     # unlisted: boolean, true or false
@@ -377,68 +422,83 @@ models:
     # - processes have 5 seconds to shutdown until forceful termination is attempted
     cmdStop: docker stop ${MODEL_ID}
 
-# groups: a dictionary of group settings
-# - optional, default: empty dictionary
-# - provides advanced controls over model swapping behaviour
-# - using groups some models can be kept loaded indefinitely, while others are swapped out
-# - model IDs must be defined in the Models section
-# - a model can only be a member of one group
-# - group behaviour is controlled via the `swap`, `exclusive` and `persistent` fields
-# - see issue #109 for details
+# =============================================================================
+# matrix: run concurrent models with a solver-based swap DSL
+# =============================================================================
 #
-# NOTE: the example below uses model names that are not defined above for demonstration purposes
-groups:
-  # group1 works the same as the default behaviour of llama-swap where only one model is allowed
-  # to run a time across the whole llama-swap instance
-  "group1":
-    # swap: controls the model swapping behaviour in within the group
-    # - optional, default: true
-    # - true : only one model is allowed to run at a time
-    # - false: all models can run together, no swapping
-    swap: true
+# Note:
+# A config must use either a matrix or legacy groups, not both. A configuration error
+# will occur if both are defined. Configuration examples for legacy Groups can be found:
+# https://github.com/mostlygeek/llama-swap/blob/40e39f7/config.example.yaml#L334-L396
+#
+# The matrix declares valid combinations of models that can run concurrently.
+# When a model is requested, the solver finds the cheapest way to make it
+# available by evicting as few (and least costly) running models as possible.
+#
+# Solver behavior:
+#   1. Request arrives for model X
+#   2. If X is already running, forward immediately. Done.
+#   3. Find all sets containing X
+#   4. For each candidate set, compute cost: sum of evict_costs for
+#      every running model NOT in that set
+#   5. Pick lowest cost candidate. Ties broken by definition order.
+#   6. Evict what needs to stop. Start X. Forward request.
+#
+# Subset semantics: a set [a, b, c] means any subset is valid.
+# Only the requested model is started — others are not preloaded.
+#
+# A model not appearing in any set can only run alone.
+#
+matrix:
+  # vars: short names for models (alphanumeric, 1-8 chars)
+  # - required for sets and evict_costs settings
+  # - each entry is a short name to a real model ID. Do not use an alias
+  # - used to keep set DSL logic short and easier to read
+  # - sets and evict_costs only use identifiers defined in vars
+  vars:
+    g: gemma-model
+    q: qwen-model
+    m: mistral-model
+    v: voxtral-model
+    e: reranker-model
+    L: llama-70B
+    sd: stable-diffusion
 
-    # exclusive: controls how the group affects other groups
-    # - optional, default: true
-    # - true: causes all other groups to unload when this group runs a model
-    # - false: does not affect other groups
-    exclusive: true
+  # evict_costs: relative cost of losing a running model (default: 1)
+  evict_costs:
+    v: 50 # vllm backend, slow cold start
+    L: 30 # 70B weights, slow to load
 
-    # members references the models defined above
-    # required
-    members:
-      - "llama"
-      - "qwen-unlisted"
+  # sets: named sets of concurrent model combinations
+  # Values are DSL strings with operators:
+  #   &     AND (models run together)
+  #   |     OR  (alternatives)
+  #   ()    grouping
+  #   +ref  inline another set's expression
+  #
+  # Expansion examples:
+  #   "L"                  → [L]
+  #   "a & b"              → [a, b]
+  #   "a | b"              → [a], [b]
+  #   "(a | b) & c"        → [a, c], [b, c]
+  #   "(a | b) & (c | d)"  → [a,c], [a,d], [b,c], [b,d]
+  #   "+llms & v"          → expands llms inline, then applies & v
+  sets:
+    # LLM + TTS: switching between g/q/m won't evict v
+    # expands to: [g,v], [q,v], [m,v]
+    standard: "(g | q | m) & v"
 
-  # Example:
-  # - in group2 all models can run at the same time
-  # - when a different group is loaded it causes all running models in this group to unload
-  "group2":
-    swap: false
+    # LLM + TTS + reranker
+    # expands to: [g,v,e], [q,v,e]
+    with_rerank: "(g | q) & v & e"
 
-    # exclusive: false does not unload other groups when a model in group2 is requested
-    # - the models in group2 will be loaded but will not unload any other groups
-    exclusive: false
-    members:
-      - "docker-llama"
-      - "modelA"
-      - "modelB"
+    # LLM + image generation, no TTS
+    # expands to: [g,sd], [q,sd]
+    creative: "(g | q) & sd"
 
-  # Example:
-  # - a persistent group, prevents other groups from unloading it
-  "forever":
-    # persistent: prevents over groups from unloading the models in this group
-    # - optional, default: false
-    # - does not affect individual model behaviour
-    persistent: true
-
-    # set swap/exclusive to false to prevent swapping inside the group
-    # and the unloading of other groups
-    swap: false
-    exclusive: false
-    members:
-      - "forever-modelA"
-      - "forever-modelB"
-      - "forever-modelc"
+    # 70B model uses all GPUs, can only run alone
+    # expands to: [L]
+    full: "L"
 
 # hooks: a dictionary of event triggers and actions
 # - optional, default: empty dictionary
@@ -467,17 +527,6 @@ peers:
     # - required
     # - requested path to llama-swap will be appended to the end of the proxy value
     proxy: http://192.168.1.23
-
-    # timeouts: configure proxy connection timeouts for this peer
-    # - optional, defaults shown below
-    # - useful when the peer runs on slower hardware
-    # - set any value to 0 to disable that timeout (not recommended)
-    timeouts:
-      connect: 30
-      responseHeader: 60
-      tlsHandshake: 10
-      idleConn: 90
-
     # models: a list of models served by the peer
     # - required
     models:
@@ -490,7 +539,8 @@ peers:
     # - optional, default: ""
     # - if blank, no key will be added to the request
     # - key will be injected into headers: Authorization: Bearer <key> and x-api-key: <key>
-    apiKey: sk-your-openrouter-key
+    # - can be a string or a macro
+    apiKey: ${env.OPENROUTER_API_KEY}
     models:
       - meta-llama/llama-3.1-8b-instruct
       - qwen/qwen3-235b-a22b-2507
@@ -498,4 +548,35 @@ peers:
       - z-ai/glm-4.7
       - moonshotai/kimi-k2-0905
       - minimax/minimax-m2.1
+    # timeouts: configure proxy connection timeouts for this peer
+    # - optional, defaults shown below
+    # - useful when the peer runs on slower hardware
+    # - set any value to 0 to disable that timeout (not recommended)
+    timeouts:
+      connect: 30
+      keepalive: 30
+      responseHeader: 60
+      tlsHandshake: 10
+      idleConn: 90
+
+    # filters: a dictionary of filter settings for peer requests
+    # - optional, default: empty dictionary
+    # - same capabilities as model filters (stripParams, setParams)
+    filters:
+      # stripParams: a comma separated list of parameters to remove from the request
+      # - optional, default: ""
+      # - useful for removing parameters that the peer doesn't support
+      # - the `model` parameter can never be removed
+      stripParams: "temperature, top_p"
+
+      # setParams: a dictionary of parameters to set/override in requests to this peer
+      # - optional, default: empty dictionary
+      # - useful for injecting provider-specific settings like data retention policies
+      # - protected params like "model" cannot be overridden
+      # - values can be strings, numbers, booleans, arrays, or objects
+      setParams:
+        # Example: enforce zero-data-retention for OpenRouter
+        provider:
+          data_collection: "deny"
+          zdr: true
 ```

go.mod

@@ -4,8 +4,8 @@ go 1.26.1
 
 require (
 	github.com/billziss-gh/golib v0.2.0
-	github.com/fsnotify/fsnotify v1.9.0
 	github.com/gin-gonic/gin v1.10.0
+	github.com/klauspost/compress v1.18.5
 	github.com/stretchr/testify v1.9.0
 	github.com/tidwall/gjson v1.18.0
 	github.com/tidwall/sjson v1.2.5

go.sum

@@ -11,8 +11,6 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
-github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -34,6 +32,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
 github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=

llama-swap.go

@@ -9,14 +9,15 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"runtime"
 	"syscall"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
 	"github.com/gin-gonic/gin"
 	"github.com/mostlygeek/llama-swap/event"
 	"github.com/mostlygeek/llama-swap/proxy"
 	"github.com/mostlygeek/llama-swap/proxy/config"
+	"github.com/mostlygeek/llama-swap/proxy/configwatcher"
 )
 
 var (
@@ -79,6 +80,17 @@ func main() {
 	sigChan := make(chan os.Signal, 1)
 	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
 
+	// Reload signals (SIGHUP on POSIX, none on Windows — Windows does not
+	// deliver SIGHUP). Always wired up so `kill -HUP` works regardless of
+	// --watch-config.
+	reloadChan := make(chan os.Signal, 1)
+	if runtime.GOOS != "windows" {
+		signal.Notify(reloadChan, syscall.SIGHUP)
+	}
+
+	// Context that bounds the lifetime of background watcher goroutines.
+	watcherCtx, watcherCancel := context.WithCancel(context.Background())
+
 	// Create server with initial handler
 	srv := &http.Server{
 		Addr: *listenStr,
@@ -121,52 +133,45 @@ func main() {
 	// load the initial proxy manager
 	reloadProxyManager()
 	debouncedReload := debounce(time.Second, reloadProxyManager)
-	if *watchConfig {
-		defer event.On(func(e proxy.ConfigFileChangedEvent) {
-			if e.ReloadingState == proxy.ReloadingStateStart {
-				debouncedReload()
-			}
-		})()
 
-		fmt.Println("Watching Configuration for changes")
+	// Listen for ConfigFileChangedEvent unconditionally so SIGHUP and the
+	// poll-based watcher both feed the same debounced reload pipeline. The
+	// UI also listens for the matching ReloadingStateEnd emitted from
+	// reloadProxyManager.
+	defer event.On(func(e proxy.ConfigFileChangedEvent) {
+		if e.ReloadingState == proxy.ReloadingStateStart {
+			debouncedReload()
+		}
+	})()
+
+	// SIGHUP (or platform-equivalent) → reload. Back-to-back signals collapse
+	// to one reload via the debounce window, which is the desired behavior.
+	go func() {
+		for range reloadChan {
+			fmt.Println("Received reload signal, reloading configuration")
+			event.Emit(proxy.ConfigFileChangedEvent{
+				ReloadingState: proxy.ReloadingStateStart,
+			})
+		}
+	}()
+
+	if *watchConfig {
 		go func() {
 			absConfigPath, err := filepath.Abs(*configPath)
 			if err != nil {
 				fmt.Printf("Error getting absolute path for watching config file: %v\n", err)
 				return
 			}
-			watcher, err := fsnotify.NewWatcher()
-			if err != nil {
-				fmt.Printf("Error creating file watcher: %v. File watching disabled.\n", err)
-				return
-			}
-
-			configDir := filepath.Dir(absConfigPath)
-			err = watcher.Add(configDir)
-			if err != nil {
-				fmt.Printf("Error adding config path directory (%s) to watcher: %v. File watching disabled.", configDir, err)
-				return
-			}
-
-			defer watcher.Close()
-			for {
-				select {
-				case changeEvent := <-watcher.Events:
-					if changeEvent.Name == absConfigPath && (changeEvent.Has(fsnotify.Write) || changeEvent.Has(fsnotify.Create) || changeEvent.Has(fsnotify.Remove)) {
-						event.Emit(proxy.ConfigFileChangedEvent{
-							ReloadingState: proxy.ReloadingStateStart,
-						})
-					} else if changeEvent.Name == filepath.Join(configDir, "..data") && changeEvent.Has(fsnotify.Create) {
-						// the change for k8s configmap
-						event.Emit(proxy.ConfigFileChangedEvent{
-							ReloadingState: proxy.ReloadingStateStart,
-						})
-					}
-
-				case err := <-watcher.Errors:
-					log.Printf("File watcher error: %v", err)
-				}
-			}
+			fmt.Println("Watching configuration for changes (poll-based, 2s interval)")
+			(&configwatcher.Watcher{
+				Path:     absConfigPath,
+				Interval: configwatcher.DefaultInterval,
+				OnChange: func() {
+					event.Emit(proxy.ConfigFileChangedEvent{
+						ReloadingState: proxy.ReloadingStateStart,
+					})
+				},
+			}).Run(watcherCtx)
 		}()
 	}
 
@@ -174,6 +179,7 @@ func main() {
 	go func() {
 		sig := <-sigChan
 		fmt.Printf("Received signal %v, shutting down...\n", sig)
+		watcherCancel()
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 		defer cancel()
 

proxy/config/config.go

@@ -129,6 +129,12 @@ type Config struct {
 	Profiles           map[string][]string    `yaml:"profiles"`
 	Groups             map[string]GroupConfig `yaml:"groups"` /* key is group ID */
 
+	// swap matrix: solver-based alternative to groups
+	Matrix *MatrixConfig `yaml:"matrix"`
+
+	// populated during validation when matrix is configured
+	ExpandedSets []ExpandedSet `yaml:"-"`
+
 	// for key/value replacements in model's cmd, cmdStop, proxy, checkEndPoint
 	Macros MacroList `yaml:"macros"`
 
@@ -438,22 +444,35 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 		config.Models[modelId] = modelConfig
 	}
 
-	config = AddDefaultGroupToConfig(config)
+	// groups XOR matrix
+	if config.Matrix != nil && len(config.Groups) > 0 {
+		return Config{}, fmt.Errorf("config cannot use both 'groups' and 'matrix'")
+	}
 
-	// Validate group members
-	memberUsage := make(map[string]string)
-	for groupID, groupConfig := range config.Groups {
-		prevSet := make(map[string]bool)
-		for _, member := range groupConfig.Members {
-			if _, found := prevSet[member]; found {
-				return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID)
-			}
-			prevSet[member] = true
+	if config.Matrix != nil {
+		expandedSets, err := ValidateMatrix(*config.Matrix, config.Models)
+		if err != nil {
+			return Config{}, fmt.Errorf("matrix: %w", err)
+		}
+		config.ExpandedSets = expandedSets
+	} else {
+		config = AddDefaultGroupToConfig(config)
 
-			if existingGroup, exists := memberUsage[member]; exists {
-				return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID)
+		// Validate group members
+		memberUsage := make(map[string]string)
+		for groupID, groupConfig := range config.Groups {
+			prevSet := make(map[string]bool)
+			for _, member := range groupConfig.Members {
+				if _, found := prevSet[member]; found {
+					return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID)
+				}
+				prevSet[member] = true
+
+				if existingGroup, exists := memberUsage[member]; exists {
+					return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID)
+				}
+				memberUsage[member] = groupID
 			}
-			memberUsage[member] = groupID
 		}
 	}
 

proxy/config/config_posix_test.go

@@ -163,6 +163,15 @@ groups:
 
 	modelLoadingState := false
 
+	defaultTimeout := TimeoutsConfig{
+		Connect:        30,
+		KeepAlive:      30,
+		ResponseHeader: 0,
+		TLSHandshake:   10,
+		ExpectContinue: 1,
+		IdleConn:       90,
+	}
+
 	expected := Config{
 		LogLevel:      "info",
 		LogTimeFormat: "",
@@ -187,13 +196,7 @@ groups:
 				Name:             "Model 1",
 				Description:      "This is model 1",
 				SendLoadingState: &modelLoadingState,
-				Timeouts: TimeoutsConfig{
-					Connect:        30,
-					ResponseHeader: 60,
-					TLSHandshake:   10,
-					ExpectContinue: 1,
-					IdleConn:       90,
-				},
+				Timeouts:         defaultTimeout,
 			},
 			"model2": {
 				Cmd:              "path/to/server --arg1 one",
@@ -202,13 +205,7 @@ groups:
 				Env:              []string{},
 				CheckEndpoint:    "/",
 				SendLoadingState: &modelLoadingState,
-				Timeouts: TimeoutsConfig{
-					Connect:        30,
-					ResponseHeader: 60,
-					TLSHandshake:   10,
-					ExpectContinue: 1,
-					IdleConn:       90,
-				},
+				Timeouts:         defaultTimeout,
 			},
 			"model3": {
 				Cmd:              "path/to/cmd --arg1 one",
@@ -217,13 +214,7 @@ groups:
 				Env:              []string{},
 				CheckEndpoint:    "/",
 				SendLoadingState: &modelLoadingState,
-				Timeouts: TimeoutsConfig{
-					Connect:        30,
-					ResponseHeader: 60,
-					TLSHandshake:   10,
-					ExpectContinue: 1,
-					IdleConn:       90,
-				},
+				Timeouts:         defaultTimeout,
 			},
 			"model4": {
 				Cmd:              "path/to/cmd --arg1 one",
@@ -232,13 +223,7 @@ groups:
 				Aliases:          []string{},
 				Env:              []string{},
 				SendLoadingState: &modelLoadingState,
-				Timeouts: TimeoutsConfig{
-					Connect:        30,
-					ResponseHeader: 60,
-					TLSHandshake:   10,
-					ExpectContinue: 1,
-					IdleConn:       90,
-				},
+				Timeouts:         defaultTimeout,
 			},
 		},
 		HealthCheckTimeout: 15,

proxy/config/config_test.go

@@ -1475,7 +1475,7 @@ models:
 
 	// Default values should be set during unmarshaling
 	assert.Equal(t, 30, modelConfig.Timeouts.Connect)
-	assert.Equal(t, 60, modelConfig.Timeouts.ResponseHeader)
+	assert.Equal(t, 0, modelConfig.Timeouts.ResponseHeader)
 	assert.Equal(t, 10, modelConfig.Timeouts.TLSHandshake)
 	assert.Equal(t, 1, modelConfig.Timeouts.ExpectContinue)
 	assert.Equal(t, 90, modelConfig.Timeouts.IdleConn)

proxy/config/config_windows_test.go

@@ -155,6 +155,15 @@ groups:
 
 	modelLoadingState := false
 
+	defaultTimeout := TimeoutsConfig{
+		Connect:        30,
+		KeepAlive:      30,
+		ResponseHeader: 0,
+		TLSHandshake:   10,
+		ExpectContinue: 1,
+		IdleConn:       90,
+	}
+
 	expected := Config{
 		LogLevel:      "info",
 		LogTimeFormat: "",
@@ -173,13 +182,7 @@ groups:
 				Env:              []string{"VAR1=value1", "VAR2=value2"},
 				CheckEndpoint:    "/health",
 				SendLoadingState: &modelLoadingState,
-				Timeouts: TimeoutsConfig{
-					Connect:        30,
-					ResponseHeader: 60,
-					TLSHandshake:   10,
-					ExpectContinue: 1,
-					IdleConn:       90,
-				},
+				Timeouts:         defaultTimeout,
 			},
 			"model2": {
 				Cmd:              "path/to/server --arg1 one",
@@ -189,13 +192,7 @@ groups:
 				Env:              []string{},
 				CheckEndpoint:    "/",
 				SendLoadingState: &modelLoadingState,
-				Timeouts: TimeoutsConfig{
-					Connect:        30,
-					ResponseHeader: 60,
-					TLSHandshake:   10,
-					ExpectContinue: 1,
-					IdleConn:       90,
-				},
+				Timeouts:         defaultTimeout,
 			},
 			"model3": {
 				Cmd:              "path/to/cmd --arg1 one",
@@ -205,13 +202,7 @@ groups:
 				Env:              []string{},
 				CheckEndpoint:    "/",
 				SendLoadingState: &modelLoadingState,
-				Timeouts: TimeoutsConfig{
-					Connect:        30,
-					ResponseHeader: 60,
-					TLSHandshake:   10,
-					ExpectContinue: 1,
-					IdleConn:       90,
-				},
+				Timeouts:         defaultTimeout,
 			},
 			"model4": {
 				Cmd:              "path/to/cmd --arg1 one",
@@ -221,13 +212,7 @@ groups:
 				Aliases:          []string{},
 				Env:              []string{},
 				SendLoadingState: &modelLoadingState,
-				Timeouts: TimeoutsConfig{
-					Connect:        30,
-					ResponseHeader: 60,
-					TLSHandshake:   10,
-					ExpectContinue: 1,
-					IdleConn:       90,
-				},
+				Timeouts:         defaultTimeout,
 			},
 		},
 		HealthCheckTimeout: 15,

proxy/config/matrix.go

@@ -0,0 +1,226 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+
+	"gopkg.in/yaml.v3"
+)
+
+var varKeyPattern = regexp.MustCompile(`^[a-zA-Z0-9]{1,8}$`)
+
+// MatrixConfig represents the swap matrix configuration block.
+type MatrixConfig struct {
+	Var        map[string]string `yaml:"vars"`
+	EvictCosts map[string]int    `yaml:"evict_costs"`
+	Sets       OrderedSets       `yaml:"sets"`
+}
+
+// SetEntry is a single named set with its DSL expression.
+type SetEntry struct {
+	Name string
+	DSL  string
+}
+
+// OrderedSets preserves YAML definition order of sets (used for tie-breaking).
+type OrderedSets []SetEntry
+
+func (os *OrderedSets) UnmarshalYAML(value *yaml.Node) error {
+	if value.Kind != yaml.MappingNode {
+		return fmt.Errorf("sets must be a mapping")
+	}
+
+	entries := make([]SetEntry, 0, len(value.Content)/2)
+	for i := 0; i < len(value.Content); i += 2 {
+		keyNode := value.Content[i]
+		valueNode := value.Content[i+1]
+
+		var name string
+		if err := keyNode.Decode(&name); err != nil {
+			return fmt.Errorf("failed to decode set name: %w", err)
+		}
+
+		var dsl string
+		if err := valueNode.Decode(&dsl); err != nil {
+			return fmt.Errorf("failed to decode DSL for set %q: %w", name, err)
+		}
+
+		entries = append(entries, SetEntry{Name: name, DSL: dsl})
+	}
+
+	*os = entries
+	return nil
+}
+
+// ExpandedSet is one valid combination of concurrent models (real model names).
+type ExpandedSet struct {
+	SetName string
+	DSL     string
+	Models  []string // real model names, sorted
+}
+
+// ValidateMatrix validates the matrix config and returns all expanded sets.
+func ValidateMatrix(matrix MatrixConfig, models map[string]ModelConfig) ([]ExpandedSet, error) {
+	if len(matrix.Sets) == 0 {
+		return nil, fmt.Errorf("matrix must define at least one set")
+	}
+
+	if len(matrix.Var) == 0 {
+		return nil, fmt.Errorf("matrix must define at least one var")
+	}
+
+	// Validate var entries
+	if matrix.Var != nil {
+		for id, modelName := range matrix.Var {
+			if !varKeyPattern.MatchString(id) {
+				return nil, fmt.Errorf("var key %q must be alphanumeric and 1-8 characters", id)
+			}
+			if _, exists := models[modelName]; !exists {
+				return nil, fmt.Errorf("var key %q references unknown model %q", id, modelName)
+			}
+		}
+	}
+
+	// Validate evict_costs
+	if matrix.EvictCosts != nil {
+		for key, cost := range matrix.EvictCosts {
+			if cost <= 0 {
+				return nil, fmt.Errorf("evict_cost for %q must be a positive integer, got %d", key, cost)
+			}
+			if _, ok := matrix.Var[key]; !ok {
+				return nil, fmt.Errorf("evict_costs: unknown var ID %q", key)
+			}
+		}
+	}
+
+	// Build dependency graph for +ref topological sort
+	setNames := make(map[string]bool)
+	for _, entry := range matrix.Sets {
+		setNames[entry.Name] = true
+	}
+
+	deps := make(map[string][]string) // setName -> set names it depends on
+	for _, entry := range matrix.Sets {
+		refs, err := extractRefs(entry.DSL)
+		if err != nil {
+			return nil, fmt.Errorf("set %q: %w", entry.Name, err)
+		}
+		for _, ref := range refs {
+			if !setNames[ref] {
+				return nil, fmt.Errorf("set %q references undefined set %q", entry.Name, ref)
+			}
+		}
+		deps[entry.Name] = refs
+	}
+
+	// Topological sort with cycle detection
+	order, err := topologicalSort(matrix.Sets, deps)
+	if err != nil {
+		return nil, err
+	}
+
+	// Expand sets in topological order
+	resolvedRefs := make(map[string][][]string) // set name -> expanded alias-level combos
+	var allExpanded []ExpandedSet
+	totalCombinations := 0
+
+	// Build ordered map for efficient lookup
+	setDSL := make(map[string]string)
+	for _, entry := range matrix.Sets {
+		setDSL[entry.Name] = entry.DSL
+	}
+
+	for _, name := range order {
+		dsl := setDSL[name]
+		combos, err := ParseAndExpandDSL(dsl, resolvedRefs)
+		if err != nil {
+			return nil, fmt.Errorf("set %q: %w", name, err)
+		}
+
+		resolvedRefs[name] = combos
+
+		// Resolve var IDs to real model names
+		for _, combo := range combos {
+			resolved := make([]string, len(combo))
+			for i, ident := range combo {
+				realName, ok := matrix.Var[ident]
+				if !ok {
+					return nil, fmt.Errorf("set %q: unknown var ID %q", name, ident)
+				}
+				resolved[i] = realName
+			}
+			sort.Strings(resolved)
+			allExpanded = append(allExpanded, ExpandedSet{
+				SetName: name,
+				DSL:     dsl,
+				Models:  resolved,
+			})
+		}
+
+		totalCombinations += len(combos)
+		if totalCombinations > maxDSLExpansions {
+			return nil, fmt.Errorf("total expanded combinations (%d) exceed limit of %d", totalCombinations, maxDSLExpansions)
+		}
+	}
+
+	return allExpanded, nil
+}
+
+// topologicalSort returns set names in dependency order.
+// Returns an error if a cycle is detected.
+func topologicalSort(sets OrderedSets, deps map[string][]string) ([]string, error) {
+	// States: 0 = unvisited, 1 = visiting, 2 = visited
+	state := make(map[string]int)
+	var order []string
+
+	var visit func(name string) error
+	visit = func(name string) error {
+		switch state[name] {
+		case 1:
+			return fmt.Errorf("circular reference detected involving set %q", name)
+		case 2:
+			return nil
+		}
+		state[name] = 1
+
+		for _, dep := range deps[name] {
+			if err := visit(dep); err != nil {
+				return err
+			}
+		}
+
+		state[name] = 2
+		order = append(order, name)
+		return nil
+	}
+
+	// Visit in definition order for deterministic output
+	for _, entry := range sets {
+		if state[entry.Name] == 0 {
+			if err := visit(entry.Name); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return order, nil
+}
+
+// ResolvedEvictCosts returns a map of real model name -> evict cost,
+// resolving var IDs. Models not listed default to 1.
+func (m *MatrixConfig) ResolvedEvictCosts() map[string]int {
+	costs := make(map[string]int)
+	if m.EvictCosts == nil {
+		return costs
+	}
+	for key, cost := range m.EvictCosts {
+		// Resolve var ID if present
+		if realName, ok := m.Var[key]; ok {
+			costs[realName] = cost
+		} else {
+			costs[key] = cost
+		}
+	}
+	return costs
+}

proxy/config/matrix_dsl.go

@@ -0,0 +1,376 @@
+package config
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"unicode"
+)
+
+const maxDSLExpansions = 1000
+
+// Token types for the DSL lexer
+type tokenType int
+
+const (
+	tokIdent  tokenType = iota // model alias or name
+	tokAnd                     // &
+	tokOr                      // |
+	tokLParen                  // (
+	tokRParen                  // )
+	tokRef                     // +setName
+	tokEOF
+)
+
+type token struct {
+	typ tokenType
+	val string
+}
+
+// tokenize splits a DSL string into tokens.
+func tokenize(input string) ([]token, error) {
+	var tokens []token
+	i := 0
+	runes := []rune(input)
+
+	for i < len(runes) {
+		ch := runes[i]
+
+		// skip whitespace
+		if unicode.IsSpace(ch) {
+			i++
+			continue
+		}
+
+		switch ch {
+		case '&':
+			tokens = append(tokens, token{tokAnd, "&"})
+			i++
+		case '|':
+			tokens = append(tokens, token{tokOr, "|"})
+			i++
+		case '(':
+			tokens = append(tokens, token{tokLParen, "("})
+			i++
+		case ')':
+			tokens = append(tokens, token{tokRParen, ")"})
+			i++
+		case '+':
+			// +ref: read the identifier that follows
+			i++
+			start := i
+			for i < len(runes) && isIdentChar(runes[i]) {
+				i++
+			}
+			if i == start {
+				return nil, fmt.Errorf("expected set name after '+' at position %d", start)
+			}
+			tokens = append(tokens, token{tokRef, string(runes[start:i])})
+		default:
+			if isIdentChar(ch) {
+				start := i
+				for i < len(runes) && isIdentChar(runes[i]) {
+					i++
+				}
+				tokens = append(tokens, token{tokIdent, string(runes[start:i])})
+			} else {
+				return nil, fmt.Errorf("unexpected character %q at position %d", ch, i)
+			}
+		}
+	}
+
+	tokens = append(tokens, token{tokEOF, ""})
+	return tokens, nil
+}
+
+func isIdentChar(ch rune) bool {
+	return unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '_' || ch == '-' || ch == '.'
+}
+
+// AST node types
+type dslNode interface {
+	dslNode()
+}
+
+type andNode struct {
+	children []dslNode
+}
+
+type orNode struct {
+	children []dslNode
+}
+
+type leafNode struct {
+	name string
+}
+
+type refNode struct {
+	setName string
+}
+
+func (andNode) dslNode()  {}
+func (orNode) dslNode()   {}
+func (leafNode) dslNode() {}
+func (refNode) dslNode()  {}
+
+// parser holds state for recursive-descent parsing.
+type parser struct {
+	tokens []token
+	pos    int
+}
+
+func (p *parser) peek() token {
+	if p.pos < len(p.tokens) {
+		return p.tokens[p.pos]
+	}
+	return token{tokEOF, ""}
+}
+
+func (p *parser) next() token {
+	t := p.peek()
+	if t.typ != tokEOF {
+		p.pos++
+	}
+	return t
+}
+
+func (p *parser) expect(typ tokenType) (token, error) {
+	t := p.next()
+	if t.typ != typ {
+		return t, fmt.Errorf("expected token type %d, got %q", typ, t.val)
+	}
+	return t, nil
+}
+
+// Grammar:
+//
+//	expr    = andExpr
+//	andExpr = orExpr ('&' orExpr)*
+//	orExpr  = atom ('|' atom)*
+//	atom    = ident | '+' ident | '(' expr ')'
+//
+// & binds tighter than |, so "a | b & c" means "a | (b & c)"
+func parse(tokens []token) (dslNode, error) {
+	p := &parser{tokens: tokens}
+	node, err := p.parseExpr()
+	if err != nil {
+		return nil, err
+	}
+	if p.peek().typ != tokEOF {
+		return nil, fmt.Errorf("unexpected token %q after expression", p.peek().val)
+	}
+	return node, nil
+}
+
+func (p *parser) parseExpr() (dslNode, error) {
+	return p.parseOrExpr()
+}
+
+func (p *parser) parseOrExpr() (dslNode, error) {
+	left, err := p.parseAndExpr()
+	if err != nil {
+		return nil, err
+	}
+
+	if p.peek().typ == tokOr {
+		children := []dslNode{left}
+		for p.peek().typ == tokOr {
+			p.next() // consume |
+			right, err := p.parseAndExpr()
+			if err != nil {
+				return nil, err
+			}
+			children = append(children, right)
+		}
+		return orNode{children: children}, nil
+	}
+
+	return left, nil
+}
+
+func (p *parser) parseAndExpr() (dslNode, error) {
+	left, err := p.parseAtom()
+	if err != nil {
+		return nil, err
+	}
+
+	if p.peek().typ == tokAnd {
+		children := []dslNode{left}
+		for p.peek().typ == tokAnd {
+			p.next() // consume &
+			right, err := p.parseAtom()
+			if err != nil {
+				return nil, err
+			}
+			children = append(children, right)
+		}
+		return andNode{children: children}, nil
+	}
+
+	return left, nil
+}
+
+func (p *parser) parseAtom() (dslNode, error) {
+	t := p.peek()
+
+	switch t.typ {
+	case tokIdent:
+		p.next()
+		return leafNode{name: t.val}, nil
+
+	case tokRef:
+		p.next()
+		return refNode{setName: t.val}, nil
+
+	case tokLParen:
+		p.next() // consume (
+		node, err := p.parseExpr()
+		if err != nil {
+			return nil, err
+		}
+		if _, err := p.expect(tokRParen); err != nil {
+			return nil, fmt.Errorf("missing closing parenthesis")
+		}
+		return node, nil
+
+	default:
+		return nil, fmt.Errorf("unexpected token %q", t.val)
+	}
+}
+
+// expand walks the AST and produces all combinations.
+// resolvedRefs contains previously expanded sets for +ref resolution.
+func expand(node dslNode, resolvedRefs map[string][][]string) ([][]string, error) {
+	switch n := node.(type) {
+	case leafNode:
+		return [][]string{{n.name}}, nil
+
+	case refNode:
+		expanded, ok := resolvedRefs[n.setName]
+		if !ok {
+			return nil, fmt.Errorf("unknown set reference +%s", n.setName)
+		}
+		// Return a copy
+		result := make([][]string, len(expanded))
+		for i, combo := range expanded {
+			result[i] = make([]string, len(combo))
+			copy(result[i], combo)
+		}
+		return result, nil
+
+	case orNode:
+		// Union of all children's expansions
+		var result [][]string
+		for _, child := range n.children {
+			childResult, err := expand(child, resolvedRefs)
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, childResult...)
+			if len(result) > maxDSLExpansions {
+				return nil, fmt.Errorf("DSL expansion exceeded %d combinations", maxDSLExpansions)
+			}
+		}
+		return result, nil
+
+	case andNode:
+		// Cartesian product across children
+		result := [][]string{{}} // start with one empty combo
+		for _, child := range n.children {
+			childResult, err := expand(child, resolvedRefs)
+			if err != nil {
+				return nil, err
+			}
+			result, err = cartesianProduct(result, childResult, maxDSLExpansions)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return result, nil
+
+	default:
+		return nil, fmt.Errorf("unknown node type %T", node)
+	}
+}
+
+// cartesianProduct computes the cartesian product of two sets of combinations.
+// It returns an error if the product would exceed cap.
+func cartesianProduct(left, right [][]string, cap int) ([][]string, error) {
+	if int64(len(left))*int64(len(right)) > int64(cap) {
+		return nil, fmt.Errorf("DSL expansion exceeded %d combinations", cap)
+	}
+	result := make([][]string, 0, len(left)*len(right))
+	for _, l := range left {
+		for _, r := range right {
+			combo := make([]string, 0, len(l)+len(r))
+			combo = append(combo, l...)
+			combo = append(combo, r...)
+			result = append(result, combo)
+		}
+	}
+	return result, nil
+}
+
+// ParseAndExpandDSL tokenizes, parses, and expands a DSL string.
+// resolvedRefs contains previously expanded sets for +ref inlining.
+func ParseAndExpandDSL(dsl string, resolvedRefs map[string][][]string) ([][]string, error) {
+	dsl = strings.TrimSpace(dsl)
+	if dsl == "" {
+		return nil, fmt.Errorf("empty DSL expression")
+	}
+
+	tokens, err := tokenize(dsl)
+	if err != nil {
+		return nil, fmt.Errorf("tokenize: %w", err)
+	}
+
+	tree, err := parse(tokens)
+	if err != nil {
+		return nil, fmt.Errorf("parse: %w", err)
+	}
+
+	result, err := expand(tree, resolvedRefs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Deduplicate models within each combination and sort for consistency
+	for i, combo := range result {
+		result[i] = dedupAndSort(combo)
+	}
+
+	return result, nil
+}
+
+// dedupAndSort removes duplicate entries and sorts alphabetically.
+func dedupAndSort(items []string) []string {
+	seen := make(map[string]bool, len(items))
+	var unique []string
+	for _, item := range items {
+		if !seen[item] {
+			seen[item] = true
+			unique = append(unique, item)
+		}
+	}
+	sort.Strings(unique)
+	return unique
+}
+
+// extractRefs scans a DSL string for +ref tokens without full parsing.
+// Used for building the dependency graph for topological sorting.
+func extractRefs(dsl string) ([]string, error) {
+	tokens, err := tokenize(dsl)
+	if err != nil {
+		return nil, err
+	}
+
+	var refs []string
+	seen := make(map[string]bool)
+	for _, t := range tokens {
+		if t.typ == tokRef && !seen[t.val] {
+			seen[t.val] = true
+			refs = append(refs, t.val)
+		}
+	}
+	return refs, nil
+}

proxy/config/matrix_dsl_test.go

@@ -0,0 +1,300 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDSL_Tokenize(t *testing.T) {
+	tests := []struct {
+		name   string
+		input  string
+		expect []token
+		errMsg string
+	}{
+		{
+			name:  "single identifier",
+			input: "abc",
+			expect: []token{
+				{tokIdent, "abc"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "identifier with hyphens and dots",
+			input: "model-name.v2",
+			expect: []token{
+				{tokIdent, "model-name.v2"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "and expression",
+			input: "a & b",
+			expect: []token{
+				{tokIdent, "a"},
+				{tokAnd, "&"},
+				{tokIdent, "b"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "or expression",
+			input: "a | b",
+			expect: []token{
+				{tokIdent, "a"},
+				{tokOr, "|"},
+				{tokIdent, "b"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "parentheses",
+			input: "(a | b) & c",
+			expect: []token{
+				{tokLParen, "("},
+				{tokIdent, "a"},
+				{tokOr, "|"},
+				{tokIdent, "b"},
+				{tokRParen, ")"},
+				{tokAnd, "&"},
+				{tokIdent, "c"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "ref token",
+			input: "+llms & v",
+			expect: []token{
+				{tokRef, "llms"},
+				{tokAnd, "&"},
+				{tokIdent, "v"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:  "no whitespace",
+			input: "(a|b)&c",
+			expect: []token{
+				{tokLParen, "("},
+				{tokIdent, "a"},
+				{tokOr, "|"},
+				{tokIdent, "b"},
+				{tokRParen, ")"},
+				{tokAnd, "&"},
+				{tokIdent, "c"},
+				{tokEOF, ""},
+			},
+		},
+		{
+			name:   "empty ref",
+			input:  "+",
+			errMsg: "expected set name after '+'",
+		},
+		{
+			name:   "invalid character",
+			input:  "a @ b",
+			errMsg: "unexpected character",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tokens, err := tokenize(tt.input)
+			if tt.errMsg != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.errMsg)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tt.expect, tokens)
+			}
+		})
+	}
+}
+
+func TestDSL_ParseAndExpand(t *testing.T) {
+	tests := []struct {
+		name   string
+		dsl    string
+		refs   map[string][][]string
+		expect [][]string
+		errMsg string
+	}{
+		{
+			name:   "single model",
+			dsl:    "L",
+			expect: [][]string{{"L"}},
+		},
+		{
+			name:   "two models with AND",
+			dsl:    "a & b",
+			expect: [][]string{{"a", "b"}},
+		},
+		{
+			name:   "two models with OR",
+			dsl:    "a | b",
+			expect: [][]string{{"a"}, {"b"}},
+		},
+		{
+			name:   "three models with OR",
+			dsl:    "a | b | c",
+			expect: [][]string{{"a"}, {"b"}, {"c"}},
+		},
+		{
+			name: "cartesian product (a|b) & (c|d)",
+			dsl:  "(a | b) & (c | d)",
+			expect: [][]string{
+				{"a", "c"},
+				{"a", "d"},
+				{"b", "c"},
+				{"b", "d"},
+			},
+		},
+		{
+			name: "three-way AND",
+			dsl:  "a & b & c",
+			expect: [][]string{
+				{"a", "b", "c"},
+			},
+		},
+		{
+			name: "(g | q | m) & v",
+			dsl:  "(g | q | m) & v",
+			expect: [][]string{
+				{"g", "v"},
+				{"q", "v"},
+				{"m", "v"},
+			},
+		},
+		{
+			name: "(g | q) & v & e",
+			dsl:  "(g | q) & v & e",
+			expect: [][]string{
+				{"e", "g", "v"},
+				{"e", "q", "v"},
+			},
+		},
+		{
+			name: "precedence: a | b & c means a | (b & c)",
+			dsl:  "a | b & c",
+			expect: [][]string{
+				{"a"},
+				{"b", "c"},
+			},
+		},
+		{
+			name: "+ref inlining",
+			dsl:  "+llms & v",
+			refs: map[string][][]string{
+				"llms": {{"g"}, {"q"}, {"m"}},
+			},
+			expect: [][]string{
+				{"g", "v"},
+				{"q", "v"},
+				{"m", "v"},
+			},
+		},
+		{
+			name: "+ref chained",
+			dsl:  "+with_tts & e",
+			refs: map[string][][]string{
+				"with_tts": {{"g", "v"}, {"q", "v"}, {"m", "v"}},
+			},
+			expect: [][]string{
+				{"e", "g", "v"},
+				{"e", "q", "v"},
+				{"e", "m", "v"},
+			},
+		},
+		{
+			name: "dedup within combination",
+			dsl:  "a & a",
+			expect: [][]string{
+				{"a"},
+			},
+		},
+		{
+			name:   "empty expression",
+			dsl:    "",
+			errMsg: "empty DSL expression",
+		},
+		{
+			name:   "unmatched open paren",
+			dsl:    "(a | b",
+			errMsg: "missing closing parenthesis",
+		},
+		{
+			name:   "unmatched close paren",
+			dsl:    "a | b)",
+			errMsg: "unexpected token",
+		},
+		{
+			name:   "unknown ref",
+			dsl:    "+unknown",
+			errMsg: "unknown set reference +unknown",
+		},
+		{
+			name:   "empty parens",
+			dsl:    "()",
+			errMsg: "unexpected token",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			refs := tt.refs
+			if refs == nil {
+				refs = map[string][][]string{}
+			}
+			result, err := ParseAndExpandDSL(tt.dsl, refs)
+			if tt.errMsg != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.errMsg)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tt.expect, result)
+			}
+		})
+	}
+}
+
+func TestDSL_ExpansionCap(t *testing.T) {
+	// Build an expression that would exceed 1000 combinations:
+	// (a1|a2|...|a32) & (b1|b2|...|b32) = 1024 combos
+	var aItems, bItems []string
+	for i := 0; i < 32; i++ {
+		aItems = append(aItems, fmt.Sprintf("a%d", i))
+		bItems = append(bItems, fmt.Sprintf("b%d", i))
+	}
+	dsl := fmt.Sprintf("(%s) & (%s)",
+		join(aItems, " | "),
+		join(bItems, " | "),
+	)
+	_, err := ParseAndExpandDSL(dsl, map[string][][]string{})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "exceeded")
+}
+
+func TestDSL_ExtractRefs(t *testing.T) {
+	refs, err := extractRefs("+llms & v & +other")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"llms", "other"}, refs)
+
+	refs, err = extractRefs("a & b")
+	require.NoError(t, err)
+	assert.Empty(t, refs)
+}
+
+func join(items []string, sep string) string {
+	result := ""
+	for i, item := range items {
+		if i > 0 {
+			result += sep
+		}
+		result += item
+	}
+	return result
+}

proxy/config/matrix_test.go

@@ -0,0 +1,305 @@
+package config
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func makeModels(names ...string) map[string]ModelConfig {
+	m := make(map[string]ModelConfig)
+	for _, name := range names {
+		m[name] = ModelConfig{Cmd: "echo " + name}
+	}
+	return m
+}
+
+func TestValidateMatrix_Basic(t *testing.T) {
+	models := makeModels("gemma", "qwen", "mistral", "voxtral", "llama70B")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{
+			"g": "gemma",
+			"q": "qwen",
+			"m": "mistral",
+			"v": "voxtral",
+			"L": "llama70B",
+		},
+		EvictCosts: map[string]int{
+			"L": 30,
+			"v": 50,
+		},
+		Sets: OrderedSets{
+			{Name: "standard", DSL: "(g | q | m) & v"},
+			{Name: "full", DSL: "L"},
+		},
+	}
+
+	expanded, err := ValidateMatrix(matrix, models)
+	require.NoError(t, err)
+
+	// standard expands to [gemma,voxtral], [qwen,voxtral], [mistral,voxtral]
+	// full expands to [llama70B]
+	assert.Len(t, expanded, 4)
+
+	assert.Equal(t, "standard", expanded[0].SetName)
+	assert.Equal(t, []string{"gemma", "voxtral"}, expanded[0].Models)
+
+	assert.Equal(t, "standard", expanded[1].SetName)
+	assert.Equal(t, []string{"qwen", "voxtral"}, expanded[1].Models)
+
+	assert.Equal(t, "standard", expanded[2].SetName)
+	assert.Equal(t, []string{"mistral", "voxtral"}, expanded[2].Models)
+
+	assert.Equal(t, "full", expanded[3].SetName)
+	assert.Equal(t, []string{"llama70B"}, expanded[3].Models)
+}
+
+func TestValidateMatrix_WithRef(t *testing.T) {
+	models := makeModels("gemma", "qwen", "mistral", "voxtral", "reranker")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{
+			"g": "gemma",
+			"q": "qwen",
+			"m": "mistral",
+			"v": "voxtral",
+			"e": "reranker",
+		},
+		Sets: OrderedSets{
+			{Name: "llms", DSL: "g | q | m"},
+			{Name: "with_tts", DSL: "+llms & v"},
+			{Name: "mega", DSL: "+with_tts & e"},
+		},
+	}
+
+	expanded, err := ValidateMatrix(matrix, models)
+	require.NoError(t, err)
+
+	// llms: [gemma], [qwen], [mistral]
+	// with_tts: [gemma,voxtral], [qwen,voxtral], [mistral,voxtral]
+	// mega: [gemma,reranker,voxtral], [qwen,reranker,voxtral], [mistral,reranker,voxtral]
+	assert.Len(t, expanded, 9)
+
+	// Check mega entries
+	megaEntries := filterBySetName(expanded, "mega")
+	assert.Len(t, megaEntries, 3)
+	assert.Equal(t, []string{"gemma", "reranker", "voxtral"}, megaEntries[0].Models)
+}
+
+func TestValidateMatrix_MapIDRequired(t *testing.T) {
+	// DSL cannot use real model names directly — must use var IDs
+	models := makeModels("gemma", "voxtral")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{"g": "gemma"},
+		Sets: OrderedSets{
+			{Name: "combo", DSL: "g & voxtral"},
+		},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unknown var ID")
+}
+
+func TestValidateMatrix_InvalidAliasKey(t *testing.T) {
+	models := makeModels("gemma")
+
+	tests := []struct {
+		name   string
+		alias  string
+		errMsg string
+	}{
+		{"too long", "abcdefghi", "alphanumeric and 1-8 characters"},
+		{"has underscore", "a_b", "alphanumeric and 1-8 characters"},
+		{"has hyphen", "a-b", "alphanumeric and 1-8 characters"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			matrix := MatrixConfig{
+				Var:  map[string]string{tt.alias: "gemma"},
+				Sets: OrderedSets{{Name: "s", DSL: tt.alias}},
+			}
+			_, err := ValidateMatrix(matrix, models)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tt.errMsg)
+		})
+	}
+}
+
+func TestValidateMatrix_AliasReferencesUnknownModel(t *testing.T) {
+	models := makeModels("gemma")
+
+	matrix := MatrixConfig{
+		Var:  map[string]string{"x": "nonexistent"},
+		Sets: OrderedSets{{Name: "s", DSL: "x"}},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unknown model")
+}
+
+func TestValidateMatrix_EvictCostInvalid(t *testing.T) {
+	models := makeModels("gemma")
+
+	t.Run("zero cost", func(t *testing.T) {
+		matrix := MatrixConfig{
+			Var:        map[string]string{"g": "gemma"},
+			EvictCosts: map[string]int{"g": 0},
+			Sets:       OrderedSets{{Name: "s", DSL: "g"}},
+		}
+		_, err := ValidateMatrix(matrix, models)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "positive integer")
+	})
+
+	t.Run("negative cost", func(t *testing.T) {
+		matrix := MatrixConfig{
+			Var:        map[string]string{"g": "gemma"},
+			EvictCosts: map[string]int{"g": -1},
+			Sets:       OrderedSets{{Name: "s", DSL: "g"}},
+		}
+		_, err := ValidateMatrix(matrix, models)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "positive integer")
+	})
+
+	t.Run("unknown var ID in evict_costs", func(t *testing.T) {
+		matrix := MatrixConfig{
+			Var:        map[string]string{"g": "gemma"},
+			EvictCosts: map[string]int{"unknown": 5},
+			Sets:       OrderedSets{{Name: "s", DSL: "g"}},
+		}
+		_, err := ValidateMatrix(matrix, models)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "unknown var ID")
+	})
+}
+
+func TestValidateMatrix_CycleDetection(t *testing.T) {
+	models := makeModels("gemma")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{"g": "gemma"},
+		Sets: OrderedSets{
+			{Name: "a", DSL: "+b"},
+			{Name: "b", DSL: "+a"},
+		},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "circular reference")
+}
+
+func TestValidateMatrix_UndefinedRefTarget(t *testing.T) {
+	models := makeModels("gemma")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{"g": "gemma"},
+		Sets: OrderedSets{
+			{Name: "a", DSL: "+nonexistent"},
+		},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "references undefined set")
+}
+
+func TestValidateMatrix_NoSets(t *testing.T) {
+	_, err := ValidateMatrix(MatrixConfig{}, makeModels("gemma"))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "at least one set")
+}
+
+func TestValidateMatrix_UnknownMapIDInDSL(t *testing.T) {
+	models := makeModels("gemma")
+
+	matrix := MatrixConfig{
+		Var: map[string]string{"g": "gemma"},
+		Sets: OrderedSets{
+			{Name: "s", DSL: "g & nonexistent"},
+		},
+	}
+
+	_, err := ValidateMatrix(matrix, models)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "unknown var ID")
+}
+
+func TestValidateMatrix_ResolvedEvictCosts(t *testing.T) {
+	mc := &MatrixConfig{
+		Var: map[string]string{
+			"g": "gemma",
+			"L": "llama70B",
+		},
+		EvictCosts: map[string]int{
+			"L": 30,
+			"g": 5,
+		},
+	}
+
+	costs := mc.ResolvedEvictCosts()
+	assert.Equal(t, 30, costs["llama70B"])
+	assert.Equal(t, 5, costs["gemma"])
+}
+
+func TestValidateMatrix_ConfigXOR(t *testing.T) {
+	// groups and matrix both defined
+	yaml := `
+models:
+  model1:
+    cmd: echo model1
+    proxy: http://localhost:8080
+groups:
+  group1:
+    members:
+      - model1
+matrix:
+  sets:
+    s: "model1"
+`
+	_, err := LoadConfigFromReader(strings.NewReader(yaml))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "cannot use both")
+}
+
+func TestValidateMatrix_ConfigMatrixOnly(t *testing.T) {
+	yaml := `
+models:
+  gemma:
+    cmd: echo gemma
+    proxy: http://localhost:8080
+  qwen:
+    cmd: echo qwen
+    proxy: http://localhost:8081
+matrix:
+  vars:
+    g: gemma
+    q: qwen
+  sets:
+    combo: "g | q"
+`
+	cfg, err := LoadConfigFromReader(strings.NewReader(yaml))
+	require.NoError(t, err)
+	assert.NotNil(t, cfg.Matrix)
+	assert.Len(t, cfg.ExpandedSets, 2)
+	// Groups should be empty when matrix is used
+	assert.Empty(t, cfg.Groups)
+}
+
+func filterBySetName(sets []ExpandedSet, name string) []ExpandedSet {
+	var result []ExpandedSet
+	for _, s := range sets {
+		if s.SetName == name {
+			result = append(result, s)
+		}
+	}
+	return result
+}

proxy/config/model_config.go

@@ -10,12 +10,14 @@ const (
 )
 
 // TimeoutsConfig holds timeout settings for proxy connections
+// 0 = no timeout
 type TimeoutsConfig struct {
-	Connect        int `yaml:"connect"`        // seconds, 0 = no timeout (not recommended)
-	ResponseHeader int `yaml:"responseHeader"` // seconds, 0 = no timeout (not recommended)
-	TLSHandshake   int `yaml:"tlsHandshake"`   // seconds, 0 = no timeout (not recommended)
-	ExpectContinue int `yaml:"expectContinue"` // seconds, 0 = no timeout (not recommended)
-	IdleConn       int `yaml:"idleConn"`       // seconds, 0 = no timeout (not recommended)
+	Connect        int `yaml:"connect"`
+	KeepAlive      int `yaml:"keepalive"`
+	ResponseHeader int `yaml:"responseHeader"`
+	TLSHandshake   int `yaml:"tlsHandshake"`
+	ExpectContinue int `yaml:"expectContinue"`
+	IdleConn       int `yaml:"idleConn"`
 }
 
 type ModelConfig struct {
@@ -69,9 +71,12 @@ func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		ConcurrencyLimit: 0,
 		Name:             "",
 		Description:      "",
+
+		// matches http.DefaultTransport
 		Timeouts: TimeoutsConfig{
 			Connect:        30,
-			ResponseHeader: 60,
+			KeepAlive:      30,
+			ResponseHeader: 0,
 			TLSHandshake:   10,
 			ExpectContinue: 1,
 			IdleConn:       90,

proxy/config/peer.go

@@ -24,8 +24,12 @@ func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		ApiKey:  "",
 		Models:  []string{},
 		Filters: Filters{},
+
+		// mostly matches http.DefaultTransport but with a 60s ResponseHeader timeout
+		// to match the pre PR #619 functionality
 		Timeouts: TimeoutsConfig{
 			Connect:        30,
+			KeepAlive:      30,
 			ResponseHeader: 60,
 			TLSHandshake:   10,
 			ExpectContinue: 1,

proxy/configwatcher/watcher.go

@@ -0,0 +1,85 @@
+// Package configwatcher provides a simple cross-platform file watcher based
+// on os.Stat polling. It works correctly inside Docker containers where the
+// config file is bind-mounted as an individual file, and for k8s ConfigMap
+// projections (which present the file as a symlink to an atomically swapped
+// target) — both cases where inotify-based watchers are unreliable.
+package configwatcher
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"log"
+	"os"
+	"time"
+)
+
+const DefaultInterval = 2 * time.Second
+
+type Watcher struct {
+	Path     string
+	Interval time.Duration
+	OnChange func()
+}
+
+type snapshot struct {
+	exists  bool
+	modTime time.Time
+	size    int64
+}
+
+// Run blocks until ctx is canceled. It polls Path on Interval and invokes
+// OnChange whenever the file's modification time or size changes, or when
+// the file reappears after being missing. The baseline poll establishes
+// initial state and does not fire OnChange.
+func (w *Watcher) Run(ctx context.Context) {
+	interval := w.Interval
+	if interval <= 0 {
+		interval = DefaultInterval
+	}
+
+	prev := stat(w.Path)
+
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			cur := stat(w.Path)
+			if changed(prev, cur) && w.OnChange != nil {
+				w.OnChange()
+			}
+			prev = cur
+		}
+	}
+}
+
+func stat(path string) snapshot {
+	fi, err := os.Stat(path)
+	if err != nil {
+		if !errors.Is(err, fs.ErrNotExist) {
+			log.Printf("configwatcher: stat %s: %v", path, err)
+		}
+		return snapshot{}
+	}
+	return snapshot{
+		exists:  true,
+		modTime: fi.ModTime(),
+		size:    fi.Size(),
+	}
+}
+
+func changed(prev, cur snapshot) bool {
+	// Present → missing: stay quiet (likely a transient rename-style write).
+	// Missing → present: fire so we reload as soon as the file comes back.
+	if !cur.exists {
+		return false
+	}
+	if !prev.exists {
+		return true
+	}
+	return !prev.modTime.Equal(cur.modTime) || prev.size != cur.size
+}

proxy/configwatcher/watcher_test.go

@@ -0,0 +1,191 @@
+package configwatcher
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+const testInterval = 25 * time.Millisecond
+
+// startWatcher launches w.Run in a goroutine and returns a function that
+// cancels the context and waits for Run to return.
+func startWatcher(t *testing.T, w *Watcher) func() {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		w.Run(ctx)
+		close(done)
+	}()
+	return func() {
+		cancel()
+		select {
+		case <-done:
+		case <-time.After(2 * time.Second):
+			t.Fatal("watcher did not stop within 2s of cancel")
+		}
+	}
+}
+
+// waitForCount blocks until counter reaches want or timeout elapses.
+func waitForCount(t *testing.T, counter *int64, want int64, timeout time.Duration) bool {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if atomic.LoadInt64(counter) >= want {
+			return true
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	return false
+}
+
+func TestWatcher_NoFireOnBaseline(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("a"), 0o644))
+
+	var n int64
+	stop := startWatcher(t, &Watcher{
+		Path:     path,
+		Interval: testInterval,
+		OnChange: func() { atomic.AddInt64(&n, 1) },
+	})
+	defer stop()
+
+	time.Sleep(testInterval * 5)
+	require.Equal(t, int64(0), atomic.LoadInt64(&n), "baseline poll must not fire")
+}
+
+func TestWatcher_DetectsModTimeChange(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("a"), 0o644))
+
+	// Force a known baseline mtime.
+	base := time.Now().Add(-1 * time.Hour).Truncate(time.Second)
+	require.NoError(t, os.Chtimes(path, base, base))
+
+	var n int64
+	stop := startWatcher(t, &Watcher{
+		Path:     path,
+		Interval: testInterval,
+		OnChange: func() { atomic.AddInt64(&n, 1) },
+	})
+	defer stop()
+
+	// Let the baseline settle.
+	time.Sleep(testInterval * 2)
+
+	// Bump mtime well above the baseline so low-resolution filesystems still notice.
+	require.NoError(t, os.Chtimes(path, base.Add(10*time.Second), base.Add(10*time.Second)))
+
+	require.True(t, waitForCount(t, &n, 1, time.Second), "callback should fire after mtime change")
+}
+
+func TestWatcher_DetectsSizeChangeWithSameModTime(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("a"), 0o644))
+
+	fi, err := os.Stat(path)
+	require.NoError(t, err)
+	originalMtime := fi.ModTime()
+
+	var n int64
+	stop := startWatcher(t, &Watcher{
+		Path:     path,
+		Interval: testInterval,
+		OnChange: func() { atomic.AddInt64(&n, 1) },
+	})
+	defer stop()
+	time.Sleep(testInterval * 2)
+
+	require.NoError(t, os.WriteFile(path, []byte("aaaaa"), 0o644))
+	// Reset mtime back to the original so size is the only signal.
+	require.NoError(t, os.Chtimes(path, originalMtime, originalMtime))
+
+	require.True(t, waitForCount(t, &n, 1, time.Second), "callback should fire on size change")
+}
+
+func TestWatcher_SymlinkTargetSwap(t *testing.T) {
+	dir := t.TempDir()
+	targetA := filepath.Join(dir, "targetA")
+	targetB := filepath.Join(dir, "targetB")
+	link := filepath.Join(dir, "config.yaml")
+
+	require.NoError(t, os.WriteFile(targetA, []byte("AAAA"), 0o644))
+	require.NoError(t, os.WriteFile(targetB, []byte("BBBBBBBB"), 0o644))
+
+	if err := os.Symlink(targetA, link); err != nil {
+		if runtime.GOOS == "windows" {
+			t.Skipf("symlink creation requires privilege on Windows: %v", err)
+		}
+		t.Fatalf("os.Symlink: %v", err)
+	}
+
+	var n int64
+	stop := startWatcher(t, &Watcher{
+		Path:     link,
+		Interval: testInterval,
+		OnChange: func() { atomic.AddInt64(&n, 1) },
+	})
+	defer stop()
+	time.Sleep(testInterval * 2)
+
+	// Atomic symlink swap (k8s ConfigMap pattern): create new symlink at a
+	// temp name, then rename over the existing one.
+	tmpLink := filepath.Join(dir, "config.yaml.tmp")
+	require.NoError(t, os.Symlink(targetB, tmpLink))
+	require.NoError(t, os.Rename(tmpLink, link))
+
+	require.True(t, waitForCount(t, &n, 1, time.Second), "callback should fire after symlink target swap")
+}
+
+func TestWatcher_FileMissingThenReturns(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("a"), 0o644))
+
+	var n int64
+	stop := startWatcher(t, &Watcher{
+		Path:     path,
+		Interval: testInterval,
+		OnChange: func() { atomic.AddInt64(&n, 1) },
+	})
+	defer stop()
+	time.Sleep(testInterval * 2)
+
+	require.NoError(t, os.Remove(path))
+	time.Sleep(testInterval * 3)
+	require.Equal(t, int64(0), atomic.LoadInt64(&n), "removal alone must not fire")
+
+	require.NoError(t, os.WriteFile(path, []byte("b"), 0o644))
+	require.True(t, waitForCount(t, &n, 1, time.Second), "callback should fire when file returns")
+}
+
+func TestWatcher_ContextCancelStopsRun(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("a"), 0o644))
+
+	w := &Watcher{Path: path, Interval: testInterval}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { w.Run(ctx); close(done) }()
+
+	time.Sleep(testInterval * 2)
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Run did not return within 2s of cancel")
+	}
+}

proxy/helpers_test.go

@@ -1,15 +1,22 @@
 package proxy
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/mostlygeek/llama-swap/proxy/config"
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/gjson"
 	"gopkg.in/yaml.v3"
 )
 
@@ -66,6 +73,16 @@ func getTestPort() int {
 	return port
 }
 
+// testConfigFromYAML substitutes {{RESPONDER}} with the simple-responder path and
+// loads through the real config pipeline (env vars, macros, port assignment, etc.)
+func testConfigFromYAML(t *testing.T, yamlTmpl string) config.Config {
+	t.Helper()
+	yamlStr := strings.ReplaceAll(yamlTmpl, "{{RESPONDER}}", filepath.ToSlash(simpleResponderPath))
+	cfg, err := config.LoadConfigFromReader(strings.NewReader(yamlStr))
+	require.NoError(t, err)
+	return cfg
+}
+
 func getTestSimpleResponderConfig(expectedMessage string) config.ModelConfig {
 	return getTestSimpleResponderConfigPort(expectedMessage, getTestPort())
 }
@@ -88,3 +105,188 @@ proxy: "http://127.0.0.1:%d"
 
 	return cfg
 }
+
+// injectTestHandlers sets a testHandler on every Process in every ProcessGroup
+// of the given ProxyManager, bypassing subprocess launches. modelResponses maps
+// model IDs to their respond strings; if a model ID is not in the map, the model
+// ID itself is used.
+func injectTestHandlers(pm *ProxyManager, modelResponses map[string]string) {
+	for _, pg := range pm.processGroups {
+		for modelID, process := range pg.processes {
+			respond := modelID
+			if r, ok := modelResponses[modelID]; ok {
+				respond = r
+			}
+			process.testHandler = newTestHandler(respond)
+		}
+	}
+}
+
+// newTestHandler returns an http.Handler that mimics simple-responder's API.
+// It supports the endpoints that routing tests depend on, without launching
+// any subprocess or binding any port.
+func newTestHandler(respond string) http.Handler {
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
+		bodyBytes, _ := io.ReadAll(r.Body)
+		isStreaming := r.URL.Query().Get("stream") == "true"
+
+		if wait := r.URL.Query().Get("wait"); wait != "" {
+			if d, err := time.ParseDuration(wait); err == nil {
+				time.Sleep(d)
+			}
+		}
+
+		if isStreaming {
+			w.Header().Set("Content-Type", "text/event-stream")
+			w.Header().Set("Cache-Control", "no-cache")
+			w.Header().Set("Connection", "keep-alive")
+			flusher := w.(http.Flusher)
+
+			for i := 0; i < 10; i++ {
+				data, _ := json.Marshal(map[string]any{
+					"created": time.Now().Unix(),
+					"choices": []map[string]any{
+						{"index": 0, "delta": map[string]any{"content": "asdf"}, "finish_reason": nil},
+					},
+				})
+				fmt.Fprintf(w, "event: message\ndata: %s\n\n", data)
+				flusher.Flush()
+			}
+
+			finalData, _ := json.Marshal(map[string]any{
+				"usage": map[string]any{
+					"completion_tokens": 10, "prompt_tokens": 25, "total_tokens": 35,
+				},
+				"timings": map[string]any{
+					"prompt_n": 25, "prompt_ms": 13, "predicted_n": 10,
+					"predicted_ms": 17, "predicted_per_second": 10,
+				},
+			})
+			fmt.Fprintf(w, "event: message\ndata: %s\n\n", finalData)
+			flusher.Flush()
+
+			fmt.Fprintf(w, "event: message\ndata: [DONE]\n\n")
+			flusher.Flush()
+		} else {
+			w.Header().Set("Content-Type", "application/json")
+			json.NewEncoder(w).Encode(map[string]any{
+				"responseMessage":  respond,
+				"h_content_length": r.Header.Get("Content-Length"),
+				"request_body":     string(bodyBytes),
+				"usage": map[string]any{
+					"completion_tokens": 10, "prompt_tokens": 25, "total_tokens": 35,
+				},
+				"timings": map[string]any{
+					"prompt_n": 25, "prompt_ms": 13, "predicted_n": 10,
+					"predicted_ms": 17, "predicted_per_second": 10,
+				},
+			})
+		}
+	})
+
+	mux.HandleFunc("/v1/audio/speech", func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		modelName := gjson.GetBytes(body, "model").String()
+		if modelName != respond {
+			w.WriteHeader(http.StatusBadRequest)
+			json.NewEncoder(w).Encode(map[string]string{"error": fmt.Sprintf("Invalid model: %s, expected: %s", modelName, respond)})
+			return
+		}
+		json.NewEncoder(w).Encode(map[string]string{"message": "ok"})
+	})
+
+	mux.HandleFunc("/v1/completions", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(map[string]any{
+			"responseMessage": respond,
+			"usage": map[string]any{
+				"completion_tokens": 10, "prompt_tokens": 25, "total_tokens": 35,
+			},
+		})
+	})
+
+	mux.HandleFunc("/completion", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(map[string]any{
+			"responseMessage": respond,
+			"usage": map[string]any{
+				"completion_tokens": 10, "prompt_tokens": 25, "total_tokens": 35,
+			},
+		})
+	})
+
+	mux.HandleFunc("/v1/audio/transcriptions", func(w http.ResponseWriter, r *http.Request) {
+		if err := r.ParseMultipartForm(10 << 20); err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			json.NewEncoder(w).Encode(map[string]string{"error": fmt.Sprintf("Error parsing multipart form: %s", err)})
+			return
+		}
+		model := r.FormValue("model")
+		if model == "" {
+			w.WriteHeader(http.StatusBadRequest)
+			json.NewEncoder(w).Encode(map[string]string{"error": "Missing model parameter"})
+			return
+		}
+		file, _, err := r.FormFile("file")
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			json.NewEncoder(w).Encode(map[string]string{"error": fmt.Sprintf("Error getting file: %s", err)})
+			return
+		}
+		fileBytes, _ := io.ReadAll(file)
+		file.Close()
+		json.NewEncoder(w).Encode(map[string]any{
+			"text":             fmt.Sprintf("The length of the file is %d bytes", len(fileBytes)),
+			"model":            model,
+			"h_content_type":   r.Header.Get("Content-Type"),
+			"h_content_length": r.Header.Get("Content-Length"),
+		})
+	})
+
+	mux.HandleFunc("/v1/audio/voices", func(w http.ResponseWriter, r *http.Request) {
+		model := r.URL.Query().Get("model")
+		json.NewEncoder(w).Encode(map[string]any{
+			"voices": []string{"voice1"}, "model": model,
+		})
+	})
+
+	mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		fmt.Fprint(w, respond)
+	})
+
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		fmt.Fprintf(w, "%s %s", r.Method, r.URL.Path)
+	})
+
+	mux.HandleFunc("/sdapi/v1/txt2img", func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		modelName := gjson.GetBytes(body, "model").String()
+		json.NewEncoder(w).Encode(map[string]any{
+			"model": modelName, "images": []string{},
+		})
+	})
+
+	mux.HandleFunc("/sdapi/v1/img2img", func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		modelName := gjson.GetBytes(body, "model").String()
+		json.NewEncoder(w).Encode(map[string]any{
+			"model": modelName, "images": []string{},
+		})
+	})
+
+	mux.HandleFunc("/sdapi/v1/loras", func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(map[string]any{
+			"loras": []string{},
+		})
+	})
+
+	return mux
+}

proxy/matrix.go

@@ -0,0 +1,329 @@
+package proxy
+
+import (
+	"fmt"
+	"net/http"
+	"slices"
+	"sort"
+	"sync"
+
+	"github.com/mostlygeek/llama-swap/proxy/config"
+)
+
+// MatrixSolver contains pure swap-decision logic with no Process dependencies.
+// It is safe for concurrent reads after construction.
+type MatrixSolver struct {
+	expandedSets []config.ExpandedSet // all valid model combinations
+	evictCosts   map[string]int       // real model name -> eviction cost (default 1)
+	modelToSets  map[string][]int     // model name -> indices into expandedSets
+}
+
+// NewMatrixSolver builds a solver from expanded sets and eviction costs.
+func NewMatrixSolver(expandedSets []config.ExpandedSet, evictCosts map[string]int) *MatrixSolver {
+	modelToSets := make(map[string][]int)
+	for i, es := range expandedSets {
+		for _, model := range es.Models {
+			modelToSets[model] = append(modelToSets[model], i)
+		}
+	}
+
+	return &MatrixSolver{
+		expandedSets: expandedSets,
+		evictCosts:   evictCosts,
+		modelToSets:  modelToSets,
+	}
+}
+
+// SolveResult describes what the solver decided.
+type SolveResult struct {
+	Evict     []string // running models that must be stopped
+	TargetSet []string // the chosen set of models (for informational purposes)
+	SetName   string   // name of the chosen set
+	DSL       string   // original DSL expression for the chosen set
+	TotalCost int      // total eviction cost
+}
+
+// Solve determines which models to evict when a model is requested.
+//
+// Algorithm:
+//  1. If requestedModel is already running, no eviction needed.
+//  2. Find all sets containing requestedModel.
+//  3. If no sets found, the model runs alone; evict all running models.
+//  4. For each candidate set, compute cost = sum of evict_costs for running
+//     models NOT in that set.
+//  5. Pick lowest cost. Ties broken by definition order (index in expandedSets).
+//  6. Return models to evict and the chosen set.
+func (s *MatrixSolver) Solve(requestedModel string, runningModels []string) (SolveResult, error) {
+	// If already running, nothing to do (but fill in set info for logging)
+	if slices.Contains(runningModels, requestedModel) {
+		setName, dsl := s.findMatchingSet(requestedModel, runningModels)
+		return SolveResult{
+			TargetSet: runningModels,
+			SetName:   setName,
+			DSL:       dsl,
+		}, nil
+	}
+
+	candidateIndices := s.modelToSets[requestedModel]
+
+	// Model not in any set: runs alone, evict everything
+	if len(candidateIndices) == 0 {
+		evict := make([]string, len(runningModels))
+		copy(evict, runningModels)
+		return SolveResult{
+			Evict:     evict,
+			TargetSet: []string{requestedModel},
+		}, nil
+	}
+
+	// Find the cheapest candidate set
+	bestCost := -1
+	bestIdx := -1
+
+	for _, idx := range candidateIndices {
+		setModels := s.expandedSets[idx].Models
+		cost := 0
+		for _, running := range runningModels {
+			if !slices.Contains(setModels, running) {
+				cost += s.evictCost(running)
+			}
+		}
+
+		if bestCost < 0 || cost < bestCost || (cost == bestCost && idx < bestIdx) {
+			bestCost = cost
+			bestIdx = idx
+		}
+	}
+
+	// Determine which running models to evict
+	chosen := s.expandedSets[bestIdx]
+	var evict []string
+	for _, running := range runningModels {
+		if !slices.Contains(chosen.Models, running) {
+			evict = append(evict, running)
+		}
+	}
+
+	return SolveResult{
+		Evict:     evict,
+		TargetSet: chosen.Models,
+		SetName:   chosen.SetName,
+		DSL:       chosen.DSL,
+		TotalCost: bestCost,
+	}, nil
+}
+
+// findMatchingSet finds the expanded set that contains all running models.
+// Returns the set name and DSL, or empty strings if no match.
+func (s *MatrixSolver) findMatchingSet(requestedModel string, runningModels []string) (string, string) {
+	for _, idx := range s.modelToSets[requestedModel] {
+		set := s.expandedSets[idx]
+		allInSet := true
+		for _, m := range runningModels {
+			if !slices.Contains(set.Models, m) {
+				allInSet = false
+				break
+			}
+		}
+		if allInSet {
+			return set.SetName, set.DSL
+		}
+	}
+	return "", ""
+}
+
+func (s *MatrixSolver) evictCost(model string) int {
+	if cost, ok := s.evictCosts[model]; ok {
+		return cost
+	}
+	return 1
+}
+
+// Matrix manages processes using solver-based swap logic.
+type Matrix struct {
+	sync.Mutex
+	solver         *MatrixSolver
+	processes      map[string]*Process // all processes keyed by real model name
+	config         config.Config
+	proxyLogger    *LogMonitor
+	upstreamLogger *LogMonitor
+
+	// inflight tracks ProxyRequest calls that have released m.Lock but may
+	// not yet have incremented Process.inFlightRequests. A concurrent
+	// request that needs to evict models waits for inflight to drain under
+	// m.Lock before stopping anything. Without this, a request that
+	// released m.Lock but has not yet reached Process.inFlightRequests.Add(1)
+	// races with Stop()'s Wait() and can be killed mid-request.
+	inflight sync.WaitGroup
+
+	// testDelayFastPath is a test-only hook invoked in the no-eviction path
+	// after m.Lock is released but before the request is dispatched to
+	// Process.ProxyRequest. Tests use it to park a request at the exact
+	// race window to deterministically reproduce the race.
+	testDelayFastPath func()
+}
+
+// NewMatrix creates a Matrix from config. It creates a Process for every
+// model defined in the config (any model can run alone even if not in a set).
+func NewMatrix(cfg config.Config, proxyLogger, upstreamLogger *LogMonitor) *Matrix {
+	processes := make(map[string]*Process)
+	for modelID, modelConfig := range cfg.Models {
+		processLogger := NewLogMonitorWriter(upstreamLogger)
+		process := NewProcess(modelID, cfg.HealthCheckTimeout, modelConfig, processLogger, proxyLogger)
+		processes[modelID] = process
+	}
+
+	evictCosts := cfg.Matrix.ResolvedEvictCosts()
+
+	return &Matrix{
+		solver:         NewMatrixSolver(cfg.ExpandedSets, evictCosts),
+		processes:      processes,
+		config:         cfg,
+		proxyLogger:    proxyLogger,
+		upstreamLogger: upstreamLogger,
+	}
+}
+
+// ProxyRequest handles the swap logic and proxies the request to the model.
+func (m *Matrix) ProxyRequest(modelID string, w http.ResponseWriter, r *http.Request) error {
+	process, ok := m.processes[modelID]
+	if !ok {
+		return fmt.Errorf("model %s not found in matrix", modelID)
+	}
+
+	m.Lock()
+	running := m.runningModels()
+	result, err := m.solver.Solve(modelID, running)
+	if err != nil {
+		m.Unlock()
+		return fmt.Errorf("matrix solver error: %w", err)
+	}
+
+	// Log solver decision
+	if len(result.Evict) > 0 {
+		m.proxyLogger.Infof("Matrix: model=%s set=%s dsl=%q evict=%v target=%v cost=%d",
+			modelID, result.SetName, result.DSL, result.Evict, result.TargetSet, result.TotalCost)
+	} else if len(running) == 0 {
+		m.proxyLogger.Infof("Matrix: model=%s starting (no models running)", modelID)
+	} else {
+		m.proxyLogger.Debugf("Matrix: model=%s already running in set=%s dsl=%q", modelID, result.SetName, result.DSL)
+	}
+
+	// Evict models that need to be stopped
+	if len(result.Evict) > 0 {
+		// Wait for any in-flight ProxyRequest calls to register on their
+		// Process before stopping anything. Without this, a request that
+		// released m.Lock but has not yet incremented
+		// Process.inFlightRequests races with Stop() and can be killed
+		// mid-request.
+		m.inflight.Wait()
+
+		var wg sync.WaitGroup
+		for _, evictModel := range result.Evict {
+			if p, exists := m.processes[evictModel]; exists {
+				wg.Add(1)
+				go func(p *Process) {
+					defer wg.Done()
+					p.Stop()
+				}(p)
+			}
+		}
+		wg.Wait()
+	}
+
+	// Register this request in inflight before releasing m.Lock so a
+	// concurrent eviction will wait for it to complete.
+	m.inflight.Add(1)
+	defer m.inflight.Done()
+	isFastPath := len(result.Evict) == 0
+	m.Unlock()
+
+	if isFastPath && m.testDelayFastPath != nil {
+		m.testDelayFastPath()
+	}
+
+	// Proxy the request (Process handles on-demand start)
+	process.ProxyRequest(w, r)
+	return nil
+}
+
+// StopProcesses stops all running processes.
+func (m *Matrix) StopProcesses(strategy StopStrategy) {
+	m.Lock()
+	defer m.Unlock()
+
+	var wg sync.WaitGroup
+	for _, process := range m.processes {
+		wg.Add(1)
+		go func(p *Process) {
+			defer wg.Done()
+			switch strategy {
+			case StopImmediately:
+				p.StopImmediately()
+			default:
+				p.Stop()
+			}
+		}(process)
+	}
+	wg.Wait()
+}
+
+// StopProcess stops a single process by model ID.
+func (m *Matrix) StopProcess(modelID string, strategy StopStrategy) error {
+	process, ok := m.processes[modelID]
+	if !ok {
+		return fmt.Errorf("process not found for %s", modelID)
+	}
+
+	switch strategy {
+	case StopImmediately:
+		process.StopImmediately()
+	default:
+		process.Stop()
+	}
+	return nil
+}
+
+// Shutdown shuts down all processes.
+func (m *Matrix) Shutdown() {
+	var wg sync.WaitGroup
+	for _, process := range m.processes {
+		wg.Add(1)
+		go func(p *Process) {
+			defer wg.Done()
+			p.Shutdown()
+		}(process)
+	}
+	wg.Wait()
+}
+
+// RunningModels returns model names currently in an active (non-stopped) state.
+func (m *Matrix) RunningModels() []string {
+	m.Lock()
+	defer m.Unlock()
+	return m.runningModels()
+}
+
+// runningModels returns running model names (caller must hold lock).
+func (m *Matrix) runningModels() []string {
+	var running []string
+	for id, process := range m.processes {
+		if process.CurrentState() != StateStopped && process.CurrentState() != StateShutdown {
+			running = append(running, id)
+		}
+	}
+	sort.Strings(running)
+	return running
+}
+
+// GetProcess returns the Process for a model.
+func (m *Matrix) GetProcess(modelID string) (*Process, bool) {
+	p, ok := m.processes[modelID]
+	return p, ok
+}
+
+// HasModel returns true if the model is managed by this matrix.
+func (m *Matrix) HasModel(modelID string) bool {
+	_, ok := m.processes[modelID]
+	return ok
+}

proxy/matrix_test.go

@@ -0,0 +1,349 @@
+package proxy
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/mostlygeek/llama-swap/proxy/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Helper to build expanded sets for solver tests
+func makeExpandedSets(sets ...struct {
+	name   string
+	models []string
+}) []config.ExpandedSet {
+	var result []config.ExpandedSet
+	for _, s := range sets {
+		result = append(result, config.ExpandedSet{
+			SetName: s.name,
+			Models:  s.models,
+		})
+	}
+	return result
+}
+
+func es(name string, models ...string) struct {
+	name   string
+	models []string
+} {
+	return struct {
+		name   string
+		models []string
+	}{name, models}
+}
+
+func TestMatrixSolver_AlreadyRunning(t *testing.T) {
+	solver := NewMatrixSolver(
+		makeExpandedSets(es("s1", "a", "b")),
+		nil,
+	)
+
+	result, err := solver.Solve("a", []string{"a"})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"a"}, result.TargetSet)
+	assert.Equal(t, "s1", result.SetName)
+}
+
+func TestMatrixSolver_NotInAnySet_RunsAlone(t *testing.T) {
+	solver := NewMatrixSolver(
+		makeExpandedSets(es("s1", "a", "b")),
+		nil,
+	)
+
+	// Model "c" not in any set
+	result, err := solver.Solve("c", []string{"a", "b"})
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"a", "b"}, result.Evict)
+	assert.Equal(t, []string{"c"}, result.TargetSet)
+}
+
+func TestMatrixSolver_NotInAnySet_NothingRunning(t *testing.T) {
+	solver := NewMatrixSolver(
+		makeExpandedSets(es("s1", "a", "b")),
+		nil,
+	)
+
+	result, err := solver.Solve("c", []string{})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"c"}, result.TargetSet)
+}
+
+func TestMatrixSolver_SingleSet_EvictsNonMembers(t *testing.T) {
+	// Set: [a, b]. Request a when b and c are running.
+	solver := NewMatrixSolver(
+		makeExpandedSets(es("s1", "a", "b")),
+		nil,
+	)
+
+	result, err := solver.Solve("a", []string{"b", "c"})
+	require.NoError(t, err)
+	// c is not in the set, so it gets evicted. b is in the set, so it stays.
+	assert.Equal(t, []string{"c"}, result.Evict)
+	assert.Equal(t, []string{"a", "b"}, result.TargetSet)
+}
+
+func TestMatrixSolver_PicksLowestCost(t *testing.T) {
+	// Two sets containing model "a":
+	// s1: [a, v]  — if v is running, cost=0; if L is running, cost=30
+	// s2: [a, L]  — if L is running, cost=0; if v is running, cost=50
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("s1", "a", "v"),
+			es("s2", "a", "L"),
+		),
+		map[string]int{"v": 50, "L": 30},
+	)
+
+	// v is running. Switching to a:
+	// s1 cost: v is in s1, so 0
+	// s2 cost: v is NOT in s2, so 50
+	// => pick s1
+	result, err := solver.Solve("a", []string{"v"})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"a", "v"}, result.TargetSet)
+
+	// L is running. Switching to a:
+	// s1 cost: L is NOT in s1, so 30
+	// s2 cost: L is in s2, so 0
+	// => pick s2
+	result, err = solver.Solve("a", []string{"L"})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"a", "L"}, result.TargetSet)
+}
+
+func TestMatrixSolver_TieBreakingByDefinitionOrder(t *testing.T) {
+	// Two sets with identical cost. Definition order should win.
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("s1", "a", "x"),
+			es("s2", "a", "y"),
+		),
+		nil,
+	)
+
+	// Nothing running, both sets cost 0. s1 is first.
+	result, err := solver.Solve("a", []string{})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"a", "x"}, result.TargetSet)
+}
+
+func TestMatrixSolver_EvictCostPreservesExpensive(t *testing.T) {
+	// Model "v" costs 50 to evict, "m" costs 1 (default).
+	// Sets: [g,v], [g,m]
+	// Running: v, m. Request g.
+	// s1=[g,v]: evict m (cost 1), keep v
+	// s2=[g,m]: evict v (cost 50), keep m
+	// => pick s1
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("s1", "g", "v"),
+			es("s2", "g", "m"),
+		),
+		map[string]int{"v": 50},
+	)
+
+	result, err := solver.Solve("g", []string{"v", "m"})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"m"}, result.Evict)
+	assert.Equal(t, []string{"g", "v"}, result.TargetSet)
+}
+
+func TestMatrixSolver_NothingRunning(t *testing.T) {
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("s1", "g", "v"),
+			es("s2", "q", "v"),
+		),
+		nil,
+	)
+
+	result, err := solver.Solve("g", []string{})
+	require.NoError(t, err)
+	assert.Empty(t, result.Evict)
+	assert.Equal(t, []string{"g", "v"}, result.TargetSet)
+}
+
+// TestMatrix_ProxyRequestSwapRaceAgainstFastPath verifies that an eviction
+// cannot stop a process while an in-flight ProxyRequest for that process is
+// still in the [m.Unlock, Process.inFlightRequests.Add(1)] window. Without
+// matrix-level inflight tracking, the eviction's Stop() races with the
+// pending request and kills it mid-start.
+func TestMatrix_ProxyRequestSwapRaceAgainstFastPath(t *testing.T) {
+	cfg := config.Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]config.ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+		},
+		ExpandedSets: []config.ExpandedSet{
+			{SetName: "s1", Models: []string{"model1"}},
+			{SetName: "s2", Models: []string{"model2"}},
+		},
+		Matrix: &config.MatrixConfig{},
+	}
+
+	m := NewMatrix(cfg, testLogger, testLogger)
+	defer m.StopProcesses(StopImmediately)
+
+	// Bypass real subprocesses so the test is fast and deterministic.
+	m.processes["model1"].testHandler = newTestHandler("model1")
+	m.processes["model2"].testHandler = newTestHandler("model2")
+
+	// Prime: run a request through model1 so it reaches StateReady and
+	// subsequent requests take the no-eviction path.
+	primeReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
+	primeW := httptest.NewRecorder()
+	require.NoError(t, m.ProxyRequest("model1", primeW, primeReq))
+	require.Equal(t, http.StatusOK, primeW.Code)
+	require.Equal(t, StateReady, m.processes["model1"].CurrentState())
+	require.Equal(t, StateStopped, m.processes["model2"].CurrentState())
+
+	// Install fast-path hook that signals arrival and waits for release.
+	// This parks R2 at the race window — after m.Lock is released but
+	// before Process.inFlightRequests.Add(1).
+	r2Reached := make(chan struct{})
+	r2Release := make(chan struct{})
+	m.testDelayFastPath = func() {
+		close(r2Reached)
+		<-r2Release
+	}
+
+	// R2: no-eviction request for model1. Will pause at the hook.
+	r2Done := make(chan struct{})
+	w2 := httptest.NewRecorder()
+	go func() {
+		defer close(r2Done)
+		req := httptest.NewRequest("POST", "/v1/chat/completions", nil)
+		assert.NoError(t, m.ProxyRequest("model1", w2, req))
+	}()
+
+	// Deterministically wait for R2 to reach the race window.
+	<-r2Reached
+
+	// R3: request for model2 which requires evicting model1. Must wait for
+	// R2 to finish before touching model1.
+	r3Done := make(chan struct{})
+	w3 := httptest.NewRecorder()
+	go func() {
+		defer close(r3Done)
+		req := httptest.NewRequest("POST", "/v1/chat/completions", nil)
+		assert.NoError(t, m.ProxyRequest("model2", w3, req))
+	}()
+
+	// Spin until R3 has acquired m.Lock and entered the eviction path. In
+	// the fixed code, R3 then blocks on m.inflight.Wait() while still
+	// holding the lock, so TryLock keeps failing.
+	for m.TryLock() {
+		m.Unlock()
+		runtime.Gosched()
+	}
+
+	// Bounded poll: give R3 a chance to demonstrate the bug by mutating
+	// state. In the fixed code R3 is blocked and nothing changes; in the
+	// buggy code R3 will Stop() model1 and start model2 within microseconds.
+	deadline := time.Now().Add(100 * time.Millisecond)
+	for time.Now().Before(deadline) {
+		if m.processes["model1"].CurrentState() != StateReady ||
+			m.processes["model2"].CurrentState() != StateStopped {
+			break
+		}
+		done := false
+		select {
+		case <-r3Done:
+			done = true
+		default:
+		}
+		if done {
+			break
+		}
+		runtime.Gosched()
+	}
+
+	// Invariant: R3 must be blocked while R2 is still in flight.
+	select {
+	case <-r3Done:
+		t.Fatal("eviction completed while in-flight request was still pending — race not prevented")
+	default:
+	}
+	assert.Equal(t, StateReady, m.processes["model1"].CurrentState(),
+		"model1 must stay Ready while an in-flight request is pending")
+	assert.Equal(t, StateStopped, m.processes["model2"].CurrentState(),
+		"model2 must not be started until R2 finishes and model1 is evicted")
+
+	// Release R2 and let both requests finish.
+	close(r2Release)
+	<-r2Done
+	<-r3Done
+
+	assert.Equal(t, http.StatusOK, w2.Code)
+	assert.Contains(t, w2.Body.String(), "model1")
+	assert.Equal(t, http.StatusOK, w3.Code)
+	assert.Contains(t, w3.Body.String(), "model2")
+}
+
+func TestMatrixSolver_FullScenario(t *testing.T) {
+	// Simulates the example config:
+	// standard: [g,v], [q,v], [m,v]
+	// with_rerank: [g,v,e], [q,v,e]
+	// creative: [g,sd], [q,sd]
+	// full: [L]
+	solver := NewMatrixSolver(
+		makeExpandedSets(
+			es("standard", "g", "v"),
+			es("standard", "q", "v"),
+			es("standard", "m", "v"),
+			es("with_rerank", "e", "g", "v"),
+			es("with_rerank", "e", "q", "v"),
+			es("creative", "g", "sd"),
+			es("creative", "q", "sd"),
+			es("full", "L"),
+		),
+		map[string]int{"v": 50, "L": 30, "whisper": 10},
+	)
+
+	// Running: g, v. Request q.
+	// standard[q,v]: evict g (cost 1), keep v. Total: 1.
+	// with_rerank[q,v,e]: evict g (cost 1), keep v. Total: 1.
+	// => tie, pick first by definition order = standard[q,v]
+	result, err := solver.Solve("q", []string{"g", "v"})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"g"}, result.Evict)
+	assert.Equal(t, []string{"q", "v"}, result.TargetSet)
+
+	// Running: g, v. Request L.
+	// full[L]: evict g (cost 1) + v (cost 50). Total: 51.
+	// Only one set contains L, so pick it.
+	result, err = solver.Solve("L", []string{"g", "v"})
+	require.NoError(t, err)
+	assert.ElementsMatch(t, []string{"g", "v"}, result.Evict)
+	assert.Equal(t, []string{"L"}, result.TargetSet)
+
+	// Running: g, v. Request sd.
+	// creative[g,sd]: evict v (cost 50). Total: 50.
+	// creative[q,sd]: evict g (cost 1) + v (cost 50). Total: 51.
+	// => pick creative[g,sd]
+	result, err = solver.Solve("sd", []string{"g", "v"})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"v"}, result.Evict)
+	assert.Equal(t, []string{"g", "sd"}, result.TargetSet)
+
+	// Running: q, v, e. Request g.
+	// standard[g,v]: evict q (1) + e (1). Total: 2.
+	// with_rerank[g,v,e]: evict q (1). Total: 1.
+	// creative[g,sd]: evict q (1) + v (50) + e (1). Total: 52.
+	// => pick with_rerank[g,v,e]
+	result, err = solver.Solve("g", []string{"e", "q", "v"})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"q"}, result.Evict)
+	assert.Equal(t, []string{"e", "g", "v"}, result.TargetSet)
+}

proxy/metrics_monitor.go

@@ -13,10 +13,54 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/klauspost/compress/zstd"
 	"github.com/mostlygeek/llama-swap/event"
 	"github.com/tidwall/gjson"
 )
 
+// zstdEncOptions are the shared zstd encoder options for maximum compression.
+var zstdEncOptions = []zstd.EOption{
+	zstd.WithEncoderLevel(zstd.SpeedBetterCompression),
+}
+
+// zstdDecOptions are the shared zstd decoder options.
+var zstdDecOptions = []zstd.DOption{}
+
+// zstdEncPool pools zstd.Encoder instances to reduce allocations.
+var zstdEncPool = &sync.Pool{
+	New: func() interface{} {
+		enc, _ := zstd.NewWriter(nil, zstdEncOptions...)
+		return enc
+	},
+}
+
+// zstdDecPool pools zstd.Decoder instances to reduce allocations.
+var zstdDecPool = &sync.Pool{
+	New: func() interface{} {
+		dec, _ := zstd.NewReader(nil, zstdDecOptions...)
+		return dec
+	},
+}
+
+// compressCapture marshals a ReqRespCapture to JSON and compresses it with zstd.
+// Returns compressed bytes and the original JSON byte count for logging.
+func compressCapture(c *ReqRespCapture) ([]byte, int, error) {
+	jsonBytes, err := json.Marshal(c)
+	if err != nil {
+		return nil, 0, fmt.Errorf("marshal capture: %w", err)
+	}
+	enc := zstdEncPool.Get().(*zstd.Encoder)
+	defer zstdEncPool.Put(enc)
+	return enc.EncodeAll(jsonBytes, nil), len(jsonBytes), nil
+}
+
+// decompressCapture decompresses zstd-compressed JSON and returns it.
+func decompressCapture(data []byte) ([]byte, error) {
+	dec := zstdDecPool.Get().(*zstd.Decoder)
+	defer zstdDecPool.Put(dec)
+	return dec.DecodeAll(data, nil)
+}
+
 // TokenMetrics represents parsed token statistics from llama-server logs
 type TokenMetrics struct {
 	ID              int       `json:"id"`
@@ -40,18 +84,6 @@ type ReqRespCapture struct {
 	RespBody    []byte            `json:"resp_body"`
 }
 
-// Size returns the approximate memory usage of this capture in bytes
-func (c *ReqRespCapture) Size() int {
-	size := len(c.ReqPath) + len(c.ReqBody) + len(c.RespBody)
-	for k, v := range c.ReqHeaders {
-		size += len(k) + len(v)
-	}
-	for k, v := range c.RespHeaders {
-		size += len(k) + len(v)
-	}
-	return size
-}
-
 // TokenMetricsEvent represents a token metrics event
 type TokenMetricsEvent struct {
 	Metrics TokenMetrics
@@ -71,10 +103,10 @@ type metricsMonitor struct {
 
 	// capture fields
 	enableCaptures bool
-	captures       map[int]ReqRespCapture // map for O(1) lookup by ID
-	captureOrder   []int                  // track insertion order for FIFO eviction
-	captureSize    int                    // current total size in bytes
-	maxCaptureSize int                    // max bytes for captures
+	captures       map[int][]byte // zstd-compressed JSON of ReqRespCapture
+	captureOrder   []int          // track insertion order for FIFO eviction
+	captureSize    int            // current total compressed size in bytes
+	maxCaptureSize int            // max bytes for captures (uncompressed)
 }
 
 // newMetricsMonitor creates a new metricsMonitor. captureBufferMB is the
@@ -84,7 +116,7 @@ func newMetricsMonitor(logger *LogMonitor, maxMetrics int, captureBufferMB int)
 		logger:         logger,
 		maxMetrics:     maxMetrics,
 		enableCaptures: captureBufferMB > 0,
-		captures:       make(map[int]ReqRespCapture),
+		captures:       make(map[int][]byte),
 		captureOrder:   make([]int, 0),
 		captureSize:    0,
 		maxCaptureSize: captureBufferMB * 1024 * 1024,
@@ -108,45 +140,80 @@ func (mp *metricsMonitor) addMetrics(metric TokenMetrics) int {
 }
 
 // addCapture adds a new capture to the buffer with size-based eviction.
-// Captures are skipped if enableCaptures is false or if capture exceeds maxCaptureSize.
+// Captures are skipped if enableCaptures is false or if compressed data exceeds maxCaptureSize.
 func (mp *metricsMonitor) addCapture(capture ReqRespCapture) {
 	if !mp.enableCaptures {
 		return
 	}
 
-	mp.mu.Lock()
-	defer mp.mu.Unlock()
-
-	captureSize := capture.Size()
-	if captureSize > mp.maxCaptureSize {
-		mp.logger.Warnf("capture size %d exceeds max %d, skipping", captureSize, mp.maxCaptureSize)
+	compressed, uncompressedBytes, err := compressCapture(&capture)
+	if err != nil {
+		mp.logger.Warnf("failed to compress capture: %v, skipping", err)
 		return
 	}
 
-	// Evict oldest (FIFO) until room available
+	captureSize := len(compressed)
+	if captureSize > mp.maxCaptureSize {
+		mp.logger.Warnf("compressed capture size %d exceeds max %d, skipping", captureSize, mp.maxCaptureSize)
+		return
+	}
+
+	compressionRatio := (1 - float64(captureSize)/float64(uncompressedBytes)) * 100
+
+	mp.mu.Lock()
+	defer mp.mu.Unlock()
+
+	// Evict oldest (FIFO) until room available for the compressed data
 	for mp.captureSize+captureSize > mp.maxCaptureSize && len(mp.captureOrder) > 0 {
 		oldestID := mp.captureOrder[0]
 		mp.captureOrder = mp.captureOrder[1:]
 		if evicted, exists := mp.captures[oldestID]; exists {
-			mp.captureSize -= evicted.Size()
+			l := len(evicted)
+			mp.captureSize -= l
 			delete(mp.captures, oldestID)
+			mp.logger.Debugf("Capture %d evicted to make space: %d bytes", oldestID, l)
 		}
 	}
 
-	mp.captures[capture.ID] = capture
+	mp.captures[capture.ID] = compressed
 	mp.captureOrder = append(mp.captureOrder, capture.ID)
 	mp.captureSize += captureSize
+
+	mp.logger.Debugf("Capture %d compressed and saved: %d bytes -> %d bytes (%.1f%% compression)", capture.ID, uncompressedBytes, len(compressed), compressionRatio)
 }
 
-// getCaptureByID returns a capture by its ID, or nil if not found.
-func (mp *metricsMonitor) getCaptureByID(id int) *ReqRespCapture {
+// getCompressedBytes returns the raw compressed bytes for a capture by ID.
+func (mp *metricsMonitor) getCompressedBytes(id int) ([]byte, bool) {
 	mp.mu.RLock()
 	defer mp.mu.RUnlock()
 
-	if capture, exists := mp.captures[id]; exists {
-		return &capture
+	data, exists := mp.captures[id]
+	return data, exists
+}
+
+// getCaptureByID returns decompressed capture bytes if found and decompress=true.
+// If decompress=false, returns the raw zstd-compressed bytes.
+// Returns nil if the capture is not found.
+func (mp *metricsMonitor) getCaptureByID(id int, decompress bool) []byte {
+	mp.mu.RLock()
+	defer mp.mu.RUnlock()
+
+	data, exists := mp.captures[id]
+	if !exists {
+		return nil
 	}
-	return nil
+
+	if !decompress {
+		return data
+	}
+
+	decompressed, err := decompressCapture(data)
+	if err != nil {
+		mp.logger.Warnf("failed to decompress capture %d: %v", id, err)
+		return nil
+	}
+
+	return decompressed
 }
 
 // getMetrics returns a copy of the current metrics
@@ -290,8 +357,8 @@ func (mp *metricsMonitor) wrapHandler(
 			RespHeaders: respHeaders,
 			RespBody:    body,
 		}
-		// Only set HasCapture if the capture will actually be stored (not too large)
-		if capture.Size() <= mp.maxCaptureSize {
+		compressed, _, err := compressCapture(capture)
+		if err == nil && len(compressed) <= mp.maxCaptureSize {
 			tm.HasCapture = true
 		}
 	}

proxy/metrics_monitor_test.go

@@ -5,6 +5,7 @@ import (
 	"compress/flate"
 	"compress/gzip"
 	"encoding/json"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"sync"
@@ -953,28 +954,27 @@ func TestMetricsMonitor_WrapHandler_Compression(t *testing.T) {
 	})
 }
 
-func TestReqRespCapture_Size(t *testing.T) {
-	t.Run("calculates size correctly", func(t *testing.T) {
+func TestReqRespCapture_CompressedSize(t *testing.T) {
+	t.Run("compressed size is smaller than uncompressed", func(t *testing.T) {
 		capture := ReqRespCapture{
-			ID:      1,
-			ReqPath: "/v1/chat/completions", // 20 bytes
-			ReqHeaders: map[string]string{
-				"Content-Type": "application/json", // 12 + 16 = 28
-			},
-			ReqBody: []byte("request body"), // 12 bytes
-			RespHeaders: map[string]string{
-				"X-Test": "value", // 6 + 5 = 11
-			},
-			RespBody: []byte("response body"), // 13 bytes
+			ID:       1,
+			ReqPath:  "/v1/chat/completions",
+			ReqBody:  []byte(`{"model":"test","prompt":"hello world this is a test request body that is reasonably long"}`),
+			RespBody: []byte(`{"id":"resp-123","object":"chat.completion","created":1234567890,"model":"test-model","choices":[{"index":0,"message":{"role":"assistant","content":"This is a test response body with some meaningful content to compress"}},{"index":1,"message":{"role":"user","content":"Another message here"}}]}`),
 		}
 
-		// Expected: 20 + 12 + 13 + 28 + 11 = 84
-		assert.Equal(t, 84, capture.Size())
+		compressed, uncompressed, err := compressCapture(&capture)
+		assert.NoError(t, err)
+		assert.Greater(t, uncompressed, 0)
+		assert.True(t, len(compressed) < uncompressed, "compressed (%d bytes) should be smaller than uncompressed JSON (%d bytes)", len(compressed), uncompressed)
 	})
 
-	t.Run("handles empty capture", func(t *testing.T) {
+	t.Run("empty capture produces compressed output", func(t *testing.T) {
 		capture := ReqRespCapture{}
-		assert.Equal(t, 0, capture.Size())
+		compressed, _, err := compressCapture(&capture)
+		assert.NoError(t, err)
+		assert.NotNil(t, compressed)
+		assert.True(t, len(compressed) > 0)
 	})
 }
 
@@ -989,7 +989,7 @@ func TestMetricsMonitor_AddCapture(t *testing.T) {
 		mm.addCapture(capture)
 
 		// Should not store capture
-		assert.Nil(t, mm.getCaptureByID(0))
+		assert.Nil(t, mm.getCaptureByID(0, false))
 	})
 
 	t.Run("adds capture when enabled", func(t *testing.T) {
@@ -1002,41 +1002,55 @@ func TestMetricsMonitor_AddCapture(t *testing.T) {
 		}
 		mm.addCapture(capture)
 
-		retrieved := mm.getCaptureByID(0)
+		retrieved := mm.getCaptureByID(0, true)
 		assert.NotNil(t, retrieved)
-		assert.Equal(t, 0, retrieved.ID)
-		assert.Equal(t, []byte("test request"), retrieved.ReqBody)
-		assert.Equal(t, []byte("test response"), retrieved.RespBody)
+
+		var decoded ReqRespCapture
+		err := json.Unmarshal(retrieved, &decoded)
+		assert.NoError(t, err)
+		assert.Equal(t, 0, decoded.ID)
+		assert.Equal(t, []byte("test request"), decoded.ReqBody)
+		assert.Equal(t, []byte("test response"), decoded.RespBody)
 	})
 
 	t.Run("evicts oldest when exceeding max size", func(t *testing.T) {
 		mm := newMetricsMonitor(testLogger, 10, 5)
-		mm.maxCaptureSize = 100 // Set small limit for test
+		// Each full ReqRespCapture with 80 bytes random data compresses to ~185 bytes.
+		// 2 captures = ~370 bytes, 3 captures = ~555 bytes. Set limit so only 2 fit.
+		mm.maxCaptureSize = 450
 
-		// Add captures that will exceed the limit
-		capture1 := ReqRespCapture{ID: 0, ReqBody: make([]byte, 40)}
-		capture2 := ReqRespCapture{ID: 1, ReqBody: make([]byte, 40)}
-		capture3 := ReqRespCapture{ID: 2, ReqBody: make([]byte, 40)}
+		// Use random-looking data that doesn't compress well with zstd
+		rng := rand.New(rand.NewSource(42))
+		capture1 := ReqRespCapture{ID: 0, ReqBody: make([]byte, 80)}
+		rng.Read(capture1.ReqBody)
+		capture2 := ReqRespCapture{ID: 1, ReqBody: make([]byte, 80)}
+		rng.Read(capture2.ReqBody)
+		capture3 := ReqRespCapture{ID: 2, ReqBody: make([]byte, 80)}
+		rng.Read(capture3.ReqBody)
 
 		mm.addCapture(capture1)
 		mm.addCapture(capture2)
 		// Adding capture3 should evict capture1
 		mm.addCapture(capture3)
 
-		assert.Nil(t, mm.getCaptureByID(0), "capture 0 should be evicted")
-		assert.NotNil(t, mm.getCaptureByID(1), "capture 1 should exist")
-		assert.NotNil(t, mm.getCaptureByID(2), "capture 2 should exist")
+		assert.Nil(t, mm.getCaptureByID(0, true), "capture 0 should be evicted")
+		retrieved := mm.getCaptureByID(1, true)
+		assert.NotNil(t, retrieved, "capture 1 should exist")
+		retrieved = mm.getCaptureByID(2, true)
+		assert.NotNil(t, retrieved, "capture 2 should exist")
 	})
 
 	t.Run("skips capture larger than max size", func(t *testing.T) {
 		mm := newMetricsMonitor(testLogger, 10, 5)
 		mm.maxCaptureSize = 100
 
-		// Add a capture larger than max
-		largeCapture := ReqRespCapture{ID: 0, ReqBody: make([]byte, 200)}
+		// Use random data that doesn't compress well to create an oversized capture
+		rng := rand.New(rand.NewSource(99))
+		largeCapture := ReqRespCapture{ID: 0, ReqBody: make([]byte, 300)}
+		rng.Read(largeCapture.ReqBody)
 		mm.addCapture(largeCapture)
 
-		assert.Nil(t, mm.getCaptureByID(0), "oversized capture should not be stored")
+		assert.Nil(t, mm.getCaptureByID(0, false), "oversized capture should not be stored")
 	})
 }
 
@@ -1044,21 +1058,44 @@ func TestMetricsMonitor_GetCaptureByID(t *testing.T) {
 	t.Run("returns nil for non-existent ID", func(t *testing.T) {
 		mm := newMetricsMonitor(testLogger, 10, 5)
 
-		assert.Nil(t, mm.getCaptureByID(999))
+		assert.Nil(t, mm.getCaptureByID(999, false))
 	})
 
-	t.Run("returns capture by ID", func(t *testing.T) {
+	t.Run("returns decompressed capture by ID", func(t *testing.T) {
 		mm := newMetricsMonitor(testLogger, 10, 5)
 
 		capture := ReqRespCapture{
-			ID:      42,
-			ReqBody: []byte("test"),
+			ID:       42,
+			ReqBody:  []byte("test request"),
+			RespBody: []byte("test response"),
 		}
 		mm.addCapture(capture)
 
-		retrieved := mm.getCaptureByID(42)
+		retrieved := mm.getCaptureByID(42, true)
 		assert.NotNil(t, retrieved)
-		assert.Equal(t, 42, retrieved.ID)
+
+		var decoded ReqRespCapture
+		err := json.Unmarshal(retrieved, &decoded)
+		assert.NoError(t, err)
+		assert.Equal(t, 42, decoded.ID)
+		assert.Equal(t, []byte("test request"), decoded.ReqBody)
+		assert.Equal(t, []byte("test response"), decoded.RespBody)
+	})
+
+	t.Run("returns compressed bytes when decompress=false", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10, 5)
+
+		capture := ReqRespCapture{
+			ID:       42,
+			ReqBody:  []byte("test request body"),
+			RespBody: []byte("test response body"),
+		}
+		mm.addCapture(capture)
+
+		compressed := mm.getCaptureByID(42, false)
+		assert.NotNil(t, compressed)
+		// Compressed data should not be valid JSON (it's zstd-compressed)
+		assert.False(t, gjson.ValidBytes(compressed))
 	})
 }
 
@@ -1135,9 +1172,13 @@ func TestMetricsMonitor_WrapHandler_Capture(t *testing.T) {
 		assert.Equal(t, 1, len(metrics))
 		metricID := metrics[0].ID
 
-		// Check capture was stored with same ID
-		capture := mm.getCaptureByID(metricID)
-		assert.NotNil(t, capture)
+		// Check capture was stored with same ID (decompressed)
+		captureData := mm.getCaptureByID(metricID, true)
+		assert.NotNil(t, captureData)
+
+		var capture ReqRespCapture
+		err = json.Unmarshal(captureData, &capture)
+		assert.NoError(t, err)
 		assert.Equal(t, metricID, capture.ID)
 		assert.Equal(t, []byte(requestBody), capture.ReqBody)
 		assert.Equal(t, []byte(responseBody), capture.RespBody)
@@ -1173,7 +1214,7 @@ func TestMetricsMonitor_WrapHandler_Capture(t *testing.T) {
 		assert.Equal(t, 1, len(metrics))
 
 		// But no capture
-		capture := mm.getCaptureByID(metrics[0].ID)
+		capture := mm.getCaptureByID(metrics[0].ID, false)
 		assert.Nil(t, capture)
 	})
 }

proxy/peerproxy.go

@@ -42,7 +42,7 @@ func NewPeerProxy(peers config.PeerDictionaryConfig, proxyLogger *LogMonitor) (*
 			Proxy: http.ProxyFromEnvironment,
 			DialContext: (&net.Dialer{
 				Timeout:   time.Duration(peer.Timeouts.Connect) * time.Second,
-				KeepAlive: 30 * time.Second,
+				KeepAlive: time.Duration(peer.Timeouts.KeepAlive) * time.Second,
 			}).DialContext,
 			TLSHandshakeTimeout:   time.Duration(peer.Timeouts.TLSHandshake) * time.Second,
 			ResponseHeaderTimeout: time.Duration(peer.Timeouts.ResponseHeader) * time.Second,

proxy/process.go

@@ -77,6 +77,9 @@ type Process struct {
 	// used for testing to override the default value
 	gracefulStopTimeout time.Duration
 
+	// used for testing to bypass subprocess and reverse proxy
+	testHandler http.Handler
+
 	// track the number of failed starts
 	failedStartCount int
 }
@@ -102,7 +105,7 @@ func NewProcess(ID string, healthCheckTimeout int, config config.ModelConfig, pr
 			Proxy: http.ProxyFromEnvironment,
 			DialContext: (&net.Dialer{
 				Timeout:   time.Duration(config.Timeouts.Connect) * time.Second,
-				KeepAlive: 30 * time.Second,
+				KeepAlive: time.Duration(config.Timeouts.KeepAlive) * time.Second,
 			}).DialContext,
 			TLSHandshakeTimeout:   time.Duration(config.Timeouts.TLSHandshake) * time.Second,
 			ResponseHeaderTimeout: time.Duration(config.Timeouts.ResponseHeader) * time.Second,
@@ -236,6 +239,49 @@ func (p *Process) forceState(newState ProcessState) {
 // at any time.
 func (p *Process) start() error {
 
+	// test-only fast path: skip subprocess, health check, and TTL goroutine
+	if p.testHandler != nil {
+		if curState, err := p.swapState(StateStopped, StateStarting); err != nil {
+			if err == ErrExpectedStateMismatch {
+				if curState == StateStarting {
+					p.waitStarting.Wait()
+					curState = p.CurrentState()
+					if curState == StateReady {
+						return nil
+					}
+					return fmt.Errorf("process was already starting but wound up in state %v", curState)
+				}
+				return fmt.Errorf("process was in state %v when start() was called", curState)
+			}
+			return fmt.Errorf("failed to set Process state to starting: current state: %v, error: %v", curState, err)
+		}
+		defer p.waitStarting.Done()
+
+		// Mimic the real stop path: cancelUpstream transitions
+		// StateStopping -> StateStopped and closes cmdWaitChan,
+		// matching what waitForCmd does for real subprocesses.
+		ch := make(chan struct{})
+		p.cmdMutex.Lock()
+		p.cancelUpstream = func() {
+			if curState := p.CurrentState(); curState == StateStopping {
+				if _, err := p.swapState(StateStopping, StateStopped); err != nil {
+					p.forceState(StateStopped)
+				}
+			} else {
+				p.forceState(StateStopped)
+			}
+			close(ch)
+		}
+		p.cmdWaitChan = ch
+		p.cmdMutex.Unlock()
+
+		if curState, err := p.swapState(StateStarting, StateReady); err != nil {
+			return fmt.Errorf("failed to set Process state to ready: current state: %v, error: %v", curState, err)
+		}
+		p.failedStartCount = 0
+		return nil
+	}
+
 	if p.config.Proxy == "" {
 		return fmt.Errorf("can not start(), upstream proxy missing")
 	}
@@ -386,7 +432,10 @@ func (p *Process) start() error {
 
 // Stop will wait for inflight requests to complete before stopping the process.
 func (p *Process) Stop() {
+
+	// guard to prevent multiple goroutines from stopping
 	if !isValidTransition(p.CurrentState(), StateStopping) {
+		p.proxyLogger.Debugf("<%s> Stop() suppressing invalid transition from %s to StateStopping", p.ID, p.CurrentState())
 		return
 	}
 
@@ -399,13 +448,17 @@ func (p *Process) Stop() {
 // StopImmediately will transition the process to the stopping state and stop the process with a SIGTERM.
 // If the process does not stop within the specified timeout, it will be forcefully stopped with a SIGKILL.
 func (p *Process) StopImmediately() {
-	if !isValidTransition(p.CurrentState(), StateStopping) {
+
+	// guard to prevent multiple goroutines from stopping the process
+	enterState := p.CurrentState()
+	if !isValidTransition(enterState, StateStopping) {
+		p.proxyLogger.Debugf("<%s> StopImmediate() suppressing invalid transition from %s to StateStopping", p.ID, p.CurrentState())
 		return
 	}
 
-	p.proxyLogger.Debugf("<%s> Stopping process, current state: %s", p.ID, p.CurrentState())
-	if curState, err := p.swapState(StateReady, StateStopping); err != nil {
-		p.proxyLogger.Infof("<%s> Stop() Ready -> StateStopping err: %v, current state: %v", p.ID, err, curState)
+	p.proxyLogger.Debugf("<%s> Stopping process, enter state: %s", p.ID, enterState)
+	if curState, err := p.swapState(enterState, StateStopping); err != nil {
+		p.proxyLogger.Infof("<%s> Stop() %s -> StateStopping err: %v, current state: %v", p.ID, enterState, err, curState)
 		return
 	}
 
@@ -577,6 +630,11 @@ func (p *Process) ProxyRequest(w http.ResponseWriter, r *http.Request) {
 		if !srw.waitForCompletion(completionTimeout) {
 			p.proxyLogger.Warnf("<%s> status updates goroutine did not complete within %v, proceeding with proxy request", p.ID, completionTimeout)
 		}
+	}
+
+	if p.testHandler != nil {
+		p.testHandler.ServeHTTP(w, r)
+	} else if srw != nil {
 		p.reverseProxy.ServeHTTP(srw, r)
 	} else {
 		p.reverseProxy.ServeHTTP(w, r)

proxy/processgroup.go

@@ -24,6 +24,22 @@ type ProcessGroup struct {
 	// map of current processes
 	processes       map[string]*Process
 	lastUsedProcess string
+
+	// inflight tracks fast-path requests (requests for the already-selected
+	// model in a swap group). Fast-path requests Add(1) while holding pg.Lock
+	// and Done() on completion; a concurrent swap request calls inflight.Wait()
+	// under pg.Lock before stopping the current process. Without this tracking,
+	// a fast-path request that has released pg.Lock but has not yet called
+	// Process.inFlightRequests.Add(1) races with Stop()'s Wait() and can be
+	// killed mid-request.
+	inflight sync.WaitGroup
+
+	// testDelayFastPath is a test-only hook that, when non-nil, is invoked in
+	// the fast path after pg.Lock is released but before the request is
+	// dispatched to Process.ProxyRequest. Tests use it to park a fast-path
+	// request at the exact race window to deterministically reproduce the
+	// fast-path vs swap race.
+	testDelayFastPath func()
 }
 
 func NewProcessGroup(id string, config config.Config, proxyLogger *LogMonitor, upstreamLogger *LogMonitor) *ProcessGroup {
@@ -64,6 +80,13 @@ func (pg *ProcessGroup) ProxyRequest(modelID string, writer http.ResponseWriter,
 		pg.Lock()
 		if pg.lastUsedProcess != modelID {
 
+			// Wait for in-flight fast-path requests to drain before stopping
+			// the previous process. Without this, a fast-path request that has
+			// released pg.Lock but has not yet incremented
+			// Process.inFlightRequests races with Stop() and can be killed
+			// mid-request.
+			pg.inflight.Wait()
+
 			// is there something already running?
 			if pg.lastUsedProcess != "" {
 				pg.processes[pg.lastUsedProcess].Stop()
@@ -78,7 +101,16 @@ func (pg *ProcessGroup) ProxyRequest(modelID string, writer http.ResponseWriter,
 			pg.Unlock()
 			return nil
 		}
+
+		// Fast path: register this request in inflight before releasing
+		// pg.Lock so a concurrent swap will wait for it to complete.
+		pg.inflight.Add(1)
+		defer pg.inflight.Done()
 		pg.Unlock()
+
+		if pg.testDelayFastPath != nil {
+			pg.testDelayFastPath()
+		}
 	}
 
 	pg.processes[modelID].ProxyRequest(writer, request)
@@ -123,6 +155,10 @@ func (pg *ProcessGroup) StopProcesses(strategy StopStrategy) {
 	pg.Lock()
 	defer pg.Unlock()
 
+	if strategy != StopImmediately {
+		pg.inflight.Wait()
+	}
+
 	if len(pg.processes) == 0 {
 		return
 	}

proxy/processgroup_test.go

@@ -4,11 +4,14 @@ import (
 	"bytes"
 	"net/http"
 	"net/http/httptest"
+	"runtime"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/mostlygeek/llama-swap/proxy/config"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 var processGroupTestConfig = config.AddDefaultGroupToConfig(config.Config{
@@ -95,6 +98,229 @@ func TestProcessGroup_ProxyRequestSwapIsTrueParallel(t *testing.T) {
 	wg.Wait()
 }
 
+// TestProcessGroup_ProxyRequestSwapRaceAgainstFastPath verifies that a swap
+// request cannot stop the current process while a fast-path request (for the
+// already-selected model) is in flight. Without ProcessGroup-level inflight
+// tracking, a fast-path request that has released pg.Lock but has not yet
+// incremented Process.inFlightRequests races with Stop()'s Wait() and the
+// process is killed mid-request.
+func TestProcessGroup_ProxyRequestSwapRaceAgainstFastPath(t *testing.T) {
+	cfg := config.AddDefaultGroupToConfig(config.Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]config.ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+		},
+		Groups: map[string]config.GroupConfig{
+			"G1": {
+				Swap:    true,
+				Members: []string{"model1", "model2"},
+			},
+		},
+	})
+
+	pg := NewProcessGroup("G1", cfg, testLogger, testLogger)
+	defer pg.StopProcesses(StopImmediately)
+
+	// Bypass real subprocesses so the test is fast and deterministic.
+	pg.processes["model1"].testHandler = newTestHandler("model1")
+	pg.processes["model2"].testHandler = newTestHandler("model2")
+
+	// Prime: run a request through model1 via the swap path so that
+	// lastUsedProcess == "model1" and subsequent model1 requests take the
+	// fast path.
+	primeReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
+	primeW := httptest.NewRecorder()
+	require.NoError(t, pg.ProxyRequest("model1", primeW, primeReq))
+	require.Equal(t, http.StatusOK, primeW.Code)
+	require.Equal(t, StateReady, pg.processes["model1"].CurrentState())
+	require.Equal(t, StateStopped, pg.processes["model2"].CurrentState())
+
+	// Fast-path hook: signal arrival at the race window, then wait for
+	// release. This parks R2 deterministically at the point where pg.Lock
+	// has been released but Process.inFlightRequests has not yet been
+	// incremented — the exact window the race exploits.
+	r2Reached := make(chan struct{})
+	r2Release := make(chan struct{})
+	pg.testDelayFastPath = func() {
+		close(r2Reached)
+		<-r2Release
+	}
+
+	// R2: fast-path request for model1. Will pause at the test hook.
+	r2Done := make(chan struct{})
+	w2 := httptest.NewRecorder()
+	go func() {
+		defer close(r2Done)
+		req := httptest.NewRequest("POST", "/v1/chat/completions", nil)
+		assert.NoError(t, pg.ProxyRequest("model1", w2, req))
+	}()
+
+	// Deterministically wait for R2 to reach the race window.
+	<-r2Reached
+
+	// R3: swap request for model2. Must wait for R2 to finish before touching
+	// model1, otherwise model1 gets killed mid-request.
+	r3Done := make(chan struct{})
+	w3 := httptest.NewRecorder()
+	go func() {
+		defer close(r3Done)
+		req := httptest.NewRequest("POST", "/v1/chat/completions", nil)
+		assert.NoError(t, pg.ProxyRequest("model2", w3, req))
+	}()
+
+	// Spin until R3 has acquired pg.Lock and entered the swap critical
+	// section. In the fixed code, R3 then blocks on pg.inflight.Wait() while
+	// still holding the lock, so TryLock keeps failing.
+	for pg.TryLock() {
+		pg.Unlock()
+		runtime.Gosched()
+	}
+
+	// Bounded poll: give R3 a chance to demonstrate the bug by mutating
+	// state. In the fixed code, R3 is blocked on pg.inflight.Wait() and
+	// nothing changes, so we wait the full window. In the buggy code, R3
+	// will Stop() model1 and start serving via model2 within microseconds —
+	// we exit early once the mutation is observable.
+	deadline := time.Now().Add(100 * time.Millisecond)
+	for time.Now().Before(deadline) {
+		if pg.processes["model1"].CurrentState() != StateReady ||
+			pg.processes["model2"].CurrentState() != StateStopped {
+			break
+		}
+		done := false
+		select {
+		case <-r3Done:
+			done = true
+		default:
+		}
+		if done {
+			break
+		}
+		runtime.Gosched()
+	}
+
+	// Invariant: R3 must be blocked while R2 is still in flight.
+	select {
+	case <-r3Done:
+		t.Fatal("swap completed while fast-path request was still in flight — race not prevented")
+	default:
+	}
+	assert.Equal(t, StateReady, pg.processes["model1"].CurrentState(),
+		"model1 must stay Ready while a fast-path request is in flight")
+	assert.Equal(t, StateStopped, pg.processes["model2"].CurrentState(),
+		"model2 must not be started until R2 finishes and model1 is swapped out")
+
+	// Release R2 and let both requests finish.
+	close(r2Release)
+	<-r2Done
+	<-r3Done
+
+	assert.Equal(t, http.StatusOK, w2.Code)
+	assert.Contains(t, w2.Body.String(), "model1")
+	assert.Equal(t, http.StatusOK, w3.Code)
+	assert.Contains(t, w3.Body.String(), "model2")
+}
+
+// TestProcessGroup_StopProcessesWaitsForInflight verifies that StopProcesses
+// (called externally, e.g. from ProxyManager.swapProcessGroup) cannot stop a
+// process while a fast-path ProxyRequest is in the [pg.Unlock,
+// Process.inFlightRequests.Add(1)] window. Without pg.inflight.Wait() in
+// StopProcesses, the external caller bypasses the inflight guard and kills the
+// process mid-request.
+func TestProcessGroup_StopProcessesWaitsForInflight(t *testing.T) {
+	cfg := config.AddDefaultGroupToConfig(config.Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]config.ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+		},
+		Groups: map[string]config.GroupConfig{
+			"G1": {
+				Swap:    true,
+				Members: []string{"model1", "model2"},
+			},
+		},
+	})
+
+	pg := NewProcessGroup("G1", cfg, testLogger, testLogger)
+	defer pg.StopProcesses(StopImmediately)
+
+	pg.processes["model1"].testHandler = newTestHandler("model1")
+	pg.processes["model2"].testHandler = newTestHandler("model2")
+
+	// Prime: model1 is active so subsequent model1 requests take the fast path.
+	primeReq := httptest.NewRequest("POST", "/v1/chat/completions", nil)
+	primeW := httptest.NewRecorder()
+	require.NoError(t, pg.ProxyRequest("model1", primeW, primeReq))
+	require.Equal(t, http.StatusOK, primeW.Code)
+	require.Equal(t, StateReady, pg.processes["model1"].CurrentState())
+
+	// Park a fast-path request at the race window.
+	r2Reached := make(chan struct{})
+	r2Release := make(chan struct{})
+	pg.testDelayFastPath = func() {
+		close(r2Reached)
+		<-r2Release
+	}
+
+	r2Done := make(chan struct{})
+	w2 := httptest.NewRecorder()
+	go func() {
+		defer close(r2Done)
+		req := httptest.NewRequest("POST", "/v1/chat/completions", nil)
+		assert.NoError(t, pg.ProxyRequest("model1", w2, req))
+	}()
+
+	<-r2Reached
+
+	// Simulate an external caller (e.g. ProxyManager.swapProcessGroup) stopping
+	// the group while a fast-path request is in flight.
+	r3Done := make(chan struct{})
+	go func() {
+		defer close(r3Done)
+		pg.StopProcesses(StopWaitForInflightRequest)
+	}()
+
+	// Spin until StopProcesses has acquired pg.Lock.
+	for pg.TryLock() {
+		pg.Unlock()
+		runtime.Gosched()
+	}
+
+	// Bounded poll: in the fixed code StopProcesses blocks on pg.inflight.Wait()
+	// and model1 stays Ready. In the buggy code it proceeds immediately and
+	// kills model1.
+	deadline := time.Now().Add(100 * time.Millisecond)
+	for time.Now().Before(deadline) {
+		if pg.processes["model1"].CurrentState() != StateReady {
+			break
+		}
+		select {
+		case <-r3Done:
+			goto done
+		default:
+		}
+		runtime.Gosched()
+	}
+done:
+
+	select {
+	case <-r3Done:
+		t.Fatal("StopProcesses completed while a fast-path request was still in flight — race not prevented")
+	default:
+	}
+	assert.Equal(t, StateReady, pg.processes["model1"].CurrentState(),
+		"model1 must stay Ready while a fast-path request is in flight")
+
+	close(r2Release)
+	<-r2Done
+	<-r3Done
+
+	assert.Equal(t, http.StatusOK, w2.Code)
+	assert.Contains(t, w2.Body.String(), "model1")
+}
+
 func TestProcessGroup_ProxyRequestSwapIsFalse(t *testing.T) {
 	pg := NewProcessGroup("G2", processGroupTestConfig, testLogger, testLogger)
 	defer pg.StopProcesses(StopWaitForInflightRequest)

proxy/proxymanager.go

@@ -77,6 +77,9 @@ type ProxyManager struct {
 
 	processGroups map[string]*ProcessGroup
 
+	// matrix-based swap (mutually exclusive with processGroups)
+	matrix *Matrix
+
 	inFlightCounter *InflightCounter
 
 	// shutdown signaling
@@ -203,10 +206,14 @@ func New(proxyConfig config.Config) *ProxyManager {
 		peerProxy: peerProxy,
 	}
 
-	// create the process groups
-	for groupID := range proxyConfig.Groups {
-		processGroup := NewProcessGroup(groupID, proxyConfig, proxyLogger, upstreamLogger)
-		pm.processGroups[groupID] = processGroup
+	// create either matrix or process groups (mutually exclusive)
+	if proxyConfig.Matrix != nil {
+		pm.matrix = NewMatrix(proxyConfig, proxyLogger, upstreamLogger)
+	} else {
+		for groupID := range proxyConfig.Groups {
+			processGroup := NewProcessGroup(groupID, proxyConfig, proxyLogger, upstreamLogger)
+			pm.processGroups[groupID] = processGroup
+		}
 	}
 
 	pm.setupGinEngine()
@@ -225,18 +232,29 @@ func New(proxyConfig config.Config) *ProxyManager {
 				}
 
 				proxyLogger.Infof("Preloading model: %s", modelID)
-				processGroup, err := pm.swapProcessGroup(modelID)
 
-				if err != nil {
+				var preloadErr error
+				req, _ := http.NewRequest("GET", "/", nil)
+
+				if pm.matrix != nil {
+					preloadErr = pm.matrix.ProxyRequest(modelID, discardWriter, req)
+				} else {
+					processGroup, err := pm.swapProcessGroup(modelID)
+					if err != nil {
+						preloadErr = err
+					} else {
+						preloadErr = processGroup.ProxyRequest(modelID, discardWriter, req)
+					}
+				}
+
+				if preloadErr != nil {
 					event.Emit(ModelPreloadedEvent{
 						ModelName: modelID,
 						Success:   false,
 					})
-					proxyLogger.Errorf("Failed to preload model %s: %v", modelID, err)
+					proxyLogger.Errorf("Failed to preload model %s: %v", modelID, preloadErr)
 					continue
 				} else {
-					req, _ := http.NewRequest("GET", "/", nil)
-					processGroup.ProxyRequest(modelID, discardWriter, req)
 					event.Emit(ModelPreloadedEvent{
 						ModelName: modelID,
 						Success:   true,
@@ -453,6 +471,11 @@ func (pm *ProxyManager) StopProcesses(strategy StopStrategy) {
 	pm.Lock()
 	defer pm.Unlock()
 
+	if pm.matrix != nil {
+		pm.matrix.StopProcesses(strategy)
+		return
+	}
+
 	// stop Processes in parallel
 	var wg sync.WaitGroup
 	for _, processGroup := range pm.processGroups {
@@ -473,6 +496,12 @@ func (pm *ProxyManager) Shutdown() {
 
 	pm.proxyLogger.Debug("Shutdown() called in proxy manager")
 
+	if pm.matrix != nil {
+		pm.matrix.Shutdown()
+		pm.shutdownCancel()
+		return
+	}
+
 	var wg sync.WaitGroup
 	// Send shutdown signal to all process in groups
 	for _, processGroup := range pm.processGroups {
@@ -639,10 +668,16 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
 		return
 	}
 
-	processGroup, err := pm.swapProcessGroup(modelID)
-	if err != nil {
-		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
-		return
+	var handler func(string, http.ResponseWriter, *http.Request) error
+	if pm.matrix != nil {
+		handler = pm.matrix.ProxyRequest
+	} else {
+		processGroup, err := pm.swapProcessGroup(modelID)
+		if err != nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+			return
+		}
+		handler = processGroup.ProxyRequest
 	}
 
 	// rewrite the path
@@ -651,13 +686,13 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
 
 	// attempt to record metrics if it is a POST request
 	if pm.metricsMonitor != nil && c.Request.Method == "POST" {
-		if err := pm.metricsMonitor.wrapHandler(modelID, c.Writer, c.Request, processGroup.ProxyRequest); err != nil {
+		if err := pm.metricsMonitor.wrapHandler(modelID, c.Writer, c.Request, handler); err != nil {
 			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying metrics wrapped request: %s", err.Error()))
 			pm.proxyLogger.Errorf("Error proxying wrapped upstream request for model %s, path=%s", modelID, originalPath)
 			return
 		}
 	} else {
-		if err := processGroup.ProxyRequest(modelID, c.Writer, c.Request); err != nil {
+		if err := handler(modelID, c.Writer, c.Request); err != nil {
 			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error()))
 			pm.proxyLogger.Errorf("Error proxying upstream request for model %s, path=%s", modelID, originalPath)
 			return
@@ -683,10 +718,16 @@ func (pm *ProxyManager) proxyInferenceHandler(c *gin.Context) {
 
 	modelID, found := pm.config.RealModelName(requestedModel)
 	if found {
-		processGroup, err := pm.swapProcessGroup(modelID)
-		if err != nil {
-			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
-			return
+		var localHandler func(string, http.ResponseWriter, *http.Request) error
+		if pm.matrix != nil {
+			localHandler = pm.matrix.ProxyRequest
+		} else {
+			processGroup, err := pm.swapProcessGroup(modelID)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+				return
+			}
+			localHandler = processGroup.ProxyRequest
 		}
 
 		// issue #69 allow custom model names to be sent to upstream
@@ -737,7 +778,7 @@ func (pm *ProxyManager) proxyInferenceHandler(c *gin.Context) {
 		}
 
 		pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
-		nextHandler = processGroup.ProxyRequest
+		nextHandler = localHandler
 	} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
 		pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
 		modelID = requestedModel
@@ -823,15 +864,19 @@ func (pm *ProxyManager) proxyOAIPostFormHandler(c *gin.Context) {
 
 	modelID, found := pm.config.RealModelName(requestedModel)
 	if found {
-		processGroup, err := pm.swapProcessGroup(modelID)
-		if err != nil {
-			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
-			return
+		if pm.matrix != nil {
+			nextHandler = pm.matrix.ProxyRequest
+		} else {
+			processGroup, err := pm.swapProcessGroup(modelID)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+				return
+			}
+			nextHandler = processGroup.ProxyRequest
 		}
 
 		useModelName = pm.config.Models[modelID].UseModelName
 		pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
-		nextHandler = processGroup.ProxyRequest
 	} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
 		pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
 		modelID = requestedModel
@@ -942,14 +987,18 @@ func (pm *ProxyManager) proxyGETModelHandler(c *gin.Context) {
 	var modelID string
 
 	if realModelID, found := pm.config.RealModelName(requestedModel); found {
-		processGroup, err := pm.swapProcessGroup(realModelID)
-		if err != nil {
-			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
-			return
-		}
 		modelID = realModelID
+		if pm.matrix != nil {
+			nextHandler = pm.matrix.ProxyRequest
+		} else {
+			processGroup, err := pm.swapProcessGroup(realModelID)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+				return
+			}
+			nextHandler = processGroup.ProxyRequest
+		}
 		pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
-		nextHandler = processGroup.ProxyRequest
 	} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
 		modelID = requestedModel
 		pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
@@ -1048,9 +1097,9 @@ func (pm *ProxyManager) listRunningProcessesHandler(context *gin.Context) {
 	context.Header("Content-Type", "application/json")
 	runningProcesses := make([]gin.H, 0) // Default to an empty response.
 
-	for _, processGroup := range pm.processGroups {
-		for _, process := range processGroup.processes {
-			if process.CurrentState() == StateReady {
+	if pm.matrix != nil {
+		for _, modelID := range pm.matrix.RunningModels() {
+			if process, ok := pm.matrix.GetProcess(modelID); ok {
 				runningProcesses = append(runningProcesses, gin.H{
 					"model":       process.ID,
 					"state":       process.state,
@@ -1062,6 +1111,22 @@ func (pm *ProxyManager) listRunningProcessesHandler(context *gin.Context) {
 				})
 			}
 		}
+	} else {
+		for _, processGroup := range pm.processGroups {
+			for _, process := range processGroup.processes {
+				if process.CurrentState() == StateReady {
+					runningProcesses = append(runningProcesses, gin.H{
+						"model":       process.ID,
+						"state":       process.state,
+						"cmd":         process.config.Cmd,
+						"proxy":       process.config.Proxy,
+						"ttl":         process.config.UnloadAfter,
+						"name":        process.config.Name,
+						"description": process.config.Description,
+					})
+				}
+			}
+		}
 	}
 
 	// Put the results under the `running` key.

proxy/proxymanager_api.go

@@ -55,27 +55,28 @@ func (pm *ProxyManager) getModelStatus() []Model {
 	// Iterate over sorted keys
 	for _, modelID := range modelIDs {
 		// Get process state
-		processGroup := pm.findGroupByModelName(modelID)
 		state := "unknown"
-		if processGroup != nil {
-			process := processGroup.processes[modelID]
-			if process != nil {
-				var stateStr string
-				switch process.CurrentState() {
-				case StateReady:
-					stateStr = "ready"
-				case StateStarting:
-					stateStr = "starting"
-				case StateStopping:
-					stateStr = "stopping"
-				case StateShutdown:
-					stateStr = "shutdown"
-				case StateStopped:
-					stateStr = "stopped"
-				default:
-					stateStr = "unknown"
-				}
-				state = stateStr
+		var process *Process
+		if pm.matrix != nil {
+			process, _ = pm.matrix.GetProcess(modelID)
+		} else {
+			processGroup := pm.findGroupByModelName(modelID)
+			if processGroup != nil {
+				process = processGroup.processes[modelID]
+			}
+		}
+		if process != nil {
+			switch process.CurrentState() {
+			case StateReady:
+				state = "ready"
+			case StateStarting:
+				state = "starting"
+			case StateStopping:
+				state = "stopping"
+			case StateShutdown:
+				state = "shutdown"
+			case StateStopped:
+				state = "stopped"
 			}
 		}
 		models = append(models, Model{
@@ -254,18 +255,23 @@ func (pm *ProxyManager) apiUnloadSingleModelHandler(c *gin.Context) {
 		return
 	}
 
-	processGroup := pm.findGroupByModelName(realModelName)
-	if processGroup == nil {
-		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("process group not found for model %s", requestedModel))
-		return
+	var stopErr error
+	if pm.matrix != nil {
+		stopErr = pm.matrix.StopProcess(realModelName, StopImmediately)
+	} else {
+		processGroup := pm.findGroupByModelName(realModelName)
+		if processGroup == nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("process group not found for model %s", requestedModel))
+			return
+		}
+		stopErr = processGroup.StopProcess(realModelName, StopImmediately)
 	}
 
-	if err := processGroup.StopProcess(realModelName, StopImmediately); err != nil {
-		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error stopping process: %s", err.Error()))
+	if stopErr != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error stopping process: %s", stopErr.Error()))
 		return
-	} else {
-		c.String(http.StatusOK, "OK")
 	}
+	c.String(http.StatusOK, "OK")
 }
 
 func (pm *ProxyManager) apiGetVersion(c *gin.Context) {
@@ -284,11 +290,26 @@ func (pm *ProxyManager) apiGetCapture(c *gin.Context) {
 		return
 	}
 
-	capture := pm.metricsMonitor.getCaptureByID(id)
-	if capture == nil {
+	data, exists := pm.metricsMonitor.getCompressedBytes(id)
+	if !exists {
 		c.JSON(http.StatusNotFound, gin.H{"error": "capture not found"})
 		return
 	}
 
-	c.JSON(http.StatusOK, capture)
+	c.Header("Vary", "Accept-Encoding")
+
+	// ¯\_(ツ)_/¯ quality weights are too fancy for us anyway
+	hasZstd := strings.Contains(c.GetHeader("Accept-Encoding"), "zstd")
+
+	if hasZstd {
+		c.Header("Content-Encoding", "zstd")
+		c.Data(http.StatusOK, "application/json", data)
+	} else {
+		decompressed, err := decompressCapture(data)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to decompress capture"})
+			return
+		}
+		c.Data(http.StatusOK, "application/json", decompressed)
+	}
 }

ui-svelte/package-lock.json

@@ -37,21 +37,21 @@
       }
     },
     "node_modules/@emnapi/core": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.0.tgz",
-      "integrity": "sha512-0DQ98G9ZQZOxfUcQn1waV2yS8aWdZ6kJMbYCJB3oUBecjWYO1fqJ+a1DRfPF3O5JEkwqwP1A9QEN/9mYm2Yd0w==",
+      "version": "1.9.2",
+      "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz",
+      "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==",
       "dev": true,
       "license": "MIT",
       "optional": true,
       "dependencies": {
-        "@emnapi/wasi-threads": "1.2.0",
+        "@emnapi/wasi-threads": "1.2.1",
         "tslib": "^2.4.0"
       }
     },
     "node_modules/@emnapi/runtime": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.0.tgz",
-      "integrity": "sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==",
+      "version": "1.9.2",
+      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz",
+      "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==",
       "dev": true,
       "license": "MIT",
       "optional": true,
@@ -60,9 +60,9 @@
       }
     },
     "node_modules/@emnapi/wasi-threads": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.0.tgz",
-      "integrity": "sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz",
+      "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==",
       "dev": true,
       "license": "MIT",
       "optional": true,
@@ -121,36 +121,28 @@
       }
     },
     "node_modules/@napi-rs/wasm-runtime": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz",
-      "integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==",
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.3.tgz",
+      "integrity": "sha512-xK9sGVbJWYb08+mTJt3/YV24WxvxpXcXtP6B172paPZ+Ts69Re9dAr7lKwJoeIx8OoeuimEiRZ7umkiUVClmmQ==",
       "dev": true,
       "license": "MIT",
       "optional": true,
       "dependencies": {
-        "@emnapi/core": "^1.7.1",
-        "@emnapi/runtime": "^1.7.1",
         "@tybys/wasm-util": "^0.10.1"
       },
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/Brooooooklyn"
-      }
-    },
-    "node_modules/@oxc-project/runtime": {
-      "version": "0.115.0",
-      "resolved": "https://registry.npmjs.org/@oxc-project/runtime/-/runtime-0.115.0.tgz",
-      "integrity": "sha512-Rg8Wlt5dCbXhQnsXPrkOjL1DTSvXLgb2R/KYfnf1/K+R0k6UMLEmbQXPM+kwrWqSmWA2t0B1EtHy2/3zikQpvQ==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": "^20.19.0 || >=22.12.0"
+      },
+      "peerDependencies": {
+        "@emnapi/core": "^1.7.1",
+        "@emnapi/runtime": "^1.7.1"
       }
     },
     "node_modules/@oxc-project/types": {
-      "version": "0.115.0",
-      "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.115.0.tgz",
-      "integrity": "sha512-4n91DKnebUS4yjUHl2g3/b2T+IUdCfmoZGhmwsovZCDaJSs+QkVAM+0AqqTxHSsHfeiMuueT75cZaZcT/m0pSw==",
+      "version": "0.124.0",
+      "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.124.0.tgz",
+      "integrity": "sha512-VBFWMTBvHxS11Z5Lvlr3IWgrwhMTXV+Md+EQF0Xf60+wAdsGFTBx7X7K/hP4pi8N7dcm1RvcHwDxZ16Qx8keUg==",
       "dev": true,
       "license": "MIT",
       "funding": {
@@ -158,9 +150,9 @@
       }
     },
     "node_modules/@rolldown/binding-android-arm64": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.9.tgz",
-      "integrity": "sha512-lcJL0bN5hpgJfSIz/8PIf02irmyL43P+j1pTCfbD1DbLkmGRuFIA4DD3B3ZOvGqG0XiVvRznbKtN0COQVaKUTg==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.15.tgz",
+      "integrity": "sha512-YYe6aWruPZDtHNpwu7+qAHEMbQ/yRl6atqb/AhznLTnD3UY99Q1jE7ihLSahNWkF4EqRPVC4SiR4O0UkLK02tA==",
       "cpu": [
         "arm64"
       ],
@@ -175,9 +167,9 @@
       }
     },
     "node_modules/@rolldown/binding-darwin-arm64": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.9.tgz",
-      "integrity": "sha512-J7Zk3kLYFsLtuH6U+F4pS2sYVzac0qkjcO5QxHS7OS7yZu2LRs+IXo+uvJ/mvpyUljDJ3LROZPoQfgBIpCMhdQ==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.15.tgz",
+      "integrity": "sha512-oArR/ig8wNTPYsXL+Mzhs0oxhxfuHRfG7Ikw7jXsw8mYOtk71W0OkF2VEVh699pdmzjPQsTjlD1JIOoHkLP1Fg==",
       "cpu": [
         "arm64"
       ],
@@ -192,9 +184,9 @@
       }
     },
     "node_modules/@rolldown/binding-darwin-x64": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.9.tgz",
-      "integrity": "sha512-iwtmmghy8nhfRGeNAIltcNXzD0QMNaaA5U/NyZc1Ia4bxrzFByNMDoppoC+hl7cDiUq5/1CnFthpT9n+UtfFyg==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.15.tgz",
+      "integrity": "sha512-YzeVqOqjPYvUbJSWJ4EDL8ahbmsIXQpgL3JVipmN+MX0XnXMeWomLN3Fb+nwCmP/jfyqte5I3XRSm7OfQrbyxw==",
       "cpu": [
         "x64"
       ],
@@ -209,9 +201,9 @@
       }
     },
     "node_modules/@rolldown/binding-freebsd-x64": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.9.tgz",
-      "integrity": "sha512-DLFYI78SCiZr5VvdEplsVC2Vx53lnA4/Ga5C65iyldMVaErr86aiqCoNBLl92PXPfDtUYjUh+xFFor40ueNs4Q==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.15.tgz",
+      "integrity": "sha512-9Erhx956jeQ0nNTyif1+QWAXDRD38ZNjr//bSHrt6wDwB+QkAfl2q6Mn1k6OBPerznjRmbM10lgRb1Pli4xZPw==",
       "cpu": [
         "x64"
       ],
@@ -226,9 +218,9 @@
       }
     },
     "node_modules/@rolldown/binding-linux-arm-gnueabihf": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.9.tgz",
-      "integrity": "sha512-CsjTmTwd0Hri6iTw/DRMK7kOZ7FwAkrO4h8YWKoX/kcj833e4coqo2wzIFywtch/8Eb5enQ/lwLM7w6JX1W5RQ==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.15.tgz",
+      "integrity": "sha512-cVwk0w8QbZJGTnP/AHQBs5yNwmpgGYStL88t4UIaqcvYJWBfS0s3oqVLZPwsPU6M0zlW4GqjP0Zq5MnAGwFeGA==",
       "cpu": [
         "arm"
       ],
@@ -243,9 +235,9 @@
       }
     },
     "node_modules/@rolldown/binding-linux-arm64-gnu": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.9.tgz",
-      "integrity": "sha512-2x9O2JbSPxpxMDhP9Z74mahAStibTlrBMW0520+epJH5sac7/LwZW5Bmg/E6CXuEF53JJFW509uP+lSedaUNxg==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.15.tgz",
+      "integrity": "sha512-eBZ/u8iAK9SoHGanqe/jrPnY0JvBN6iXbVOsbO38mbz+ZJsaobExAm1Iu+rxa4S1l2FjG0qEZn4Rc6X8n+9M+w==",
       "cpu": [
         "arm64"
       ],
@@ -260,9 +252,9 @@
       }
     },
     "node_modules/@rolldown/binding-linux-arm64-musl": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.9.tgz",
-      "integrity": "sha512-JA1QRW31ogheAIRhIg9tjMfsYbglXXYGNPLdPEYrwFxdbkQCAzvpSCSHCDWNl4hTtrol8WeboCSEpjdZK8qrCg==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.15.tgz",
+      "integrity": "sha512-ZvRYMGrAklV9PEkgt4LQM6MjQX2P58HPAuecwYObY2DhS2t35R0I810bKi0wmaYORt6m/2Sm+Z+nFgb0WhXNcQ==",
       "cpu": [
         "arm64"
       ],
@@ -277,9 +269,9 @@
       }
     },
     "node_modules/@rolldown/binding-linux-ppc64-gnu": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.9.tgz",
-      "integrity": "sha512-aOKU9dJheda8Kj8Y3w9gnt9QFOO+qKPAl8SWd7JPHP+Cu0EuDAE5wokQubLzIDQWg2myXq2XhTpOVS07qqvT+w==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.15.tgz",
+      "integrity": "sha512-VDpgGBzgfg5hLg+uBpCLoFG5kVvEyafmfxGUV0UHLcL5irxAK7PKNeC2MwClgk6ZAiNhmo9FLhRYgvMmedLtnQ==",
       "cpu": [
         "ppc64"
       ],
@@ -294,9 +286,9 @@
       }
     },
     "node_modules/@rolldown/binding-linux-s390x-gnu": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.9.tgz",
-      "integrity": "sha512-OalO94fqj7IWRn3VdXWty75jC5dk4C197AWEuMhIpvVv2lw9fiPhud0+bW2ctCxb3YoBZor71QHbY+9/WToadA==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.15.tgz",
+      "integrity": "sha512-y1uXY3qQWCzcPgRJATPSOUP4tCemh4uBdY7e3EZbVwCJTY3gLJWnQABgeUetvED+bt1FQ01OeZwvhLS2bpNrAQ==",
       "cpu": [
         "s390x"
       ],
@@ -311,9 +303,9 @@
       }
     },
     "node_modules/@rolldown/binding-linux-x64-gnu": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.9.tgz",
-      "integrity": "sha512-cVEl1vZtBsBZna3YMjGXNvnYYrOJ7RzuWvZU0ffvJUexWkukMaDuGhUXn0rjnV0ptzGVkvc+vW9Yqy6h8YX4pg==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.15.tgz",
+      "integrity": "sha512-023bTPBod7J3Y/4fzAN6QtpkSABR0rigtrwaP+qSEabUh5zf6ELr9Nc7GujaROuPY3uwdSIXWrvhn1KxOvurWA==",
       "cpu": [
         "x64"
       ],
@@ -328,9 +320,9 @@
       }
     },
     "node_modules/@rolldown/binding-linux-x64-musl": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.9.tgz",
-      "integrity": "sha512-UzYnKCIIc4heAKgI4PZ3dfBGUZefGCJ1TPDuLHoCzgrMYPb5Rv6TLFuYtyM4rWyHM7hymNdsg5ik2C+UD9VDbA==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.15.tgz",
+      "integrity": "sha512-witB2O0/hU4CgfOOKUoeFgQ4GktPi1eEbAhaLAIpgD6+ZnhcPkUtPsoKKHRzmOoWPZue46IThdSgdo4XneOLYw==",
       "cpu": [
         "x64"
       ],
@@ -345,9 +337,9 @@
       }
     },
     "node_modules/@rolldown/binding-openharmony-arm64": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.9.tgz",
-      "integrity": "sha512-+6zoiF+RRyf5cdlFQP7nm58mq7+/2PFaY2DNQeD4B87N36JzfF/l9mdBkkmTvSYcYPE8tMh/o3cRlsx1ldLfog==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.15.tgz",
+      "integrity": "sha512-UCL68NJ0Ud5zRipXZE9dF5PmirzJE4E4BCIOOssEnM7wLDsxjc6Qb0sGDxTNRTP53I6MZpygyCpY8Aa8sPfKPg==",
       "cpu": [
         "arm64"
       ],
@@ -362,9 +354,9 @@
       }
     },
     "node_modules/@rolldown/binding-wasm32-wasi": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.9.tgz",
-      "integrity": "sha512-rgFN6sA/dyebil3YTlL2evvi/M+ivhfnyxec7AccTpRPccno/rPoNlqybEZQBkcbZu8Hy+eqNJCqfBR8P7Pg8g==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.15.tgz",
+      "integrity": "sha512-ApLruZq/ig+nhaE7OJm4lDjayUnOHVUa77zGeqnqZ9pn0ovdVbbNPerVibLXDmWeUZXjIYIT8V3xkT58Rm9u5Q==",
       "cpu": [
         "wasm32"
       ],
@@ -372,16 +364,18 @@
       "license": "MIT",
       "optional": true,
       "dependencies": {
-        "@napi-rs/wasm-runtime": "^1.1.1"
+        "@emnapi/core": "1.9.2",
+        "@emnapi/runtime": "1.9.2",
+        "@napi-rs/wasm-runtime": "^1.1.3"
       },
       "engines": {
         "node": ">=14.0.0"
       }
     },
     "node_modules/@rolldown/binding-win32-arm64-msvc": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.9.tgz",
-      "integrity": "sha512-lHVNUG/8nlF1IQk1C0Ci574qKYyty2goMiPlRqkC5R+3LkXDkL5Dhx8ytbxq35m+pkHVIvIxviD+TWLdfeuadA==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.15.tgz",
+      "integrity": "sha512-KmoUoU7HnN+Si5YWJigfTws1jz1bKBYDQKdbLspz0UaqjjFkddHsqorgiW1mxcAj88lYUE6NC/zJNwT+SloqtA==",
       "cpu": [
         "arm64"
       ],
@@ -396,9 +390,9 @@
       }
     },
     "node_modules/@rolldown/binding-win32-x64-msvc": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.9.tgz",
-      "integrity": "sha512-G0oA4+w1iY5AGi5HcDTxWsoxF509hrFIPB2rduV5aDqS9FtDg1CAfa7V34qImbjfhIcA8C+RekocJZA96EarwQ==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.15.tgz",
+      "integrity": "sha512-3P2A8L+x75qavWLe/Dll3EYBJLQmtkJN8rfh+U/eR3MqMgL/h98PhYI+JFfXuDPgPeCB7iZAKiqii5vqOvnA0g==",
       "cpu": [
         "x64"
       ],
@@ -413,9 +407,9 @@
       }
     },
     "node_modules/@rolldown/pluginutils": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.9.tgz",
-      "integrity": "sha512-w6oiRWgEBl04QkFZgmW+jnU1EC9b57Oihi2ot3HNWIQRqgHp5PnYDia5iZ5FF7rpa4EQdiqMDXjlqKGXBhsoXw==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.15.tgz",
+      "integrity": "sha512-UromN0peaE53IaBRe9W7CjrZgXl90fqGpK+mIZbA3qSTeYqg3pqpROBdIPvOG3F5ereDHNwoHBI2e50n1BDr1g==",
       "dev": true,
       "license": "MIT"
     },
@@ -2972,14 +2966,14 @@
       }
     },
     "node_modules/rolldown": {
-      "version": "1.0.0-rc.9",
-      "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.9.tgz",
-      "integrity": "sha512-9EbgWge7ZH+yqb4d2EnELAntgPTWbfL8ajiTW+SyhJEC4qhBbkCKbqFV4Ge4zmu5ziQuVbWxb/XwLZ+RIO7E8Q==",
+      "version": "1.0.0-rc.15",
+      "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.15.tgz",
+      "integrity": "sha512-Ff31guA5zT6WjnGp0SXw76X6hzGRk/OQq2hE+1lcDe+lJdHSgnSX6nK3erbONHyCbpSj9a9E+uX/OvytZoWp2g==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@oxc-project/types": "=0.115.0",
-        "@rolldown/pluginutils": "1.0.0-rc.9"
+        "@oxc-project/types": "=0.124.0",
+        "@rolldown/pluginutils": "1.0.0-rc.15"
       },
       "bin": {
         "rolldown": "bin/cli.mjs"
@@ -2988,21 +2982,21 @@
         "node": "^20.19.0 || >=22.12.0"
       },
       "optionalDependencies": {
-        "@rolldown/binding-android-arm64": "1.0.0-rc.9",
-        "@rolldown/binding-darwin-arm64": "1.0.0-rc.9",
-        "@rolldown/binding-darwin-x64": "1.0.0-rc.9",
-        "@rolldown/binding-freebsd-x64": "1.0.0-rc.9",
-        "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.9",
-        "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.9",
-        "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.9",
-        "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.9",
-        "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.9",
-        "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.9",
-        "@rolldown/binding-linux-x64-musl": "1.0.0-rc.9",
-        "@rolldown/binding-openharmony-arm64": "1.0.0-rc.9",
-        "@rolldown/binding-wasm32-wasi": "1.0.0-rc.9",
-        "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.9",
-        "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.9"
+        "@rolldown/binding-android-arm64": "1.0.0-rc.15",
+        "@rolldown/binding-darwin-arm64": "1.0.0-rc.15",
+        "@rolldown/binding-darwin-x64": "1.0.0-rc.15",
+        "@rolldown/binding-freebsd-x64": "1.0.0-rc.15",
+        "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.15",
+        "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.15",
+        "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.15",
+        "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.15",
+        "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.15",
+        "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.15",
+        "@rolldown/binding-linux-x64-musl": "1.0.0-rc.15",
+        "@rolldown/binding-openharmony-arm64": "1.0.0-rc.15",
+        "@rolldown/binding-wasm32-wasi": "1.0.0-rc.15",
+        "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.15",
+        "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.15"
       }
     },
     "node_modules/sade": {
@@ -3416,17 +3410,16 @@
       }
     },
     "node_modules/vite": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.0.tgz",
-      "integrity": "sha512-fPGaRNj9Zytaf8LEiBhY7Z6ijnFKdzU/+mL8EFBaKr7Vw1/FWcTBAMW0wLPJAGMPX38ZPVCVgLceWiEqeoqL2Q==",
+      "version": "8.0.8",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.8.tgz",
+      "integrity": "sha512-dbU7/iLVa8KZALJyLOBOQ88nOXtNG8vxKuOT4I2mD+Ya70KPceF4IAmDsmU0h1Qsn5bPrvsY9HJstCRh3hG6Uw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@oxc-project/runtime": "0.115.0",
         "lightningcss": "^1.32.0",
-        "picomatch": "^4.0.3",
+        "picomatch": "^4.0.4",
         "postcss": "^8.5.8",
-        "rolldown": "1.0.0-rc.9",
+        "rolldown": "1.0.0-rc.15",
         "tinyglobby": "^0.2.15"
       },
       "bin": {
@@ -3443,8 +3436,8 @@
       },
       "peerDependencies": {
         "@types/node": "^20.19.0 || >=22.12.0",
-        "@vitejs/devtools": "^0.0.0-alpha.31",
-        "esbuild": "^0.27.0",
+        "@vitejs/devtools": "^0.1.0",
+        "esbuild": "^0.27.0 || ^0.28.0",
         "jiti": ">=1.21.0",
         "less": "^4.0.0",
         "sass": "^1.70.0",

ui-svelte/src/components/ActivityStats.svelte

@@ -0,0 +1,72 @@
+<script lang="ts">
+  import { inFlightRequests, metrics } from "../stores/api";
+  import { persistentStore } from "../stores/persistent";
+  import { calculateHistogramData } from "../lib/histogram";
+  import TokenHistogram from "./TokenHistogram.svelte";
+
+  const nf = new Intl.NumberFormat();
+  const histogramCollapsed = persistentStore<boolean>("activity-histogram-collapsed", false);
+
+  let stats = $derived.by(() => {
+    const totalRequests = $metrics.length;
+    const totalInputTokens = $metrics.reduce((sum, m) => sum + m.input_tokens, 0);
+    const totalOutputTokens = $metrics.reduce((sum, m) => sum + m.output_tokens, 0);
+
+    const tokensPerSecond = $metrics
+      .filter((m) => m.tokens_per_second > 0)
+      .map((m) => m.tokens_per_second);
+
+    const histogramData = tokensPerSecond.length > 0
+      ? calculateHistogramData(tokensPerSecond, { minBins: 20, maxBins: 80, binScaling: 3 })
+      : null;
+
+    return {
+      totalRequests,
+      totalInputTokens,
+      totalOutputTokens,
+      inFlightRequests: $inFlightRequests,
+      histogramData,
+    };
+  });
+</script>
+
+<div class="card">
+  <button
+    class="flex items-center gap-1 px-4 pt-3 text-xs font-medium text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-200 transition-colors"
+    onclick={() => $histogramCollapsed = !$histogramCollapsed}
+  >
+    <svg
+      class="w-3 h-3 transition-transform"
+      style="transform: rotate({$histogramCollapsed ? -90 : 0}deg)"
+      viewBox="0 0 16 16"
+      fill="currentColor"
+    >
+      <path d="M4.5 6l3.5 4 3.5-4H4.5z" />
+    </svg>
+    Tokens/sec Distribution
+  </button>
+  {#if !$histogramCollapsed}
+    {#if stats.histogramData}
+      <TokenHistogram data={stats.histogramData} />
+    {:else}
+      <div class="px-4 py-6 text-center text-sm text-gray-500 dark:text-gray-400">
+        No token speed data yet
+      </div>
+    {/if}
+  {/if}
+  <div class="grid grid-cols-3 gap-x-6 gap-y-1 px-4 pb-3 text-sm">
+    <div class="text-xs uppercase tracking-wider text-gray-500 dark:text-gray-400">Requests</div>
+    <div class="text-xs uppercase tracking-wider text-gray-500 dark:text-gray-400">Processed</div>
+    <div class="text-xs uppercase tracking-wider text-gray-500 dark:text-gray-400">Generated</div>
+    <div class="text-sm text-gray-700 dark:text-gray-300">
+      <span class="font-semibold">{nf.format(stats.totalRequests)}</span> completed,
+      <span class="font-semibold">{nf.format(stats.inFlightRequests)}</span> waiting
+    </div>
+    <div class="text-sm text-gray-700 dark:text-gray-300">
+      <span class="font-semibold">{nf.format(stats.totalInputTokens)}</span> tokens
+    </div>
+    <div class="text-sm text-gray-700 dark:text-gray-300">
+      <span class="font-semibold">{nf.format(stats.totalOutputTokens)}</span> tokens
+    </div>
+  </div>
+</div>

ui-svelte/src/components/StatsPanel.svelte

@@ -1,167 +0,0 @@
-<script lang="ts">
-  import { inFlightRequests, metrics } from "../stores/api";
-  import TokenHistogram from "./TokenHistogram.svelte";
-
-  interface HistogramData {
-    bins: number[];
-    min: number;
-    max: number;
-    binSize: number;
-    p99: number;
-    p95: number;
-    p50: number;
-  }
-
-  let stats = $derived.by(() => {
-    const totalRequests = $metrics.length;
-    if (totalRequests === 0) {
-      return {
-        totalRequests: 0,
-        totalInputTokens: 0,
-        totalOutputTokens: 0,
-        inFlightRequests: $inFlightRequests,
-        tokenStats: { p99: "0", p95: "0", p50: "0" },
-        histogramData: null,
-      };
-    }
-
-    const totalInputTokens = $metrics.reduce((sum, m) => sum + m.input_tokens, 0);
-    const totalOutputTokens = $metrics.reduce((sum, m) => sum + m.output_tokens, 0);
-
-    // Calculate token statistics using output_tokens and duration_ms
-    const validMetrics = $metrics.filter((m) => m.duration_ms > 0 && m.output_tokens > 0);
-    if (validMetrics.length === 0) {
-      return {
-        totalRequests,
-        totalInputTokens,
-        totalOutputTokens,
-        inFlightRequests: $inFlightRequests,
-        tokenStats: { p99: "0", p95: "0", p50: "0" },
-        histogramData: null,
-      };
-    }
-
-    // Calculate tokens/second for each valid metric
-    const tokensPerSecond = validMetrics.map((m) => m.output_tokens / (m.duration_ms / 1000));
-
-    // Sort for percentile calculation
-    const sortedTokensPerSecond = [...tokensPerSecond].sort((a, b) => a - b);
-
-    const p99 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.99)];
-    const p95 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.95)];
-    const p50 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.5)];
-
-    // Create histogram data
-    const min = Math.min(...tokensPerSecond);
-    const max = Math.max(...tokensPerSecond);
-    const binCount = Math.min(30, Math.max(10, Math.floor(tokensPerSecond.length / 5)));
-    const binSize = (max - min) / binCount;
-
-    const bins = Array(binCount).fill(0);
-    tokensPerSecond.forEach((value) => {
-      const binIndex = Math.min(Math.floor((value - min) / binSize), binCount - 1);
-      bins[binIndex]++;
-    });
-
-    const histogramData: HistogramData = {
-      bins,
-      min,
-      max,
-      binSize,
-      p99,
-      p95,
-      p50,
-    };
-
-    return {
-      totalRequests,
-      totalInputTokens,
-      totalOutputTokens,
-      inFlightRequests: $inFlightRequests,
-      tokenStats: {
-        p99: p99.toFixed(2),
-        p95: p95.toFixed(2),
-        p50: p50.toFixed(2),
-      },
-      histogramData,
-    };
-  });
-
-  const nf = new Intl.NumberFormat();
-</script>
-
-<div class="card">
-  <div class="rounded-lg overflow-hidden border border-card-border-inner">
-    <table class="min-w-full divide-y divide-card-border-inner">
-      <thead class="bg-secondary">
-        <tr>
-          <th class="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain">Requests</th>
-          <th class="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
-            Processed
-          </th>
-          <th class="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
-            Generated
-          </th>
-          <th class="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
-            Token Stats (tokens/sec)
-          </th>
-        </tr>
-      </thead>
-
-      <tbody class="bg-surface divide-y divide-card-border-inner">
-        <tr class="hover:bg-secondary">
-          <td class="px-4 py-4 text-sm font-semibold text-gray-900 dark:text-white">
-            <div class="flex flex-col gap-1">
-              <span class="text-xs font-medium text-gray-500 dark:text-gray-400">Completed: {nf.format(stats.totalRequests)}</span>
-              <span class="text-xs font-medium text-gray-500 dark:text-gray-400">Waiting: {nf.format(stats.inFlightRequests)}</span>
-            </div>
-          </td>
-
-          <td class="px-4 py-4 text-sm text-gray-700 dark:text-gray-300 border-l border-gray-200 dark:border-white/10">
-            <div class="flex items-center gap-2">
-              <span class="text-sm font-medium">{nf.format(stats.totalInputTokens)}</span>
-              <span class="text-xs text-gray-500 dark:text-gray-400">tokens</span>
-            </div>
-          </td>
-
-          <td class="px-4 py-4 text-sm text-gray-700 dark:text-gray-300 border-l border-gray-200 dark:border-white/10">
-            <div class="flex items-center gap-2">
-              <span class="text-sm font-medium">{nf.format(stats.totalOutputTokens)}</span>
-              <span class="text-xs text-gray-500 dark:text-gray-400">tokens</span>
-            </div>
-          </td>
-
-          <td class="px-4 py-4 border-l border-gray-200 dark:border-white/10">
-            <div class="space-y-3">
-              <div class="grid grid-cols-3 gap-2 items-center">
-                <div class="text-center">
-                  <div class="text-xs text-gray-500 dark:text-gray-400">P50</div>
-                  <div class="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
-                    {stats.tokenStats.p50}
-                  </div>
-                </div>
-
-                <div class="text-center">
-                  <div class="text-xs text-gray-500 dark:text-gray-400">P95</div>
-                  <div class="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
-                    {stats.tokenStats.p95}
-                  </div>
-                </div>
-
-                <div class="text-center">
-                  <div class="text-xs text-gray-500 dark:text-gray-400">P99</div>
-                  <div class="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
-                    {stats.tokenStats.p99}
-                  </div>
-                </div>
-              </div>
-              {#if stats.histogramData}
-                <TokenHistogram data={stats.histogramData} />
-              {/if}
-            </div>
-          </td>
-        </tr>
-      </tbody>
-    </table>
-  </div>
-</div>

ui-svelte/src/components/TokenHistogram.svelte

@@ -1,23 +1,11 @@
 <script lang="ts">
-  interface HistogramData {
-    bins: number[];
-    min: number;
-    max: number;
-    binSize: number;
-    p99: number;
-    p95: number;
-    p50: number;
-  }
+  import type { HistogramData } from "../lib/types";
 
-  interface Props {
-    data: HistogramData;
-  }
+  let { data }: { data: HistogramData } = $props();
 
-  let { data }: Props = $props();
-
-  const height = 120;
-  const padding = { top: 10, right: 15, bottom: 25, left: 45 };
-  const viewBoxWidth = 600;
+  const height = 55;
+  const padding = { top: 5, right: 45, bottom: 15, left: 45 };
+  const viewBoxWidth = 1200;
   const chartWidth = viewBoxWidth - padding.left - padding.right;
   const chartHeight = height - padding.top - padding.bottom;
 
@@ -121,9 +109,5 @@
       {data.max.toFixed(1)}
     </text>
 
-    <!-- X-axis label -->
-    <text x={padding.left + chartWidth / 2} y={height - 2} font-size="10" fill="currentColor" opacity="0.6" text-anchor="middle">
-      Tokens/Second Distribution
-    </text>
   </svg>
 </div>

ui-svelte/src/lib/histogram.test.ts

@@ -0,0 +1,163 @@
+import { describe, it, expect } from "vitest";
+import { calculateHistogramData } from "./histogram";
+
+describe("calculateHistogramData", () => {
+  describe("edge cases", () => {
+    it("returns null for empty input", () => {
+      expect(calculateHistogramData([])).toBeNull();
+    });
+
+    it("handles single value", () => {
+      const result = calculateHistogramData([42]);
+      expect(result).not.toBeNull();
+      expect(result!.bins).toEqual([1]);
+      expect(result!.min).toBe(42);
+      expect(result!.max).toBe(42);
+      expect(result!.binSize).toBe(0);
+      expect(result!.p50).toBe(42);
+      expect(result!.p95).toBe(42);
+      expect(result!.p99).toBe(42);
+    });
+
+    it("handles all identical values", () => {
+      const result = calculateHistogramData([10, 10, 10, 10, 10]);
+      expect(result).not.toBeNull();
+      expect(result!.bins).toEqual([5]);
+      expect(result!.min).toBe(10);
+      expect(result!.max).toBe(10);
+      expect(result!.binSize).toBe(0);
+    });
+
+    it("handles two distinct values", () => {
+      const result = calculateHistogramData([10, 20]);
+      expect(result).not.toBeNull();
+      expect(result!.min).toBe(10);
+      expect(result!.max).toBe(20);
+      expect(result!.p50).toBe(15);
+      const binSum = result!.bins.reduce((s, b) => s + b, 0);
+      expect(binSum).toBe(2);
+    });
+  });
+
+  describe("bin distribution", () => {
+    it("bins sum to total number of values", () => {
+      const values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+      const result = calculateHistogramData(values);
+      expect(result).not.toBeNull();
+      const binSum = result!.bins.reduce((s, b) => s + b, 0);
+      expect(binSum).toBe(values.length);
+    });
+
+    it("distributes uniform values across bins", () => {
+      const values = Array.from({ length: 100 }, (_, i) => i);
+      const result = calculateHistogramData(values);
+      expect(result).not.toBeNull();
+      expect(result!.bins.length).toBe(20);
+      const binSum = result!.bins.reduce((s, b) => s + b, 0);
+      expect(binSum).toBe(100);
+    });
+
+    it("places values in correct bins", () => {
+      const values = [1, 1, 1, 5, 5, 9, 9, 9];
+      const result = calculateHistogramData(values, { minBins: 3, maxBins: 3, binScaling: 1 });
+      expect(result).not.toBeNull();
+      expect(result!.bins.length).toBe(3);
+      expect(result!.bins.reduce((s, b) => s + b, 0)).toBe(8);
+    });
+
+    it("handles skewed distribution", () => {
+      const values = [1, 1, 1, 1, 1, 100];
+      const result = calculateHistogramData(values);
+      expect(result).not.toBeNull();
+      const binSum = result!.bins.reduce((s, b) => s + b, 0);
+      expect(binSum).toBe(6);
+    });
+  });
+
+  describe("percentiles", () => {
+    it("calculates correct p50 for even-length array", () => {
+      const values = [1, 2, 3, 4];
+      const result = calculateHistogramData(values);
+      expect(result).not.toBeNull();
+      expect(result!.p50).toBe(2.5);
+    });
+
+    it("calculates correct p50 for odd-length array", () => {
+      const values = [1, 2, 3, 4, 5];
+      const result = calculateHistogramData(values);
+      expect(result).not.toBeNull();
+      expect(result!.p50).toBe(3);
+    });
+
+    it("calculates p99 with interpolation", () => {
+      const values = Array.from({ length: 100 }, (_, i) => i + 1);
+      const result = calculateHistogramData(values);
+      expect(result).not.toBeNull();
+      expect(result!.p99).toBeCloseTo(99.01);
+    });
+
+    it("calculates p95 with interpolation", () => {
+      const values = Array.from({ length: 100 }, (_, i) => i + 1);
+      const result = calculateHistogramData(values);
+      expect(result).not.toBeNull();
+      expect(result!.p95).toBeCloseTo(95.05);
+    });
+
+    it("percentiles are monotonically increasing", () => {
+      const values = Array.from({ length: 200 }, () => Math.random() * 100);
+      const result = calculateHistogramData(values);
+      expect(result).not.toBeNull();
+      expect(result!.p50).toBeLessThanOrEqual(result!.p95);
+      expect(result!.p95).toBeLessThanOrEqual(result!.p99);
+    });
+  });
+
+  describe("bin count adaptation", () => {
+    it("uses minimum bins for small datasets", () => {
+      const values = Array.from({ length: 20 }, (_, i) => i);
+      const result = calculateHistogramData(values);
+      expect(result!.bins.length).toBe(10);
+    });
+
+    it("scales bins with dataset size", () => {
+      const values = Array.from({ length: 100 }, (_, i) => i);
+      const result = calculateHistogramData(values);
+      expect(result!.bins.length).toBe(20);
+    });
+
+    it("caps bins at maximum", () => {
+      const values = Array.from({ length: 200 }, (_, i) => i);
+      const result = calculateHistogramData(values);
+      expect(result!.bins.length).toBe(30);
+    });
+
+    it("respects custom options", () => {
+      const values = Array.from({ length: 100 }, (_, i) => i);
+      const result = calculateHistogramData(values, { minBins: 5, maxBins: 10, binScaling: 2 });
+      expect(result!.bins.length).toBe(10);
+    });
+  });
+
+  describe("min and max", () => {
+    it("correctly identifies min and max", () => {
+      const values = [5, 3, 8, 1, 9, 2];
+      const result = calculateHistogramData(values);
+      expect(result!.min).toBe(1);
+      expect(result!.max).toBe(9);
+    });
+
+    it("handles negative values", () => {
+      const values = [-10, -5, 0, 5, 10];
+      const result = calculateHistogramData(values);
+      expect(result!.min).toBe(-10);
+      expect(result!.max).toBe(10);
+    });
+
+    it("handles floating point values", () => {
+      const values = [1.5, 2.7, 3.14, 0.5, 4.99];
+      const result = calculateHistogramData(values);
+      expect(result!.min).toBe(0.5);
+      expect(result!.max).toBe(4.99);
+    });
+  });
+});

ui-svelte/src/lib/histogram.ts

@@ -0,0 +1,72 @@
+import type { HistogramData } from "./types";
+
+export interface HistogramOptions {
+  minBins?: number;
+  maxBins?: number;
+  binScaling?: number;
+}
+
+const DEFAULT_OPTIONS: HistogramOptions = {
+  minBins: 10,
+  maxBins: 30,
+  binScaling: 5,
+};
+
+function percentile(sorted: number[], p: number): number {
+  if (sorted.length === 0) return 0;
+  if (sorted.length === 1) return sorted[0];
+
+  const rank = (p / 100) * (sorted.length - 1);
+  const lower = Math.floor(rank);
+  const upper = Math.ceil(rank);
+  const fraction = rank - lower;
+
+  return sorted[lower] + fraction * (sorted[upper] - sorted[lower]);
+}
+
+export function calculateHistogramData(
+  values: number[],
+  options: HistogramOptions = DEFAULT_OPTIONS,
+): HistogramData | null {
+  if (values.length === 0) return null;
+
+  const sorted = [...values].sort((a, b) => a - b);
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+
+  const p50 = percentile(sorted, 50);
+  const p95 = percentile(sorted, 95);
+  const p99 = percentile(sorted, 99);
+
+  if (min === max) {
+    return {
+      bins: [values.length],
+      min,
+      max,
+      binSize: 0,
+      p50,
+      p95,
+      p99,
+    };
+  }
+
+  const { minBins = 10, maxBins = 30, binScaling = 5 } = options;
+  const binCount = Math.min(maxBins, Math.max(minBins, Math.floor(values.length / binScaling)));
+  const binSize = (max - min) / binCount;
+
+  const bins = new Array(binCount).fill(0);
+  for (const value of values) {
+    const binIndex = Math.min(Math.floor((value - min) / binSize), binCount - 1);
+    bins[binIndex]++;
+  }
+
+  return {
+    bins,
+    min,
+    max,
+    binSize,
+    p50,
+    p95,
+    p99,
+  };
+}

ui-svelte/src/lib/types.ts

@@ -48,6 +48,16 @@ export interface APIEventEnvelope {
   data: string;
 }
 
+export interface HistogramData {
+  bins: number[];
+  min: number;
+  max: number;
+  binSize: number;
+  p99: number;
+  p95: number;
+  p50: number;
+}
+
 export interface VersionInfo {
   build_date: string;
   commit: string;

ui-svelte/src/routes/Activity.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
   import { metrics, getCapture } from "../stores/api";
+  import ActivityStats from "../components/ActivityStats.svelte";
   import Tooltip from "../components/Tooltip.svelte";
   import CaptureDialog from "../components/CaptureDialog.svelte";
   import type { ReqRespCapture } from "../lib/types";
@@ -63,33 +64,38 @@
 
 <div class="p-2">
   <h1 class="text-2xl font-bold">Activity</h1>
+  <div class="mt-4 mb-4">
+    <ActivityStats />
+  </div>
 
-  {#if $metrics.length === 0}
-    <div class="text-center py-8">
-      <p class="text-gray-600">No metrics data available</p>
-    </div>
-  {:else}
-    <div class="card overflow-auto">
-      <table class="min-w-full divide-y">
-        <thead class="border-gray-200 dark:border-white/10">
-          <tr class="text-left text-xs uppercase tracking-wider">
-            <th class="px-6 py-3">ID</th>
-            <th class="px-6 py-3">Time</th>
-            <th class="px-6 py-3">Model</th>
-            <th class="px-6 py-3">
-              Cached <Tooltip content="prompt tokens from cache" />
-            </th>
-            <th class="px-6 py-3">
-              Prompt <Tooltip content="new prompt tokens processed" />
-            </th>
-            <th class="px-6 py-3">Generated</th>
-            <th class="px-6 py-3">Prompt Processing</th>
-            <th class="px-6 py-3">Generation Speed</th>
-            <th class="px-6 py-3">Duration</th>
-            <th class="px-6 py-3">Capture</th>
+  <div class="card overflow-auto">
+    <table class="min-w-full divide-y">
+      <thead class="border-gray-200 dark:border-white/10">
+        <tr class="text-left text-xs uppercase tracking-wider">
+          <th class="px-6 py-3">ID</th>
+          <th class="px-6 py-3">Time</th>
+          <th class="px-6 py-3">Model</th>
+          <th class="px-6 py-3">
+            Cached <Tooltip content="prompt tokens from cache" />
+          </th>
+          <th class="px-6 py-3">
+            Prompt <Tooltip content="new prompt tokens processed" />
+          </th>
+          <th class="px-6 py-3">Generated</th>
+          <th class="px-6 py-3">Prompt Processing</th>
+          <th class="px-6 py-3">Generation Speed</th>
+          <th class="px-6 py-3">Duration</th>
+          <th class="px-6 py-3">Capture</th>
+        </tr>
+      </thead>
+      <tbody class="divide-y">
+        {#if sortedMetrics.length === 0}
+          <tr>
+            <td colspan="10" class="px-6 py-8 text-center text-sm text-gray-500 dark:text-gray-400">
+              No activity recorded
+            </td>
           </tr>
-        </thead>
-        <tbody class="divide-y">
+        {:else}
           {#each sortedMetrics as metric (metric.id)}
             <tr class="whitespace-nowrap text-sm border-gray-200 dark:border-white/10">
               <td class="px-4 py-4">{metric.id + 1}</td>
@@ -116,10 +122,10 @@
               </td>
             </tr>
           {/each}
-        </tbody>
-      </table>
-    </div>
-  {/if}
+        {/if}
+      </tbody>
+    </table>
+  </div>
 </div>
 
 <CaptureDialog capture={selectedCapture} open={dialogOpen} onclose={closeDialog} />

ui-svelte/src/routes/Models.svelte

@@ -2,7 +2,6 @@
   import { isNarrow } from "../stores/theme";
   import { upstreamLogs } from "../stores/api";
   import ModelsPanel from "../components/ModelsPanel.svelte";
-  import StatsPanel from "../components/StatsPanel.svelte";
   import LogPanel from "../components/LogPanel.svelte";
   import ResizablePanels from "../components/ResizablePanels.svelte";
 
@@ -14,13 +13,6 @@
     <ModelsPanel />
   {/snippet}
   {#snippet rightPanel()}
-    <div class="flex flex-col h-full space-y-4">
-      {#if direction === "horizontal"}
-        <StatsPanel />
-      {/if}
-      <div class="flex-1 min-h-0">
-        <LogPanel id="modelsupstream" title="Upstream Logs" logData={$upstreamLogs} />
-      </div>
-    </div>
+    <LogPanel id="modelsupstream" title="Upstream Logs" logData={$upstreamLogs} />
   {/snippet}
 </ResizablePanels>