steve/llama-swap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: steve/llama-swap:v200
Branches Tags
steve/llama-swap:main steve/llama-swap:claude/ui-svelte-shading-migration-w30ta6 steve/llama-swap:coderabbitai/utg/6cf1317
steve/llama-swap:v230 steve/llama-swap:v229 steve/llama-swap:v228 steve/llama-swap:v227 steve/llama-swap:v226 steve/llama-swap:v225 steve/llama-swap:v224 steve/llama-swap:v223 steve/llama-swap:v222 steve/llama-swap:v221 steve/llama-swap:v220 steve/llama-swap:v219 steve/llama-swap:v218 steve/llama-swap:v217 steve/llama-swap:v216 steve/llama-swap:v215 steve/llama-swap:v214 steve/llama-swap:v213 steve/llama-swap:v212 steve/llama-swap:v211 steve/llama-swap:v210 steve/llama-swap:v209 steve/llama-swap:v208 steve/llama-swap:v206 steve/llama-swap:v207 steve/llama-swap:v205 steve/llama-swap:v204 steve/llama-swap:v203 steve/llama-swap:v202 steve/llama-swap:v201 steve/llama-swap:v200 steve/llama-swap:v199 steve/llama-swap:v198 steve/llama-swap:v197 steve/llama-swap:v196 steve/llama-swap:v195 steve/llama-swap:v194 steve/llama-swap:v193 steve/llama-swap:v192 steve/llama-swap:v191 steve/llama-swap:v190 steve/llama-swap:v189 steve/llama-swap:v188 steve/llama-swap:v187 steve/llama-swap:v186 steve/llama-swap:v185 steve/llama-swap:v184 steve/llama-swap:v183 steve/llama-swap:v182 steve/llama-swap:v181 steve/llama-swap:v180 steve/llama-swap:v179 steve/llama-swap:v178 steve/llama-swap:v177 steve/llama-swap:v176 steve/llama-swap:v175 steve/llama-swap:v174 steve/llama-swap:v173 steve/llama-swap:v172 steve/llama-swap:v171 steve/llama-swap:v170 steve/llama-swap:v169 steve/llama-swap:v168 steve/llama-swap:v167 steve/llama-swap:v166 steve/llama-swap:v165 steve/llama-swap:v164 steve/llama-swap:v163 steve/llama-swap:v162 steve/llama-swap:v161 steve/llama-swap:v160 steve/llama-swap:v159 steve/llama-swap:v158 steve/llama-swap:v157 steve/llama-swap:v156 steve/llama-swap:v155 steve/llama-swap:v154 steve/llama-swap:v153 steve/llama-swap:v152 steve/llama-swap:v151 steve/llama-swap:v150 steve/llama-swap:v149 steve/llama-swap:v148 steve/llama-swap:v147 steve/llama-swap:v146 steve/llama-swap:v145 steve/llama-swap:v144 steve/llama-swap:v143 steve/llama-swap:v142 steve/llama-swap:v141 steve/llama-swap:v140 steve/llama-swap:v139 steve/llama-swap:v138 steve/llama-swap:v137 steve/llama-swap:v136 steve/llama-swap:v135 steve/llama-swap:v134 steve/llama-swap:v133 steve/llama-swap:v132 steve/llama-swap:v131 steve/llama-swap:v130 steve/llama-swap:v129 steve/llama-swap:v128 steve/llama-swap:v127 steve/llama-swap:v126 steve/llama-swap:v125 steve/llama-swap:v124 steve/llama-swap:v123 steve/llama-swap:v122 steve/llama-swap:v121 steve/llama-swap:v120 steve/llama-swap:v119 steve/llama-swap:v118 steve/llama-swap:v117 steve/llama-swap:v116 steve/llama-swap:v115 steve/llama-swap:v114 steve/llama-swap:v113 steve/llama-swap:v112 steve/llama-swap:v111 steve/llama-swap:v110 steve/llama-swap:v109 steve/llama-swap:v108 steve/llama-swap:v107 steve/llama-swap:v106 steve/llama-swap:v105 steve/llama-swap:v104 steve/llama-swap:v103 steve/llama-swap:v102 steve/llama-swap:v101 steve/llama-swap:v100 steve/llama-swap:v99 steve/llama-swap:v98 steve/llama-swap:v97 steve/llama-swap:v96 steve/llama-swap:v95 steve/llama-swap:v94 steve/llama-swap:v93 steve/llama-swap:v92 steve/llama-swap:v91 steve/llama-swap:v90 steve/llama-swap:v0.0.51 steve/llama-swap:v89 steve/llama-swap:v88 steve/llama-swap:v87 steve/llama-swap:v86 steve/llama-swap:v85 steve/llama-swap:v84 steve/llama-swap:v83 steve/llama-swap:v82 steve/llama-swap:v81 steve/llama-swap:v80 steve/llama-swap:v79 steve/llama-swap:v78 steve/llama-swap:v77 steve/llama-swap:v76 steve/llama-swap:v75 steve/llama-swap:v74 steve/llama-swap:v72 steve/llama-swap:v0.1.5 steve/llama-swap:v0.1.4 steve/llama-swap:v0.1.3 steve/llama-swap:v0.1.2 steve/llama-swap:v0.1.1 steve/llama-swap:v0.1.0 steve/llama-swap:v0.0.10 steve/llama-swap:v0.0.9 steve/llama-swap:v0.0.8 steve/llama-swap:v0.0.7 steve/llama-swap:v0.0.6 steve/llama-swap:v0.0.5 steve/llama-swap:v0.0.4 steve/llama-swap:v0.0.3 steve/llama-swap:v0.0.1 steve/llama-swap:v0.0.2
..
compare: steve/llama-swap:v202
Branches Tags
steve/llama-swap:main steve/llama-swap:claude/ui-svelte-shading-migration-w30ta6 steve/llama-swap:coderabbitai/utg/6cf1317
steve/llama-swap:v230 steve/llama-swap:v229 steve/llama-swap:v228 steve/llama-swap:v227 steve/llama-swap:v226 steve/llama-swap:v225 steve/llama-swap:v224 steve/llama-swap:v223 steve/llama-swap:v222 steve/llama-swap:v221 steve/llama-swap:v220 steve/llama-swap:v219 steve/llama-swap:v218 steve/llama-swap:v217 steve/llama-swap:v216 steve/llama-swap:v215 steve/llama-swap:v214 steve/llama-swap:v213 steve/llama-swap:v212 steve/llama-swap:v211 steve/llama-swap:v210 steve/llama-swap:v209 steve/llama-swap:v208 steve/llama-swap:v206 steve/llama-swap:v207 steve/llama-swap:v205 steve/llama-swap:v204 steve/llama-swap:v203 steve/llama-swap:v202 steve/llama-swap:v201 steve/llama-swap:v200 steve/llama-swap:v199 steve/llama-swap:v198 steve/llama-swap:v197 steve/llama-swap:v196 steve/llama-swap:v195 steve/llama-swap:v194 steve/llama-swap:v193 steve/llama-swap:v192 steve/llama-swap:v191 steve/llama-swap:v190 steve/llama-swap:v189 steve/llama-swap:v188 steve/llama-swap:v187 steve/llama-swap:v186 steve/llama-swap:v185 steve/llama-swap:v184 steve/llama-swap:v183 steve/llama-swap:v182 steve/llama-swap:v181 steve/llama-swap:v180 steve/llama-swap:v179 steve/llama-swap:v178 steve/llama-swap:v177 steve/llama-swap:v176 steve/llama-swap:v175 steve/llama-swap:v174 steve/llama-swap:v173 steve/llama-swap:v172 steve/llama-swap:v171 steve/llama-swap:v170 steve/llama-swap:v169 steve/llama-swap:v168 steve/llama-swap:v167 steve/llama-swap:v166 steve/llama-swap:v165 steve/llama-swap:v164 steve/llama-swap:v163 steve/llama-swap:v162 steve/llama-swap:v161 steve/llama-swap:v160 steve/llama-swap:v159 steve/llama-swap:v158 steve/llama-swap:v157 steve/llama-swap:v156 steve/llama-swap:v155 steve/llama-swap:v154 steve/llama-swap:v153 steve/llama-swap:v152 steve/llama-swap:v151 steve/llama-swap:v150 steve/llama-swap:v149 steve/llama-swap:v148 steve/llama-swap:v147 steve/llama-swap:v146 steve/llama-swap:v145 steve/llama-swap:v144 steve/llama-swap:v143 steve/llama-swap:v142 steve/llama-swap:v141 steve/llama-swap:v140 steve/llama-swap:v139 steve/llama-swap:v138 steve/llama-swap:v137 steve/llama-swap:v136 steve/llama-swap:v135 steve/llama-swap:v134 steve/llama-swap:v133 steve/llama-swap:v132 steve/llama-swap:v131 steve/llama-swap:v130 steve/llama-swap:v129 steve/llama-swap:v128 steve/llama-swap:v127 steve/llama-swap:v126 steve/llama-swap:v125 steve/llama-swap:v124 steve/llama-swap:v123 steve/llama-swap:v122 steve/llama-swap:v121 steve/llama-swap:v120 steve/llama-swap:v119 steve/llama-swap:v118 steve/llama-swap:v117 steve/llama-swap:v116 steve/llama-swap:v115 steve/llama-swap:v114 steve/llama-swap:v113 steve/llama-swap:v112 steve/llama-swap:v111 steve/llama-swap:v110 steve/llama-swap:v109 steve/llama-swap:v108 steve/llama-swap:v107 steve/llama-swap:v106 steve/llama-swap:v105 steve/llama-swap:v104 steve/llama-swap:v103 steve/llama-swap:v102 steve/llama-swap:v101 steve/llama-swap:v100 steve/llama-swap:v99 steve/llama-swap:v98 steve/llama-swap:v97 steve/llama-swap:v96 steve/llama-swap:v95 steve/llama-swap:v94 steve/llama-swap:v93 steve/llama-swap:v92 steve/llama-swap:v91 steve/llama-swap:v90 steve/llama-swap:v0.0.51 steve/llama-swap:v89 steve/llama-swap:v88 steve/llama-swap:v87 steve/llama-swap:v86 steve/llama-swap:v85 steve/llama-swap:v84 steve/llama-swap:v83 steve/llama-swap:v82 steve/llama-swap:v81 steve/llama-swap:v80 steve/llama-swap:v79 steve/llama-swap:v78 steve/llama-swap:v77 steve/llama-swap:v76 steve/llama-swap:v75 steve/llama-swap:v74 steve/llama-swap:v72 steve/llama-swap:v0.1.5 steve/llama-swap:v0.1.4 steve/llama-swap:v0.1.3 steve/llama-swap:v0.1.2 steve/llama-swap:v0.1.1 steve/llama-swap:v0.1.0 steve/llama-swap:v0.0.10 steve/llama-swap:v0.0.9 steve/llama-swap:v0.0.8 steve/llama-swap:v0.0.7 steve/llama-swap:v0.0.6 steve/llama-swap:v0.0.5 steve/llama-swap:v0.0.4 steve/llama-swap:v0.0.3 steve/llama-swap:v0.0.1 steve/llama-swap:v0.0.2

6 Commits
v200 .. v202

Author SHA1 Message Date
Benson Wong 17233e9278 docs: update configuration.md for matrix 2026-04-14 22:01:03 -07:00
Benson Wong 4866d16c3e README.md: update to use matrix instead of groups 2026-04-14 21:57:49 -07:00
Benson Wong 35193f82f1 proxy: add swap matrix with solver-based model swapping (#646) 
Add a new swap matrix to supersede groups for running concurrent models.
The matrix uses a solver that picks the lowest cost evictions to make a
requested model available. This simple approach along with a very basic
DSL grammar can enable very complex swapping scenarios.

- add DSL parser for set expressions with & (AND), | (OR), (), +ref
- add MatrixConfig structs, validation, and topological sort for +ref
- add MatrixSolver with cost-minimizing swap decisions
- add Matrix runtime integrating solver with Process lifecycle
- integrate matrix into ProxyManager with if-branches at all endpoints
- update config.example.yaml and config-schema.json with matrix schema
- config enforces groups XOR matrix (cannot use both)

fixes #643
 2026-04-14 21:55:30 -07:00
Benson Wong 40e39f7a86 ui-svelte: fix security issues (#649) 2026-04-12 16:21:31 -07:00
Benson Wong a9d840ffd7 proxy,proxy/config: restore timeouts to pre PR 619 (#648) 
Reset the default ResponseHeader timeout to 0 (no timeout) which was set
to 60 seconds in PR #619.

Fixes #647
 2026-04-11 20:42:13 -07:00
Benson Wong 7b2b82777f docker/unified: derive rootless image from root container (#644) 
Build the root image once, then derive the rootless variant from it
using a small inline Dockerfile that adds the non-root user and chowns
the writable directories. This halves the number of CI jobs (4 → 2) and
eliminates the redundant full CUDA compilation for the rootless variant.

- remove RUN_UID build arg from build-image.sh
- derive rootless image inline after root build completes
- collapse variant matrix out of unified-docker.yml
- push both root and rootless tags in a single CI job

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
 2026-04-10 22:59:54 -07:00
24 changed files with 2487 additions and 508 deletions
Expand all files Collapse all files
.github/workflows/go-ci.yml
+46 -47
View File
@@ -2,69 +2,68 @@ name: Linux CI
on: on:
push: push:
branches: [ "main" ] branches: ["main"]
# only run when backend source changes # only run when backend source changes
# cmd/ is excluded because it contains utilities without tests # cmd/ is excluded because it contains utilities without tests
paths: paths:
- '**/*.go' - "**/*.go"
- '!cmd/**' - "!cmd/**"
- 'go.mod' - "go.mod"
- 'go.sum' - "go.sum"
- 'Makefile' - "Makefile"
- '.github/workflows/go-ci.yml' - ".github/workflows/go-ci.yml"
pull_request: pull_request:
branches: [ "main" ] branches: ["main"]
paths: paths:
- '**/*.go' - "**/*.go"
- '!cmd/**' - "!cmd/**"
- 'go.mod' - "go.mod"
- 'go.sum' - "go.sum"
- 'Makefile' - "Makefile"
- '.github/workflows/go-ci.yml' - ".github/workflows/go-ci.yml"
# Allows manual triggering of the workflow # Allows manual triggering of the workflow
workflow_dispatch: workflow_dispatch:
jobs: jobs:
run-tests: run-tests:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: Set up Go - name: Set up Go
uses: actions/setup-go@v4 uses: actions/setup-go@v4
with: with:
go-version-file: go.mod go-version-file: go.mod
# Only run in this linux based runner # Only run in this linux based runner
- name: Check Formatting - name: Check Formatting
run: | run: |
if [ "$(gofmt -l . | grep -v 'event/.*_test.go' | wc -l)" -gt 0 ]; then if [ "$(gofmt -l . | grep -v 'event/.*_test.go' | wc -l)" -gt 0 ]; then
gofmt -l . | grep -v 'event/.*_test.go' gofmt -l . | grep -v 'event/.*_test.go'
exit 1 exit 1
fi fi
# cache simple-responder to save the build time # cache simple-responder to save the build time
- name: Restore Simple Responder - name: Restore Simple Responder
id: restore-simple-responder id: restore-simple-responder
uses: actions/cache/restore@v4 uses: actions/cache/restore@v4
with: with:
path: ./build path: ./build
key: ${{ runner.os }}-simple-responder-${{ hashFiles('cmd/simple-responder/simple-responder.go') }} key: ${{ runner.os }}-simple-responder-${{ hashFiles('cmd/simple-responder/simple-responder.go') }}
# necessary for testing proxy/Process swapping # necessary for testing proxy/Process swapping
- name: Create simple-responder - name: Create simple-responder
run: make simple-responder run: make simple-responder
- name: Save Simple Responder - name: Save Simple Responder
# nothing new to save ... skip this step # nothing new to save ... skip this step
if: steps.restore-simple-responder.outputs.cache-hit != 'true' if: steps.restore-simple-responder.outputs.cache-hit != 'true'
id: save-simple-responder id: save-simple-responder
uses: actions/cache/save@v4 uses: actions/cache/save@v4
with: with:
path: ./build path: ./build
key: ${{ runner.os }}-simple-responder-${{ hashFiles('misc/simple-responder/simple-responder.go') }} key: ${{ runner.os }}-simple-responder-${{ hashFiles('cmd/simple-responder/simple-responder.go') }}
- name: Test all - name: Test all
run: make test-all run: make test-all
.github/workflows/unified-docker.yml
+12 -14
View File
@@ -68,13 +68,6 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
backend: ${{ fromJSON(needs.setup.outputs.matrix) }} backend: ${{ fromJSON(needs.setup.outputs.matrix) }}
variant:
- name: root
uid: "0"
suffix: ""
- name: rootless
uid: "10001"
suffix: "-rootless"
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -106,15 +99,14 @@ jobs:
username: ${{ github.actor }} username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Build unified Docker image (${{ matrix.backend }}, ${{ matrix.variant.name }}) - name: Build unified Docker image (${{ matrix.backend }})
env: env:
LLAMA_REF: ${{ inputs.llama_cpp_ref || 'master' }} LLAMA_REF: ${{ inputs.llama_cpp_ref || 'master' }}
WHISPER_REF: ${{ inputs.whisper_ref || 'master' }} WHISPER_REF: ${{ inputs.whisper_ref || 'master' }}
SD_REF: ${{ inputs.sd_ref || 'master' }} SD_REF: ${{ inputs.sd_ref || 'master' }}
IK_LLAMA_REF: ${{ inputs.ik_llama_ref || 'main' }} IK_LLAMA_REF: ${{ inputs.ik_llama_ref || 'main' }}
LS_VERSION: ${{ inputs.llama_swap_version || 'main' }} LS_VERSION: ${{ inputs.llama_swap_version || 'main' }}
RUN_UID: ${{ matrix.variant.uid }} DOCKER_IMAGE_TAG: ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}
DOCKER_IMAGE_TAG: ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}${{ matrix.variant.suffix }}
# When running under act, use the local builder that has warm ccache. # When running under act, use the local builder that has warm ccache.
# On GitHub Actions, BUILDX_BUILDER is unset so docker uses the builder # On GitHub Actions, BUILDX_BUILDER is unset so docker uses the builder
# created by setup-buildx-action above. # created by setup-buildx-action above.
@@ -126,8 +118,14 @@ jobs:
- name: Push to GitHub Container Registry - name: Push to GitHub Container Registry
if: ${{ !env.ACT }} if: ${{ !env.ACT }}
run: | run: |
TAG="ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}${{ matrix.variant.suffix }}" BASE_TAG="ghcr.io/mostlygeek/llama-swap:unified-${{ matrix.backend }}"
docker push "${TAG}"
DATE_TAG=$(date -u +%Y-%m-%d) DATE_TAG=$(date -u +%Y-%m-%d)
docker tag "${TAG}" "${TAG}-${DATE_TAG}"
docker push "${TAG}-${DATE_TAG}" docker push "${BASE_TAG}"
docker tag "${BASE_TAG}" "${BASE_TAG}-${DATE_TAG}"
docker push "${BASE_TAG}-${DATE_TAG}"
ROOTLESS_TAG="${BASE_TAG}-rootless"
docker push "${ROOTLESS_TAG}"
docker tag "${ROOTLESS_TAG}" "${ROOTLESS_TAG}-${DATE_TAG}"
docker push "${ROOTLESS_TAG}-${DATE_TAG}"
README.md
+7 -9
View File
@@ -5,7 +5,7 @@
# llama-swap # llama-swap
Run multiple generative AI models on your machine and hot-swap between them on demand. llama-swap works with any OpenAI and Anthropic API compatible server and is used by thousands of people to power their local AI workflows. Run multiple generative AI models on your machine and hot-swap between them on demand. llama-swap works with any OpenAI and Anthropic API compatible server and is used by thousands of people to power their local AI workflows.
Built in Go for performance and simplicity, llama-swap has zero dependencies and is incredibly easy to set up. Get started in minutes - just one binary and one configuration file. Built in Go for performance and simplicity, llama-swap has zero dependencies and is incredibly easy to set up. Get started in minutes - just one binary and one configuration file.
@@ -45,14 +45,14 @@ Built in Go for performance and simplicity, llama-swap has zero dependencies and
- `/health` - just returns "OK" - `/health` - just returns "OK"
- ✅ API Key support - define keys to restrict access to API endpoints - ✅ API Key support - define keys to restrict access to API endpoints
- ✅ Customizable - ✅ Customizable
- Run multiple models at once with `Groups` ([#107](https://github.com/mostlygeek/llama-swap/issues/107)) - Run concurrent models with a custom DSL swap matrix ([#643](https://github.com/mostlygeek/llama-swap/issues/643))
- Automatic unloading of models after timeout by setting a `ttl` - Automatic unloading of models after timeout by setting a `ttl`
- Reliable Docker and Podman support using `cmd` and `cmdStop` together - Reliable Docker and Podman support using `cmd` and `cmdStop` together
- Preload models on startup with `hooks` ([#235](https://github.com/mostlygeek/llama-swap/pull/235)) - Preload models on startup with `hooks` ([#235](https://github.com/mostlygeek/llama-swap/pull/235))
### Web UI ### Web UI
llama-swap includes a real time web interface with a playground for testing out all sorts of local models: llama-swap includes a real time web interface with a playground for testing out all sorts of local models:
<img width="1125" height="876" alt="image" src="https://github.com/user-attachments/assets/8ee41947-97af-463d-b0f0-8e9c478fac07" /> <img width="1125" height="876" alt="image" src="https://github.com/user-attachments/assets/8ee41947-97af-463d-b0f0-8e9c478fac07" />
@@ -64,16 +64,14 @@ Inspect request and responses:
<img width="1111" height="720" alt="image" src="https://github.com/user-attachments/assets/24fe4aca-1448-4d7c-b9e8-a967589bda6c" /> <img width="1111" height="720" alt="image" src="https://github.com/user-attachments/assets/24fe4aca-1448-4d7c-b9e8-a967589bda6c" />
Manually load and unload models: Manually load and unload models:
<img width="1109" height="719" alt="image" src="https://github.com/user-attachments/assets/02b1e1f2-abd0-4050-84ae-facd66ff01c4" /> <img width="1109" height="719" alt="image" src="https://github.com/user-attachments/assets/02b1e1f2-abd0-4050-84ae-facd66ff01c4" />
Real time log streaming:
Real time log streaming:
<img width="1107" height="559" alt="image" src="https://github.com/user-attachments/assets/39669a10-cff2-409e-836a-5bad8bd0140c" /> <img width="1107" height="559" alt="image" src="https://github.com/user-attachments/assets/39669a10-cff2-409e-836a-5bad8bd0140c" />
## Installation ## Installation
llama-swap can be installed in multiple ways llama-swap can be installed in multiple ways
@@ -182,7 +180,7 @@ That's all you need to get started:
Almost all configuration settings are optional and can be added one step at a time: Almost all configuration settings are optional and can be added one step at a time:
- Advanced features - Advanced features
- `groups` to run multiple models at once - `matrix` to run concurrent models with a custom swap logic DSL
- `hooks` to run things on startup - `hooks` to run things on startup
- `macros` reusable snippets - `macros` reusable snippets
- Model customization - Model customization
@@ -200,7 +198,7 @@ See the [configuration documentation](docs/configuration.md) for all options.
When a request is made to an OpenAI compatible endpoint, llama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to handle the request correctly. When a request is made to an OpenAI compatible endpoint, llama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to handle the request correctly.
In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `groups` feature allows multiple models to be loaded at the same time. You have complete control over how your system resources are used. In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, using a `matrix` allows multiple models to be loaded at the same time. You have complete control over how your system resources are used.
## Reverse Proxy Configuration (nginx) ## Reverse Proxy Configuration (nginx)
config-schema.json
+84 -12
View File
@@ -47,31 +47,37 @@
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 30, "default": 30,
"description": "TCP connection timeout in seconds. Set to 0 to disable (not recommended)." "description": "TCP connection timeout in seconds. Set to 0 to disable."
},
"keepalive": {
"type": "integer",
"minimum": 0,
"default": 30,
"description": "TCP keepalive timeout in seconds. Set to 0 to disable."
}, },
"responseHeader": { "responseHeader": {
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 60, "default": 0,
"description": "Time to wait for response headers in seconds. Set to 0 to disable (not recommended)." "description": "Time to wait for response headers in seconds. Set to 0 to disable."
}, },
"tlsHandshake": { "tlsHandshake": {
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 10, "default": 10,
"description": "TLS handshake timeout in seconds. Set to 0 to disable (not recommended)." "description": "TLS handshake timeout in seconds. Set to 0 to disable."
}, },
"expectContinue": { "expectContinue": {
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 1, "default": 1,
"description": "Expect-Continue timeout in seconds. Set to 0 to disable (not recommended)." "description": "Expect-Continue timeout in seconds. Set to 0 to disable."
}, },
"idleConn": { "idleConn": {
"type": "integer", "type": "integer",
"minimum": 0, "minimum": 0,
"default": 90, "default": 90,
"description": "Idle connection timeout in seconds. Set to 0 to disable (not recommended)." "description": "Idle connection timeout in seconds. Set to 0 to disable."
} }
}, },
"additionalProperties": false, "additionalProperties": false,
@@ -319,6 +325,44 @@
}, },
"description": "A dictionary of group settings. Provides advanced controls over model swapping behaviour. Model IDs must be defined in models. A model can only be a member of one group. Behaviour controlled via swap, exclusive, persistent." "description": "A dictionary of group settings. Provides advanced controls over model swapping behaviour. Model IDs must be defined in models. A model can only be a member of one group. Behaviour controlled via swap, exclusive, persistent."
}, },
"matrix": {
"type": "object",
"description": "Solver-based alternative to groups. Declares valid combinations of concurrent models. The solver minimizes eviction cost when swapping. A config must use either groups or matrix, not both.",
"required": [
"vars",
"sets"
],
"properties": {
"vars": {
"type": "object",
"description": "Short names for models. Keys must be alphanumeric, 1-8 characters. All sets and evict_costs must use these IDs.",
"minProperties": 1,
"additionalProperties": {
"type": "string"
},
"propertyNames": {
"pattern": "^[a-zA-Z0-9]{1,8}$"
}
},
"evict_costs": {
"type": "object",
"description": "Relative cost of evicting a running model. Models not listed default to 1. Values must be positive integers.",
"additionalProperties": {
"type": "integer",
"minimum": 1
}
},
"sets": {
"type": "object",
"description": "Named sets of concurrent model combinations. Values are DSL strings using & (AND), | (OR), () (grouping), and +ref (inline another set). Definition order is used for tie-breaking.",
"minProperties": 1,
"additionalProperties": {
"type": "string"
}
}
},
"additionalProperties": false
},
"hooks": { "hooks": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -413,25 +457,31 @@
"properties": { "properties": {
"connect": { "connect": {
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 0,
"default": 30, "default": 30,
"description": "TCP connection timeout in seconds." "description": "TCP connection timeout in seconds."
}, },
"keepalive": {
"type": "integer",
"minimum": 0,
"default": 30,
"description": "TCP keepalive connection timeout in seconds."
},
"responseHeader": { "responseHeader": {
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 0,
"default": 60, "default": 0,
"description": "Time to wait for response headers in seconds." "description": "Time to wait for response headers in seconds."
}, },
"tlsHandshake": { "tlsHandshake": {
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 0,
"default": 10, "default": 10,
"description": "TLS handshake timeout in seconds." "description": "TLS handshake timeout in seconds."
}, },
"idleConn": { "idleConn": {
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 0,
"default": 90, "default": 90,
"description": "Idle connection timeout in seconds." "description": "Idle connection timeout in seconds."
} }
@@ -444,5 +494,27 @@
"default": {}, "default": {},
"description": "A dictionary of remote peers and models they provide. Peers can be another llama-swap or any server that provides the /v1/ generative API endpoints supported by llama-swap." "description": "A dictionary of remote peers and models they provide. Peers can be another llama-swap or any server that provides the /v1/ generative API endpoints supported by llama-swap."
} }
} },
"allOf": [
{
"if": {
"required": ["groups"]
},
"then": {
"not": {
"required": ["matrix"]
}
}
},
{
"if": {
"required": ["matrix"]
},
"then": {
"not": {
"required": ["groups"]
}
}
}
]
} }
config.example.yaml
+78 -61
View File
@@ -287,14 +287,15 @@ models:
# timeouts: configure proxy connection timeouts for this model # timeouts: configure proxy connection timeouts for this model
# - optional, defaults shown below # - optional, defaults shown below
# - useful for models running on slower hardware that need longer timeouts # - useful for models running on slower hardware that need longer timeouts
# - connect: TCP connection timeout in seconds # - connect: TCP dial connection timeout in seconds, default: 30 seconds
# - responseHeader: time to wait for response headers in seconds # - keepalive: TCP connection keepalive timeout, default: 30 seconds
# (increasing this helps avoid 502 errors on slow hardware) # - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
# - tlsHandshake: TLS handshake timeout in seconds # - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
# - idleConn: idle connection timeout in seconds # - idleConn: idle connection timeout in seconds, default: 90 seconds
# - set any value to 0 to disable that timeout (not recommended) # - set any value to 0 to disable that timeout (not recommended)
timeouts: timeouts:
connect: 30 connect: 30
keepalive: 0
responseHeader: 60 responseHeader: 60
tlsHandshake: 10 tlsHandshake: 10
idleConn: 90 idleConn: 90
@@ -330,68 +331,83 @@ models:
# - processes have 5 seconds to shutdown until forceful termination is attempted # - processes have 5 seconds to shutdown until forceful termination is attempted
cmdStop: docker stop ${MODEL_ID} cmdStop: docker stop ${MODEL_ID}
# groups: a dictionary of group settings # =============================================================================
# - optional, default: empty dictionary # matrix: run concurrent models with a solver-based swap DSL
# - provides advanced controls over model swapping behaviour # =============================================================================
# - using groups some models can be kept loaded indefinitely, while others are swapped out
# - model IDs must be defined in the Models section
# - a model can only be a member of one group
# - group behaviour is controlled via the `swap`, `exclusive` and `persistent` fields
# - see issue #109 for details
# #
# NOTE: the example below uses model names that are not defined above for demonstration purposes # Note:
groups: # A config must use either a matrix or legacy groups, not both. A configuration error
# group1 works the same as the default behaviour of llama-swap where only one model is allowed # will occur if both are defined. Configuration examples for legacy Groups can be found:
# to run a time across the whole llama-swap instance # https://github.com/mostlygeek/llama-swap/blob/40e39f7/config.example.yaml#L334-L396
"group1": #
# swap: controls the model swapping behaviour in within the group # The matrix declares valid combinations of models that can run concurrently.
# - optional, default: true # When a model is requested, the solver finds the cheapest way to make it
# - true : only one model is allowed to run at a time # available by evicting as few (and least costly) running models as possible.
# - false: all models can run together, no swapping #
swap: true # Solver behavior:
# 1. Request arrives for model X
# 2. If X is already running, forward immediately. Done.
# 3. Find all sets containing X
# 4. For each candidate set, compute cost: sum of evict_costs for
# every running model NOT in that set
# 5. Pick lowest cost candidate. Ties broken by definition order.
# 6. Evict what needs to stop. Start X. Forward request.
#
# Subset semantics: a set [a, b, c] means any subset is valid.
# Only the requested model is started — others are not preloaded.
#
# A model not appearing in any set can only run alone.
#
matrix:
# vars: short names for models (alphanumeric, 1-8 chars)
# - required for sets and evict_costs settings
# - each entry is a short name to a real model ID. Do not use an alias
# - used to keep set DSL logic short and easier to read
# - sets and evict_costs only use identifiers defined in vars
vars:
g: gemma-model
q: qwen-model
m: mistral-model
v: voxtral-model
e: reranker-model
L: llama-70B
sd: stable-diffusion
# exclusive: controls how the group affects other groups # evict_costs: relative cost of losing a running model (default: 1)
# - optional, default: true evict_costs:
# - true: causes all other groups to unload when this group runs a model v: 50 # vllm backend, slow cold start
# - false: does not affect other groups L: 30 # 70B weights, slow to load
exclusive: true
# members references the models defined above # sets: named sets of concurrent model combinations
# required # Values are DSL strings with operators:
members: # & AND (models run together)
- "llama" # | OR (alternatives)
- "qwen-unlisted" # () grouping
# +ref inline another set's expression
#
# Expansion examples:
# "L" → [L]
# "a & b" → [a, b]
# "a | b" → [a], [b]
# "(a | b) & c" → [a, c], [b, c]
# "(a | b) & (c | d)" → [a,c], [a,d], [b,c], [b,d]
# "+llms & v" → expands llms inline, then applies & v
sets:
# LLM + TTS: switching between g/q/m won't evict v
# expands to: [g,v], [q,v], [m,v]
standard: "(g | q | m) & v"
# Example: # LLM + TTS + reranker
# - in group2 all models can run at the same time # expands to: [g,v,e], [q,v,e]
# - when a different group is loaded it causes all running models in this group to unload with_rerank: "(g | q) & v & e"
"group2":
swap: false
# exclusive: false does not unload other groups when a model in group2 is requested # LLM + image generation, no TTS
# - the models in group2 will be loaded but will not unload any other groups # expands to: [g,sd], [q,sd]
exclusive: false creative: "(g | q) & sd"
members:
- "docker-llama"
- "modelA"
- "modelB"
# Example: # 70B model uses all GPUs, can only run alone
# - a persistent group, prevents other groups from unloading it # expands to: [L]
"forever": full: "L"
# persistent: prevents over groups from unloading the models in this group
# - optional, default: false
# - does not affect individual model behaviour
persistent: true
# set swap/exclusive to false to prevent swapping inside the group
# and the unloading of other groups
swap: false
exclusive: false
members:
- "forever-modelA"
- "forever-modelB"
- "forever-modelc"
# hooks: a dictionary of event triggers and actions # hooks: a dictionary of event triggers and actions
# - optional, default: empty dictionary # - optional, default: empty dictionary
@@ -447,6 +463,7 @@ peers:
# - set any value to 0 to disable that timeout (not recommended) # - set any value to 0 to disable that timeout (not recommended)
timeouts: timeouts:
connect: 30 connect: 30
keepalive: 30
responseHeader: 60 responseHeader: 60
tlsHandshake: 10 tlsHandshake: 10
idleConn: 90 idleConn: 90
docker/unified/build-image.sh
+22 -2
View File
@@ -201,7 +201,6 @@ BUILD_ARGS=(
--build-arg "SD_COMMIT_HASH=${SD_HASH}" --build-arg "SD_COMMIT_HASH=${SD_HASH}"
--build-arg "IK_LLAMA_COMMIT_HASH=${IK_LLAMA_HASH}" --build-arg "IK_LLAMA_COMMIT_HASH=${IK_LLAMA_HASH}"
--build-arg "LS_VERSION=${LS_HASH}" --build-arg "LS_VERSION=${LS_HASH}"
--build-arg "RUN_UID=${RUN_UID:-0}"
-t "${DOCKER_IMAGE_TAG}" -t "${DOCKER_IMAGE_TAG}"
-f "${SCRIPT_DIR}/Dockerfile" -f "${SCRIPT_DIR}/Dockerfile"
) )
@@ -255,12 +254,33 @@ if [[ "$BACKEND" == "cuda" ]]; then
fi fi
echo "All expected binaries verified: ${VERIFIED_LIST}" echo "All expected binaries verified: ${VERIFIED_LIST}"
echo ""
echo "=========================================="
echo "Building rootless image..."
echo "=========================================="
echo ""
ROOTLESS_TAG="${DOCKER_IMAGE_TAG}-rootless"
docker buildx build --load -t "${ROOTLESS_TAG}" - <<EOF
FROM ${DOCKER_IMAGE_TAG}
USER root
RUN groupadd --system --gid 10001 llama-swap && \\
useradd --system --uid 10001 --gid 10001 \\
--home /app --shell /sbin/nologin llama-swap && \\
chown -R 10001:10001 /etc/llama-swap /models
USER 10001
EOF
echo "Rootless image built: ${ROOTLESS_TAG}"
echo "" echo ""
echo "==========================================" echo "=========================================="
echo "Build complete!" echo "Build complete!"
echo "==========================================" echo "=========================================="
echo "" echo ""
echo "Image tag: ${DOCKER_IMAGE_TAG}" echo "Image tags:"
echo " ${DOCKER_IMAGE_TAG}"
echo " ${ROOTLESS_TAG}"
echo "" echo ""
echo "Built with:" echo "Built with:"
echo " llama.cpp: ${LLAMA_HASH}" echo " llama.cpp: ${LLAMA_HASH}"
docs/configuration.md
+199 -118
View File
@@ -22,7 +22,7 @@ models:
cmd: llama-server --port ${PORT} -m /path/to/third_model.gguf cmd: llama-server --port ${PORT} -m /path/to/third_model.gguf
``` ```
With this configuration models will be hot swapped and loaded on demand. The special `${PORT}` macro provides a unique port per model. Useful if you want to run multiple models at the same time with the `groups` feature. With this configuration models will be hot swapped and loaded on demand. The special `${PORT}` macro provides a unique port per model which is useful if you want to run multiple models at the same time with the `matrix` feature.
## Advanced control with `cmd` ## Advanced control with `cmd`
@@ -76,7 +76,7 @@ llama-swap supports many more features to customize how you want to manage your
| --------- | ---------------------------------------------- | | --------- | ---------------------------------------------- |
| `ttl` | automatic unloading of models after a timeout | | `ttl` | automatic unloading of models after a timeout |
| `macros` | reusable snippets to use in configurations | | `macros` | reusable snippets to use in configurations |
| `groups` | run multiple models at a time | | `matrix` | run multiple models at a time |
| `hooks` | event driven functionality | | `hooks` | event driven functionality |
| `env` | define environment variables per model | | `env` | define environment variables per model |
| `aliases` | serve a model with different names | | `aliases` | serve a model with different names |
@@ -141,6 +141,11 @@ logToStdout: "proxy"
# - useful for limiting memory usage when processing large volumes of metrics # - useful for limiting memory usage when processing large volumes of metrics
metricsMaxInMemory: 1000 metricsMaxInMemory: 1000
# captureBuffer: how many MBs to allocate for storing request/response captures
# - optional, default: 10
# - set to 0 to disable
captureBuffer: 15
# startPort: sets the starting port number for the automatic ${PORT} macro. # startPort: sets the starting port number for the automatic ${PORT} macro.
# - optional, default: 5800 # - optional, default: 5800
# - the ${PORT} macro can be used in model.cmd and model.proxy settings # - the ${PORT} macro can be used in model.cmd and model.proxy settings
@@ -161,15 +166,10 @@ sendLoadingState: true
# all fields except for Id so chat UIs can use the alias equivalent to the original. # all fields except for Id so chat UIs can use the alias equivalent to the original.
includeAliasesInList: false includeAliasesInList: false
# apiKeys: require an API key when making requests to inference endpoints # globalTTL: the default TTL in seconds before unloading a model
# - optional, default: [] # - optional, default: 0 (never automatically unload)
# - when empty (the default) authorization will not be checked as llama-swap is default-allow # - must be >= 0
# - each key is a non-empty string globalTTL: 0
apiKeys:
- "sk-hunter2"
# hint, one liner: printf "sk-%s\n" "$(head -c 48 /dev/urandom | base64 )"
- "sk-gyCPiKUcIfPlaM4OSMZekkprgijPx6+OsmQs8Rsg0xZ9qpy6gKWsIKqHOk+cgXVx"
- "sk-+QtIn0Zjj4UHjiaZYiZEnru4mrwKM9RzhmJeK5SobNXLl8QMFXxGz1/2lEuvQpkb"
# macros: a dictionary of string substitutions # macros: a dictionary of string substitutions
# - optional, default: empty dictionary # - optional, default: empty dictionary
@@ -181,6 +181,9 @@ apiKeys:
# - macro names must not be a reserved name: PORT or MODEL_ID # - macro names must not be a reserved name: PORT or MODEL_ID
# - macro values can be numbers, bools, or strings # - macro values can be numbers, bools, or strings
# - macros can contain other macros, but they must be defined before they are used # - macros can contain other macros, but they must be defined before they are used
# - environment variables can be referenced with ${env.VAR_NAME} syntax
# - env macros are substituted first, before regular macros
# - if the env var is not set, config loading will fail with an error
macros: macros:
# Example of a multi-line macro # Example of a multi-line macro
"latest-llama": > "latest-llama": >
@@ -193,6 +196,24 @@ macros:
# but they must be previously declared. # but they must be previously declared.
"default_args": "--ctx-size ${default_ctx}" "default_args": "--ctx-size ${default_ctx}"
# Example of environment variable macros
# - ${env.VAR_NAME} pulls the value from the system environment
# - useful for paths, secrets, or machine-specific configuration
"models_dir": "${env.HOME}/models"
# apiKeys: require an API key when making requests to inference endpoints
# - optional, default: []
# - when empty (the default) authorization will not be checked as llama-swap is default-allow
# - each key is a non-empty string
apiKeys:
- "sk-hunter2"
# tip, one liner: printf "sk-%s\n" "$(head -c 48 /dev/urandom | base64 )"
- "sk-gyCPiKUcIfPlaM4OSMZekkprgijPx6+OsmQs8Rsg0xZ9qpy6gKWsIKqHOk+cgXVx"
# use environment variable macros to keep secrets out of the config
- "${env.API_KEY_1}"
- "${env.API_KEY_2}"
# models: a dictionary of model configurations # models: a dictionary of model configurations
# - required # - required
# - each key is the model's ID, used in API requests # - each key is the model's ID, used in API requests
@@ -201,7 +222,7 @@ macros:
# - below are examples of the all the settings a model can have # - below are examples of the all the settings a model can have
models: models:
# keys are the model names used in API requests # keys are the model names used in API requests
"llama": "gpt-oss-120b":
# macros: a dictionary of string substitutions specific to this model # macros: a dictionary of string substitutions specific to this model
# - optional, default: empty dictionary # - optional, default: empty dictionary
# - macros defined here override macros defined in the global macros section # - macros defined here override macros defined in the global macros section
@@ -218,7 +239,7 @@ models:
cmd: | cmd: |
# ${latest-llama} is a macro that is defined above # ${latest-llama} is a macro that is defined above
${latest-llama} ${latest-llama}
--model path/to/llama-8B-Q4_K_M.gguf --model path/to/gpt-oss-120B.gguf
--ctx-size ${default_ctx} --ctx-size ${default_ctx}
--temperature ${temp} --temperature ${temp}
@@ -226,13 +247,13 @@ models:
# - optional, default: empty string # - optional, default: empty string
# - if set, it will be used in the v1/models API response # - if set, it will be used in the v1/models API response
# - if not set, it will be omitted in the JSON model record # - if not set, it will be omitted in the JSON model record
name: "llama 3.1 8B" name: "gpt-oss 120B"
# description: a description for the model # description: a description for the model
# - optional, default: empty string # - optional, default: empty string
# - if set, it will be used in the v1/models API response # - if set, it will be used in the v1/models API response
# - if not set, it will be omitted in the JSON model record # - if not set, it will be omitted in the JSON model record
description: "A small but capable model used for quick testing" description: "A thinking model from OpenAI"
# env: define an array of environment variables to inject into cmd's environment # env: define an array of environment variables to inject into cmd's environment
# - optional, default: empty array # - optional, default: empty array
@@ -247,14 +268,6 @@ models:
# - if you use a custom port in cmd this *must* be set # - if you use a custom port in cmd this *must* be set
proxy: http://127.0.0.1:8999 proxy: http://127.0.0.1:8999
# aliases: alternative model names that this model configuration is used for
# - optional, default: empty array
# - aliases must be unique globally
# - useful for impersonating a specific model
aliases:
- "gpt-4o-mini"
- "gpt-3.5-turbo"
# checkEndpoint: URL path to check if the server is ready # checkEndpoint: URL path to check if the server is ready
# - optional, default: /health # - optional, default: /health
# - endpoint is expected to return an HTTP 200 response # - endpoint is expected to return an HTTP 200 response
@@ -263,8 +276,10 @@ models:
checkEndpoint: /custom-endpoint checkEndpoint: /custom-endpoint
# ttl: automatically unload the model after ttl seconds # ttl: automatically unload the model after ttl seconds
# - optional, default: 0 # - optional, default: -1 (use global default)
# - ttl values must be a value greater than 0 # - ttl values must be a value greater than or equal to 0
# - a ttl of -1 will use the global TTL value as the default
# - a ttl of 0 will mean never unload
# - a value of 0 disables automatic unloading of the model # - a value of 0 disables automatic unloading of the model
ttl: 60 ttl: 60
@@ -272,11 +287,11 @@ models:
# - optional, default: "" # - optional, default: ""
# - useful for when the upstream server expects a specific model name that # - useful for when the upstream server expects a specific model name that
# is different from the model's ID # is different from the model's ID
useModelName: "qwen:qwq" useModelName: "openai/gpt-oss-120B"
# filters: a dictionary of filter settings # filters: a dictionary of filter settings
# - optional, default: empty dictionary # - optional, default: empty dictionary
# - only stripParams is currently supported # - same capabilities as peer filters (stripParams, setParams)
filters: filters:
# stripParams: a comma separated list of parameters to remove from the request # stripParams: a comma separated list of parameters to remove from the request
# - optional, default: "" # - optional, default: ""
@@ -286,6 +301,43 @@ models:
# - recommended to stick to sampling parameters # - recommended to stick to sampling parameters
stripParams: "temperature, top_p, top_k" stripParams: "temperature, top_p, top_k"
# setParams: a dictionary of parameters to set/override in requests
# - optional, default: empty dictionary
# - useful for enforcing specific parameter values
# - protected params like "model" cannot be overridden
# - values can be strings, numbers, booleans, arrays, or objects
# - always runs for the model
setParams:
# Example: enforce specific sampling parameters
temperature: 0.7
top_p: 0.9
# setParamsByID: a dictionary of parameters to set based the model ID
# - optional, default: empty dictionary
# - combine with aliases to create variant behaviour without reloading the model
# - parameters are set in the request body JSON
# - run after setParams so it will override any settings
# - protected params like "model" cannot be overridden
# - values can be strings, numbers, booleans, arrays, or objects
# - model aliases will be automatically created for each key
setParamsByID:
"${MODEL_ID}":
chat_template_kwargs:
reasoning_effort: medium
"${MODEL_ID}:high":
chat_template_kwargs:
reasoning_effort: high
"${MODEL_ID}:low":
chat_template_kwargs:
reasoning_effort: low
# aliases: alternative model names that this model configuration is used for
# - optional, default: empty array
# - aliases must be unique globally
# - useful for impersonating a specific model
aliases:
- "gpt-4o-mini"
# metadata: a dictionary of arbitrary values that are included in /v1/models # metadata: a dictionary of arbitrary values that are included in /v1/models
# - optional, default: empty dictionary # - optional, default: empty dictionary
# - while metadata can contains complex types it is recommended to keep it simple # - while metadata can contains complex types it is recommended to keep it simple
@@ -319,33 +371,26 @@ models:
# - recommended to be omitted and the default used # - recommended to be omitted and the default used
concurrencyLimit: 0 concurrencyLimit: 0
# timeouts: configure proxy connection timeouts for this model
# - optional, defaults shown below
# - useful for models on slower hardware that need longer timeouts
# - increase responseHeader to avoid "timeout awaiting response headers" errors
# - set any value to 0 to disable that timeout (not recommended)
timeouts:
# connect: TCP connection timeout in seconds
# - default: 30
connect: 30
# responseHeader: time to wait for response headers in seconds
# - default: 60
# - for slow image generation or large models, consider increasing to 300+ seconds
responseHeader: 60
# tlsHandshake: TLS handshake timeout in seconds
# - default: 10
tlsHandshake: 10
# idleConn: idle connection timeout in seconds
# - default: 90
idleConn: 90
# sendLoadingState: overrides the global sendLoadingState setting for this model # sendLoadingState: overrides the global sendLoadingState setting for this model
# - optional, default: undefined (use global setting) # - optional, default: undefined (use global setting)
sendLoadingState: false sendLoadingState: false
# timeouts: configure proxy connection timeouts for this model
# - optional, defaults shown below
# - useful for models running on slower hardware that need longer timeouts
# - connect: TCP dial connection timeout in seconds, default: 30 seconds
# - keepalive: TCP connection keepalive timeout, default: 30 seconds
# - responseHeader: time to wait for response headers in seconds, default: 0 (no timeout)
# - tlsHandshake: TLS handshake timeout in seconds, default: 10 seconds
# - idleConn: idle connection timeout in seconds, default: 90 seconds
# - set any value to 0 to disable that timeout (not recommended)
timeouts:
connect: 30
keepalive: 0
responseHeader: 60
tlsHandshake: 10
idleConn: 90
# Unlisted model example: # Unlisted model example:
"qwen-unlisted": "qwen-unlisted":
# unlisted: boolean, true or false # unlisted: boolean, true or false
@@ -377,68 +422,83 @@ models:
# - processes have 5 seconds to shutdown until forceful termination is attempted # - processes have 5 seconds to shutdown until forceful termination is attempted
cmdStop: docker stop ${MODEL_ID} cmdStop: docker stop ${MODEL_ID}
# groups: a dictionary of group settings # =============================================================================
# - optional, default: empty dictionary # matrix: run concurrent models with a solver-based swap DSL
# - provides advanced controls over model swapping behaviour # =============================================================================
# - using groups some models can be kept loaded indefinitely, while others are swapped out
# - model IDs must be defined in the Models section
# - a model can only be a member of one group
# - group behaviour is controlled via the `swap`, `exclusive` and `persistent` fields
# - see issue #109 for details
# #
# NOTE: the example below uses model names that are not defined above for demonstration purposes # Note:
groups: # A config must use either a matrix or legacy groups, not both. A configuration error
# group1 works the same as the default behaviour of llama-swap where only one model is allowed # will occur if both are defined. Configuration examples for legacy Groups can be found:
# to run a time across the whole llama-swap instance # https://github.com/mostlygeek/llama-swap/blob/40e39f7/config.example.yaml#L334-L396
"group1": #
# swap: controls the model swapping behaviour in within the group # The matrix declares valid combinations of models that can run concurrently.
# - optional, default: true # When a model is requested, the solver finds the cheapest way to make it
# - true : only one model is allowed to run at a time # available by evicting as few (and least costly) running models as possible.
# - false: all models can run together, no swapping #
swap: true # Solver behavior:
# 1. Request arrives for model X
# 2. If X is already running, forward immediately. Done.
# 3. Find all sets containing X
# 4. For each candidate set, compute cost: sum of evict_costs for
# every running model NOT in that set
# 5. Pick lowest cost candidate. Ties broken by definition order.
# 6. Evict what needs to stop. Start X. Forward request.
#
# Subset semantics: a set [a, b, c] means any subset is valid.
# Only the requested model is started — others are not preloaded.
#
# A model not appearing in any set can only run alone.
#
matrix:
# vars: short names for models (alphanumeric, 1-8 chars)
# - required for sets and evict_costs settings
# - each entry is a short name to a real model ID. Do not use an alias
# - used to keep set DSL logic short and easier to read
# - sets and evict_costs only use identifiers defined in vars
vars:
g: gemma-model
q: qwen-model
m: mistral-model
v: voxtral-model
e: reranker-model
L: llama-70B
sd: stable-diffusion
# exclusive: controls how the group affects other groups # evict_costs: relative cost of losing a running model (default: 1)
# - optional, default: true evict_costs:
# - true: causes all other groups to unload when this group runs a model v: 50 # vllm backend, slow cold start
# - false: does not affect other groups L: 30 # 70B weights, slow to load
exclusive: true
# members references the models defined above # sets: named sets of concurrent model combinations
# required # Values are DSL strings with operators:
members: # & AND (models run together)
- "llama" # | OR (alternatives)
- "qwen-unlisted" # () grouping
# +ref inline another set's expression
#
# Expansion examples:
# "L" → [L]
# "a & b" → [a, b]
# "a | b" → [a], [b]
# "(a | b) & c" → [a, c], [b, c]
# "(a | b) & (c | d)" → [a,c], [a,d], [b,c], [b,d]
# "+llms & v" → expands llms inline, then applies & v
sets:
# LLM + TTS: switching between g/q/m won't evict v
# expands to: [g,v], [q,v], [m,v]
standard: "(g | q | m) & v"
# Example: # LLM + TTS + reranker
# - in group2 all models can run at the same time # expands to: [g,v,e], [q,v,e]
# - when a different group is loaded it causes all running models in this group to unload with_rerank: "(g | q) & v & e"
"group2":
swap: false
# exclusive: false does not unload other groups when a model in group2 is requested # LLM + image generation, no TTS
# - the models in group2 will be loaded but will not unload any other groups # expands to: [g,sd], [q,sd]
exclusive: false creative: "(g | q) & sd"
members:
- "docker-llama"
- "modelA"
- "modelB"
# Example: # 70B model uses all GPUs, can only run alone
# - a persistent group, prevents other groups from unloading it # expands to: [L]
"forever": full: "L"
# persistent: prevents over groups from unloading the models in this group
# - optional, default: false
# - does not affect individual model behaviour
persistent: true
# set swap/exclusive to false to prevent swapping inside the group
# and the unloading of other groups
swap: false
exclusive: false
members:
- "forever-modelA"
- "forever-modelB"
- "forever-modelc"
# hooks: a dictionary of event triggers and actions # hooks: a dictionary of event triggers and actions
# - optional, default: empty dictionary # - optional, default: empty dictionary
@@ -467,17 +527,6 @@ peers:
# - required # - required
# - requested path to llama-swap will be appended to the end of the proxy value # - requested path to llama-swap will be appended to the end of the proxy value
proxy: http://192.168.1.23 proxy: http://192.168.1.23
# timeouts: configure proxy connection timeouts for this peer
# - optional, defaults shown below
# - useful when the peer runs on slower hardware
# - set any value to 0 to disable that timeout (not recommended)
timeouts:
connect: 30
responseHeader: 60
tlsHandshake: 10
idleConn: 90
# models: a list of models served by the peer # models: a list of models served by the peer
# - required # - required
models: models:
@@ -490,7 +539,8 @@ peers:
# - optional, default: "" # - optional, default: ""
# - if blank, no key will be added to the request # - if blank, no key will be added to the request
# - key will be injected into headers: Authorization: Bearer <key> and x-api-key: <key> # - key will be injected into headers: Authorization: Bearer <key> and x-api-key: <key>
apiKey: sk-your-openrouter-key # - can be a string or a macro
apiKey: ${env.OPENROUTER_API_KEY}
models: models:
- meta-llama/llama-3.1-8b-instruct - meta-llama/llama-3.1-8b-instruct
- qwen/qwen3-235b-a22b-2507 - qwen/qwen3-235b-a22b-2507
@@ -498,4 +548,35 @@ peers:
- z-ai/glm-4.7 - z-ai/glm-4.7
- moonshotai/kimi-k2-0905 - moonshotai/kimi-k2-0905
- minimax/minimax-m2.1 - minimax/minimax-m2.1
# timeouts: configure proxy connection timeouts for this peer
# - optional, defaults shown below
# - useful when the peer runs on slower hardware
# - set any value to 0 to disable that timeout (not recommended)
timeouts:
connect: 30
keepalive: 30
responseHeader: 60
tlsHandshake: 10
idleConn: 90
# filters: a dictionary of filter settings for peer requests
# - optional, default: empty dictionary
# - same capabilities as model filters (stripParams, setParams)
filters:
# stripParams: a comma separated list of parameters to remove from the request
# - optional, default: ""
# - useful for removing parameters that the peer doesn't support
# - the `model` parameter can never be removed
stripParams: "temperature, top_p"
# setParams: a dictionary of parameters to set/override in requests to this peer
# - optional, default: empty dictionary
# - useful for injecting provider-specific settings like data retention policies
# - protected params like "model" cannot be overridden
# - values can be strings, numbers, booleans, arrays, or objects
setParams:
# Example: enforce zero-data-retention for OpenRouter
provider:
data_collection: "deny"
zdr: true
``` ```
proxy/config/config.go
+32 -13
View File
@@ -129,6 +129,12 @@ type Config struct {
Profiles map[string][]string `yaml:"profiles"` Profiles map[string][]string `yaml:"profiles"`
Groups map[string]GroupConfig `yaml:"groups"` /* key is group ID */ Groups map[string]GroupConfig `yaml:"groups"` /* key is group ID */
// swap matrix: solver-based alternative to groups
Matrix *MatrixConfig `yaml:"matrix"`
// populated during validation when matrix is configured
ExpandedSets []ExpandedSet `yaml:"-"`
// for key/value replacements in model's cmd, cmdStop, proxy, checkEndPoint // for key/value replacements in model's cmd, cmdStop, proxy, checkEndPoint
Macros MacroList `yaml:"macros"` Macros MacroList `yaml:"macros"`
@@ -438,22 +444,35 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
config.Models[modelId] = modelConfig config.Models[modelId] = modelConfig
} }
config = AddDefaultGroupToConfig(config) // groups XOR matrix
if config.Matrix != nil && len(config.Groups) > 0 {
return Config{}, fmt.Errorf("config cannot use both 'groups' and 'matrix'")
}
// Validate group members if config.Matrix != nil {
memberUsage := make(map[string]string) expandedSets, err := ValidateMatrix(*config.Matrix, config.Models)
for groupID, groupConfig := range config.Groups { if err != nil {
prevSet := make(map[string]bool) return Config{}, fmt.Errorf("matrix: %w", err)
for _, member := range groupConfig.Members { }
if _, found := prevSet[member]; found { config.ExpandedSets = expandedSets
return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID) } else {
} config = AddDefaultGroupToConfig(config)
prevSet[member] = true
if existingGroup, exists := memberUsage[member]; exists { // Validate group members
return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID) memberUsage := make(map[string]string)
for groupID, groupConfig := range config.Groups {
prevSet := make(map[string]bool)
for _, member := range groupConfig.Members {
if _, found := prevSet[member]; found {
return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID)
}
prevSet[member] = true
if existingGroup, exists := memberUsage[member]; exists {
return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID)
}
memberUsage[member] = groupID
} }
memberUsage[member] = groupID
} }
} }
proxy/config/config_posix_test.go
+13 -28
View File
@@ -163,6 +163,15 @@ groups:
modelLoadingState := false modelLoadingState := false
defaultTimeout := TimeoutsConfig{
Connect: 30,
KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
}
expected := Config{ expected := Config{
LogLevel: "info", LogLevel: "info",
LogTimeFormat: "", LogTimeFormat: "",
@@ -187,13 +196,7 @@ groups:
Name: "Model 1", Name: "Model 1",
Description: "This is model 1", Description: "This is model 1",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model2": { "model2": {
Cmd: "path/to/server --arg1 one", Cmd: "path/to/server --arg1 one",
@@ -202,13 +205,7 @@ groups:
Env: []string{}, Env: []string{},
CheckEndpoint: "/", CheckEndpoint: "/",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model3": { "model3": {
Cmd: "path/to/cmd --arg1 one", Cmd: "path/to/cmd --arg1 one",
@@ -217,13 +214,7 @@ groups:
Env: []string{}, Env: []string{},
CheckEndpoint: "/", CheckEndpoint: "/",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model4": { "model4": {
Cmd: "path/to/cmd --arg1 one", Cmd: "path/to/cmd --arg1 one",
@@ -232,13 +223,7 @@ groups:
Aliases: []string{}, Aliases: []string{},
Env: []string{}, Env: []string{},
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
}, },
HealthCheckTimeout: 15, HealthCheckTimeout: 15,
proxy/config/config_test.go
+1 -1
View File
@@ -1475,7 +1475,7 @@ models:
// Default values should be set during unmarshaling // Default values should be set during unmarshaling
assert.Equal(t, 30, modelConfig.Timeouts.Connect) assert.Equal(t, 30, modelConfig.Timeouts.Connect)
assert.Equal(t, 60, modelConfig.Timeouts.ResponseHeader) assert.Equal(t, 0, modelConfig.Timeouts.ResponseHeader)
assert.Equal(t, 10, modelConfig.Timeouts.TLSHandshake) assert.Equal(t, 10, modelConfig.Timeouts.TLSHandshake)
assert.Equal(t, 1, modelConfig.Timeouts.ExpectContinue) assert.Equal(t, 1, modelConfig.Timeouts.ExpectContinue)
assert.Equal(t, 90, modelConfig.Timeouts.IdleConn) assert.Equal(t, 90, modelConfig.Timeouts.IdleConn)
proxy/config/config_windows_test.go
+13 -28
View File
@@ -155,6 +155,15 @@ groups:
modelLoadingState := false modelLoadingState := false
defaultTimeout := TimeoutsConfig{
Connect: 30,
KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
}
expected := Config{ expected := Config{
LogLevel: "info", LogLevel: "info",
LogTimeFormat: "", LogTimeFormat: "",
@@ -173,13 +182,7 @@ groups:
Env: []string{"VAR1=value1", "VAR2=value2"}, Env: []string{"VAR1=value1", "VAR2=value2"},
CheckEndpoint: "/health", CheckEndpoint: "/health",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model2": { "model2": {
Cmd: "path/to/server --arg1 one", Cmd: "path/to/server --arg1 one",
@@ -189,13 +192,7 @@ groups:
Env: []string{}, Env: []string{},
CheckEndpoint: "/", CheckEndpoint: "/",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model3": { "model3": {
Cmd: "path/to/cmd --arg1 one", Cmd: "path/to/cmd --arg1 one",
@@ -205,13 +202,7 @@ groups:
Env: []string{}, Env: []string{},
CheckEndpoint: "/", CheckEndpoint: "/",
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
"model4": { "model4": {
Cmd: "path/to/cmd --arg1 one", Cmd: "path/to/cmd --arg1 one",
@@ -221,13 +212,7 @@ groups:
Aliases: []string{}, Aliases: []string{},
Env: []string{}, Env: []string{},
SendLoadingState: &modelLoadingState, SendLoadingState: &modelLoadingState,
Timeouts: TimeoutsConfig{ Timeouts: defaultTimeout,
Connect: 30,
ResponseHeader: 60,
TLSHandshake: 10,
ExpectContinue: 1,
IdleConn: 90,
},
}, },
}, },
HealthCheckTimeout: 15, HealthCheckTimeout: 15,
proxy/config/matrix.go
+226
View File
@@ -0,0 +1,226 @@
package config
import (
"fmt"
"regexp"
"sort"
"gopkg.in/yaml.v3"
)
var varKeyPattern = regexp.MustCompile(`^[a-zA-Z0-9]{1,8}$`)
// MatrixConfig represents the swap matrix configuration block.
type MatrixConfig struct {
Var map[string]string `yaml:"vars"`
EvictCosts map[string]int `yaml:"evict_costs"`
Sets OrderedSets `yaml:"sets"`
}
// SetEntry is a single named set with its DSL expression.
type SetEntry struct {
Name string
DSL string
}
// OrderedSets preserves YAML definition order of sets (used for tie-breaking).
type OrderedSets []SetEntry
func (os *OrderedSets) UnmarshalYAML(value *yaml.Node) error {
if value.Kind != yaml.MappingNode {
return fmt.Errorf("sets must be a mapping")
}
entries := make([]SetEntry, 0, len(value.Content)/2)
for i := 0; i < len(value.Content); i += 2 {
keyNode := value.Content[i]
valueNode := value.Content[i+1]
var name string
if err := keyNode.Decode(&name); err != nil {
return fmt.Errorf("failed to decode set name: %w", err)
}
var dsl string
if err := valueNode.Decode(&dsl); err != nil {
return fmt.Errorf("failed to decode DSL for set %q: %w", name, err)
}
entries = append(entries, SetEntry{Name: name, DSL: dsl})
}
*os = entries
return nil
}
// ExpandedSet is one valid combination of concurrent models (real model names).
type ExpandedSet struct {
SetName string
DSL string
Models []string // real model names, sorted
}
// ValidateMatrix validates the matrix config and returns all expanded sets.
func ValidateMatrix(matrix MatrixConfig, models map[string]ModelConfig) ([]ExpandedSet, error) {
if len(matrix.Sets) == 0 {
return nil, fmt.Errorf("matrix must define at least one set")
}
if len(matrix.Var) == 0 {
return nil, fmt.Errorf("matrix must define at least one var")
}
// Validate var entries
if matrix.Var != nil {
for id, modelName := range matrix.Var {
if !varKeyPattern.MatchString(id) {
return nil, fmt.Errorf("var key %q must be alphanumeric and 1-8 characters", id)
}
if _, exists := models[modelName]; !exists {
return nil, fmt.Errorf("var key %q references unknown model %q", id, modelName)
}
}
}
// Validate evict_costs
if matrix.EvictCosts != nil {
for key, cost := range matrix.EvictCosts {
if cost <= 0 {
return nil, fmt.Errorf("evict_cost for %q must be a positive integer, got %d", key, cost)
}
if _, ok := matrix.Var[key]; !ok {
return nil, fmt.Errorf("evict_costs: unknown var ID %q", key)
}
}
}
// Build dependency graph for +ref topological sort
setNames := make(map[string]bool)
for _, entry := range matrix.Sets {
setNames[entry.Name] = true
}
deps := make(map[string][]string) // setName -> set names it depends on
for _, entry := range matrix.Sets {
refs, err := extractRefs(entry.DSL)
if err != nil {
return nil, fmt.Errorf("set %q: %w", entry.Name, err)
}
for _, ref := range refs {
if !setNames[ref] {
return nil, fmt.Errorf("set %q references undefined set %q", entry.Name, ref)
}
}
deps[entry.Name] = refs
}
// Topological sort with cycle detection
order, err := topologicalSort(matrix.Sets, deps)
if err != nil {
return nil, err
}
// Expand sets in topological order
resolvedRefs := make(map[string][][]string) // set name -> expanded alias-level combos
var allExpanded []ExpandedSet
totalCombinations := 0
// Build ordered map for efficient lookup
setDSL := make(map[string]string)
for _, entry := range matrix.Sets {
setDSL[entry.Name] = entry.DSL
}
for _, name := range order {
dsl := setDSL[name]
combos, err := ParseAndExpandDSL(dsl, resolvedRefs)
if err != nil {
return nil, fmt.Errorf("set %q: %w", name, err)
}
resolvedRefs[name] = combos
// Resolve var IDs to real model names
for _, combo := range combos {
resolved := make([]string, len(combo))
for i, ident := range combo {
realName, ok := matrix.Var[ident]
if !ok {
return nil, fmt.Errorf("set %q: unknown var ID %q", name, ident)
}
resolved[i] = realName
}
sort.Strings(resolved)
allExpanded = append(allExpanded, ExpandedSet{
SetName: name,
DSL: dsl,
Models: resolved,
})
}
totalCombinations += len(combos)
if totalCombinations > maxDSLExpansions {
return nil, fmt.Errorf("total expanded combinations (%d) exceed limit of %d", totalCombinations, maxDSLExpansions)
}
}
return allExpanded, nil
}
// topologicalSort returns set names in dependency order.
// Returns an error if a cycle is detected.
func topologicalSort(sets OrderedSets, deps map[string][]string) ([]string, error) {
// States: 0 = unvisited, 1 = visiting, 2 = visited
state := make(map[string]int)
var order []string
var visit func(name string) error
visit = func(name string) error {
switch state[name] {
case 1:
return fmt.Errorf("circular reference detected involving set %q", name)
case 2:
return nil
}
state[name] = 1
for _, dep := range deps[name] {
if err := visit(dep); err != nil {
return err
}
}
state[name] = 2
order = append(order, name)
return nil
}
// Visit in definition order for deterministic output
for _, entry := range sets {
if state[entry.Name] == 0 {
if err := visit(entry.Name); err != nil {
return nil, err
}
}
}
return order, nil
}
// ResolvedEvictCosts returns a map of real model name -> evict cost,
// resolving var IDs. Models not listed default to 1.
func (m *MatrixConfig) ResolvedEvictCosts() map[string]int {
costs := make(map[string]int)
if m.EvictCosts == nil {
return costs
}
for key, cost := range m.EvictCosts {
// Resolve var ID if present
if realName, ok := m.Var[key]; ok {
costs[realName] = cost
} else {
costs[key] = cost
}
}
return costs
}
proxy/config/matrix_dsl.go
+376
View File
@@ -0,0 +1,376 @@
package config
import (
"fmt"
"sort"
"strings"
"unicode"
)
const maxDSLExpansions = 1000
// Token types for the DSL lexer
type tokenType int
const (
tokIdent tokenType = iota // model alias or name
tokAnd // &
tokOr // |
tokLParen // (
tokRParen // )
tokRef // +setName
tokEOF
)
type token struct {
typ tokenType
val string
}
// tokenize splits a DSL string into tokens.
func tokenize(input string) ([]token, error) {
var tokens []token
i := 0
runes := []rune(input)
for i < len(runes) {
ch := runes[i]
// skip whitespace
if unicode.IsSpace(ch) {
i++
continue
}
switch ch {
case '&':
tokens = append(tokens, token{tokAnd, "&"})
i++
case '|':
tokens = append(tokens, token{tokOr, "|"})
i++
case '(':
tokens = append(tokens, token{tokLParen, "("})
i++
case ')':
tokens = append(tokens, token{tokRParen, ")"})
i++
case '+':
// +ref: read the identifier that follows
i++
start := i
for i < len(runes) && isIdentChar(runes[i]) {
i++
}
if i == start {
return nil, fmt.Errorf("expected set name after '+' at position %d", start)
}
tokens = append(tokens, token{tokRef, string(runes[start:i])})
default:
if isIdentChar(ch) {
start := i
for i < len(runes) && isIdentChar(runes[i]) {
i++
}
tokens = append(tokens, token{tokIdent, string(runes[start:i])})
} else {
return nil, fmt.Errorf("unexpected character %q at position %d", ch, i)
}
}
}
tokens = append(tokens, token{tokEOF, ""})
return tokens, nil
}
func isIdentChar(ch rune) bool {
return unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '_' || ch == '-' || ch == '.'
}
// AST node types
type dslNode interface {
dslNode()
}
type andNode struct {
children []dslNode
}
type orNode struct {
children []dslNode
}
type leafNode struct {
name string
}
type refNode struct {
setName string
}
func (andNode) dslNode() {}
func (orNode) dslNode() {}
func (leafNode) dslNode() {}
func (refNode) dslNode() {}
// parser holds state for recursive-descent parsing.
type parser struct {
tokens []token
pos int
}
func (p *parser) peek() token {
if p.pos < len(p.tokens) {
return p.tokens[p.pos]
}
return token{tokEOF, ""}
}
func (p *parser) next() token {
t := p.peek()
if t.typ != tokEOF {
p.pos++
}
return t
}
func (p *parser) expect(typ tokenType) (token, error) {
t := p.next()
if t.typ != typ {
return t, fmt.Errorf("expected token type %d, got %q", typ, t.val)
}
return t, nil
}
// Grammar:
//
// expr = andExpr
// andExpr = orExpr ('&' orExpr)*
// orExpr = atom ('|' atom)*
// atom = ident | '+' ident | '(' expr ')'
//
// & binds tighter than |, so "a | b & c" means "a | (b & c)"
func parse(tokens []token) (dslNode, error) {
p := &parser{tokens: tokens}
node, err := p.parseExpr()
if err != nil {
return nil, err
}
if p.peek().typ != tokEOF {
return nil, fmt.Errorf("unexpected token %q after expression", p.peek().val)
}
return node, nil
}
func (p *parser) parseExpr() (dslNode, error) {
return p.parseOrExpr()
}
func (p *parser) parseOrExpr() (dslNode, error) {
left, err := p.parseAndExpr()
if err != nil {
return nil, err
}
if p.peek().typ == tokOr {
children := []dslNode{left}
for p.peek().typ == tokOr {
p.next() // consume |
right, err := p.parseAndExpr()
if err != nil {
return nil, err
}
children = append(children, right)
}
return orNode{children: children}, nil
}
return left, nil
}
func (p *parser) parseAndExpr() (dslNode, error) {
left, err := p.parseAtom()
if err != nil {
return nil, err
}
if p.peek().typ == tokAnd {
children := []dslNode{left}
for p.peek().typ == tokAnd {
p.next() // consume &
right, err := p.parseAtom()
if err != nil {
return nil, err
}
children = append(children, right)
}
return andNode{children: children}, nil
}
return left, nil
}
func (p *parser) parseAtom() (dslNode, error) {
t := p.peek()
switch t.typ {
case tokIdent:
p.next()
return leafNode{name: t.val}, nil
case tokRef:
p.next()
return refNode{setName: t.val}, nil
case tokLParen:
p.next() // consume (
node, err := p.parseExpr()
if err != nil {
return nil, err
}
if _, err := p.expect(tokRParen); err != nil {
return nil, fmt.Errorf("missing closing parenthesis")
}
return node, nil
default:
return nil, fmt.Errorf("unexpected token %q", t.val)
}
}
// expand walks the AST and produces all combinations.
// resolvedRefs contains previously expanded sets for +ref resolution.
func expand(node dslNode, resolvedRefs map[string][][]string) ([][]string, error) {
switch n := node.(type) {
case leafNode:
return [][]string{{n.name}}, nil
case refNode:
expanded, ok := resolvedRefs[n.setName]
if !ok {
return nil, fmt.Errorf("unknown set reference +%s", n.setName)
}
// Return a copy
result := make([][]string, len(expanded))
for i, combo := range expanded {
result[i] = make([]string, len(combo))
copy(result[i], combo)
}
return result, nil
case orNode:
// Union of all children's expansions
var result [][]string
for _, child := range n.children {
childResult, err := expand(child, resolvedRefs)
if err != nil {
return nil, err
}
result = append(result, childResult...)
if len(result) > maxDSLExpansions {
return nil, fmt.Errorf("DSL expansion exceeded %d combinations", maxDSLExpansions)
}
}
return result, nil
case andNode:
// Cartesian product across children
result := [][]string{{}} // start with one empty combo
for _, child := range n.children {
childResult, err := expand(child, resolvedRefs)
if err != nil {
return nil, err
}
result, err = cartesianProduct(result, childResult, maxDSLExpansions)
if err != nil {
return nil, err
}
}
return result, nil
default:
return nil, fmt.Errorf("unknown node type %T", node)
}
}
// cartesianProduct computes the cartesian product of two sets of combinations.
// It returns an error if the product would exceed cap.
func cartesianProduct(left, right [][]string, cap int) ([][]string, error) {
if int64(len(left))*int64(len(right)) > int64(cap) {
return nil, fmt.Errorf("DSL expansion exceeded %d combinations", cap)
}
result := make([][]string, 0, len(left)*len(right))
for _, l := range left {
for _, r := range right {
combo := make([]string, 0, len(l)+len(r))
combo = append(combo, l...)
combo = append(combo, r...)
result = append(result, combo)
}
}
return result, nil
}
// ParseAndExpandDSL tokenizes, parses, and expands a DSL string.
// resolvedRefs contains previously expanded sets for +ref inlining.
func ParseAndExpandDSL(dsl string, resolvedRefs map[string][][]string) ([][]string, error) {
dsl = strings.TrimSpace(dsl)
if dsl == "" {
return nil, fmt.Errorf("empty DSL expression")
}
tokens, err := tokenize(dsl)
if err != nil {
return nil, fmt.Errorf("tokenize: %w", err)
}
tree, err := parse(tokens)
if err != nil {
return nil, fmt.Errorf("parse: %w", err)
}
result, err := expand(tree, resolvedRefs)
if err != nil {
return nil, err
}
// Deduplicate models within each combination and sort for consistency
for i, combo := range result {
result[i] = dedupAndSort(combo)
}
return result, nil
}
// dedupAndSort removes duplicate entries and sorts alphabetically.
func dedupAndSort(items []string) []string {
seen := make(map[string]bool, len(items))
var unique []string
for _, item := range items {
if !seen[item] {
seen[item] = true
unique = append(unique, item)
}
}
sort.Strings(unique)
return unique
}
// extractRefs scans a DSL string for +ref tokens without full parsing.
// Used for building the dependency graph for topological sorting.
func extractRefs(dsl string) ([]string, error) {
tokens, err := tokenize(dsl)
if err != nil {
return nil, err
}
var refs []string
seen := make(map[string]bool)
for _, t := range tokens {
if t.typ == tokRef && !seen[t.val] {
seen[t.val] = true
refs = append(refs, t.val)
}
}
return refs, nil
}
proxy/config/matrix_dsl_test.go
+300
View File
@@ -0,0 +1,300 @@
package config
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDSL_Tokenize(t *testing.T) {
tests := []struct {
name string
input string
expect []token
errMsg string
}{
{
name: "single identifier",
input: "abc",
expect: []token{
{tokIdent, "abc"},
{tokEOF, ""},
},
},
{
name: "identifier with hyphens and dots",
input: "model-name.v2",
expect: []token{
{tokIdent, "model-name.v2"},
{tokEOF, ""},
},
},
{
name: "and expression",
input: "a & b",
expect: []token{
{tokIdent, "a"},
{tokAnd, "&"},
{tokIdent, "b"},
{tokEOF, ""},
},
},
{
name: "or expression",
input: "a | b",
expect: []token{
{tokIdent, "a"},
{tokOr, "|"},
{tokIdent, "b"},
{tokEOF, ""},
},
},
{
name: "parentheses",
input: "(a | b) & c",
expect: []token{
{tokLParen, "("},
{tokIdent, "a"},
{tokOr, "|"},
{tokIdent, "b"},
{tokRParen, ")"},
{tokAnd, "&"},
{tokIdent, "c"},
{tokEOF, ""},
},
},
{
name: "ref token",
input: "+llms & v",
expect: []token{
{tokRef, "llms"},
{tokAnd, "&"},
{tokIdent, "v"},
{tokEOF, ""},
},
},
{
name: "no whitespace",
input: "(a|b)&c",
expect: []token{
{tokLParen, "("},
{tokIdent, "a"},
{tokOr, "|"},
{tokIdent, "b"},
{tokRParen, ")"},
{tokAnd, "&"},
{tokIdent, "c"},
{tokEOF, ""},
},
},
{
name: "empty ref",
input: "+",
errMsg: "expected set name after '+'",
},
{
name: "invalid character",
input: "a @ b",
errMsg: "unexpected character",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tokens, err := tokenize(tt.input)
if tt.errMsg != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.errMsg)
} else {
require.NoError(t, err)
assert.Equal(t, tt.expect, tokens)
}
})
}
}
func TestDSL_ParseAndExpand(t *testing.T) {
tests := []struct {
name string
dsl string
refs map[string][][]string
expect [][]string
errMsg string
}{
{
name: "single model",
dsl: "L",
expect: [][]string{{"L"}},
},
{
name: "two models with AND",
dsl: "a & b",
expect: [][]string{{"a", "b"}},
},
{
name: "two models with OR",
dsl: "a | b",
expect: [][]string{{"a"}, {"b"}},
},
{
name: "three models with OR",
dsl: "a | b | c",
expect: [][]string{{"a"}, {"b"}, {"c"}},
},
{
name: "cartesian product (a|b) & (c|d)",
dsl: "(a | b) & (c | d)",
expect: [][]string{
{"a", "c"},
{"a", "d"},
{"b", "c"},
{"b", "d"},
},
},
{
name: "three-way AND",
dsl: "a & b & c",
expect: [][]string{
{"a", "b", "c"},
},
},
{
name: "(g | q | m) & v",
dsl: "(g | q | m) & v",
expect: [][]string{
{"g", "v"},
{"q", "v"},
{"m", "v"},
},
},
{
name: "(g | q) & v & e",
dsl: "(g | q) & v & e",
expect: [][]string{
{"e", "g", "v"},
{"e", "q", "v"},
},
},
{
name: "precedence: a | b & c means a | (b & c)",
dsl: "a | b & c",
expect: [][]string{
{"a"},
{"b", "c"},
},
},
{
name: "+ref inlining",
dsl: "+llms & v",
refs: map[string][][]string{
"llms": {{"g"}, {"q"}, {"m"}},
},
expect: [][]string{
{"g", "v"},
{"q", "v"},
{"m", "v"},
},
},
{
name: "+ref chained",
dsl: "+with_tts & e",
refs: map[string][][]string{
"with_tts": {{"g", "v"}, {"q", "v"}, {"m", "v"}},
},
expect: [][]string{
{"e", "g", "v"},
{"e", "q", "v"},
{"e", "m", "v"},
},
},
{
name: "dedup within combination",
dsl: "a & a",
expect: [][]string{
{"a"},
},
},
{
name: "empty expression",
dsl: "",
errMsg: "empty DSL expression",
},
{
name: "unmatched open paren",
dsl: "(a | b",
errMsg: "missing closing parenthesis",
},
{
name: "unmatched close paren",
dsl: "a | b)",
errMsg: "unexpected token",
},
{
name: "unknown ref",
dsl: "+unknown",
errMsg: "unknown set reference +unknown",
},
{
name: "empty parens",
dsl: "()",
errMsg: "unexpected token",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
refs := tt.refs
if refs == nil {
refs = map[string][][]string{}
}
result, err := ParseAndExpandDSL(tt.dsl, refs)
if tt.errMsg != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.errMsg)
} else {
require.NoError(t, err)
assert.Equal(t, tt.expect, result)
}
})
}
}
func TestDSL_ExpansionCap(t *testing.T) {
// Build an expression that would exceed 1000 combinations:
// (a1|a2|...|a32) & (b1|b2|...|b32) = 1024 combos
var aItems, bItems []string
for i := 0; i < 32; i++ {
aItems = append(aItems, fmt.Sprintf("a%d", i))
bItems = append(bItems, fmt.Sprintf("b%d", i))
}
dsl := fmt.Sprintf("(%s) & (%s)",
join(aItems, " | "),
join(bItems, " | "),
)
_, err := ParseAndExpandDSL(dsl, map[string][][]string{})
require.Error(t, err)
assert.Contains(t, err.Error(), "exceeded")
}
func TestDSL_ExtractRefs(t *testing.T) {
refs, err := extractRefs("+llms & v & +other")
require.NoError(t, err)
assert.Equal(t, []string{"llms", "other"}, refs)
refs, err = extractRefs("a & b")
require.NoError(t, err)
assert.Empty(t, refs)
}
func join(items []string, sep string) string {
result := ""
for i, item := range items {
if i > 0 {
result += sep
}
result += item
}
return result
}
proxy/config/matrix_test.go
+305
View File
@@ -0,0 +1,305 @@
package config
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func makeModels(names ...string) map[string]ModelConfig {
m := make(map[string]ModelConfig)
for _, name := range names {
m[name] = ModelConfig{Cmd: "echo " + name}
}
return m
}
func TestValidateMatrix_Basic(t *testing.T) {
models := makeModels("gemma", "qwen", "mistral", "voxtral", "llama70B")
matrix := MatrixConfig{
Var: map[string]string{
"g": "gemma",
"q": "qwen",
"m": "mistral",
"v": "voxtral",
"L": "llama70B",
},
EvictCosts: map[string]int{
"L": 30,
"v": 50,
},
Sets: OrderedSets{
{Name: "standard", DSL: "(g | q | m) & v"},
{Name: "full", DSL: "L"},
},
}
expanded, err := ValidateMatrix(matrix, models)
require.NoError(t, err)
// standard expands to [gemma,voxtral], [qwen,voxtral], [mistral,voxtral]
// full expands to [llama70B]
assert.Len(t, expanded, 4)
assert.Equal(t, "standard", expanded[0].SetName)
assert.Equal(t, []string{"gemma", "voxtral"}, expanded[0].Models)
assert.Equal(t, "standard", expanded[1].SetName)
assert.Equal(t, []string{"qwen", "voxtral"}, expanded[1].Models)
assert.Equal(t, "standard", expanded[2].SetName)
assert.Equal(t, []string{"mistral", "voxtral"}, expanded[2].Models)
assert.Equal(t, "full", expanded[3].SetName)
assert.Equal(t, []string{"llama70B"}, expanded[3].Models)
}
func TestValidateMatrix_WithRef(t *testing.T) {
models := makeModels("gemma", "qwen", "mistral", "voxtral", "reranker")
matrix := MatrixConfig{
Var: map[string]string{
"g": "gemma",
"q": "qwen",
"m": "mistral",
"v": "voxtral",
"e": "reranker",
},
Sets: OrderedSets{
{Name: "llms", DSL: "g | q | m"},
{Name: "with_tts", DSL: "+llms & v"},
{Name: "mega", DSL: "+with_tts & e"},
},
}
expanded, err := ValidateMatrix(matrix, models)
require.NoError(t, err)
// llms: [gemma], [qwen], [mistral]
// with_tts: [gemma,voxtral], [qwen,voxtral], [mistral,voxtral]
// mega: [gemma,reranker,voxtral], [qwen,reranker,voxtral], [mistral,reranker,voxtral]
assert.Len(t, expanded, 9)
// Check mega entries
megaEntries := filterBySetName(expanded, "mega")
assert.Len(t, megaEntries, 3)
assert.Equal(t, []string{"gemma", "reranker", "voxtral"}, megaEntries[0].Models)
}
func TestValidateMatrix_MapIDRequired(t *testing.T) {
// DSL cannot use real model names directly — must use var IDs
models := makeModels("gemma", "voxtral")
matrix := MatrixConfig{
Var: map[string]string{"g": "gemma"},
Sets: OrderedSets{
{Name: "combo", DSL: "g & voxtral"},
},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), "unknown var ID")
}
func TestValidateMatrix_InvalidAliasKey(t *testing.T) {
models := makeModels("gemma")
tests := []struct {
name string
alias string
errMsg string
}{
{"too long", "abcdefghi", "alphanumeric and 1-8 characters"},
{"has underscore", "a_b", "alphanumeric and 1-8 characters"},
{"has hyphen", "a-b", "alphanumeric and 1-8 characters"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
matrix := MatrixConfig{
Var: map[string]string{tt.alias: "gemma"},
Sets: OrderedSets{{Name: "s", DSL: tt.alias}},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), tt.errMsg)
})
}
}
func TestValidateMatrix_AliasReferencesUnknownModel(t *testing.T) {
models := makeModels("gemma")
matrix := MatrixConfig{
Var: map[string]string{"x": "nonexistent"},
Sets: OrderedSets{{Name: "s", DSL: "x"}},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), "unknown model")
}
func TestValidateMatrix_EvictCostInvalid(t *testing.T) {
models := makeModels("gemma")
t.Run("zero cost", func(t *testing.T) {
matrix := MatrixConfig{
Var: map[string]string{"g": "gemma"},
EvictCosts: map[string]int{"g": 0},
Sets: OrderedSets{{Name: "s", DSL: "g"}},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), "positive integer")
})
t.Run("negative cost", func(t *testing.T) {
matrix := MatrixConfig{
Var: map[string]string{"g": "gemma"},
EvictCosts: map[string]int{"g": -1},
Sets: OrderedSets{{Name: "s", DSL: "g"}},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), "positive integer")
})
t.Run("unknown var ID in evict_costs", func(t *testing.T) {
matrix := MatrixConfig{
Var: map[string]string{"g": "gemma"},
EvictCosts: map[string]int{"unknown": 5},
Sets: OrderedSets{{Name: "s", DSL: "g"}},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), "unknown var ID")
})
}
func TestValidateMatrix_CycleDetection(t *testing.T) {
models := makeModels("gemma")
matrix := MatrixConfig{
Var: map[string]string{"g": "gemma"},
Sets: OrderedSets{
{Name: "a", DSL: "+b"},
{Name: "b", DSL: "+a"},
},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), "circular reference")
}
func TestValidateMatrix_UndefinedRefTarget(t *testing.T) {
models := makeModels("gemma")
matrix := MatrixConfig{
Var: map[string]string{"g": "gemma"},
Sets: OrderedSets{
{Name: "a", DSL: "+nonexistent"},
},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), "references undefined set")
}
func TestValidateMatrix_NoSets(t *testing.T) {
_, err := ValidateMatrix(MatrixConfig{}, makeModels("gemma"))
require.Error(t, err)
assert.Contains(t, err.Error(), "at least one set")
}
func TestValidateMatrix_UnknownMapIDInDSL(t *testing.T) {
models := makeModels("gemma")
matrix := MatrixConfig{
Var: map[string]string{"g": "gemma"},
Sets: OrderedSets{
{Name: "s", DSL: "g & nonexistent"},
},
}
_, err := ValidateMatrix(matrix, models)
require.Error(t, err)
assert.Contains(t, err.Error(), "unknown var ID")
}
func TestValidateMatrix_ResolvedEvictCosts(t *testing.T) {
mc := &MatrixConfig{
Var: map[string]string{
"g": "gemma",
"L": "llama70B",
},
EvictCosts: map[string]int{
"L": 30,
"g": 5,
},
}
costs := mc.ResolvedEvictCosts()
assert.Equal(t, 30, costs["llama70B"])
assert.Equal(t, 5, costs["gemma"])
}
func TestValidateMatrix_ConfigXOR(t *testing.T) {
// groups and matrix both defined
yaml := `
models:
model1:
cmd: echo model1
proxy: http://localhost:8080
groups:
group1:
members:
- model1
matrix:
sets:
s: "model1"
`
_, err := LoadConfigFromReader(strings.NewReader(yaml))
require.Error(t, err)
assert.Contains(t, err.Error(), "cannot use both")
}
func TestValidateMatrix_ConfigMatrixOnly(t *testing.T) {
yaml := `
models:
gemma:
cmd: echo gemma
proxy: http://localhost:8080
qwen:
cmd: echo qwen
proxy: http://localhost:8081
matrix:
vars:
g: gemma
q: qwen
sets:
combo: "g | q"
`
cfg, err := LoadConfigFromReader(strings.NewReader(yaml))
require.NoError(t, err)
assert.NotNil(t, cfg.Matrix)
assert.Len(t, cfg.ExpandedSets, 2)
// Groups should be empty when matrix is used
assert.Empty(t, cfg.Groups)
}
func filterBySetName(sets []ExpandedSet, name string) []ExpandedSet {
var result []ExpandedSet
for _, s := range sets {
if s.SetName == name {
result = append(result, s)
}
}
return result
}
proxy/config/model_config.go
+11 -6
View File
@@ -10,12 +10,14 @@ const (
) )
// TimeoutsConfig holds timeout settings for proxy connections // TimeoutsConfig holds timeout settings for proxy connections
// 0 = no timeout
type TimeoutsConfig struct { type TimeoutsConfig struct {
Connect int `yaml:"connect"` // seconds, 0 = no timeout (not recommended) Connect int `yaml:"connect"`
ResponseHeader int `yaml:"responseHeader"` // seconds, 0 = no timeout (not recommended) KeepAlive int `yaml:"keepalive"`
TLSHandshake int `yaml:"tlsHandshake"` // seconds, 0 = no timeout (not recommended) ResponseHeader int `yaml:"responseHeader"`
ExpectContinue int `yaml:"expectContinue"` // seconds, 0 = no timeout (not recommended) TLSHandshake int `yaml:"tlsHandshake"`
IdleConn int `yaml:"idleConn"` // seconds, 0 = no timeout (not recommended) ExpectContinue int `yaml:"expectContinue"`
IdleConn int `yaml:"idleConn"`
} }
type ModelConfig struct { type ModelConfig struct {
@@ -69,9 +71,12 @@ func (m *ModelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
ConcurrencyLimit: 0, ConcurrencyLimit: 0,
Name: "", Name: "",
Description: "", Description: "",
// matches http.DefaultTransport
Timeouts: TimeoutsConfig{ Timeouts: TimeoutsConfig{
Connect: 30, Connect: 30,
ResponseHeader: 60, KeepAlive: 30,
ResponseHeader: 0,
TLSHandshake: 10, TLSHandshake: 10,
ExpectContinue: 1, ExpectContinue: 1,
IdleConn: 90, IdleConn: 90,
proxy/config/peer.go
+4
View File
@@ -24,8 +24,12 @@ func (c *PeerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
ApiKey: "", ApiKey: "",
Models: []string{}, Models: []string{},
Filters: Filters{}, Filters: Filters{},
// mostly matches http.DefaultTransport but with a 60s ResponseHeader timeout
// to match the pre PR #619 functionality
Timeouts: TimeoutsConfig{ Timeouts: TimeoutsConfig{
Connect: 30, Connect: 30,
KeepAlive: 30,
ResponseHeader: 60, ResponseHeader: 60,
TLSHandshake: 10, TLSHandshake: 10,
ExpectContinue: 1, ExpectContinue: 1,
proxy/matrix.go
+298
View File
@@ -0,0 +1,298 @@
package proxy
import (
"fmt"
"net/http"
"slices"
"sort"
"sync"
"github.com/mostlygeek/llama-swap/proxy/config"
)
// MatrixSolver contains pure swap-decision logic with no Process dependencies.
// It is safe for concurrent reads after construction.
type MatrixSolver struct {
expandedSets []config.ExpandedSet // all valid model combinations
evictCosts map[string]int // real model name -> eviction cost (default 1)
modelToSets map[string][]int // model name -> indices into expandedSets
}
// NewMatrixSolver builds a solver from expanded sets and eviction costs.
func NewMatrixSolver(expandedSets []config.ExpandedSet, evictCosts map[string]int) *MatrixSolver {
modelToSets := make(map[string][]int)
for i, es := range expandedSets {
for _, model := range es.Models {
modelToSets[model] = append(modelToSets[model], i)
}
}
return &MatrixSolver{
expandedSets: expandedSets,
evictCosts: evictCosts,
modelToSets: modelToSets,
}
}
// SolveResult describes what the solver decided.
type SolveResult struct {
Evict []string // running models that must be stopped
TargetSet []string // the chosen set of models (for informational purposes)
SetName string // name of the chosen set
DSL string // original DSL expression for the chosen set
TotalCost int // total eviction cost
}
// Solve determines which models to evict when a model is requested.
//
// Algorithm:
// 1. If requestedModel is already running, no eviction needed.
// 2. Find all sets containing requestedModel.
// 3. If no sets found, the model runs alone; evict all running models.
// 4. For each candidate set, compute cost = sum of evict_costs for running
// models NOT in that set.
// 5. Pick lowest cost. Ties broken by definition order (index in expandedSets).
// 6. Return models to evict and the chosen set.
func (s *MatrixSolver) Solve(requestedModel string, runningModels []string) (SolveResult, error) {
// If already running, nothing to do (but fill in set info for logging)
if slices.Contains(runningModels, requestedModel) {
setName, dsl := s.findMatchingSet(requestedModel, runningModels)
return SolveResult{
TargetSet: runningModels,
SetName: setName,
DSL: dsl,
}, nil
}
candidateIndices := s.modelToSets[requestedModel]
// Model not in any set: runs alone, evict everything
if len(candidateIndices) == 0 {
evict := make([]string, len(runningModels))
copy(evict, runningModels)
return SolveResult{
Evict: evict,
TargetSet: []string{requestedModel},
}, nil
}
// Find the cheapest candidate set
bestCost := -1
bestIdx := -1
for _, idx := range candidateIndices {
setModels := s.expandedSets[idx].Models
cost := 0
for _, running := range runningModels {
if !slices.Contains(setModels, running) {
cost += s.evictCost(running)
}
}
if bestCost < 0 || cost < bestCost || (cost == bestCost && idx < bestIdx) {
bestCost = cost
bestIdx = idx
}
}
// Determine which running models to evict
chosen := s.expandedSets[bestIdx]
var evict []string
for _, running := range runningModels {
if !slices.Contains(chosen.Models, running) {
evict = append(evict, running)
}
}
return SolveResult{
Evict: evict,
TargetSet: chosen.Models,
SetName: chosen.SetName,
DSL: chosen.DSL,
TotalCost: bestCost,
}, nil
}
// findMatchingSet finds the expanded set that contains all running models.
// Returns the set name and DSL, or empty strings if no match.
func (s *MatrixSolver) findMatchingSet(requestedModel string, runningModels []string) (string, string) {
for _, idx := range s.modelToSets[requestedModel] {
set := s.expandedSets[idx]
allInSet := true
for _, m := range runningModels {
if !slices.Contains(set.Models, m) {
allInSet = false
break
}
}
if allInSet {
return set.SetName, set.DSL
}
}
return "", ""
}
func (s *MatrixSolver) evictCost(model string) int {
if cost, ok := s.evictCosts[model]; ok {
return cost
}
return 1
}
// Matrix manages processes using solver-based swap logic.
type Matrix struct {
sync.Mutex
solver *MatrixSolver
processes map[string]*Process // all processes keyed by real model name
config config.Config
proxyLogger *LogMonitor
upstreamLogger *LogMonitor
}
// NewMatrix creates a Matrix from config. It creates a Process for every
// model defined in the config (any model can run alone even if not in a set).
func NewMatrix(cfg config.Config, proxyLogger, upstreamLogger *LogMonitor) *Matrix {
processes := make(map[string]*Process)
for modelID, modelConfig := range cfg.Models {
processLogger := NewLogMonitorWriter(upstreamLogger)
process := NewProcess(modelID, cfg.HealthCheckTimeout, modelConfig, processLogger, proxyLogger)
processes[modelID] = process
}
evictCosts := cfg.Matrix.ResolvedEvictCosts()
return &Matrix{
solver: NewMatrixSolver(cfg.ExpandedSets, evictCosts),
processes: processes,
config: cfg,
proxyLogger: proxyLogger,
upstreamLogger: upstreamLogger,
}
}
// ProxyRequest handles the swap logic and proxies the request to the model.
func (m *Matrix) ProxyRequest(modelID string, w http.ResponseWriter, r *http.Request) error {
process, ok := m.processes[modelID]
if !ok {
return fmt.Errorf("model %s not found in matrix", modelID)
}
m.Lock()
running := m.runningModels()
result, err := m.solver.Solve(modelID, running)
if err != nil {
m.Unlock()
return fmt.Errorf("matrix solver error: %w", err)
}
// Log solver decision
if len(result.Evict) > 0 {
m.proxyLogger.Infof("Matrix: model=%s set=%s dsl=%q evict=%v target=%v cost=%d",
modelID, result.SetName, result.DSL, result.Evict, result.TargetSet, result.TotalCost)
} else if len(running) == 0 {
m.proxyLogger.Infof("Matrix: model=%s starting (no models running)", modelID)
} else {
m.proxyLogger.Debugf("Matrix: model=%s already running in set=%s dsl=%q", modelID, result.SetName, result.DSL)
}
// Evict models that need to be stopped
if len(result.Evict) > 0 {
var wg sync.WaitGroup
for _, evictModel := range result.Evict {
if p, exists := m.processes[evictModel]; exists {
wg.Add(1)
go func(p *Process) {
defer wg.Done()
p.Stop()
}(p)
}
}
wg.Wait()
}
m.Unlock()
// Proxy the request (Process handles on-demand start)
process.ProxyRequest(w, r)
return nil
}
// StopProcesses stops all running processes.
func (m *Matrix) StopProcesses(strategy StopStrategy) {
m.Lock()
defer m.Unlock()
var wg sync.WaitGroup
for _, process := range m.processes {
wg.Add(1)
go func(p *Process) {
defer wg.Done()
switch strategy {
case StopImmediately:
p.StopImmediately()
default:
p.Stop()
}
}(process)
}
wg.Wait()
}
// StopProcess stops a single process by model ID.
func (m *Matrix) StopProcess(modelID string, strategy StopStrategy) error {
process, ok := m.processes[modelID]
if !ok {
return fmt.Errorf("process not found for %s", modelID)
}
switch strategy {
case StopImmediately:
process.StopImmediately()
default:
process.Stop()
}
return nil
}
// Shutdown shuts down all processes.
func (m *Matrix) Shutdown() {
var wg sync.WaitGroup
for _, process := range m.processes {
wg.Add(1)
go func(p *Process) {
defer wg.Done()
p.Shutdown()
}(process)
}
wg.Wait()
}
// RunningModels returns model names currently in StateReady.
func (m *Matrix) RunningModels() []string {
m.Lock()
defer m.Unlock()
return m.runningModels()
}
// runningModels returns running model names (caller must hold lock).
func (m *Matrix) runningModels() []string {
var running []string
for id, process := range m.processes {
if process.CurrentState() == StateReady {
running = append(running, id)
}
}
sort.Strings(running)
return running
}
// GetProcess returns the Process for a model.
func (m *Matrix) GetProcess(modelID string) (*Process, bool) {
p, ok := m.processes[modelID]
return p, ok
}
// HasModel returns true if the model is managed by this matrix.
func (m *Matrix) HasModel(modelID string) bool {
_, ok := m.processes[modelID]
return ok
}
proxy/matrix_test.go
+227
View File
@@ -0,0 +1,227 @@
package proxy
import (
"testing"
"github.com/mostlygeek/llama-swap/proxy/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Helper to build expanded sets for solver tests
func makeExpandedSets(sets ...struct {
name string
models []string
}) []config.ExpandedSet {
var result []config.ExpandedSet
for _, s := range sets {
result = append(result, config.ExpandedSet{
SetName: s.name,
Models: s.models,
})
}
return result
}
func es(name string, models ...string) struct {
name string
models []string
} {
return struct {
name string
models []string
}{name, models}
}
func TestMatrixSolver_AlreadyRunning(t *testing.T) {
solver := NewMatrixSolver(
makeExpandedSets(es("s1", "a", "b")),
nil,
)
result, err := solver.Solve("a", []string{"a"})
require.NoError(t, err)
assert.Empty(t, result.Evict)
assert.Equal(t, []string{"a"}, result.TargetSet)
assert.Equal(t, "s1", result.SetName)
}
func TestMatrixSolver_NotInAnySet_RunsAlone(t *testing.T) {
solver := NewMatrixSolver(
makeExpandedSets(es("s1", "a", "b")),
nil,
)
// Model "c" not in any set
result, err := solver.Solve("c", []string{"a", "b"})
require.NoError(t, err)
assert.ElementsMatch(t, []string{"a", "b"}, result.Evict)
assert.Equal(t, []string{"c"}, result.TargetSet)
}
func TestMatrixSolver_NotInAnySet_NothingRunning(t *testing.T) {
solver := NewMatrixSolver(
makeExpandedSets(es("s1", "a", "b")),
nil,
)
result, err := solver.Solve("c", []string{})
require.NoError(t, err)
assert.Empty(t, result.Evict)
assert.Equal(t, []string{"c"}, result.TargetSet)
}
func TestMatrixSolver_SingleSet_EvictsNonMembers(t *testing.T) {
// Set: [a, b]. Request a when b and c are running.
solver := NewMatrixSolver(
makeExpandedSets(es("s1", "a", "b")),
nil,
)
result, err := solver.Solve("a", []string{"b", "c"})
require.NoError(t, err)
// c is not in the set, so it gets evicted. b is in the set, so it stays.
assert.Equal(t, []string{"c"}, result.Evict)
assert.Equal(t, []string{"a", "b"}, result.TargetSet)
}
func TestMatrixSolver_PicksLowestCost(t *testing.T) {
// Two sets containing model "a":
// s1: [a, v] — if v is running, cost=0; if L is running, cost=30
// s2: [a, L] — if L is running, cost=0; if v is running, cost=50
solver := NewMatrixSolver(
makeExpandedSets(
es("s1", "a", "v"),
es("s2", "a", "L"),
),
map[string]int{"v": 50, "L": 30},
)
// v is running. Switching to a:
// s1 cost: v is in s1, so 0
// s2 cost: v is NOT in s2, so 50
// => pick s1
result, err := solver.Solve("a", []string{"v"})
require.NoError(t, err)
assert.Empty(t, result.Evict)
assert.Equal(t, []string{"a", "v"}, result.TargetSet)
// L is running. Switching to a:
// s1 cost: L is NOT in s1, so 30
// s2 cost: L is in s2, so 0
// => pick s2
result, err = solver.Solve("a", []string{"L"})
require.NoError(t, err)
assert.Empty(t, result.Evict)
assert.Equal(t, []string{"a", "L"}, result.TargetSet)
}
func TestMatrixSolver_TieBreakingByDefinitionOrder(t *testing.T) {
// Two sets with identical cost. Definition order should win.
solver := NewMatrixSolver(
makeExpandedSets(
es("s1", "a", "x"),
es("s2", "a", "y"),
),
nil,
)
// Nothing running, both sets cost 0. s1 is first.
result, err := solver.Solve("a", []string{})
require.NoError(t, err)
assert.Empty(t, result.Evict)
assert.Equal(t, []string{"a", "x"}, result.TargetSet)
}
func TestMatrixSolver_EvictCostPreservesExpensive(t *testing.T) {
// Model "v" costs 50 to evict, "m" costs 1 (default).
// Sets: [g,v], [g,m]
// Running: v, m. Request g.
// s1=[g,v]: evict m (cost 1), keep v
// s2=[g,m]: evict v (cost 50), keep m
// => pick s1
solver := NewMatrixSolver(
makeExpandedSets(
es("s1", "g", "v"),
es("s2", "g", "m"),
),
map[string]int{"v": 50},
)
result, err := solver.Solve("g", []string{"v", "m"})
require.NoError(t, err)
assert.Equal(t, []string{"m"}, result.Evict)
assert.Equal(t, []string{"g", "v"}, result.TargetSet)
}
func TestMatrixSolver_NothingRunning(t *testing.T) {
solver := NewMatrixSolver(
makeExpandedSets(
es("s1", "g", "v"),
es("s2", "q", "v"),
),
nil,
)
result, err := solver.Solve("g", []string{})
require.NoError(t, err)
assert.Empty(t, result.Evict)
assert.Equal(t, []string{"g", "v"}, result.TargetSet)
}
func TestMatrixSolver_FullScenario(t *testing.T) {
// Simulates the example config:
// standard: [g,v], [q,v], [m,v]
// with_rerank: [g,v,e], [q,v,e]
// creative: [g,sd], [q,sd]
// full: [L]
solver := NewMatrixSolver(
makeExpandedSets(
es("standard", "g", "v"),
es("standard", "q", "v"),
es("standard", "m", "v"),
es("with_rerank", "e", "g", "v"),
es("with_rerank", "e", "q", "v"),
es("creative", "g", "sd"),
es("creative", "q", "sd"),
es("full", "L"),
),
map[string]int{"v": 50, "L": 30, "whisper": 10},
)
// Running: g, v. Request q.
// standard[q,v]: evict g (cost 1), keep v. Total: 1.
// with_rerank[q,v,e]: evict g (cost 1), keep v. Total: 1.
// => tie, pick first by definition order = standard[q,v]
result, err := solver.Solve("q", []string{"g", "v"})
require.NoError(t, err)
assert.Equal(t, []string{"g"}, result.Evict)
assert.Equal(t, []string{"q", "v"}, result.TargetSet)
// Running: g, v. Request L.
// full[L]: evict g (cost 1) + v (cost 50). Total: 51.
// Only one set contains L, so pick it.
result, err = solver.Solve("L", []string{"g", "v"})
require.NoError(t, err)
assert.ElementsMatch(t, []string{"g", "v"}, result.Evict)
assert.Equal(t, []string{"L"}, result.TargetSet)
// Running: g, v. Request sd.
// creative[g,sd]: evict v (cost 50). Total: 50.
// creative[q,sd]: evict g (cost 1) + v (cost 50). Total: 51.
// => pick creative[g,sd]
result, err = solver.Solve("sd", []string{"g", "v"})
require.NoError(t, err)
assert.Equal(t, []string{"v"}, result.Evict)
assert.Equal(t, []string{"g", "sd"}, result.TargetSet)
// Running: q, v, e. Request g.
// standard[g,v]: evict q (1) + e (1). Total: 2.
// with_rerank[g,v,e]: evict q (1). Total: 1.
// creative[g,sd]: evict q (1) + v (50) + e (1). Total: 52.
// => pick with_rerank[g,v,e]
result, err = solver.Solve("g", []string{"e", "q", "v"})
require.NoError(t, err)
assert.Equal(t, []string{"q"}, result.Evict)
assert.Equal(t, []string{"e", "g", "v"}, result.TargetSet)
}
proxy/peerproxy.go
+1 -1
View File
@@ -42,7 +42,7 @@ func NewPeerProxy(peers config.PeerDictionaryConfig, proxyLogger *LogMonitor) (*
Proxy: http.ProxyFromEnvironment, Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{ DialContext: (&net.Dialer{
Timeout: time.Duration(peer.Timeouts.Connect) * time.Second, Timeout: time.Duration(peer.Timeouts.Connect) * time.Second,
KeepAlive: 30 * time.Second, KeepAlive: time.Duration(peer.Timeouts.KeepAlive) * time.Second,
}).DialContext, }).DialContext,
TLSHandshakeTimeout: time.Duration(peer.Timeouts.TLSHandshake) * time.Second, TLSHandshakeTimeout: time.Duration(peer.Timeouts.TLSHandshake) * time.Second,
ResponseHeaderTimeout: time.Duration(peer.Timeouts.ResponseHeader) * time.Second, ResponseHeaderTimeout: time.Duration(peer.Timeouts.ResponseHeader) * time.Second,
proxy/process.go
+1 -1
View File
@@ -102,7 +102,7 @@ func NewProcess(ID string, healthCheckTimeout int, config config.ModelConfig, pr
Proxy: http.ProxyFromEnvironment, Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{ DialContext: (&net.Dialer{
Timeout: time.Duration(config.Timeouts.Connect) * time.Second, Timeout: time.Duration(config.Timeouts.Connect) * time.Second,
KeepAlive: 30 * time.Second, KeepAlive: time.Duration(config.Timeouts.KeepAlive) * time.Second,
}).DialContext, }).DialContext,
TLSHandshakeTimeout: time.Duration(config.Timeouts.TLSHandshake) * time.Second, TLSHandshakeTimeout: time.Duration(config.Timeouts.TLSHandshake) * time.Second,
ResponseHeaderTimeout: time.Duration(config.Timeouts.ResponseHeader) * time.Second, ResponseHeaderTimeout: time.Duration(config.Timeouts.ResponseHeader) * time.Second,
proxy/proxymanager.go
+99 -34
View File
@@ -77,6 +77,9 @@ type ProxyManager struct {
processGroups map[string]*ProcessGroup processGroups map[string]*ProcessGroup
// matrix-based swap (mutually exclusive with processGroups)
matrix *Matrix
inFlightCounter *InflightCounter inFlightCounter *InflightCounter
// shutdown signaling // shutdown signaling
@@ -203,10 +206,14 @@ func New(proxyConfig config.Config) *ProxyManager {
peerProxy: peerProxy, peerProxy: peerProxy,
} }
// create the process groups // create either matrix or process groups (mutually exclusive)
for groupID := range proxyConfig.Groups { if proxyConfig.Matrix != nil {
processGroup := NewProcessGroup(groupID, proxyConfig, proxyLogger, upstreamLogger) pm.matrix = NewMatrix(proxyConfig, proxyLogger, upstreamLogger)
pm.processGroups[groupID] = processGroup } else {
for groupID := range proxyConfig.Groups {
processGroup := NewProcessGroup(groupID, proxyConfig, proxyLogger, upstreamLogger)
pm.processGroups[groupID] = processGroup
}
} }
pm.setupGinEngine() pm.setupGinEngine()
@@ -225,18 +232,29 @@ func New(proxyConfig config.Config) *ProxyManager {
} }
proxyLogger.Infof("Preloading model: %s", modelID) proxyLogger.Infof("Preloading model: %s", modelID)
processGroup, err := pm.swapProcessGroup(modelID)
if err != nil { var preloadErr error
req, _ := http.NewRequest("GET", "/", nil)
if pm.matrix != nil {
preloadErr = pm.matrix.ProxyRequest(modelID, discardWriter, req)
} else {
processGroup, err := pm.swapProcessGroup(modelID)
if err != nil {
preloadErr = err
} else {
preloadErr = processGroup.ProxyRequest(modelID, discardWriter, req)
}
}
if preloadErr != nil {
event.Emit(ModelPreloadedEvent{ event.Emit(ModelPreloadedEvent{
ModelName: modelID, ModelName: modelID,
Success: false, Success: false,
}) })
proxyLogger.Errorf("Failed to preload model %s: %v", modelID, err) proxyLogger.Errorf("Failed to preload model %s: %v", modelID, preloadErr)
continue continue
} else { } else {
req, _ := http.NewRequest("GET", "/", nil)
processGroup.ProxyRequest(modelID, discardWriter, req)
event.Emit(ModelPreloadedEvent{ event.Emit(ModelPreloadedEvent{
ModelName: modelID, ModelName: modelID,
Success: true, Success: true,
@@ -453,6 +471,11 @@ func (pm *ProxyManager) StopProcesses(strategy StopStrategy) {
pm.Lock() pm.Lock()
defer pm.Unlock() defer pm.Unlock()
if pm.matrix != nil {
pm.matrix.StopProcesses(strategy)
return
}
// stop Processes in parallel // stop Processes in parallel
var wg sync.WaitGroup var wg sync.WaitGroup
for _, processGroup := range pm.processGroups { for _, processGroup := range pm.processGroups {
@@ -473,6 +496,12 @@ func (pm *ProxyManager) Shutdown() {
pm.proxyLogger.Debug("Shutdown() called in proxy manager") pm.proxyLogger.Debug("Shutdown() called in proxy manager")
if pm.matrix != nil {
pm.matrix.Shutdown()
pm.shutdownCancel()
return
}
var wg sync.WaitGroup var wg sync.WaitGroup
// Send shutdown signal to all process in groups // Send shutdown signal to all process in groups
for _, processGroup := range pm.processGroups { for _, processGroup := range pm.processGroups {
@@ -639,10 +668,16 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
return return
} }
processGroup, err := pm.swapProcessGroup(modelID) var handler func(string, http.ResponseWriter, *http.Request) error
if err != nil { if pm.matrix != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error())) handler = pm.matrix.ProxyRequest
return } else {
processGroup, err := pm.swapProcessGroup(modelID)
if err != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
return
}
handler = processGroup.ProxyRequest
} }
// rewrite the path // rewrite the path
@@ -651,13 +686,13 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
// attempt to record metrics if it is a POST request // attempt to record metrics if it is a POST request
if pm.metricsMonitor != nil && c.Request.Method == "POST" { if pm.metricsMonitor != nil && c.Request.Method == "POST" {
if err := pm.metricsMonitor.wrapHandler(modelID, c.Writer, c.Request, processGroup.ProxyRequest); err != nil { if err := pm.metricsMonitor.wrapHandler(modelID, c.Writer, c.Request, handler); err != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying metrics wrapped request: %s", err.Error())) pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying metrics wrapped request: %s", err.Error()))
pm.proxyLogger.Errorf("Error proxying wrapped upstream request for model %s, path=%s", modelID, originalPath) pm.proxyLogger.Errorf("Error proxying wrapped upstream request for model %s, path=%s", modelID, originalPath)
return return
} }
} else { } else {
if err := processGroup.ProxyRequest(modelID, c.Writer, c.Request); err != nil { if err := handler(modelID, c.Writer, c.Request); err != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error())) pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error()))
pm.proxyLogger.Errorf("Error proxying upstream request for model %s, path=%s", modelID, originalPath) pm.proxyLogger.Errorf("Error proxying upstream request for model %s, path=%s", modelID, originalPath)
return return
@@ -683,10 +718,16 @@ func (pm *ProxyManager) proxyInferenceHandler(c *gin.Context) {
modelID, found := pm.config.RealModelName(requestedModel) modelID, found := pm.config.RealModelName(requestedModel)
if found { if found {
processGroup, err := pm.swapProcessGroup(modelID) var localHandler func(string, http.ResponseWriter, *http.Request) error
if err != nil { if pm.matrix != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error())) localHandler = pm.matrix.ProxyRequest
return } else {
processGroup, err := pm.swapProcessGroup(modelID)
if err != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
return
}
localHandler = processGroup.ProxyRequest
} }
// issue #69 allow custom model names to be sent to upstream // issue #69 allow custom model names to be sent to upstream
@@ -737,7 +778,7 @@ func (pm *ProxyManager) proxyInferenceHandler(c *gin.Context) {
} }
pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel) pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
nextHandler = processGroup.ProxyRequest nextHandler = localHandler
} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) { } else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel) pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
modelID = requestedModel modelID = requestedModel
@@ -823,15 +864,19 @@ func (pm *ProxyManager) proxyOAIPostFormHandler(c *gin.Context) {
modelID, found := pm.config.RealModelName(requestedModel) modelID, found := pm.config.RealModelName(requestedModel)
if found { if found {
processGroup, err := pm.swapProcessGroup(modelID) if pm.matrix != nil {
if err != nil { nextHandler = pm.matrix.ProxyRequest
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error())) } else {
return processGroup, err := pm.swapProcessGroup(modelID)
if err != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
return
}
nextHandler = processGroup.ProxyRequest
} }
useModelName = pm.config.Models[modelID].UseModelName useModelName = pm.config.Models[modelID].UseModelName
pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel) pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
nextHandler = processGroup.ProxyRequest
} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) { } else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel) pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
modelID = requestedModel modelID = requestedModel
@@ -942,14 +987,18 @@ func (pm *ProxyManager) proxyGETModelHandler(c *gin.Context) {
var modelID string var modelID string
if realModelID, found := pm.config.RealModelName(requestedModel); found { if realModelID, found := pm.config.RealModelName(requestedModel); found {
processGroup, err := pm.swapProcessGroup(realModelID)
if err != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
return
}
modelID = realModelID modelID = realModelID
if pm.matrix != nil {
nextHandler = pm.matrix.ProxyRequest
} else {
processGroup, err := pm.swapProcessGroup(realModelID)
if err != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
return
}
nextHandler = processGroup.ProxyRequest
}
pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel) pm.proxyLogger.Debugf("ProxyManager using local Process for model: %s", requestedModel)
nextHandler = processGroup.ProxyRequest
} else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) { } else if pm.peerProxy != nil && pm.peerProxy.HasPeerModel(requestedModel) {
modelID = requestedModel modelID = requestedModel
pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel) pm.proxyLogger.Debugf("ProxyManager using ProxyPeer for model: %s", requestedModel)
@@ -1048,9 +1097,9 @@ func (pm *ProxyManager) listRunningProcessesHandler(context *gin.Context) {
context.Header("Content-Type", "application/json") context.Header("Content-Type", "application/json")
runningProcesses := make([]gin.H, 0) // Default to an empty response. runningProcesses := make([]gin.H, 0) // Default to an empty response.
for _, processGroup := range pm.processGroups { if pm.matrix != nil {
for _, process := range processGroup.processes { for _, modelID := range pm.matrix.RunningModels() {
if process.CurrentState() == StateReady { if process, ok := pm.matrix.GetProcess(modelID); ok {
runningProcesses = append(runningProcesses, gin.H{ runningProcesses = append(runningProcesses, gin.H{
"model": process.ID, "model": process.ID,
"state": process.state, "state": process.state,
@@ -1062,6 +1111,22 @@ func (pm *ProxyManager) listRunningProcessesHandler(context *gin.Context) {
}) })
} }
} }
} else {
for _, processGroup := range pm.processGroups {
for _, process := range processGroup.processes {
if process.CurrentState() == StateReady {
runningProcesses = append(runningProcesses, gin.H{
"model": process.ID,
"state": process.state,
"cmd": process.config.Cmd,
"proxy": process.config.Proxy,
"ttl": process.config.UnloadAfter,
"name": process.config.Name,
"description": process.config.Description,
})
}
}
}
} }
// Put the results under the `running` key. // Put the results under the `running` key.
proxy/proxymanager_api.go
+34 -28
View File
@@ -55,27 +55,28 @@ func (pm *ProxyManager) getModelStatus() []Model {
// Iterate over sorted keys // Iterate over sorted keys
for _, modelID := range modelIDs { for _, modelID := range modelIDs {
// Get process state // Get process state
processGroup := pm.findGroupByModelName(modelID)
state := "unknown" state := "unknown"
if processGroup != nil { var process *Process
process := processGroup.processes[modelID] if pm.matrix != nil {
if process != nil { process, _ = pm.matrix.GetProcess(modelID)
var stateStr string } else {
switch process.CurrentState() { processGroup := pm.findGroupByModelName(modelID)
case StateReady: if processGroup != nil {
stateStr = "ready" process = processGroup.processes[modelID]
case StateStarting: }
stateStr = "starting" }
case StateStopping: if process != nil {
stateStr = "stopping" switch process.CurrentState() {
case StateShutdown: case StateReady:
stateStr = "shutdown" state = "ready"
case StateStopped: case StateStarting:
stateStr = "stopped" state = "starting"
default: case StateStopping:
stateStr = "unknown" state = "stopping"
} case StateShutdown:
state = stateStr state = "shutdown"
case StateStopped:
state = "stopped"
} }
} }
models = append(models, Model{ models = append(models, Model{
@@ -254,18 +255,23 @@ func (pm *ProxyManager) apiUnloadSingleModelHandler(c *gin.Context) {
return return
} }
processGroup := pm.findGroupByModelName(realModelName) var stopErr error
if processGroup == nil { if pm.matrix != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("process group not found for model %s", requestedModel)) stopErr = pm.matrix.StopProcess(realModelName, StopImmediately)
return } else {
processGroup := pm.findGroupByModelName(realModelName)
if processGroup == nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("process group not found for model %s", requestedModel))
return
}
stopErr = processGroup.StopProcess(realModelName, StopImmediately)
} }
if err := processGroup.StopProcess(realModelName, StopImmediately); err != nil { if stopErr != nil {
pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error stopping process: %s", err.Error())) pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error stopping process: %s", stopErr.Error()))
return return
} else {
c.String(http.StatusOK, "OK")
} }
c.String(http.StatusOK, "OK")
} }
func (pm *ProxyManager) apiGetVersion(c *gin.Context) { func (pm *ProxyManager) apiGetVersion(c *gin.Context) {
ui-svelte/package-lock.json
Generated
+98 -105
View File
@@ -37,21 +37,21 @@
} }
}, },
"node_modules/@emnapi/core": { "node_modules/@emnapi/core": {
"version": "1.9.0", "version": "1.9.2",
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.0.tgz", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz",
"integrity": "sha512-0DQ98G9ZQZOxfUcQn1waV2yS8aWdZ6kJMbYCJB3oUBecjWYO1fqJ+a1DRfPF3O5JEkwqwP1A9QEN/9mYm2Yd0w==", "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==",
"dev": true, "dev": true,
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"dependencies": { "dependencies": {
"@emnapi/wasi-threads": "1.2.0", "@emnapi/wasi-threads": "1.2.1",
"tslib": "^2.4.0" "tslib": "^2.4.0"
} }
}, },
"node_modules/@emnapi/runtime": { "node_modules/@emnapi/runtime": {
"version": "1.9.0", "version": "1.9.2",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.0.tgz", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz",
"integrity": "sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==", "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==",
"dev": true, "dev": true,
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
@@ -60,9 +60,9 @@
} }
}, },
"node_modules/@emnapi/wasi-threads": { "node_modules/@emnapi/wasi-threads": {
"version": "1.2.0", "version": "1.2.1",
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.0.tgz", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz",
"integrity": "sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg==", "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==",
"dev": true, "dev": true,
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
@@ -121,36 +121,28 @@
} }
}, },
"node_modules/@napi-rs/wasm-runtime": { "node_modules/@napi-rs/wasm-runtime": {
"version": "1.1.1", "version": "1.1.3",
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.3.tgz",
"integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==", "integrity": "sha512-xK9sGVbJWYb08+mTJt3/YV24WxvxpXcXtP6B172paPZ+Ts69Re9dAr7lKwJoeIx8OoeuimEiRZ7umkiUVClmmQ==",
"dev": true, "dev": true,
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"dependencies": { "dependencies": {
"@emnapi/core": "^1.7.1",
"@emnapi/runtime": "^1.7.1",
"@tybys/wasm-util": "^0.10.1" "@tybys/wasm-util": "^0.10.1"
}, },
"funding": { "funding": {
"type": "github", "type": "github",
"url": "https://github.com/sponsors/Brooooooklyn" "url": "https://github.com/sponsors/Brooooooklyn"
} },
}, "peerDependencies": {
"node_modules/@oxc-project/runtime": { "@emnapi/core": "^1.7.1",
"version": "0.115.0", "@emnapi/runtime": "^1.7.1"
"resolved": "https://registry.npmjs.org/@oxc-project/runtime/-/runtime-0.115.0.tgz",
"integrity": "sha512-Rg8Wlt5dCbXhQnsXPrkOjL1DTSvXLgb2R/KYfnf1/K+R0k6UMLEmbQXPM+kwrWqSmWA2t0B1EtHy2/3zikQpvQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": "^20.19.0 || >=22.12.0"
} }
}, },
"node_modules/@oxc-project/types": { "node_modules/@oxc-project/types": {
"version": "0.115.0", "version": "0.124.0",
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.115.0.tgz", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.124.0.tgz",
"integrity": "sha512-4n91DKnebUS4yjUHl2g3/b2T+IUdCfmoZGhmwsovZCDaJSs+QkVAM+0AqqTxHSsHfeiMuueT75cZaZcT/m0pSw==", "integrity": "sha512-VBFWMTBvHxS11Z5Lvlr3IWgrwhMTXV+Md+EQF0Xf60+wAdsGFTBx7X7K/hP4pi8N7dcm1RvcHwDxZ16Qx8keUg==",
"dev": true, "dev": true,
"license": "MIT", "license": "MIT",
"funding": { "funding": {
@@ -158,9 +150,9 @@
} }
}, },
"node_modules/@rolldown/binding-android-arm64": { "node_modules/@rolldown/binding-android-arm64": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.15.tgz",
"integrity": "sha512-lcJL0bN5hpgJfSIz/8PIf02irmyL43P+j1pTCfbD1DbLkmGRuFIA4DD3B3ZOvGqG0XiVvRznbKtN0COQVaKUTg==", "integrity": "sha512-YYe6aWruPZDtHNpwu7+qAHEMbQ/yRl6atqb/AhznLTnD3UY99Q1jE7ihLSahNWkF4EqRPVC4SiR4O0UkLK02tA==",
"cpu": [ "cpu": [
"arm64" "arm64"
], ],
@@ -175,9 +167,9 @@
} }
}, },
"node_modules/@rolldown/binding-darwin-arm64": { "node_modules/@rolldown/binding-darwin-arm64": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.15.tgz",
"integrity": "sha512-J7Zk3kLYFsLtuH6U+F4pS2sYVzac0qkjcO5QxHS7OS7yZu2LRs+IXo+uvJ/mvpyUljDJ3LROZPoQfgBIpCMhdQ==", "integrity": "sha512-oArR/ig8wNTPYsXL+Mzhs0oxhxfuHRfG7Ikw7jXsw8mYOtk71W0OkF2VEVh699pdmzjPQsTjlD1JIOoHkLP1Fg==",
"cpu": [ "cpu": [
"arm64" "arm64"
], ],
@@ -192,9 +184,9 @@
} }
}, },
"node_modules/@rolldown/binding-darwin-x64": { "node_modules/@rolldown/binding-darwin-x64": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.15.tgz",
"integrity": "sha512-iwtmmghy8nhfRGeNAIltcNXzD0QMNaaA5U/NyZc1Ia4bxrzFByNMDoppoC+hl7cDiUq5/1CnFthpT9n+UtfFyg==", "integrity": "sha512-YzeVqOqjPYvUbJSWJ4EDL8ahbmsIXQpgL3JVipmN+MX0XnXMeWomLN3Fb+nwCmP/jfyqte5I3XRSm7OfQrbyxw==",
"cpu": [ "cpu": [
"x64" "x64"
], ],
@@ -209,9 +201,9 @@
} }
}, },
"node_modules/@rolldown/binding-freebsd-x64": { "node_modules/@rolldown/binding-freebsd-x64": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.15.tgz",
"integrity": "sha512-DLFYI78SCiZr5VvdEplsVC2Vx53lnA4/Ga5C65iyldMVaErr86aiqCoNBLl92PXPfDtUYjUh+xFFor40ueNs4Q==", "integrity": "sha512-9Erhx956jeQ0nNTyif1+QWAXDRD38ZNjr//bSHrt6wDwB+QkAfl2q6Mn1k6OBPerznjRmbM10lgRb1Pli4xZPw==",
"cpu": [ "cpu": [
"x64" "x64"
], ],
@@ -226,9 +218,9 @@
} }
}, },
"node_modules/@rolldown/binding-linux-arm-gnueabihf": { "node_modules/@rolldown/binding-linux-arm-gnueabihf": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.15.tgz",
"integrity": "sha512-CsjTmTwd0Hri6iTw/DRMK7kOZ7FwAkrO4h8YWKoX/kcj833e4coqo2wzIFywtch/8Eb5enQ/lwLM7w6JX1W5RQ==", "integrity": "sha512-cVwk0w8QbZJGTnP/AHQBs5yNwmpgGYStL88t4UIaqcvYJWBfS0s3oqVLZPwsPU6M0zlW4GqjP0Zq5MnAGwFeGA==",
"cpu": [ "cpu": [
"arm" "arm"
], ],
@@ -243,9 +235,9 @@
} }
}, },
"node_modules/@rolldown/binding-linux-arm64-gnu": { "node_modules/@rolldown/binding-linux-arm64-gnu": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.15.tgz",
"integrity": "sha512-2x9O2JbSPxpxMDhP9Z74mahAStibTlrBMW0520+epJH5sac7/LwZW5Bmg/E6CXuEF53JJFW509uP+lSedaUNxg==", "integrity": "sha512-eBZ/u8iAK9SoHGanqe/jrPnY0JvBN6iXbVOsbO38mbz+ZJsaobExAm1Iu+rxa4S1l2FjG0qEZn4Rc6X8n+9M+w==",
"cpu": [ "cpu": [
"arm64" "arm64"
], ],
@@ -260,9 +252,9 @@
} }
}, },
"node_modules/@rolldown/binding-linux-arm64-musl": { "node_modules/@rolldown/binding-linux-arm64-musl": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.15.tgz",
"integrity": "sha512-JA1QRW31ogheAIRhIg9tjMfsYbglXXYGNPLdPEYrwFxdbkQCAzvpSCSHCDWNl4hTtrol8WeboCSEpjdZK8qrCg==", "integrity": "sha512-ZvRYMGrAklV9PEkgt4LQM6MjQX2P58HPAuecwYObY2DhS2t35R0I810bKi0wmaYORt6m/2Sm+Z+nFgb0WhXNcQ==",
"cpu": [ "cpu": [
"arm64" "arm64"
], ],
@@ -277,9 +269,9 @@
} }
}, },
"node_modules/@rolldown/binding-linux-ppc64-gnu": { "node_modules/@rolldown/binding-linux-ppc64-gnu": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.15.tgz",
"integrity": "sha512-aOKU9dJheda8Kj8Y3w9gnt9QFOO+qKPAl8SWd7JPHP+Cu0EuDAE5wokQubLzIDQWg2myXq2XhTpOVS07qqvT+w==", "integrity": "sha512-VDpgGBzgfg5hLg+uBpCLoFG5kVvEyafmfxGUV0UHLcL5irxAK7PKNeC2MwClgk6ZAiNhmo9FLhRYgvMmedLtnQ==",
"cpu": [ "cpu": [
"ppc64" "ppc64"
], ],
@@ -294,9 +286,9 @@
} }
}, },
"node_modules/@rolldown/binding-linux-s390x-gnu": { "node_modules/@rolldown/binding-linux-s390x-gnu": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.15.tgz",
"integrity": "sha512-OalO94fqj7IWRn3VdXWty75jC5dk4C197AWEuMhIpvVv2lw9fiPhud0+bW2ctCxb3YoBZor71QHbY+9/WToadA==", "integrity": "sha512-y1uXY3qQWCzcPgRJATPSOUP4tCemh4uBdY7e3EZbVwCJTY3gLJWnQABgeUetvED+bt1FQ01OeZwvhLS2bpNrAQ==",
"cpu": [ "cpu": [
"s390x" "s390x"
], ],
@@ -311,9 +303,9 @@
} }
}, },
"node_modules/@rolldown/binding-linux-x64-gnu": { "node_modules/@rolldown/binding-linux-x64-gnu": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.15.tgz",
"integrity": "sha512-cVEl1vZtBsBZna3YMjGXNvnYYrOJ7RzuWvZU0ffvJUexWkukMaDuGhUXn0rjnV0ptzGVkvc+vW9Yqy6h8YX4pg==", "integrity": "sha512-023bTPBod7J3Y/4fzAN6QtpkSABR0rigtrwaP+qSEabUh5zf6ELr9Nc7GujaROuPY3uwdSIXWrvhn1KxOvurWA==",
"cpu": [ "cpu": [
"x64" "x64"
], ],
@@ -328,9 +320,9 @@
} }
}, },
"node_modules/@rolldown/binding-linux-x64-musl": { "node_modules/@rolldown/binding-linux-x64-musl": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.15.tgz",
"integrity": "sha512-UzYnKCIIc4heAKgI4PZ3dfBGUZefGCJ1TPDuLHoCzgrMYPb5Rv6TLFuYtyM4rWyHM7hymNdsg5ik2C+UD9VDbA==", "integrity": "sha512-witB2O0/hU4CgfOOKUoeFgQ4GktPi1eEbAhaLAIpgD6+ZnhcPkUtPsoKKHRzmOoWPZue46IThdSgdo4XneOLYw==",
"cpu": [ "cpu": [
"x64" "x64"
], ],
@@ -345,9 +337,9 @@
} }
}, },
"node_modules/@rolldown/binding-openharmony-arm64": { "node_modules/@rolldown/binding-openharmony-arm64": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.15.tgz",
"integrity": "sha512-+6zoiF+RRyf5cdlFQP7nm58mq7+/2PFaY2DNQeD4B87N36JzfF/l9mdBkkmTvSYcYPE8tMh/o3cRlsx1ldLfog==", "integrity": "sha512-UCL68NJ0Ud5zRipXZE9dF5PmirzJE4E4BCIOOssEnM7wLDsxjc6Qb0sGDxTNRTP53I6MZpygyCpY8Aa8sPfKPg==",
"cpu": [ "cpu": [
"arm64" "arm64"
], ],
@@ -362,9 +354,9 @@
} }
}, },
"node_modules/@rolldown/binding-wasm32-wasi": { "node_modules/@rolldown/binding-wasm32-wasi": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.15.tgz",
"integrity": "sha512-rgFN6sA/dyebil3YTlL2evvi/M+ivhfnyxec7AccTpRPccno/rPoNlqybEZQBkcbZu8Hy+eqNJCqfBR8P7Pg8g==", "integrity": "sha512-ApLruZq/ig+nhaE7OJm4lDjayUnOHVUa77zGeqnqZ9pn0ovdVbbNPerVibLXDmWeUZXjIYIT8V3xkT58Rm9u5Q==",
"cpu": [ "cpu": [
"wasm32" "wasm32"
], ],
@@ -372,16 +364,18 @@
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"dependencies": { "dependencies": {
"@napi-rs/wasm-runtime": "^1.1.1" "@emnapi/core": "1.9.2",
"@emnapi/runtime": "1.9.2",
"@napi-rs/wasm-runtime": "^1.1.3"
}, },
"engines": { "engines": {
"node": ">=14.0.0" "node": ">=14.0.0"
} }
}, },
"node_modules/@rolldown/binding-win32-arm64-msvc": { "node_modules/@rolldown/binding-win32-arm64-msvc": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.15.tgz",
"integrity": "sha512-lHVNUG/8nlF1IQk1C0Ci574qKYyty2goMiPlRqkC5R+3LkXDkL5Dhx8ytbxq35m+pkHVIvIxviD+TWLdfeuadA==", "integrity": "sha512-KmoUoU7HnN+Si5YWJigfTws1jz1bKBYDQKdbLspz0UaqjjFkddHsqorgiW1mxcAj88lYUE6NC/zJNwT+SloqtA==",
"cpu": [ "cpu": [
"arm64" "arm64"
], ],
@@ -396,9 +390,9 @@
} }
}, },
"node_modules/@rolldown/binding-win32-x64-msvc": { "node_modules/@rolldown/binding-win32-x64-msvc": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.15.tgz",
"integrity": "sha512-G0oA4+w1iY5AGi5HcDTxWsoxF509hrFIPB2rduV5aDqS9FtDg1CAfa7V34qImbjfhIcA8C+RekocJZA96EarwQ==", "integrity": "sha512-3P2A8L+x75qavWLe/Dll3EYBJLQmtkJN8rfh+U/eR3MqMgL/h98PhYI+JFfXuDPgPeCB7iZAKiqii5vqOvnA0g==",
"cpu": [ "cpu": [
"x64" "x64"
], ],
@@ -413,9 +407,9 @@
} }
}, },
"node_modules/@rolldown/pluginutils": { "node_modules/@rolldown/pluginutils": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.15.tgz",
"integrity": "sha512-w6oiRWgEBl04QkFZgmW+jnU1EC9b57Oihi2ot3HNWIQRqgHp5PnYDia5iZ5FF7rpa4EQdiqMDXjlqKGXBhsoXw==", "integrity": "sha512-UromN0peaE53IaBRe9W7CjrZgXl90fqGpK+mIZbA3qSTeYqg3pqpROBdIPvOG3F5ereDHNwoHBI2e50n1BDr1g==",
"dev": true, "dev": true,
"license": "MIT" "license": "MIT"
}, },
@@ -2972,14 +2966,14 @@
} }
}, },
"node_modules/rolldown": { "node_modules/rolldown": {
"version": "1.0.0-rc.9", "version": "1.0.0-rc.15",
"resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.9.tgz", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.15.tgz",
"integrity": "sha512-9EbgWge7ZH+yqb4d2EnELAntgPTWbfL8ajiTW+SyhJEC4qhBbkCKbqFV4Ge4zmu5ziQuVbWxb/XwLZ+RIO7E8Q==", "integrity": "sha512-Ff31guA5zT6WjnGp0SXw76X6hzGRk/OQq2hE+1lcDe+lJdHSgnSX6nK3erbONHyCbpSj9a9E+uX/OvytZoWp2g==",
"dev": true, "dev": true,
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@oxc-project/types": "=0.115.0", "@oxc-project/types": "=0.124.0",
"@rolldown/pluginutils": "1.0.0-rc.9" "@rolldown/pluginutils": "1.0.0-rc.15"
}, },
"bin": { "bin": {
"rolldown": "bin/cli.mjs" "rolldown": "bin/cli.mjs"
@@ -2988,21 +2982,21 @@
"node": "^20.19.0 || >=22.12.0" "node": "^20.19.0 || >=22.12.0"
}, },
"optionalDependencies": { "optionalDependencies": {
"@rolldown/binding-android-arm64": "1.0.0-rc.9", "@rolldown/binding-android-arm64": "1.0.0-rc.15",
"@rolldown/binding-darwin-arm64": "1.0.0-rc.9", "@rolldown/binding-darwin-arm64": "1.0.0-rc.15",
"@rolldown/binding-darwin-x64": "1.0.0-rc.9", "@rolldown/binding-darwin-x64": "1.0.0-rc.15",
"@rolldown/binding-freebsd-x64": "1.0.0-rc.9", "@rolldown/binding-freebsd-x64": "1.0.0-rc.15",
"@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.9", "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.15",
"@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.9", "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.15",
"@rolldown/binding-linux-arm64-musl": "1.0.0-rc.9", "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.15",
"@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.9", "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.15",
"@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.9", "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.15",
"@rolldown/binding-linux-x64-gnu": "1.0.0-rc.9", "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.15",
"@rolldown/binding-linux-x64-musl": "1.0.0-rc.9", "@rolldown/binding-linux-x64-musl": "1.0.0-rc.15",
"@rolldown/binding-openharmony-arm64": "1.0.0-rc.9", "@rolldown/binding-openharmony-arm64": "1.0.0-rc.15",
"@rolldown/binding-wasm32-wasi": "1.0.0-rc.9", "@rolldown/binding-wasm32-wasi": "1.0.0-rc.15",
"@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.9", "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.15",
"@rolldown/binding-win32-x64-msvc": "1.0.0-rc.9" "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.15"
} }
}, },
"node_modules/sade": { "node_modules/sade": {
@@ -3416,17 +3410,16 @@
} }
}, },
"node_modules/vite": { "node_modules/vite": {
"version": "8.0.0", "version": "8.0.8",
"resolved": "https://registry.npmjs.org/vite/-/vite-8.0.0.tgz", "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.8.tgz",
"integrity": "sha512-fPGaRNj9Zytaf8LEiBhY7Z6ijnFKdzU/+mL8EFBaKr7Vw1/FWcTBAMW0wLPJAGMPX38ZPVCVgLceWiEqeoqL2Q==", "integrity": "sha512-dbU7/iLVa8KZALJyLOBOQ88nOXtNG8vxKuOT4I2mD+Ya70KPceF4IAmDsmU0h1Qsn5bPrvsY9HJstCRh3hG6Uw==",
"dev": true, "dev": true,
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@oxc-project/runtime": "0.115.0",
"lightningcss": "^1.32.0", "lightningcss": "^1.32.0",
"picomatch": "^4.0.3", "picomatch": "^4.0.4",
"postcss": "^8.5.8", "postcss": "^8.5.8",
"rolldown": "1.0.0-rc.9", "rolldown": "1.0.0-rc.15",
"tinyglobby": "^0.2.15" "tinyglobby": "^0.2.15"
}, },
"bin": { "bin": {
@@ -3443,8 +3436,8 @@
}, },
"peerDependencies": { "peerDependencies": {
"@types/node": "^20.19.0 || >=22.12.0", "@types/node": "^20.19.0 || >=22.12.0",
"@vitejs/devtools": "^0.0.0-alpha.31", "@vitejs/devtools": "^0.1.0",
"esbuild": "^0.27.0", "esbuild": "^0.27.0 || ^0.28.0",
"jiti": ">=1.21.0", "jiti": ">=1.21.0",
"less": "^4.0.0", "less": "^4.0.0",
"sass": "^1.70.0", "sass": "^1.70.0",