Automatic Port Numbers (#105)

Add automatic port numbers assignment in configuration file. The string `${PORT}` will be substituted in model.cmd and model.proxy for an actual port number. This also allows model.proxy to be omitted from the configuration.

.github/workflows/closeinactive.yml

@@ -0,0 +1,23 @@
+# https://docs.github.com/en/actions/use-cases-and-examples/project-management/closing-inactive-issues
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "32 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-issue-stale: 14
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 2 weeks with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 2 weeks since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/containers.yml

@@ -0,0 +1,47 @@
+name: Build Containers
+
+on:
+  # time has no specific meaning, trying to time it after
+  # the llama.cpp daily packages are published
+  # https://github.com/ggml-org/llama.cpp/blob/master/.github/workflows/docker.yml
+  schedule:
+    - cron: "37 5 * * *"
+
+  # Allows manual triggering of the workflow
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        #platform: [intel, cuda, vulkan, cpu, musa]
+        platform: [cuda, vulkan, cpu, musa]
+      fail-fast: false
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run build-container
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./docker/build-container.sh ${{ matrix.platform }} true
+
+  # note make sure mostlygeek/llama-swap has admin rights to the llama-swap package
+  # see: https://github.com/actions/delete-package-versions/issues/74
+  delete-untagged-containers:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/delete-package-versions@v5
+        with:
+          package-name: 'llama-swap'
+          package-type: 'container'
+          delete-only-untagged-versions: 'true'

.github/workflows/go-ci.yml

@@ -0,0 +1,32 @@
+# This workflow will build a golang project
+
+name: CI
+
+on:
+  push:
+    branches: [ "main" ]
+
+  pull_request:
+    branches: [ "main" ]
+
+  # Allows manual triggering of the workflow
+  workflow_dispatch:
+
+jobs:
+
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.23'
+
+    # necessary for testing proxy/Process swapping
+    - name: Create simple-responder
+      run: make simple-responder
+
+    - name: Test all
+      run: make test-all

.github/workflows/release.yml

@@ -5,6 +5,9 @@ on:
     tags:
       - '*'
 
+  # Allows manual triggering of the workflow
+  workflow_dispatch:
+
 permissions:
   contents: write
 
@@ -30,4 +33,4 @@ jobs:
           version: '~> v2'
           args: release --clean
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -2,4 +2,5 @@
 .env
 build/
 dist/
-.vscode
+.vscode
+.DS_Store

.goreleaser.yaml

@@ -6,6 +6,25 @@ builds:
     goos:
       - linux
       - darwin
+      - freebsd
+      - windows
     goarch:
       - amd64
-      - arm64
+      - arm64
+    ignore:
+      - goos: freebsd
+        goarch: arm64
+      - goos: windows
+        goarch: arm64
+
+# use zip format for windows
+archives:
+  - id: default
+    format: tar.gz
+    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
+    builds_info:
+      group: root
+      owner: root
+    format_overrides:
+      - goos: windows
+        format: zip

Makefile

@@ -2,6 +2,16 @@
 APP_NAME = llama-swap
 BUILD_DIR = build
 
+# Get the current Git hash
+GIT_HASH := $(shell git rev-parse --short HEAD)
+ifneq ($(shell git status --porcelain),)
+    # There are untracked changes
+    GIT_HASH := $(GIT_HASH)+
+endif
+
+# Capture the current build date in RFC3339 format
+BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
+
 # Default target: Builds binaries for both OSX and Linux
 all: mac linux simple-responder
 
@@ -10,26 +20,49 @@ clean:
 	rm -rf $(BUILD_DIR)
 
 test:
+	go test -short -v ./proxy
+
+test-all:
 	go test -v ./proxy
 
 # Build OSX binary
 mac:
 	@echo "Building Mac binary..."
-	GOOS=darwin GOARCH=arm64 go build -o $(BUILD_DIR)/$(APP_NAME)-darwin-arm64
+	GOOS=darwin GOARCH=arm64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-darwin-arm64
 
 # Build Linux binary
 linux:
 	@echo "Building Linux binary..."
-	GOOS=linux GOARCH=amd64 go build -o $(BUILD_DIR)/$(APP_NAME)-linux-amd64
+	GOOS=linux GOARCH=amd64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-amd64
 
-# for testing things
+# Build Windows binary
+windows:
+	@echo "Building Windows binary..."
+	GOOS=windows GOARCH=amd64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-windows-amd64.exe
+
+# for testing proxy.Process
 simple-responder:
 	@echo "Building simple responder"
-	go build -o $(BUILD_DIR)/simple-responder misc/simple-responder/simple-responder.go
+	GOOS=darwin GOARCH=arm64 go build -o $(BUILD_DIR)/simple-responder_darwin_arm64 misc/simple-responder/simple-responder.go
+	GOOS=linux GOARCH=amd64 go build -o $(BUILD_DIR)/simple-responder_linux_amd64 misc/simple-responder/simple-responder.go
 
 # Ensure build directory exists
 $(BUILD_DIR):
 	mkdir -p $(BUILD_DIR)
 
+# Create a new release tag
+release:
+	@echo "Checking for unstaged changes..."
+	@if [ -n "$(shell git status --porcelain)" ]; then \
+		echo "Error: There are unstaged changes. Please commit or stash your changes before creating a release tag." >&2; \
+		exit 1; \
+	fi
+
+# Get the highest tag in v{number} format, increment it, and create a new tag
+	@highest_tag=$$(git tag --sort=-v:refname | grep -E '^v[0-9]+$$' | head -n 1 || echo "v0"); \
+	new_tag="v$$(( $${highest_tag#v} + 1 ))"; \
+	echo "tagging new version: $$new_tag"; \
+	git tag "$$new_tag";
+
 # Phony targets
-.PHONY: all clean osx linux
+.PHONY: all clean mac linux windows simple-responder

README.md

@@ -1,89 +1,303 @@
+![llama-swap header image](header2.png)
+![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/mostlygeek/llama-swap/total)
+![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/mostlygeek/llama-swap/go-ci.yml)
+![GitHub Repo stars](https://img.shields.io/github/stars/mostlygeek/llama-swap)
+
 # llama-swap
 
-![llama-swap header image](header.jpeg)
+llama-swap is a light weight, transparent proxy server that provides automatic model swapping to llama.cpp's server.
 
-[llama.cpp's server](https://github.com/ggerganov/llama.cpp/tree/master/examples/server) can't swap models on demand. So let's swap the server on demand instead!
+Written in golang, it is very easy to install (single binary with no dependancies) and configure (single yaml file). To get started, download a pre-built binary or use the provided docker images.
 
-llama-swap is a proxy server that sits in front of llama-server. When a request for `/v1/chat/completions` comes in it will extract the `model` requested and change the underlying llama-server automatically.
+## Features:
 
-- ✅ easy to deploy: single binary with no dependencies
-- ✅ full control over llama-server's startup settings
-- ✅ ❤️ for users who are rely on llama.cpp for LLM inference
+- ✅ Easy to deploy: single binary with no dependencies
+- ✅ Easy to config: single yaml file
+- ✅ On-demand model switching
+- ✅ OpenAI API supported endpoints:
+  - `v1/completions`
+  - `v1/chat/completions`
+  - `v1/embeddings`
+  - `v1/rerank`
+  - `v1/audio/speech` ([#36](https://github.com/mostlygeek/llama-swap/issues/36))
+  - `v1/audio/transcriptions` ([docs](https://github.com/mostlygeek/llama-swap/issues/41#issuecomment-2722637867))
+- ✅ llama-swap custom API endpoints
+  - `/log` - remote log monitoring
+  - `/upstream/:model_id` - direct access to upstream HTTP server ([demo](https://github.com/mostlygeek/llama-swap/pull/31))
+  - `/unload` - manually unload running models ([#58](https://github.com/mostlygeek/llama-swap/issues/58))
+  - `/running` - list currently running models ([#61](https://github.com/mostlygeek/llama-swap/issues/61))
+- ✅ Run multiple models at once with `Groups` ([#107](https://github.com/mostlygeek/llama-swap/issues/107))
+- ✅ Automatic unloading of models after timeout by setting a `ttl`
+- ✅ Use any local OpenAI compatible server (llama.cpp, vllm, tabbyAPI, etc)
+- ✅ Docker and Podman support
+- ✅ Full control over server settings per model
+
+## How does llama-swap work?
+
+When a request is made to an OpenAI compatible endpoint, lama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to the correct one to serve the request.
+
+In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `groups` feature allows multiple models to be loaded at the same time. You have complete control over how your system resources are used.
 
 ## config.yaml
 
-llama-swap's configuration purposefully simple.
+llama-swap's configuration is purposefully simple.
+
+```yaml
+models:
+  "qwen2.5":
+    proxy: "http://127.0.0.1:9999"
+    cmd: >
+      /app/llama-server
+      -hf bartowski/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M
+      --port 9999
+
+  "smollm2":
+    proxy: "http://127.0.0.1:9999"
+    cmd: >
+      /app/llama-server
+      -hf bartowski/SmolLM2-135M-Instruct-GGUF:Q4_K_M
+      --port 9999
+```
+
+<details>
+<summary>But also very powerful ...</summary>
 
 ```yaml
 # Seconds to wait for llama.cpp to load and be ready to serve requests
 # Default (and minimum) is 15 seconds
 healthCheckTimeout: 60
 
+# Valid log levels: debug, info (default), warn, error
+logLevel: info
+
+# Automatic Port Values
+# use ${PORT} in model.cmd and model.proxy to use an automatic port number
+# when you use ${PORT} you can omit a custom model.proxy value, as it will
+# default to http://localhost:${PORT}
+
+# override the default port (5800) for automatic port values
+startPort: 10001
+
 # define valid model values and the upstream server start
 models:
   "llama":
-    cmd: llama-server --port 8999 -m Llama-3.2-1B-Instruct-Q4_K_M.gguf
-
-    # where to reach the server started by cmd
-    proxy: http://127.0.0.1:8999
-
-    # aliases model names to use this configuration for
-    aliases:
-    - "gpt-4o-mini"
-    - "gpt-3.5-turbo"
-
-    # wait for this path to return an HTTP 200 before serving requests
-    # defaults to /health to match llama.cpp
-    #
-    # use "none" to skip endpoint checking. This may cause requests to fail
-    # until the server is ready
-    checkEndpoint: /custom-endpoint
-
-  "qwen":
-    # environment variables to pass to the command
-    env:
-      - "CUDA_VISIBLE_DEVICES=0"
-
     # multiline for readability
     cmd: >
       llama-server --port 8999
       --model path/to/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf
 
+    # environment variables to pass to the command
+    env:
+      - "CUDA_VISIBLE_DEVICES=0"
+
+    # where to reach the server started by cmd, make sure the ports match
+    # can be omitted if you use an automatic ${PORT} in cmd
     proxy: http://127.0.0.1:8999
+
+    # aliases names to use this model for
+    aliases:
+      - "gpt-4o-mini"
+      - "gpt-3.5-turbo"
+
+    # check this path for an HTTP 200 OK before serving requests
+    # default: /health to match llama.cpp
+    # use "none" to skip endpoint checking, but may cause HTTP errors
+    # until the model is ready
+    checkEndpoint: /custom-endpoint
+
+    # automatically unload the model after this many seconds
+    # ttl values must be a value greater than 0
+    # default: 0 = never unload model
+    ttl: 60
+
+    # `useModelName` overrides the model name in the request
+    # and sends a specific name to the upstream server
+    useModelName: "qwen:qwq"
+
+  # unlisted models do not show up in /v1/models or /upstream lists
+  # but they can still be requested as normal
+  "qwen-unlisted":
+    unlisted: true
+    cmd: llama-server --port ${PORT} -m Llama-3.2-1B-Instruct-Q4_K_M.gguf -ngl 0
+
+  # Docker Support (v26.1.4+ required!)
+  "docker-llama":
+    proxy: "http://127.0.0.1:${PORT}"
+    cmd: >
+      docker run --name dockertest
+      --init --rm -p ${PORT}:8080 -v /mnt/nvme/models:/models
+      ghcr.io/ggerganov/llama.cpp:server
+      --model '/models/Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf'
+
+# Groups provide advanced controls over model swapping behaviour. Using groups
+# some models can be kept loaded indefinitely, while others are swapped out.
+#
+# Tips:
+#
+#  - models must be defined above in the Models section
+#  - a model can only be a member of one group
+#  - group behaviour is controlled via the `swap`, `exclusive` and `persistent` fields
+#  - see issue #109 for details
+#
+# NOTE: the example below uses model names that are not defined above for demonstration purposes
+groups:
+  # group1 is the default behaviour of llama-swap where only one model is allowed
+  # to run a time across the whole llama-swap instance
+  "group1":
+    # swap controls the model swapping behaviour in within the group
+    # - true : only one model is allowed to run at a time
+    # - false: all models can run together, no swapping
+    swap: true
+
+    # exclusive controls how the group affects other groups
+    # - true: causes all other groups to unload their models when this group runs a model
+    # - false: does not affect other groups
+    exclusive: true
+
+    # members references the models defined above
+    members:
+      - "llama"
+      - "qwen-unlisted"
+
+  # models in this group are never unloaded
+  "group2":
+    swap: false
+    exclusive: false
+    members:
+      - "docker-llama"
+      # (not defined above, here for example)
+      - "modelA"
+      - "modelB"
+
+  "forever":
+    # setting persistent to true causes the group to never be affected by the swapping behaviour of
+    # other groups. It is a shortcut to keeping some models always loaded.
+    persistent: true
+
+    # set swap/exclusive to false to prevent swapping inside the group and effect on other groups
+    swap: false
+    exclusive: false
+    members:
+      - "forever-modelA"
+      - "forever-modelB"
+      - "forever-modelc"
 ```
 
-## Installation
+### Use Case Examples
+
+- [config.example.yaml](config.example.yaml) includes example for supporting `v1/embeddings` and `v1/rerank` endpoints
+- [Speculative Decoding](examples/speculative-decoding/README.md) - using a small draft model can increase inference speeds from 20% to 40%. This example includes a configurations Qwen2.5-Coder-32B (2.5x increase) and Llama-3.1-70B (1.4x increase) in the best cases.
+- [Optimizing Code Generation](examples/benchmark-snakegame/README.md) - find the optimal settings for your machine. This example demonstrates defining multiple configurations and testing which one is fastest.
+- [Restart on Config Change](examples/restart-on-config-change/README.md) - automatically restart llama-swap when trying out different configurations.
+
+## Configuration
+
+llama-s
+
+</details>
+
+## Docker Install ([download images](https://github.com/mostlygeek/llama-swap/pkgs/container/llama-swap))
+
+Docker is the quickest way to try out llama-swap:
+
+```
+# use CPU inference
+$ docker run -it --rm -p 9292:8080 ghcr.io/mostlygeek/llama-swap:cpu
+
+
+# qwen2.5 0.5B
+$ curl -s http://localhost:9292/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer no-key" \
+    -d '{"model":"qwen2.5","messages": [{"role": "user","content": "tell me a joke"}]}' | \
+    jq -r '.choices[0].message.content'
+
+
+# SmolLM2 135M
+$ curl -s http://localhost:9292/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer no-key" \
+    -d '{"model":"smollm2","messages": [{"role": "user","content": "tell me a joke"}]}' | \
+    jq -r '.choices[0].message.content'
+```
+
+<details>
+<summary>Docker images are nightly ...</summary>
+
+They include:
+
+- `ghcr.io/mostlygeek/llama-swap:cpu`
+- `ghcr.io/mostlygeek/llama-swap:cuda`
+- `ghcr.io/mostlygeek/llama-swap:intel`
+- `ghcr.io/mostlygeek/llama-swap:vulkan`
+- ROCm disabled until fixed in llama.cpp container
+
+Specific versions are also available and are tagged with the llama-swap, architecture and llama.cpp versions. For example: `ghcr.io/mostlygeek/llama-swap:v89-cuda-b4716`
+
+Beyond the demo you will likely want to run the containers with your downloaded models and custom configuration.
+
+```
+$ docker run -it --rm --runtime nvidia -p 9292:8080 \
+  -v /path/to/models:/models \
+  -v /path/to/custom/config.yaml:/app/config.yaml \
+  ghcr.io/mostlygeek/llama-swap:cuda
+```
+
+</details>
+
+## Bare metal Install ([download](https://github.com/mostlygeek/llama-swap/releases))
+
+Pre-built binaries are available for Linux, FreeBSD and Darwin (OSX). These are automatically published and are likely a few hours ahead of the docker releases. The baremetal install works with any OpenAI compatible server, not just llama-server.
 
 1. Create a configuration file, see [config.example.yaml](config.example.yaml)
 1. Download a [release](https://github.com/mostlygeek/llama-swap/releases) appropriate for your OS and architecture.
-    * _Note: Windows currently untested._
 1. Run the binary with `llama-swap --config path/to/config.yaml`
 
+### Building from source
+
+1. Install golang for your system
+1. `git clone git@github.com:mostlygeek/llama-swap.git`
+1. `make clean all`
+1. Binaries will be in `build/` subdirectory
+
 ## Monitoring Logs
 
-The `/logs` endpoint is available to monitor what llama-swap is doing. It will send the last 10KB of logs. Useful for monitoring the output of llama-server. It also supports streaming of logs.
+Open the `http://<host>/logs` with your browser to get a web interface with streaming logs.
 
-Usage:
+Of course, CLI access is also supported:
 
 ```
-# basic, sends up to the last 10KB of logs
+# sends up to the last 10KB of logs
 curl http://host/logs'
 
-# add `stream` to stream new logs as they come in
-curl -Ns 'http://host/logs?stream'
+# streams combined logs
+curl -Ns 'http://host/logs/stream'
 
-# add `skip` to skip history (only useful if used with stream)
-curl -Ns 'http://host/logs?stream&skip'
+# just llama-swap's logs
+curl -Ns 'http://host/logs/stream/proxy'
 
-# will output nothing :)
-curl -Ns 'http://host/logs?skip'
+# just upstream's logs
+curl -Ns 'http://host/logs/stream/upstream'
+
+# stream and filter logs with linux pipes
+curl -Ns http://host/logs/stream | grep 'eval time'
+
+# skips history and just streams new log entries
+curl -Ns 'http://host/logs/stream?no-history'
 ```
 
+## Do I need to use llama.cpp's server (llama-server)?
+
+Any OpenAI compatible server would work. llama-swap was originally designed for llama-server and it is the best supported.
+
+For Python based inference servers like vllm or tabbyAPI it is recommended to run them via podman or docker. This provides clean environment isolation as well as responding correctly to `SIGTERM` signals to shutdown.
+
 ## Systemd Unit Files
 
 Use this unit file to start llama-swap on boot. This is only tested on Ubuntu.
 
 `/etc/systemd/system/llama-swap.service`
+
 ```
 [Unit]
 Description=llama-swap
@@ -104,8 +318,6 @@ StartLimitInterval=30
 WantedBy=multi-user.target
 ```
 
-## Building from Source
+## Star History
 
-1. Install golang for your system
-1. run `make clean all`
-1. binaries will be built into `build/` directory
+[![Star History Chart](https://api.star-history.com/svg?repos=mostlygeek/llama-swap&type=Date)](https://www.star-history.com/#mostlygeek/llama-swap&Date)

config.example.yaml

@@ -1,14 +1,17 @@
 # Seconds to wait for llama.cpp to be available to serve requests
 # Default (and minimum): 15 seconds
-healthCheckTimeout: 15
+healthCheckTimeout: 90
+
+# valid log levels: debug, info (default), warn, error
+logLevel: debug
 
 models:
   "llama":
     cmd: >
       models/llama-server-osx
-      --port 8999
-      -m models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
-    proxy: http://127.0.0.1:8999
+      --port 9001
+      -m models/Llama-3.2-1B-Instruct-Q4_0.gguf
+    proxy: http://127.0.0.1:9001
 
     # list of model name aliases this llama.cpp instance can serve
     aliases:
@@ -17,12 +20,48 @@ models:
     # check this path for a HTTP 200 response for the server to be ready
     checkEndpoint: /health
 
+    # unload model after 5 seconds
+    ttl: 5
+
   "qwen":
-    cmd: models/llama-server-osx --port 8999 -m models/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf
-    proxy: http://127.0.0.1:8999
+    cmd: models/llama-server-osx --port 9002 -m models/qwen2.5-0.5b-instruct-q8_0.gguf
+    proxy: http://127.0.0.1:9002
     aliases:
     - gpt-3.5-turbo
 
+  # Embedding example with Nomic
+  # https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF
+  "nomic":
+    proxy: http://127.0.0.1:9005
+    cmd: >
+      models/llama-server-osx --port 9005
+      -m models/nomic-embed-text-v1.5.Q8_0.gguf
+      --ctx-size 8192
+      --batch-size 8192
+      --rope-scaling yarn
+      --rope-freq-scale 0.75
+      -ngl 99
+      --embeddings
+
+  # Reranking example with bge-reranker
+  # https://huggingface.co/gpustack/bge-reranker-v2-m3-GGUF
+  "bge-reranker":
+    proxy: http://127.0.0.1:9006
+    cmd: >
+      models/llama-server-osx --port 9006
+      -m models/bge-reranker-v2-m3-Q4_K_M.gguf
+      --ctx-size 8192
+      --reranking
+
+  # Docker Support (v26.1.4+ required!)
+  "dockertest":
+    proxy: "http://127.0.0.1:9790"
+    cmd: >
+      docker run --name dockertest
+      --init --rm -p 9790:8080 -v /mnt/nvme/models:/models
+      ghcr.io/ggerganov/llama.cpp:server
+      --model '/models/Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf'
+
   "simple":
     # example of setting environment variables
     env:
@@ -30,6 +69,7 @@ models:
       - env1=hello
     cmd: build/simple-responder --port 8999
     proxy: http://127.0.0.1:8999
+    unlisted: true
 
     # use "none" to skip check. Caution this may cause some requests to fail
     # until the upstream server is ready for traffic
@@ -39,6 +79,14 @@ models:
   "broken":
     cmd: models/llama-server-osx --port 8999 -m models/doesnotexist.gguf
     proxy: http://127.0.0.1:8999
+    unlisted: true
   "broken_timeout":
-    cmd: models/llama-server-osx --port 8999 -m models/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf
-    proxy: http://127.0.0.1:9000
+    cmd: models/llama-server-osx --port 8999 -m models/qwen2.5-0.5b-instruct-q8_0.gguf
+    proxy: http://127.0.0.1:9000
+    unlisted: true
+
+# creating a coding profile with models for code generation and general questions
+profiles:
+  coding:
+    - "qwen"
+    - "llama"

docker/build-container.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+cd $(dirname "$0")
+
+ARCH=$1
+PUSH_IMAGES=${2:-false}
+
+# List of allowed architectures
+ALLOWED_ARCHS=("intel" "vulkan" "musa" "cuda" "cpu")
+
+# Check if ARCH is in the allowed list
+if [[ ! " ${ALLOWED_ARCHS[@]} " =~ " ${ARCH} " ]]; then
+  echo "Error: ARCH must be one of the following: ${ALLOWED_ARCHS[@]}"
+  exit 1
+fi
+
+# Check if GITHUB_TOKEN is set and not empty
+if [[ -z "$GITHUB_TOKEN" ]]; then
+  echo "Error: GITHUB_TOKEN is not set or is empty."
+  exit 1
+fi
+
+# the most recent llama-swap tag
+# have to strip out the 'v' due to .tar.gz file naming
+LS_VER=$(curl -s https://api.github.com/repos/mostlygeek/llama-swap/releases/latest | jq -r .tag_name | sed 's/v//')
+
+if [ "$ARCH" == "cpu" ]; then
+    # cpu only containers just use the latest available
+    CONTAINER_LATEST="ghcr.io/mostlygeek/llama-swap:cpu"
+    echo "Building ${CONTAINER_LATEST} $LS_VER"
+    docker build -f llama-swap.Containerfile --build-arg BASE_TAG=server --build-arg LS_VER=${LS_VER} -t ${CONTAINER_LATEST} .
+    if [ "$PUSH_IMAGES" == "true" ]; then
+      docker push ${CONTAINER_LATEST}
+    fi
+else
+    LCPP_TAG=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \
+        "https://api.github.com/users/ggml-org/packages/container/llama.cpp/versions" \
+        | jq -r --arg arch "$ARCH" '.[] | select(.metadata.container.tags[] | startswith("server-\($arch)")) | .metadata.container.tags[]' \
+        | sort -r | head -n1 | awk -F '-' '{print $3}')
+
+    # Abort if LCPP_TAG is empty.
+    if [[ -z "$LCPP_TAG" ]]; then
+        echo "Abort: Could not find llama-server container for arch: $ARCH"
+        exit 1
+    fi
+
+    CONTAINER_TAG="ghcr.io/mostlygeek/llama-swap:v${LS_VER}-${ARCH}-${LCPP_TAG}"
+    CONTAINER_LATEST="ghcr.io/mostlygeek/llama-swap:${ARCH}"
+    echo "Building ${CONTAINER_TAG} $LS_VER"
+    docker build -f llama-swap.Containerfile --build-arg BASE_TAG=server-${ARCH}-${LCPP_TAG} --build-arg LS_VER=${LS_VER} -t ${CONTAINER_TAG} -t ${CONTAINER_LATEST} .
+    if [ "$PUSH_IMAGES" == "true" ]; then
+      docker push ${CONTAINER_TAG}
+      docker push ${CONTAINER_LATEST}
+    fi
+fi

docker/config.example.yaml

@@ -0,0 +1,17 @@
+healthCheckTimeout: 300
+logRequests: true
+
+models:
+  "qwen2.5":
+    proxy: "http://127.0.0.1:9999"
+    cmd: >
+      /app/llama-server
+      -hf bartowski/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M
+      --port 9999
+
+  "smollm2":
+    proxy: "http://127.0.0.1:9999"
+    cmd: >
+      /app/llama-server
+      -hf bartowski/SmolLM2-135M-Instruct-GGUF:Q4_K_M
+      --port 9999

docker/llama-swap.Containerfile

@@ -0,0 +1,16 @@
+ARG BASE_TAG=server-cuda
+FROM ghcr.io/ggml-org/llama.cpp:${BASE_TAG}
+
+# has to be after the FROM
+ARG LS_VER=89
+
+WORKDIR /app
+RUN \
+    curl -LO https://github.com/mostlygeek/llama-swap/releases/download/v"${LS_VER}"/llama-swap_"${LS_VER}"_linux_amd64.tar.gz && \
+    tar -zxf llama-swap_"${LS_VER}"_linux_amd64.tar.gz && \
+    rm llama-swap_"${LS_VER}"_linux_amd64.tar.gz
+
+COPY config.example.yaml /app/config.yaml
+
+HEALTHCHECK CMD curl -f http://localhost:8080/ || exit 1
+ENTRYPOINT [ "/app/llama-swap", "-config", "/app/config.yaml" ]

examples/README.md

@@ -0,0 +1,6 @@
+# Example Configs and Use Cases
+
+A collections of usecases and examples for getting the most out of llama-swap.
+
+* [Speculative Decoding](speculative-decoding/README.md) - using a small draft model can increase inference speeds from 20% to 40%. This example includes a configurations Qwen2.5-Coder-32B (2.5x increase) and Llama-3.1-70B (1.4x increase) in the best cases.
+* [Optimizing Code Generation](benchmark-snakegame/README.md) - find the optimal settings for your machine. This example demonstrates defining multiple configurations and testing which one is fastest.

examples/aider-qwq-coder/README.md

@@ -0,0 +1,153 @@
+# aider, QwQ, Qwen-Coder 2.5 and llama-swap
+
+This guide show how to use aider and llama-swap to get a 100% local coding co-pilot setup. The focus is on the trickest part which is configuring aider, llama-swap and llama-server to work together.
+
+## Here's what you you need:
+
+- aider - [installation docs](https://aider.chat/docs/install.html)
+- llama-server - [download latest release](https://github.com/ggml-org/llama.cpp/releases)
+- llama-swap - [download latest release](https://github.com/mostlygeek/llama-swap/releases)
+- [QwQ 32B](https://huggingface.co/bartowski/Qwen_QwQ-32B-GGUF) and [Qwen Coder 2.5 32B](https://huggingface.co/bartowski/Qwen2.5-Coder-32B-Instruct-GGUF) models
+- 24GB VRAM video card
+
+## Running aider
+
+The goal is getting this command line to work:
+
+```sh
+aider --architect \
+    --no-show-model-warnings \
+    --model openai/QwQ \
+    --editor-model openai/qwen-coder-32B \
+    --model-settings-file aider.model.settings.yml \
+    --openai-api-key "sk-na" \
+    --openai-api-base "http://10.0.1.24:8080/v1" \
+```
+
+Set `--openai-api-base` to the IP and port where your llama-swap is running.
+
+## Create an aider model settings file
+
+```yaml
+# aider.model.settings.yml
+
+#
+# !!! important: model names must match llama-swap configuration names !!!
+#
+
+- name: "openai/QwQ"
+  edit_format: diff
+  extra_params:
+    max_tokens: 16384
+    top_p: 0.95
+    top_k: 40
+    presence_penalty: 0.1
+    repetition_penalty: 1
+    num_ctx: 16384
+  use_temperature: 0.6
+  reasoning_tag: think
+  weak_model_name: "openai/qwen-coder-32B"
+  editor_model_name: "openai/qwen-coder-32B"
+
+- name: "openai/qwen-coder-32B"
+  edit_format: diff
+  extra_params:
+    max_tokens: 16384
+    top_p: 0.8
+    top_k: 20
+    repetition_penalty: 1.05
+  use_temperature: 0.6
+  reasoning_tag: think
+  editor_edit_format: editor-diff
+  editor_model_name: "openai/qwen-coder-32B"
+```
+
+## llama-swap configuration
+
+```yaml
+# config.yaml
+
+# The parameters are tweaked to fit model+context into 24GB VRAM GPUs
+models:
+  "qwen-coder-32B":
+    proxy: "http://127.0.0.1:8999"
+    cmd: >
+      /path/to/llama-server
+      --host 127.0.0.1 --port 8999 --flash-attn --slots
+      --ctx-size 16000
+      --cache-type-k q8_0 --cache-type-v q8_0
+       -ngl 99
+      --model /path/to/Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf
+
+  "QwQ":
+    proxy: "http://127.0.0.1:9503"
+    cmd: >
+      /path/to/llama-server
+      --host 127.0.0.1 --port 9503 --flash-attn --metrics--slots
+      --cache-type-k q8_0 --cache-type-v q8_0
+      --ctx-size 32000
+      --samplers "top_k;top_p;min_p;temperature;dry;typ_p;xtc"
+      --temp 0.6 --repeat-penalty 1.1 --dry-multiplier 0.5
+      --min-p 0.01 --top-k 40 --top-p 0.95
+      -ngl 99
+      --model /mnt/nvme/models/bartowski/Qwen_QwQ-32B-Q4_K_M.gguf
+```
+
+## Advanced, Dual GPU Configuration
+
+If you have _dual 24GB GPUs_ you can use llama-swap profiles to avoid swapping between QwQ and Qwen Coder.
+
+In llama-swap's configuration file:
+
+1. add a `profiles` section with `aider` as the profile name
+2. using the `env` field to specify the GPU IDs for each model
+
+```yaml
+# config.yaml
+
+# Add a profile for aider
+profiles:
+  aider:
+    - qwen-coder-32B
+    - QwQ
+
+models:
+  "qwen-coder-32B":
+    # manually set the GPU to run on
+    env:
+      - "CUDA_VISIBLE_DEVICES=0"
+    proxy: "http://127.0.0.1:8999"
+    cmd: /path/to/llama-server ...
+
+  "QwQ":
+    # manually set the GPU to run on
+    env:
+      - "CUDA_VISIBLE_DEVICES=1"
+    proxy: "http://127.0.0.1:9503"
+    cmd: /path/to/llama-server ...
+```
+
+Append the profile tag, `aider:`, to the model names in the model settings file
+
+```yaml
+# aider.model.settings.yml
+- name: "openai/aider:QwQ"
+  weak_model_name: "openai/aider:qwen-coder-32B-aider"
+  editor_model_name: "openai/aider:qwen-coder-32B-aider"
+
+- name: "openai/aider:qwen-coder-32B"
+  editor_model_name: "openai/aider:qwen-coder-32B-aider"
+```
+
+Run aider with:
+
+```sh
+$ aider --architect \
+    --no-show-model-warnings \
+    --model openai/aider:QwQ \
+    --editor-model openai/aider:qwen-coder-32B \
+    --config aider.conf.yml \
+    --model-settings-file aider.model.settings.yml
+    --openai-api-key "sk-na" \
+    --openai-api-base "http://10.0.1.24:8080/v1"
+```

examples/aider-qwq-coder/aider.model.settings.dualgpu.yml

@@ -0,0 +1,28 @@
+# this makes use of llama-swap's profile feature to
+# keep the architect and editor models in VRAM on different GPUs
+
+- name: "openai/aider:QwQ"
+  edit_format: diff
+  extra_params:
+    max_tokens: 16384
+    top_p: 0.95
+    top_k: 40
+    presence_penalty: 0.1
+    repetition_penalty: 1
+    num_ctx: 16384
+  use_temperature: 0.6
+  reasoning_tag: think
+  weak_model_name: "openai/aider:qwen-coder-32B"
+  editor_model_name: "openai/aider:qwen-coder-32B"
+
+- name: "openai/aider:qwen-coder-32B"
+  edit_format: diff
+  extra_params:
+    max_tokens: 16384
+    top_p: 0.8
+    top_k: 20
+    repetition_penalty: 1.05
+  use_temperature: 0.6
+  reasoning_tag: think
+  editor_edit_format: editor-diff
+  editor_model_name: "openai/aider:qwen-coder-32B"

examples/aider-qwq-coder/aider.model.settings.yml

@@ -0,0 +1,26 @@
+- name: "openai/QwQ"
+  edit_format: diff
+  extra_params:
+    max_tokens: 16384
+    top_p: 0.95
+    top_k: 40
+    presence_penalty: 0.1
+    repetition_penalty: 1
+    num_ctx: 16384
+  use_temperature: 0.6
+  reasoning_tag: think
+  weak_model_name: "openai/qwen-coder-32B"
+  editor_model_name: "openai/qwen-coder-32B"
+
+- name: "openai/qwen-coder-32B"
+  edit_format: diff
+  extra_params:
+    max_tokens: 16384
+    top_p: 0.8
+    top_k: 20
+    repetition_penalty: 1.05
+  use_temperature: 0.6
+  reasoning_tag: think
+  editor_edit_format: editor-diff
+  editor_model_name: "openai/qwen-coder-32B"
+

examples/aider-qwq-coder/llama-swap.yaml

@@ -0,0 +1,49 @@
+healthCheckTimeout: 300
+logLevel: debug
+
+profiles:
+    aider:
+      - qwen-coder-32B
+      - QwQ
+
+models:
+  "qwen-coder-32B":
+    env:
+      - "CUDA_VISIBLE_DEVICES=0"
+    aliases:
+      - coder
+    proxy: "http://127.0.0.1:8999"
+
+    # set appropriate paths for your environment
+    cmd: >
+      /path/to/llama-server
+      --host 127.0.0.1 --port 8999 --flash-attn --slots
+      --ctx-size 16000
+      --ctx-size-draft 16000
+      --model /path/to/Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf
+      --model-draft /path/to/Qwen2.5-Coder-1.5B-Instruct-Q8_0.gguf
+      -ngl 99 -ngld 99
+      --draft-max 16 --draft-min 4 --draft-p-min 0.4
+      --cache-type-k q8_0 --cache-type-v q8_0
+  "QwQ":
+    env:
+      - "CUDA_VISIBLE_DEVICES=1"
+    proxy: "http://127.0.0.1:9503"
+
+    # set appropriate paths for your environment
+    cmd: >
+      /path/to/llama-server
+      --host 127.0.0.1 --port 9503
+      --flash-attn --metrics
+      --slots
+      --model /path/to/Qwen_QwQ-32B-Q4_K_M.gguf
+      --cache-type-k q8_0 --cache-type-v q8_0
+      --ctx-size 32000
+      --samplers "top_k;top_p;min_p;temperature;dry;typ_p;xtc"
+      --temp 0.6
+      --repeat-penalty 1.1
+      --dry-multiplier 0.5
+      --min-p 0.01
+      --top-k 40
+      --top-p 0.95
+      -ngl 99 -ngld 99

examples/benchmark-snakegame/README.md

@@ -0,0 +1,123 @@
+# Optimizing Code Generation with llama-swap
+
+Finding the best mix of settings for your hardware can be time consuming. This example demonstrates using a custom configuration file to automate testing different scenarios to find the an optimal configuration.
+
+The benchmark writes a snake game in Python, TypeScript, and Swift using the Qwen 2.5 Coder models. The experiments were done using a 3090 and a P40.
+
+**Benchmark Scenarios**
+
+Three scenarios are tested:
+
+- 3090-only: Just the main model on the 3090
+- 3090-with-draft: the main and draft models on the 3090
+- 3090-P40-draft: the main model on the 3090 with the draft model offloaded to the P40
+
+**Available Devices**
+
+Use the following command to list available devices IDs for the configuration:
+
+```
+$ /mnt/nvme/llama-server/llama-server-f3252055 --list-devices
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 4 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: Tesla P40, compute capability 6.1, VMM: yes
+  Device 2: Tesla P40, compute capability 6.1, VMM: yes
+  Device 3: Tesla P40, compute capability 6.1, VMM: yes
+Available devices:
+  CUDA0: NVIDIA GeForce RTX 3090 (24154 MiB, 406 MiB free)
+  CUDA1: Tesla P40 (24438 MiB, 22942 MiB free)
+  CUDA2: Tesla P40 (24438 MiB, 24144 MiB free)
+  CUDA3: Tesla P40 (24438 MiB, 24144 MiB free)
+```
+
+**Configuration**
+
+The configuration file, `benchmark-config.yaml`, defines the three scenarios:
+
+```yaml
+models:
+  "3090-only":
+    proxy: "http://127.0.0.1:9503"
+    cmd: >
+      /mnt/nvme/llama-server/llama-server-f3252055
+      --host 127.0.0.1 --port 9503
+      --flash-attn
+      --slots
+
+      --model /mnt/nvme/models/Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf
+      -ngl 99
+      --device CUDA0
+
+      --ctx-size 32768
+      --cache-type-k q8_0 --cache-type-v q8_0
+
+  "3090-with-draft":
+    proxy: "http://127.0.0.1:9503"
+    # --ctx-size 28500 max that can fit on 3090 after draft model
+    cmd: >
+      /mnt/nvme/llama-server/llama-server-f3252055
+      --host 127.0.0.1 --port 9503
+      --flash-attn
+      --slots
+
+      --model /mnt/nvme/models/Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf
+      -ngl 99
+      --device CUDA0
+
+      --model-draft /mnt/nvme/models/Qwen2.5-Coder-0.5B-Instruct-Q8_0.gguf
+      -ngld 99
+      --draft-max 16
+      --draft-min 4
+      --draft-p-min 0.4
+      --device-draft CUDA0
+
+      --ctx-size 28500
+      --cache-type-k q8_0 --cache-type-v q8_0
+
+  "3090-P40-draft":
+    proxy: "http://127.0.0.1:9503"
+    cmd: >
+      /mnt/nvme/llama-server/llama-server-f3252055
+      --host 127.0.0.1 --port 9503
+      --flash-attn --metrics
+      --slots
+      --model /mnt/nvme/models/Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf
+      -ngl 99
+      --device CUDA0
+
+      --model-draft /mnt/nvme/models/Qwen2.5-Coder-0.5B-Instruct-Q8_0.gguf
+      -ngld 99
+      --draft-max 16
+      --draft-min 4
+      --draft-p-min 0.4
+      --device-draft CUDA1
+
+      --ctx-size 32768
+      --cache-type-k q8_0 --cache-type-v q8_0
+```
+
+> Note in the `3090-with-draft` scenario the `--ctx-size` had to be reduced from 32768 to to accommodate the draft model.
+
+
+**Running the Benchmark**
+
+To run the benchmark, execute the following commands:
+
+1. `llama-swap -config benchmark-config.yaml`
+1. `./run-benchmark.sh http://localhost:8080 "3090-only" "3090-with-draft" "3090-P40-draft"`
+
+The [benchmark script](run-benchmark.sh) generates a CSV output of the results, which can be converted to a Markdown table for readability.
+
+**Results (tokens/second)**
+
+| model           | python | typescript | swift |
+|-----------------|--------|------------|-------|
+| 3090-only       | 34.03  | 34.01      | 34.01 |
+| 3090-with-draft | 106.65 | 70.48      | 57.89 |
+| 3090-P40-draft  | 81.54  | 60.35      | 46.50 |
+
+Many different factors, like the programming language, can have big impacts on the performance gains. However, with a custom configuration file for benchmarking it is easy to test the different variations to discover what's best for your hardware.
+
+Happy coding!

examples/benchmark-snakegame/run-benchmark.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# This script generates a CSV file showing the token/second for generating a Snake Game in python, typescript and swift
+# It was created to test the effects of speculative decoding and the various draft settings on performance.
+#
+# Writing code with a low temperature seems to provide fairly consistent logic.
+#
+# Usage: ./benchmark.sh <url> <model1> [model2 ...]
+# Example: ./benchmark.sh http://localhost:8080 model1 model2
+
+if [ "$#" -lt 2 ]; then
+    echo "Usage: $0 <url> <model1> [model2 ...]"
+    exit 1
+fi
+
+url=$1; shift
+
+echo "model,python,typescript,swift"
+
+for model in "$@"; do
+
+    echo -n "$model,"
+
+    for lang in "python" "typescript" "swift"; do
+        # expects a llama.cpp after PR https://github.com/ggerganov/llama.cpp/pull/10548
+        # (Dec 3rd/2024)
+        time=$(curl -s --url "$url/v1/chat/completions" -d "{\"messages\": [{\"role\": \"system\", \"content\": \"you only write code.\"}, {\"role\": \"user\", \"content\": \"write snake game in $lang\"}], \"top_k\": 1, \"timings_per_token\":true, \"model\":\"$model\"}" | jq -r .timings.predicted_per_second)
+
+        if [ $? -ne 0 ]; then
+            time="error"
+            exit 1
+        fi
+
+        if [ "$lang" != "swift" ]; then
+            printf "%0.2f tps," $time
+        else
+            printf "%0.2f tps\n" $time
+        fi
+    done
+done

examples/restart-on-config-change/README.md

@@ -0,0 +1,51 @@
+# Restart llama-swap on config change
+
+Sometimes editing the configuration file can take a bit of trail and error to get a model configuration tuned just right. The `watch-and-restart.sh` script can be used to watch `config.yaml` for changes and restart `llama-swap` when it detects a change.
+
+```bash
+#!/bin/bash
+#
+# A simple watch and restart llama-swap when its configuration
+# file changes. Useful for trying out configuration changes
+# without manually restarting the server each time.
+if [ -z "$1" ]; then
+    echo "Usage: $0 <path to config.yaml>"
+    exit 1
+fi
+
+while true; do
+    # Start the process again
+    ./llama-swap-linux-amd64 -config $1 -listen :1867 &
+    PID=$!
+    echo "Started llama-swap with PID $PID"
+
+    # Wait for modifications in the specified directory or file
+    inotifywait -e modify "$1"
+
+    # Check if process exists before sending signal
+    if kill -0 $PID 2>/dev/null; then
+        echo "Sending SIGTERM to $PID"
+        kill -SIGTERM $PID
+        wait $PID
+    else
+        echo "Process $PID no longer exists"
+    fi
+    sleep 1
+done
+```
+
+## Usage and output example
+
+```bash
+$ ./watch-and-restart.sh config.yaml
+Started llama-swap with PID 495455
+Setting up watches.
+Watches established.
+llama-swap listening on :1867
+Sending SIGTERM to 495455
+Shutting down llama-swap
+Started llama-swap with PID 495486
+Setting up watches.
+Watches established.
+llama-swap listening on :1867
+```

examples/speculative-decoding/README.md

@@ -0,0 +1,124 @@
+# Speculative Decoding
+
+Speculative decoding can significantly improve the tokens per second. However, this comes at the cost of increased VRAM usage for the draft model. The examples provided are based on a server with three P40s and one 3090.
+
+## Coding Use Case
+
+This example uses Qwen2.5 Coder 32B with the 0.5B model as a draft. A quantization of Q8_0 was chosen for the draft model, as quantization has a greater impact on smaller models.
+
+The models used are:
+
+* [Bartowski Qwen2.5-Coder-32B-Instruct](https://huggingface.co/bartowski/Qwen2.5-Coder-32B-Instruct-GGUF)
+* [Bartowski Qwen2.5-Coder-0.5B-Instruct](https://huggingface.co/bartowski/Qwen2.5-Coder-0.5B-Instruct-GGUF)
+
+The llama-swap configuration is as follows:
+
+```yaml
+models:
+  "qwen-coder-32b-q4":
+    # main model on 3090, draft on P40 #1
+    cmd: >
+      /mnt/nvme/llama-server/llama-server-be0e35
+      --host 127.0.0.1 --port 9503
+      --flash-attn --metrics
+      --slots
+      --model /mnt/nvme/models/Qwen2.5-Coder-32B-Instruct-Q4_K_M.gguf
+      -ngl 99
+      --ctx-size 19000
+      --model-draft /mnt/nvme/models/Qwen2.5-Coder-0.5B-Instruct-Q8_0.gguf
+      -ngld 99
+      --draft-max 16
+      --draft-min 4
+      --draft-p-min 0.4
+      --device CUDA0
+      --device-draft CUDA1
+    proxy: "http://127.0.0.1:9503"
+```
+
+In this configuration, two GPUs are used: a 3090 (CUDA0) for the main model and a P40 (CUDA1) for the draft model. Although both models can fit on the 3090, relocating the draft model to the P40 freed up space for a larger context size. Despite the P40 being about 1/3rd the speed of the 3090, the small model still improved tokens per second.
+
+Multiple tests were run with various parameters, and the fastest result was chosen for the configuration. In all tests, the 0.5B model produced the largest improvements to tokens per second.
+
+Baseline: 33.92 tokens/second on 3090 without a draft model.
+
+| draft-max | draft-min | draft-p-min | python | TS | swift |
+|-----------|-----------|-------------|--------|----|-------|
+| 16 | 1 | 0.9 | 71.64 | 55.55 | 48.06 |
+| 16 | 1 | 0.4 | 83.21 | 58.55 | 45.50 |
+| 16 | 1 | 0.1 | 79.72 | 55.66 | 43.94 |
+| 16 | 2 | 0.9 | 68.47 | 55.13 | 43.12 |
+| 16 | 2 | 0.4 | 82.82 | 57.42 | 48.83 |
+| 16 | 2 | 0.1 | 81.68 | 51.37 | 45.72 |
+| 16 | 4 | 0.9 | 66.44 | 48.49 | 42.40 |
+| 16 | 4 | 0.4 | _83.62_ (fastest)| _58.29_ | _50.17_ |
+| 16 | 4 | 0.1 | 82.46 | 51.45 | 40.71 |
+| 8 | 1 | 0.4 | 67.07 | 55.17 | 48.46 |
+| 4 | 1 | 0.4 | 50.13 | 44.96 | 40.79 |
+
+The test script can be found in this [gist](https://gist.github.com/mostlygeek/da429769796ac8a111142e75660820f1). It is a simple curl script that prompts generating a snake game in Python, TypeScript, or Swift. Evaluation metrics were pulled from llama.cpp's logs.
+
+```bash
+for lang in "python" "typescript" "swift"; do
+    echo "Generating Snake Game in $lang using $model"
+    curl -s --url http://localhost:8080/v1/chat/completions -d "{\"messages\": [{\"role\": \"system\", \"content\": \"you only write code.\"}, {\"role\": \"user\", \"content\": \"write snake game in $lang\"}], \"temperature\": 0.1, \"model\":\"$model\"}" > /dev/null
+done
+```
+
+Python consistently outperformed Swift in all tests, likely due to the 0.5B draft model being more proficient in generating Python code accepted by the larger 32B model.
+
+## Chat
+
+This configuration is for a regular chat use case. It produces approximately 13 tokens/second in typical use, up from ~9 tokens/second with only the 3xP40s. This is great news for P40 owners.
+
+The models used are:
+
+* [Bartowski Meta-Llama-3.1-70B-Instruct-GGUF](https://huggingface.co/bartowski/Meta-Llama-3.1-70B-Instruct-GGUF)
+* [Bartowski Llama-3.2-3B-Instruct-GGUF](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF)
+
+```yaml
+models:
+  "llama-70B":
+    cmd: >
+      /mnt/nvme/llama-server/llama-server-be0e35
+      --host 127.0.0.1 --port 9602
+      --flash-attn --metrics
+      --split-mode row
+      --ctx-size 80000
+      --model /mnt/nvme/models/Meta-Llama-3.1-70B-Instruct-Q4_K_L.gguf
+      -ngl 99
+      --model-draft /mnt/nvme/models/Llama-3.2-3B-Instruct-Q4_K_M.gguf
+      -ngld 99
+      --draft-max 16
+      --draft-min 1
+      --draft-p-min 0.4
+      --device-draft CUDA0
+      --tensor-split 0,1,1,1
+```
+
+In this configuration, Llama-3.1-70B is split across three P40s, and Llama-3.2-3B is on the 3090.
+
+Some flags deserve further explanation:
+
+* `--split-mode row` - increases inference speeds using multiple P40s by about 30%. This is a P40-specific feature.
+* `--tensor-split 0,1,1,1` - controls how the main model is split across the GPUs. This means 0% on the 3090 and an even split across the P40s. A value of `--tensor-split 0,5,4,1` would mean 0% on the 3090, 50%, 40%, and 10% respectively across the other P40s. However, this would exceed the available VRAM.
+* `--ctx-size 80000` - the maximum context size that can fit in the remaining VRAM.
+
+## What is CUDA0, CUDA1, CUDA2, CUDA3?
+
+These devices are the IDs used by llama.cpp.
+
+```bash
+$ ./llama-server --list-devices
+ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
+ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
+ggml_cuda_init: found 4 CUDA devices:
+  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
+  Device 1: Tesla P40, compute capability 6.1, VMM: yes
+  Device 2: Tesla P40, compute capability 6.1, VMM: yes
+  Device 3: Tesla P40, compute capability 6.1, VMM: yes
+Available devices:
+  CUDA0: NVIDIA GeForce RTX 3090 (24154 MiB, 23892 MiB free)
+  CUDA1: Tesla P40 (24438 MiB, 24290 MiB free)
+  CUDA2: Tesla P40 (24438 MiB, 24290 MiB free)
+  CUDA3: Tesla P40 (24438 MiB, 24290 MiB free)
+```

go.mod

@@ -3,11 +3,42 @@ module github.com/mostlygeek/llama-swap
 go 1.23.0
 
 require (
+	github.com/gin-gonic/gin v1.10.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/stretchr/testify v1.9.0
+	github.com/tidwall/gjson v1.18.0
+	github.com/tidwall/sjson v1.2.5
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.20.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	golang.org/x/arch v0.8.0 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/net v0.38.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
 )

go.sum

@@ -1,10 +1,113 @@
+github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
+github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
+github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
+github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
+github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
+github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
+github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
+github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
+github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
+github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
+golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
+golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

llama-swap.go

@@ -3,31 +3,61 @@ package main
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
 
+	"github.com/gin-gonic/gin"
 	"github.com/mostlygeek/llama-swap/proxy"
 )
 
+var version string = "0"
+var commit string = "abcd1234"
+var date = "unknown"
+
 func main() {
 	// Define a command-line flag for the port
 	configPath := flag.String("config", "config.yaml", "config file name")
 	listenStr := flag.String("listen", ":8080", "listen ip/port")
+	showVersion := flag.Bool("version", false, "show version of build")
 
 	flag.Parse() // Parse the command-line flags
 
+	if *showVersion {
+		fmt.Printf("version: %s (%s), built at %s\n", version, commit, date)
+		os.Exit(0)
+	}
+
 	config, err := proxy.LoadConfig(*configPath)
 	if err != nil {
 		fmt.Printf("Error loading config: %v\n", err)
 		os.Exit(1)
 	}
 
+	if len(config.Profiles) > 0 {
+		fmt.Println("WARNING: Profile functionality has been removed in favor of Groups. See the README for more information.")
+	}
+
+	if mode := os.Getenv("GIN_MODE"); mode != "" {
+		gin.SetMode(mode)
+	} else {
+		gin.SetMode(gin.ReleaseMode)
+	}
+
 	proxyManager := proxy.New(config)
-	http.HandleFunc("/", proxyManager.HandleFunc)
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-sigChan
+		fmt.Println("Shutting down llama-swap")
+		proxyManager.Shutdown()
+		os.Exit(0)
+	}()
 
 	fmt.Println("llama-swap listening on " + *listenStr)
-	if err := http.ListenAndServe(*listenStr, nil); err != nil {
-		fmt.Printf("Error starting server: %v\n", err)
+	if err := proxyManager.Run(*listenStr); err != nil {
+		fmt.Printf("Server error: %v\n", err)
 		os.Exit(1)
 	}
 }

misc/simple-responder/simple-responder.go

@@ -3,47 +3,208 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
+	"log"
 	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/tidwall/gjson"
 )
 
 func main() {
+	gin.SetMode(gin.TestMode)
 	// Define a command-line flag for the port
 	port := flag.String("port", "8080", "port to listen on")
+	expectedModel := flag.String("model", "TheExpectedModel", "model name to expect")
 
 	// Define a command-line flag for the response message
 	responseMessage := flag.String("respond", "hi", "message to respond with")
 
+	silent := flag.Bool("silent", false, "disable all logging")
+
 	flag.Parse() // Parse the command-line flags
 
-	// Set up the handler function using the provided response message
-	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		// Set the header to text/plain
-		w.Header().Set("Content-Type", "text/plain")
+	// Create a new Gin router
+	r := gin.New()
 
-		fmt.Fprintln(w, *responseMessage)
+	// Set up the handler function using the provided response message
+	r.POST("/v1/chat/completions", func(c *gin.Context) {
+		c.Header("Content-Type", "application/json")
+
+		// add a wait to simulate a slow query
+		if wait, err := time.ParseDuration(c.Query("wait")); err == nil {
+			time.Sleep(wait)
+		}
+
+		c.JSON(http.StatusOK, gin.H{
+			"responseMessage":  *responseMessage,
+			"h_content_length": c.Request.Header.Get("Content-Length"),
+		})
+	})
+
+	// for issue #62 to check model name strips profile slug
+	// has to be one of the openAI API endpoints that llama-swap proxies
+	// curl http://localhost:8080/v1/audio/speech -d '{"model":"profile:TheExpectedModel"}'
+	r.POST("/v1/audio/speech", func(c *gin.Context) {
+		body, err := io.ReadAll(c.Request.Body)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read request body"})
+			return
+		}
+		defer c.Request.Body.Close()
+		modelName := gjson.GetBytes(body, "model").String()
+		if modelName != *expectedModel {
+			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Invalid model: %s, expected: %s", modelName, *expectedModel)})
+			return
+		} else {
+			c.JSON(http.StatusOK, gin.H{"message": "ok"})
+		}
+	})
+
+	r.POST("/v1/completions", func(c *gin.Context) {
+		c.Header("Content-Type", "application/json")
+		c.JSON(http.StatusOK, gin.H{
+			"responseMessage": *responseMessage,
+		})
+
+	})
+
+	// issue #41
+	r.POST("/v1/audio/transcriptions", func(c *gin.Context) {
+		// Parse the multipart form
+		if err := c.Request.ParseMultipartForm(10 << 20); err != nil { // 10 MB max memory
+			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error parsing multipart form: %s", err)})
+			return
+		}
+
+		// Get the model from the form values
+		model := c.Request.FormValue("model")
+
+		if model == "" {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "Missing model parameter"})
+			return
+		}
+
+		// Get the file from the form
+		file, _, err := c.Request.FormFile("file")
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error getting file: %s", err)})
+			return
+		}
+		defer file.Close()
+
+		// Read the file content to get its size
+		fileBytes, err := io.ReadAll(file)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error reading file: %s", err)})
+			return
+		}
+
+		fileSize := len(fileBytes)
+
+		// Return a JSON response with the model and transcription text including file size
+		c.JSON(http.StatusOK, gin.H{
+			"text":  fmt.Sprintf("The length of the file is %d bytes", fileSize),
+			"model": model,
+
+			// expose some header values for testing
+			"h_content_type":   c.GetHeader("Content-Type"),
+			"h_content_length": c.GetHeader("Content-Length"),
+		})
+	})
+
+	r.GET("/slow-respond", func(c *gin.Context) {
+		echo := c.Query("echo")
+		delay := c.Query("delay")
+
+		if echo == "" {
+			echo = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+		}
+
+		// Parse the duration
+		if delay == "" {
+			delay = "100ms"
+		}
+
+		t, err := time.ParseDuration(delay)
+		if err != nil {
+			c.Header("Content-Type", "text/plain")
+			c.String(http.StatusBadRequest, fmt.Sprintf("Invalid duration: %s", err))
+			return
+		}
+
+		c.Header("Content-Type", "text/plain")
+		for _, char := range echo {
+			c.Writer.Write([]byte(string(char)))
+			c.Writer.Flush()
+
+			// wait
+			<-time.After(t)
+		}
+	})
+
+	r.GET("/test", func(c *gin.Context) {
+		c.Header("Content-Type", "text/plain")
+		c.String(200, *responseMessage)
+	})
+
+	r.GET("/env", func(c *gin.Context) {
+		c.Header("Content-Type", "text/plain")
+		c.String(200, *responseMessage)
 
 		// Get environment variables
 		envVars := os.Environ()
 
 		// Write each environment variable to the response
 		for _, envVar := range envVars {
-			fmt.Fprintln(w, envVar)
+			c.String(200, envVar)
 		}
 	})
 
 	// Set up the /health endpoint handler function
-	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/json")
-		response := `{"status": "ok"}`
-		w.Write([]byte(response))
+	r.GET("/health", func(c *gin.Context) {
+		c.Header("Content-Type", "application/json")
+		c.JSON(200, gin.H{"status": "ok"})
 	})
 
-	address := ":" + *port // Address with the specified port
-	fmt.Printf("Server is listening on port %s\n", *port)
+	r.GET("/", func(c *gin.Context) {
+		c.Header("Content-Type", "text/plain")
+		c.String(200, fmt.Sprintf("%s %s", c.Request.Method, c.Request.URL.Path))
+	})
 
-	// Start the server and log any error if it occurs
-	if err := http.ListenAndServe(address, nil); err != nil {
-		fmt.Printf("Error starting server: %s\n", err)
+	address := "127.0.0.1:" + *port // Address with the specified port
+
+	srv := &http.Server{
+		Addr:    address,
+		Handler: r.Handler(),
 	}
+
+	// Disable logging if the --silent flag is set
+	if *silent {
+		gin.SetMode(gin.ReleaseMode)
+		gin.DefaultWriter = io.Discard
+		log.SetOutput(io.Discard)
+	}
+
+	go func() {
+		log.Printf("simple-responder listening on %s\n", address)
+		// service connections
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Fatalf("simple-responder err: %s\n", err)
+		}
+	}()
+
+	// Wait for interrupt signal to gracefully shutdown the server with
+	// a timeout of 5 seconds.
+	quit := make(chan os.Signal, 1)
+	// kill (no param) default send syscall.SIGTERM
+	// kill -2 is syscall.SIGINT
+	// kill -9 is syscall.SIGKILL but can't be catch, so don't need add it
+	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+	<-quit
+	log.Println("simple-responder shutting down")
 }

misc/test-rerank/README.md

@@ -0,0 +1,4 @@
+The rerank-test.json data is from https://github.com/ggerganov/llama.cpp/pull/9510
+
+To run it:
+> curl http://127.0.0.1:8080/v1/rerank -H "Content-Type: application/json" -d @reranker-test.json  -v | jq .

misc/test-rerank/reranker-test.json

@@ -0,0 +1,17 @@
+{
+  "model": "bge-reranker",
+  "query": "Organic skincare products for sensitive skin",
+  "top_n": 3,
+  "documents": [
+    "Organic skincare for sensitive skin with aloe vera and chamomile: Imagine the soothing embrace of nature with our organic skincare range, crafted specifically for sensitive skin. Infused with the calming properties of aloe vera and chamomile, each product provides gentle nourishment and protection. Say goodbye to irritation and hello to a glowing, healthy complexion.",
+    "New makeup trends focus on bold colors and innovative techniques: Step into the world of cutting-edge beauty with this seasons makeup trends. Bold, vibrant colors and groundbreaking techniques are redefining the art of makeup. From neon eyeliners to holographic highlighters, unleash your creativity and make a statement with every look.",
+    "Bio-Hautpflege für empfindliche Haut mit Aloe Vera und Kamille: Erleben Sie die wohltuende Wirkung unserer Bio-Hautpflege, speziell für empfindliche Haut entwickelt. Mit den beruhigenden Eigenschaften von Aloe Vera und Kamille pflegen und schützen unsere Produkte Ihre Haut auf natürliche Weise. Verabschieden Sie sich von Hautirritationen und genießen Sie einen strahlenden Teint.",
+    "Neue Make-up-Trends setzen auf kräftige Farben und innovative Techniken: Tauchen Sie ein in die Welt der modernen Schönheit mit den neuesten Make-up-Trends. Kräftige, lebendige Farben und innovative Techniken setzen neue Maßstäbe. Von auffälligen Eyelinern bis hin zu holografischen Highlightern – lassen Sie Ihrer Kreativität freien Lauf und setzen Sie jedes Mal ein Statement.",
+    "Cuidado de la piel orgánico para piel sensible con aloe vera y manzanilla: Descubre el poder de la naturaleza con nuestra línea de cuidado de la piel orgánico, diseñada especialmente para pieles sensibles. Enriquecidos con aloe vera y manzanilla, estos productos ofrecen una hidratación y protección suave. Despídete de las irritaciones y saluda a una piel radiante y saludable.",
+    "Las nuevas tendencias de maquillaje se centran en colores vivos y técnicas innovadoras: Entra en el fascinante mundo del maquillaje con las tendencias más actuales. Colores vivos y técnicas innovadoras están revolucionando el arte del maquillaje. Desde delineadores neón hasta iluminadores holográficos, desata tu creatividad y destaca en cada look.",
+    "针对敏感肌专门设计的天然有机护肤产品：体验由芦荟和洋甘菊提取物带来的自然呵护。我们的护肤产品特别为敏感肌设计，温和滋润，保护您的肌肤不受刺激。让您的肌肤告别不适，迎来健康光彩。",
+    "新的化妆趋势注重鲜艳的颜色和创新的技巧：进入化妆艺术的新纪元，本季的化妆趋势以大胆的颜色和创新的技巧为主。无论是霓虹眼线还是全息高光，每一款妆容都能让您脱颖而出，展现独特魅力。",
+    "敏感肌のために特別に設計された天然有機スキンケア製品: アロエベラとカモミールのやさしい力で、自然の抱擁を感じてください。敏感肌用に特別に設計された私たちのスキンケア製品は、肌に優しく栄養を与え、保護します。肌トラブルにさようなら、輝く健康な肌にこんにちは。",
+    "新しいメイクのトレンドは鮮やかな色と革新的な技術に焦点を当てています: 今シーズンのメイクアップトレンドは、大胆な色彩と革新的な技術に注目しています。ネオンアイライナーからホログラフィックハイライターまで、クリエイティビティを解き放ち、毎回ユニークなルックを演出しましょう。"
+  ]
+}

proxy/config.go

@@ -2,64 +2,226 @@ package proxy
 
 import (
 	"fmt"
+	"io"
 	"os"
+	"sort"
+	"strconv"
 	"strings"
 
+	"github.com/google/shlex"
 	"gopkg.in/yaml.v3"
 )
 
+const DEFAULT_GROUP_ID = "(default)"
+
 type ModelConfig struct {
 	Cmd           string   `yaml:"cmd"`
 	Proxy         string   `yaml:"proxy"`
 	Aliases       []string `yaml:"aliases"`
 	Env           []string `yaml:"env"`
 	CheckEndpoint string   `yaml:"checkEndpoint"`
+	UnloadAfter   int      `yaml:"ttl"`
+	Unlisted      bool     `yaml:"unlisted"`
+	UseModelName  string   `yaml:"useModelName"`
 }
 
 func (m *ModelConfig) SanitizedCommand() ([]string, error) {
 	return SanitizeCommand(m.Cmd)
 }
 
+type GroupConfig struct {
+	Swap       bool     `yaml:"swap"`
+	Exclusive  bool     `yaml:"exclusive"`
+	Persistent bool     `yaml:"persistent"`
+	Members    []string `yaml:"members"`
+}
+
+// set default values for GroupConfig
+func (c *GroupConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawGroupConfig GroupConfig
+	defaults := rawGroupConfig{
+		Swap:       true,
+		Exclusive:  true,
+		Persistent: false,
+		Members:    []string{},
+	}
+
+	if err := unmarshal(&defaults); err != nil {
+		return err
+	}
+
+	*c = GroupConfig(defaults)
+	return nil
+}
+
 type Config struct {
-	Models             map[string]ModelConfig `yaml:"models"`
 	HealthCheckTimeout int                    `yaml:"healthCheckTimeout"`
+	LogRequests        bool                   `yaml:"logRequests"`
+	LogLevel           string                 `yaml:"logLevel"`
+	Models             map[string]ModelConfig `yaml:"models"` /* key is model ID */
+	Profiles           map[string][]string    `yaml:"profiles"`
+	Groups             map[string]GroupConfig `yaml:"groups"` /* key is group ID */
+
+	// map aliases to actual model IDs
+	aliases map[string]string
+
+	// automatic port assignments
+	StartPort int `yaml:"startPort"`
 }
 
-func (c *Config) FindConfig(modelName string) (ModelConfig, bool) {
-	modelConfig, found := c.Models[modelName]
-	if found {
-		return modelConfig, true
+func (c *Config) RealModelName(search string) (string, bool) {
+	if _, found := c.Models[search]; found {
+		return search, true
+	} else if name, found := c.aliases[search]; found {
+		return name, found
+	} else {
+		return "", false
 	}
-
-	// Search through aliases to find the right config
-	for _, config := range c.Models {
-		for _, alias := range config.Aliases {
-			if alias == modelName {
-				return config, true
-			}
-		}
-	}
-
-	return ModelConfig{}, false
 }
 
-func LoadConfig(path string) (*Config, error) {
-	data, err := os.ReadFile(path)
+func (c *Config) FindConfig(modelName string) (ModelConfig, string, bool) {
+	if realName, found := c.RealModelName(modelName); !found {
+		return ModelConfig{}, "", false
+	} else {
+		return c.Models[realName], realName, true
+	}
+}
+
+func LoadConfig(path string) (Config, error) {
+	file, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return Config{}, err
+	}
+	defer file.Close()
+	return LoadConfigFromReader(file)
+}
+
+func LoadConfigFromReader(r io.Reader) (Config, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return Config{}, err
 	}
 
 	var config Config
 	err = yaml.Unmarshal(data, &config)
 	if err != nil {
-		return nil, err
+		return Config{}, err
 	}
 
 	if config.HealthCheckTimeout < 15 {
 		config.HealthCheckTimeout = 15
 	}
 
-	return &config, nil
+	// set default port ranges
+	if config.StartPort == 0 {
+		// default to 5800
+		config.StartPort = 5800
+	} else if config.StartPort < 1 {
+		return Config{}, fmt.Errorf("startPort must be greater than 1")
+	}
+
+	// Populate the aliases map
+	config.aliases = make(map[string]string)
+	for modelName, modelConfig := range config.Models {
+		for _, alias := range modelConfig.Aliases {
+			if _, found := config.aliases[alias]; found {
+				return Config{}, fmt.Errorf("duplicate alias %s found in model: %s", alias, modelName)
+			}
+			config.aliases[alias] = modelName
+		}
+	}
+
+	// iterate over the models and replace any ${PORT} with the next available port
+	// Get and sort all model IDs first, makes testing more consistent
+	modelIds := make([]string, 0, len(config.Models))
+	for modelId := range config.Models {
+		modelIds = append(modelIds, modelId)
+	}
+	sort.Strings(modelIds) // This guarantees stable iteration order
+
+	// iterate over the sorted models
+	nextPort := config.StartPort
+	for _, modelId := range modelIds {
+		modelConfig := config.Models[modelId]
+		if strings.Contains(modelConfig.Cmd, "${PORT}") {
+			modelConfig.Cmd = strings.ReplaceAll(modelConfig.Cmd, "${PORT}", strconv.Itoa(nextPort))
+			if modelConfig.Proxy == "" {
+				modelConfig.Proxy = fmt.Sprintf("http://localhost:%d", nextPort)
+			} else {
+				modelConfig.Proxy = strings.ReplaceAll(modelConfig.Proxy, "${PORT}", strconv.Itoa(nextPort))
+			}
+			nextPort++
+			config.Models[modelId] = modelConfig
+		} else if modelConfig.Proxy == "" {
+			return Config{}, fmt.Errorf("model %s requires a proxy value when not using automatic ${PORT}", modelId)
+		}
+	}
+	config = AddDefaultGroupToConfig(config)
+	// check that members are all unique in the groups
+	memberUsage := make(map[string]string) // maps member to group it appears in
+	for groupID, groupConfig := range config.Groups {
+		prevSet := make(map[string]bool)
+		for _, member := range groupConfig.Members {
+			// Check for duplicates within this group
+			if _, found := prevSet[member]; found {
+				return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID)
+			}
+			prevSet[member] = true
+
+			// Check if member is used in another group
+			if existingGroup, exists := memberUsage[member]; exists {
+				return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID)
+			}
+			memberUsage[member] = groupID
+		}
+	}
+
+	return config, nil
+}
+
+// rewrites the yaml to include a default group with any orphaned models
+func AddDefaultGroupToConfig(config Config) Config {
+
+	if config.Groups == nil {
+		config.Groups = make(map[string]GroupConfig)
+	}
+
+	defaultGroup := GroupConfig{
+		Swap:      true,
+		Exclusive: true,
+		Members:   []string{},
+	}
+	// if groups is empty, create a default group and put
+	// all models into it
+	if len(config.Groups) == 0 {
+		for modelName := range config.Models {
+			defaultGroup.Members = append(defaultGroup.Members, modelName)
+		}
+	} else {
+		// iterate over existing group members and add non-grouped models into the default group
+		for modelName, _ := range config.Models {
+			foundModel := false
+		found:
+			// search for the model in existing groups
+			for _, groupConfig := range config.Groups {
+				for _, member := range groupConfig.Members {
+					if member == modelName {
+						foundModel = true
+						break found
+					}
+				}
+			}
+
+			if !foundModel {
+				defaultGroup.Members = append(defaultGroup.Members, modelName)
+			}
+		}
+	}
+
+	sort.Strings(defaultGroup.Members) // make consistent ordering for testing
+	config.Groups[DEFAULT_GROUP_ID] = defaultGroup
+
+	return config
 }
 
 func SanitizeCommand(cmdStr string) ([]string, error) {
@@ -68,7 +230,10 @@ func SanitizeCommand(cmdStr string) ([]string, error) {
 	cmdStr = strings.ReplaceAll(cmdStr, "\\\n", " ")
 
 	// Split the command into arguments
-	args := strings.Fields(cmdStr)
+	args, err := shlex.Split(cmdStr)
+	if err != nil {
+		return nil, err
+	}
 
 	// Ensure the command is not empty
 	if len(args) == 0 {

proxy/config_test.go

@@ -3,12 +3,13 @@ package proxy
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestLoadConfig(t *testing.T) {
+func TestConfig_Load(t *testing.T) {
 	// Create a temporary YAML file for testing
 	tempDir, err := os.MkdirTemp("", "test-config")
 	if err != nil {
@@ -17,7 +18,8 @@ func TestLoadConfig(t *testing.T) {
 	defer os.RemoveAll(tempDir)
 
 	tempFile := filepath.Join(tempDir, "config.yaml")
-	content := `models:
+	content := `
+models:
   model1:
     cmd: path/to/cmd --arg1 one
     proxy: "http://localhost:8080"
@@ -28,7 +30,38 @@ func TestLoadConfig(t *testing.T) {
       - "VAR1=value1"
       - "VAR2=value2"
     checkEndpoint: "/health"
+  model2:
+    cmd: path/to/cmd --arg1 one
+    proxy: "http://localhost:8081"
+    aliases:
+      - "m2"
+    checkEndpoint: "/"
+  model3:
+    cmd: path/to/cmd --arg1 one
+    proxy: "http://localhost:8081"
+    aliases:
+      - "mthree"
+    checkEndpoint: "/"
+  model4:
+    cmd: path/to/cmd --arg1 one
+    proxy: "http://localhost:8082"
+    checkEndpoint: "/"
+
 healthCheckTimeout: 15
+profiles:
+  test:
+    - model1
+    - model2
+groups:
+  group1:
+    swap: true
+    exclusive: false
+    members: ["model2"]
+  forever:
+    exclusive: false
+    persistent: true
+    members:
+      - "model4"
 `
 
 	if err := os.WriteFile(tempFile, []byte(content), 0644); err != nil {
@@ -41,7 +74,8 @@ healthCheckTimeout: 15
 		t.Fatalf("Failed to load config: %v", err)
 	}
 
-	expected := &Config{
+	expected := Config{
+		StartPort: 5800,
 		Models: map[string]ModelConfig{
 			"model1": {
 				Cmd:           "path/to/cmd --arg1 one",
@@ -50,14 +84,121 @@ healthCheckTimeout: 15
 				Env:           []string{"VAR1=value1", "VAR2=value2"},
 				CheckEndpoint: "/health",
 			},
+			"model2": {
+				Cmd:           "path/to/cmd --arg1 one",
+				Proxy:         "http://localhost:8081",
+				Aliases:       []string{"m2"},
+				Env:           nil,
+				CheckEndpoint: "/",
+			},
+			"model3": {
+				Cmd:           "path/to/cmd --arg1 one",
+				Proxy:         "http://localhost:8081",
+				Aliases:       []string{"mthree"},
+				Env:           nil,
+				CheckEndpoint: "/",
+			},
+			"model4": {
+				Cmd:           "path/to/cmd --arg1 one",
+				Proxy:         "http://localhost:8082",
+				CheckEndpoint: "/",
+			},
 		},
 		HealthCheckTimeout: 15,
+		Profiles: map[string][]string{
+			"test": {"model1", "model2"},
+		},
+		aliases: map[string]string{
+			"m1":        "model1",
+			"model-one": "model1",
+			"m2":        "model2",
+			"mthree":    "model3",
+		},
+		Groups: map[string]GroupConfig{
+			DEFAULT_GROUP_ID: {
+				Swap:      true,
+				Exclusive: true,
+				Members:   []string{"model1", "model3"},
+			},
+			"group1": {
+				Swap:      true,
+				Exclusive: false,
+				Members:   []string{"model2"},
+			},
+			"forever": {
+				Swap:       true,
+				Exclusive:  false,
+				Persistent: true,
+				Members:    []string{"model4"},
+			},
+		},
 	}
 
 	assert.Equal(t, expected, config)
+
+	realname, found := config.RealModelName("m1")
+	assert.True(t, found)
+	assert.Equal(t, "model1", realname)
 }
 
-func TestModelConfigSanitizedCommand(t *testing.T) {
+func TestConfig_GroupMemberIsUnique(t *testing.T) {
+	content := `
+models:
+  model1:
+    cmd: path/to/cmd --arg1 one
+    proxy: "http://localhost:8080"
+  model2:
+    cmd: path/to/cmd --arg1 one
+    proxy: "http://localhost:8081"
+    checkEndpoint: "/"
+  model3:
+    cmd: path/to/cmd --arg1 one
+    proxy: "http://localhost:8081"
+    checkEndpoint: "/"
+
+healthCheckTimeout: 15
+groups:
+  group1:
+    swap: true
+    exclusive: false
+    members: ["model2"]
+  group2:
+    swap: true
+    exclusive: false
+    members: ["model2"]
+`
+	// Load the config and verify
+	_, err := LoadConfigFromReader(strings.NewReader(content))
+
+	// a Contains as order of the map is not guaranteed
+	assert.Contains(t, err.Error(), "model member model2 is used in multiple groups:")
+}
+
+func TestConfig_ModelAliasesAreUnique(t *testing.T) {
+	content := `
+models:
+  model1:
+    cmd: path/to/cmd --arg1 one
+    proxy: "http://localhost:8080"
+    aliases:
+      - m1
+  model2:
+    cmd: path/to/cmd --arg1 one
+    proxy: "http://localhost:8081"
+    checkEndpoint: "/"
+    aliases:
+      - m1
+      - m2
+`
+	// Load the config and verify
+	_, err := LoadConfigFromReader(strings.NewReader(content))
+
+	// this is a contains because it could be `model1` or `model2` depending on the order
+	// go decided on the order of the map
+	assert.Contains(t, err.Error(), "duplicate alias m1 found in model: model")
+}
+
+func TestConfig_ModelConfigSanitizedCommand(t *testing.T) {
 	config := &ModelConfig{
 		Cmd: `python model1.py \
     --arg1 value1 \
@@ -69,7 +210,10 @@ func TestModelConfigSanitizedCommand(t *testing.T) {
 	assert.Equal(t, []string{"python", "model1.py", "--arg1", "value1", "--arg2", "value2"}, args)
 }
 
-func TestFindConfig(t *testing.T) {
+func TestConfig_FindConfig(t *testing.T) {
+
+	// TODO?
+	// make make this shared between the different tests
 	config := &Config{
 		Models: map[string]ModelConfig{
 			"model1": {
@@ -88,39 +232,130 @@ func TestFindConfig(t *testing.T) {
 			},
 		},
 		HealthCheckTimeout: 10,
+		aliases: map[string]string{
+			"m1":        "model1",
+			"model-one": "model1",
+			"m2":        "model2",
+		},
 	}
 
 	// Test finding a model by its name
-	modelConfig, found := config.FindConfig("model1")
+	modelConfig, modelId, found := config.FindConfig("model1")
 	assert.True(t, found)
+	assert.Equal(t, "model1", modelId)
 	assert.Equal(t, config.Models["model1"], modelConfig)
 
 	// Test finding a model by its alias
-	modelConfig, found = config.FindConfig("m1")
+	modelConfig, modelId, found = config.FindConfig("m1")
 	assert.True(t, found)
+	assert.Equal(t, "model1", modelId)
 	assert.Equal(t, config.Models["model1"], modelConfig)
 
 	// Test finding a model that does not exist
-	modelConfig, found = config.FindConfig("model3")
+	modelConfig, modelId, found = config.FindConfig("model3")
 	assert.False(t, found)
+	assert.Equal(t, "", modelId)
 	assert.Equal(t, ModelConfig{}, modelConfig)
 }
 
-func TestSanitizeCommand(t *testing.T) {
-	// Test a simple command
-	args, err := SanitizeCommand("python model1.py")
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"python", "model1.py"}, args)
+func TestConfig_SanitizeCommand(t *testing.T) {
 
 	// Test a command with spaces and newlines
-	args, err = SanitizeCommand(`python model1.py \
-    --arg1 value1 \
-    --arg2 value2`)
+	args, err := SanitizeCommand(`python model1.py \
+    -a "double quotes" \
+    --arg2 'single quotes'
+	-s
+	--arg3 123 \
+	--arg4 '"string in string"'
+	-c "'single quoted'"
+	`)
 	assert.NoError(t, err)
-	assert.Equal(t, []string{"python", "model1.py", "--arg1", "value1", "--arg2", "value2"}, args)
+	assert.Equal(t, []string{
+		"python", "model1.py",
+		"-a", "double quotes",
+		"--arg2", "single quotes",
+		"-s",
+		"--arg3", "123",
+		"--arg4", `"string in string"`,
+		"-c", `'single quoted'`,
+	}, args)
 
 	// Test an empty command
 	args, err = SanitizeCommand("")
 	assert.Error(t, err)
 	assert.Nil(t, args)
 }
+
+func TestConfig_AutomaticPortAssignments(t *testing.T) {
+
+	t.Run("Default Port Ranges", func(t *testing.T) {
+		content := ``
+		config, err := LoadConfigFromReader(strings.NewReader(content))
+		if !assert.NoError(t, err) {
+			t.Fatalf("Failed to load config: %v", err)
+		}
+
+		assert.Equal(t, 5800, config.StartPort)
+	})
+	t.Run("User specific port ranges", func(t *testing.T) {
+		content := `startPort: 1000`
+		config, err := LoadConfigFromReader(strings.NewReader(content))
+		if !assert.NoError(t, err) {
+			t.Fatalf("Failed to load config: %v", err)
+		}
+
+		assert.Equal(t, 1000, config.StartPort)
+	})
+
+	t.Run("Invalid start port", func(t *testing.T) {
+		content := `startPort: abcd`
+		_, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.NotNil(t, err)
+	})
+
+	t.Run("start port must be greater than 1", func(t *testing.T) {
+		content := `startPort: -99`
+		_, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.NotNil(t, err)
+	})
+
+	t.Run("Automatic port assignments", func(t *testing.T) {
+		content := `
+startPort: 5800
+models:
+  model1:
+    cmd: svr --port ${PORT}
+  model2:
+    cmd: svr --port ${PORT}
+    proxy: "http://172.11.22.33:${PORT}"
+  model3:
+    cmd: svr --port 1999
+    proxy: "http://1.2.3.4:1999"
+`
+		config, err := LoadConfigFromReader(strings.NewReader(content))
+		if !assert.NoError(t, err) {
+			t.Fatalf("Failed to load config: %v", err)
+		}
+
+		assert.Equal(t, 5800, config.StartPort)
+		assert.Equal(t, "svr --port 5800", config.Models["model1"].Cmd)
+		assert.Equal(t, "http://localhost:5800", config.Models["model1"].Proxy)
+
+		assert.Equal(t, "svr --port 5801", config.Models["model2"].Cmd)
+		assert.Equal(t, "http://172.11.22.33:5801", config.Models["model2"].Proxy)
+
+		assert.Equal(t, "svr --port 1999", config.Models["model3"].Cmd)
+		assert.Equal(t, "http://1.2.3.4:1999", config.Models["model3"].Proxy)
+
+	})
+
+	t.Run("Proxy value required if no ${PORT} in cmd", func(t *testing.T) {
+		content := `
+models:
+  model1:
+    cmd: svr --port 111
+`
+		_, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.Equal(t, "model model1 requires a proxy value when not using automatic ${PORT}", err.Error())
+	})
+}

proxy/helpers_test.go

@@ -0,0 +1,70 @@
+package proxy
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+)
+
+var (
+	nextTestPort int = 12000
+	portMutex    sync.Mutex
+	testLogger   = NewLogMonitorWriter(os.Stdout)
+)
+
+// Check if the binary exists
+func TestMain(m *testing.M) {
+	binaryPath := getSimpleResponderPath()
+	if _, err := os.Stat(binaryPath); os.IsNotExist(err) {
+		fmt.Printf("simple-responder not found at %s, did you `make simple-responder`?\n", binaryPath)
+		os.Exit(1)
+	}
+
+	gin.SetMode(gin.TestMode)
+
+	switch os.Getenv("LOG_LEVEL") {
+	case "debug":
+		testLogger.SetLogLevel(LevelDebug)
+	case "warn":
+		testLogger.SetLogLevel(LevelWarn)
+	case "info":
+		testLogger.SetLogLevel(LevelInfo)
+	default:
+		testLogger.SetLogLevel(LevelWarn)
+	}
+
+	m.Run()
+}
+
+// Helper function to get the binary path
+func getSimpleResponderPath() string {
+	goos := runtime.GOOS
+	goarch := runtime.GOARCH
+	return filepath.Join("..", "build", fmt.Sprintf("simple-responder_%s_%s", goos, goarch))
+}
+
+func getTestSimpleResponderConfig(expectedMessage string) ModelConfig {
+	portMutex.Lock()
+	defer portMutex.Unlock()
+
+	port := nextTestPort
+	nextTestPort++
+
+	return getTestSimpleResponderConfigPort(expectedMessage, port)
+}
+
+func getTestSimpleResponderConfigPort(expectedMessage string, port int) ModelConfig {
+	binaryPath := getSimpleResponderPath()
+
+	// Create a process configuration
+	return ModelConfig{
+		Cmd:           fmt.Sprintf("%s --port %d --silent --respond %s", binaryPath, port, expectedMessage),
+		Proxy:         fmt.Sprintf("http://127.0.0.1:%d", port),
+		CheckEndpoint: "/health",
+	}
+}

proxy/html/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>llama-swap</title>
+</head>
+<body>
+    <h1>llama-swap</h1>
+    <p>
+        <a href="/logs">view logs</a> | <a href="/upstream">configured models</a> | <a href="https://github.com/mostlygeek/llama-swap">github</a>
+    </p>
+</body>
+</html>

proxy/html/logs.html

@@ -0,0 +1,259 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Logs</title>
+    <style>
+        body {
+            margin: 0;
+            height: 100vh;
+            display: flex;
+            flex-direction: column;
+            font-family: "Courier New", Courier, monospace;
+        }
+        .log-container {
+            display: flex;
+            flex: 1;
+            gap: 0.5em;
+            margin: 0.5em;
+            min-height: 0;
+        }
+        .log-column {
+            display: flex;
+            flex-direction: column;
+            flex: 1;
+            min-width: 0;
+            transition: flex 0.3s ease;
+        }
+        .log-column.minimized {
+            flex: 0.1;
+            max-width: 50px;
+            border: 1px solid #777;
+            color: green;
+        }
+        .log-controls {
+            display: grid;
+            grid-template-columns: 1fr auto;
+            gap: 0.5em;
+            margin-bottom: 0.5em;
+        }
+        .log-controls input {
+            width: 100%;
+            padding: 4px;
+        }
+        .log-controls input:focus {
+           outline: none;
+        }
+        .log-stream {
+            flex: 1;
+            padding: 1em;
+            background: #f4f4f4;
+            overflow-y: auto;
+            white-space: pre-wrap;
+            word-wrap: break-word;
+            min-height: 0;
+        }
+
+        .regex-error {
+            background-color: #ff0000 !important;
+        }
+
+        /* Make headers clickable and show pointer cursor */
+        h2 {
+            cursor: pointer;
+            user-select: none;
+            margin: 0 0 0.5em 0;
+            padding: 0.5em;
+        }
+
+        h2:hover {
+            background-color: rgba(0, 0, 0, 0.05);
+        }
+
+        /* Dark mode styles */
+        @media (prefers-color-scheme: dark) {
+            body {
+                background-color: #333;
+                color: #fff;
+            }
+
+            .log-stream {
+                background: #444;
+                color: #fff;
+            }
+
+            .log-controls input {
+                background: #555;
+                color: #fff;
+                border: 1px solid #777;
+            }
+
+            .log-controls button {
+                background: #555;
+                color: #fff;
+                border: 1px solid #777;
+            }
+
+            h2:hover {
+                background-color: rgba(255, 255, 255, 0.1);
+            }
+        }
+
+        /* Hide content when minimized */
+        .log-column.minimized .log-controls,
+        .log-column.minimized .log-stream {
+            display: none;
+        }
+
+        .log-column.minimized h2 {
+            writing-mode: vertical-rl;
+            text-orientation: mixed;
+            transform: rotate(180deg);
+            white-space: nowrap;
+            margin: auto;
+        }
+    </style>
+</head>
+<body>
+    <div class="log-container">
+        <div class="log-column">
+            <h2>Proxy Logs</h2>
+            <div class="log-controls">
+                <input type="text" id="proxy-filter-input" placeholder="proxy regex filter">
+                <button id="proxy-clear-button">clear</button>
+            </div>
+            <pre class="log-stream" id="proxy-log-stream">Waiting for proxy logs...</pre>
+        </div>
+        <div class="log-column minimized">
+            <h2>Upstream Logs</h2>
+            <div class="log-controls">
+                <input type="text" id="upstream-filter-input" placeholder="upstream regex filter">
+                <button id="upstream-clear-button">clear</button>
+            </div>
+            <pre class="log-stream" id="upstream-log-stream">Waiting for upstream logs...</pre>
+        </div>
+    </div>
+    <script>
+        class LogStream {
+            constructor(streamElement, filterInput, clearButton, endpoint) {
+                this.streamElement = streamElement;
+                this.filterInput = filterInput;
+                this.clearButton = clearButton;
+                this.endpoint = endpoint;
+                this.logData = "";
+                this.regexFilter = null;
+                this.eventSource = null;
+
+                this.initialize();
+            }
+
+            initialize() {
+                this.filterInput.addEventListener('input', () => this.updateFilter());
+                this.clearButton.addEventListener('click', () => {
+                    this.filterInput.value = "";
+                    this.regexFilter = null;
+                    this.render();
+                });
+                this.setupEventSource();
+            }
+
+            setupEventSource() {
+                if (typeof(EventSource) === "undefined") {
+                    this.logData = "SSE Not supported by this browser.";
+                    this.render();
+                    return;
+                }
+
+                const connect = () => {
+                    this.eventSource = new EventSource(this.endpoint);
+
+                    this.eventSource.onmessage = (event) => {
+                        this.logData += event.data;
+                        this.logData = this.logData.slice(-1024 * 100);
+                        this.render();
+                    };
+
+                    this.eventSource.onerror = (err) => {
+                        // Close the current connection
+                        this.eventSource.close();
+
+                        this.logData += "\nConnection lost. Retrying in 5 seconds...\n";
+                        this.render();
+
+                        // Attempt to reconnect after 5 seconds
+                        setTimeout(() => {
+                            this.logData += "Attempting to reconnect...\n";
+                            this.render();
+                            connect();
+                        }, 5000);
+                    };
+                };
+
+                // Initial connection
+                connect();
+            }
+
+            render() {
+                let content = this.logData;
+
+                if (this.regexFilter) {
+                    const lines = content.split('\n');
+                    const filteredLines = lines.filter(line => this.regexFilter.test(line));
+                    content = filteredLines.length > 0 ? filteredLines.join('\n') + '\n' : "";
+                }
+
+                this.streamElement.textContent = content;
+                this.streamElement.scrollTop = this.streamElement.scrollHeight;
+            }
+
+            updateFilter() {
+                const pattern = this.filterInput.value.trim();
+                this.filterInput.classList.remove('regex-error');
+
+                if (!pattern) {
+                    this.regexFilter = null;
+                    this.render();
+                    return;
+                }
+
+                try {
+                    this.regexFilter = new RegExp(pattern);
+                } catch (e) {
+                    console.error("Invalid regex pattern:", e);
+                    this.regexFilter = null;
+                    this.filterInput.classList.add('regex-error');
+                    return;
+                }
+
+                this.render();
+            }
+        }
+
+        // Initialize both log streams
+        document.addEventListener('DOMContentLoaded', () => {
+            new LogStream(
+                document.getElementById('proxy-log-stream'),
+                document.getElementById('proxy-filter-input'),
+                document.getElementById('proxy-clear-button'),
+                "/logs/streamSSE/proxy"
+            );
+
+            new LogStream(
+                document.getElementById('upstream-log-stream'),
+                document.getElementById('upstream-filter-input'),
+                document.getElementById('upstream-clear-button'),
+                "/logs/streamSSE/upstream"
+            );
+
+            // Initialize clickable headers
+            document.querySelectorAll('h2').forEach(header => {
+                header.addEventListener('click', () => {
+                    const column = header.closest('.log-column');
+                    column.classList.toggle('minimized');
+                });
+            });
+        });
+    </script>
+</body>
+</html>

proxy/html_files.go

@@ -0,0 +1,10 @@
+package proxy
+
+import "embed"
+
+//go:embed html
+var htmlFiles embed.FS
+
+func getHTMLFile(path string) ([]byte, error) {
+	return htmlFiles.ReadFile("html/" + path)
+}

proxy/logMonitor.go

@@ -2,11 +2,21 @@ package proxy
 
 import (
 	"container/ring"
+	"fmt"
 	"io"
 	"os"
 	"sync"
 )
 
+type LogLevel int
+
+const (
+	LevelDebug LogLevel = iota
+	LevelInfo
+	LevelWarn
+	LevelError
+)
+
 type LogMonitor struct {
 	clients  map[chan []byte]bool
 	mu       sync.RWMutex
@@ -15,6 +25,10 @@ type LogMonitor struct {
 
 	// typically this can be os.Stdout
 	stdout io.Writer
+
+	// logging levels
+	level  LogLevel
+	prefix string
 }
 
 func NewLogMonitor() *LogMonitor {
@@ -26,21 +40,29 @@ func NewLogMonitorWriter(stdout io.Writer) *LogMonitor {
 		clients: make(map[chan []byte]bool),
 		buffer:  ring.New(10 * 1024), // keep 10KB of buffered logs
 		stdout:  stdout,
+		level:   LevelInfo,
+		prefix:  "",
 	}
 }
 
 func (w *LogMonitor) Write(p []byte) (n int, err error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+
 	n, err = w.stdout.Write(p)
 	if err != nil {
 		return n, err
 	}
 
 	w.bufferMu.Lock()
-	w.buffer.Value = p
+	bufferCopy := make([]byte, len(p))
+	copy(bufferCopy, p)
+	w.buffer.Value = bufferCopy
 	w.buffer = w.buffer.Next()
 	w.bufferMu.Unlock()
 
-	w.broadcast(p)
+	w.broadcast(bufferCopy)
 	return n, nil
 }
 
@@ -49,7 +71,7 @@ func (w *LogMonitor) GetHistory() []byte {
 	defer w.bufferMu.RUnlock()
 
 	var history []byte
-	w.buffer.Do(func(p interface{}) {
+	w.buffer.Do(func(p any) {
 		if p != nil {
 			if content, ok := p.([]byte); ok {
 				history = append(history, content...)
@@ -88,3 +110,77 @@ func (w *LogMonitor) broadcast(msg []byte) {
 		}
 	}
 }
+
+func (w *LogMonitor) SetPrefix(prefix string) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.prefix = prefix
+}
+
+func (w *LogMonitor) SetLogLevel(level LogLevel) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.level = level
+}
+
+func (w *LogMonitor) formatMessage(level string, msg string) []byte {
+	prefix := ""
+	if w.prefix != "" {
+		prefix = fmt.Sprintf("[%s] ", w.prefix)
+	}
+	return []byte(fmt.Sprintf("%s[%s] %s\n", prefix, level, msg))
+}
+
+func (w *LogMonitor) log(level LogLevel, msg string) {
+	if level < w.level {
+		return
+	}
+	w.Write(w.formatMessage(level.String(), msg))
+}
+
+func (w *LogMonitor) Debug(msg string) {
+	w.log(LevelDebug, msg)
+}
+
+func (w *LogMonitor) Info(msg string) {
+	w.log(LevelInfo, msg)
+}
+
+func (w *LogMonitor) Warn(msg string) {
+	w.log(LevelWarn, msg)
+}
+
+func (w *LogMonitor) Error(msg string) {
+	w.log(LevelError, msg)
+}
+
+func (w *LogMonitor) Debugf(format string, args ...interface{}) {
+	w.log(LevelDebug, fmt.Sprintf(format, args...))
+}
+
+func (w *LogMonitor) Infof(format string, args ...interface{}) {
+	w.log(LevelInfo, fmt.Sprintf(format, args...))
+}
+
+func (w *LogMonitor) Warnf(format string, args ...interface{}) {
+	w.log(LevelWarn, fmt.Sprintf(format, args...))
+}
+
+func (w *LogMonitor) Errorf(format string, args ...interface{}) {
+	w.log(LevelError, fmt.Sprintf(format, args...))
+}
+
+func (l LogLevel) String() string {
+	switch l {
+	case LevelDebug:
+		return "DEBUG"
+	case LevelInfo:
+		return "INFO"
+	case LevelWarn:
+		return "WARN"
+	case LevelError:
+		return "ERROR"
+	default:
+		return "UNKNOWN"
+	}
+}

proxy/logMonitor_test.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"bytes"
 	"io"
 	"sync"
 	"testing"
@@ -61,3 +62,34 @@ func TestLogMonitor(t *testing.T) {
 		t.Errorf("Client2 expected %s, got: %s", expectedHistory, c2Data)
 	}
 }
+
+func TestWrite_ImmutableBuffer(t *testing.T) {
+	// Create a new LogMonitor instance
+	lm := NewLogMonitorWriter(io.Discard)
+
+	// Prepare a message to write
+	msg := []byte("Hello, World!")
+	lenmsg := len(msg)
+
+	// Write the message to the LogMonitor
+	n, err := lm.Write(msg)
+	if err != nil {
+		t.Fatalf("Write failed: %v", err)
+	}
+
+	if n != lenmsg {
+		t.Errorf("Expected %d bytes written but got %d", lenmsg, n)
+	}
+
+	// Change the original message
+	msg[0] = 'B' // This should not affect the buffer
+
+	// Get the history from the LogMonitor
+	history := lm.GetHistory()
+
+	// Check that the history contains the original message, not the modified one
+	expected := []byte("Hello, World!")
+	if !bytes.Equal(history, expected) {
+		t.Errorf("Expected history to be %q, got %q", expected, history)
+	}
+}

proxy/manager.go

@@ -1,325 +0,0 @@
-package proxy
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os/exec"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-)
-
-type ProxyManager struct {
-	sync.Mutex
-
-	config        *Config
-	currentCmd    *exec.Cmd
-	currentConfig ModelConfig
-	logMonitor    *LogMonitor
-}
-
-func New(config *Config) *ProxyManager {
-	return &ProxyManager{config: config, logMonitor: NewLogMonitor()}
-}
-
-func (pm *ProxyManager) HandleFunc(w http.ResponseWriter, r *http.Request) {
-
-	// https://github.com/ggerganov/llama.cpp/blob/master/examples/server/README.md#api-endpoints
-	if r.URL.Path == "/v1/chat/completions" {
-		// extracts the `model` from json body
-		pm.proxyChatRequest(w, r)
-	} else if r.URL.Path == "/v1/models" {
-		pm.listModels(w, r)
-	} else if r.URL.Path == "/logs" {
-		pm.streamLogs(w, r)
-	} else {
-		pm.proxyRequest(w, r)
-	}
-}
-
-func (pm *ProxyManager) streamLogs(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "text/plain")
-	w.Header().Set("Transfer-Encoding", "chunked")
-	w.Header().Set("X-Content-Type-Options", "nosniff")
-
-	ch := pm.logMonitor.Subscribe()
-	defer pm.logMonitor.Unsubscribe(ch)
-
-	notify := r.Context().Done()
-	flusher, ok := w.(http.Flusher)
-	if !ok {
-		http.Error(w, "Streaming unsupported", http.StatusInternalServerError)
-		return
-	}
-
-	skipHistory := r.URL.Query().Has("skip")
-	if !skipHistory {
-		// Send history first
-		history := pm.logMonitor.GetHistory()
-		if len(history) != 0 {
-			w.Write(history)
-			flusher.Flush()
-		}
-	}
-
-	if !r.URL.Query().Has("stream") {
-		return
-	}
-
-	// Stream new logs
-	for {
-		select {
-		case msg := <-ch:
-			w.Write(msg)
-			flusher.Flush()
-		case <-notify:
-			return
-		}
-	}
-}
-
-func (pm *ProxyManager) listModels(w http.ResponseWriter, _ *http.Request) {
-	data := []interface{}{}
-	for id := range pm.config.Models {
-		data = append(data, map[string]interface{}{
-			"id":       id,
-			"object":   "model",
-			"created":  time.Now().Unix(),
-			"owned_by": "llama-swap",
-		})
-	}
-
-	// Set the Content-Type header to application/json
-	w.Header().Set("Content-Type", "application/json")
-
-	// Encode the data as JSON and write it to the response writer
-	if err := json.NewEncoder(w).Encode(map[string]interface{}{"data": data}); err != nil {
-		http.Error(w, "Error encoding JSON", http.StatusInternalServerError)
-		return
-	}
-}
-
-func (pm *ProxyManager) swapModel(requestedModel string) error {
-	pm.Lock()
-	defer pm.Unlock()
-
-	// find the model configuration matching requestedModel
-	modelConfig, found := pm.config.FindConfig(requestedModel)
-	if !found {
-		return fmt.Errorf("could not find configuration for %s", requestedModel)
-	}
-
-	// no need to swap llama.cpp instances
-	if pm.currentConfig.Cmd == modelConfig.Cmd {
-		return nil
-	}
-
-	// kill the current running one to swap it
-	if pm.currentCmd != nil {
-		pm.currentCmd.Process.Signal(syscall.SIGTERM)
-
-		// wait for it to end
-		pm.currentCmd.Process.Wait()
-	}
-
-	pm.currentConfig = modelConfig
-
-	args, err := modelConfig.SanitizedCommand()
-	if err != nil {
-		return fmt.Errorf("unable to get sanitized command: %v", err)
-	}
-	cmd := exec.Command(args[0], args[1:]...)
-
-	// logMonitor only writes to stdout
-	// so the upstream's stderr will go to os.Stdout
-	cmd.Stdout = pm.logMonitor
-	cmd.Stderr = pm.logMonitor
-
-	cmd.Env = modelConfig.Env
-
-	err = cmd.Start()
-	if err != nil {
-		return err
-	}
-	pm.currentCmd = cmd
-
-	// watch for the command to exist
-	cmdCtx, cancel := context.WithCancelCause(context.Background())
-
-	// monitor the command's exist status
-	go func() {
-		err := cmd.Wait()
-		if err != nil {
-			cancel(fmt.Errorf("command [%s] %s", strings.Join(cmd.Args, " "), err.Error()))
-		} else {
-			cancel(nil)
-		}
-	}()
-
-	// wait for checkHealthEndpoint
-	if err := pm.checkHealthEndpoint(cmdCtx); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (pm *ProxyManager) checkHealthEndpoint(cmdCtx context.Context) error {
-
-	if pm.currentConfig.Proxy == "" {
-		return fmt.Errorf("no upstream available to check /health")
-	}
-
-	checkEndpoint := strings.TrimSpace(pm.currentConfig.CheckEndpoint)
-
-	if checkEndpoint == "none" {
-		return nil
-	}
-
-	// keep default behaviour
-	if checkEndpoint == "" {
-		checkEndpoint = "/health"
-	}
-
-	proxyTo := pm.currentConfig.Proxy
-	maxDuration := time.Second * time.Duration(pm.config.HealthCheckTimeout)
-	healthURL, err := url.JoinPath(proxyTo, checkEndpoint)
-	if err != nil {
-		return fmt.Errorf("failed to create health url with with %s and path %s", proxyTo, checkEndpoint)
-	}
-
-	client := &http.Client{}
-	startTime := time.Now()
-
-	for {
-		req, err := http.NewRequest("GET", healthURL, nil)
-		if err != nil {
-			return err
-		}
-
-		ctx, cancel := context.WithTimeout(cmdCtx, 250*time.Millisecond)
-		defer cancel()
-		req = req.WithContext(ctx)
-		resp, err := client.Do(req)
-
-		ttl := (maxDuration - time.Since(startTime)).Seconds()
-
-		if err != nil {
-			// check if the context was cancelled
-			select {
-			case <-ctx.Done():
-				return context.Cause(ctx)
-			default:
-			}
-
-			// wait a bit longer for TCP connection issues
-			if strings.Contains(err.Error(), "connection refused") {
-				fmt.Fprintf(pm.logMonitor, "Connection refused on %s, ttl %.0fs\n", healthURL, ttl)
-
-				time.Sleep(5 * time.Second)
-			} else {
-				time.Sleep(time.Second)
-			}
-
-			if ttl < 0 {
-				return fmt.Errorf("failed to check health from: %s", healthURL)
-			}
-
-			continue
-		}
-
-		defer resp.Body.Close()
-		if resp.StatusCode == http.StatusOK {
-			return nil
-		}
-
-		if ttl < 0 {
-			return fmt.Errorf("failed to check health from: %s", healthURL)
-		}
-
-		time.Sleep(time.Second)
-	}
-}
-
-func (pm *ProxyManager) proxyChatRequest(w http.ResponseWriter, r *http.Request) {
-	bodyBytes, err := io.ReadAll(r.Body)
-	if err != nil {
-		http.Error(w, "Invalid JSON", http.StatusBadRequest)
-		return
-	}
-	var requestBody map[string]interface{}
-	if err := json.Unmarshal(bodyBytes, &requestBody); err != nil {
-		http.Error(w, "Invalid JSON", http.StatusBadRequest)
-		return
-	}
-	model, ok := requestBody["model"].(string)
-	if !ok {
-		http.Error(w, "Missing or invalid 'model' key", http.StatusBadRequest)
-		return
-	}
-
-	if err := pm.swapModel(model); err != nil {
-		http.Error(w, fmt.Sprintf("unable to swap to model, %s", err.Error()), http.StatusNotFound)
-		return
-	}
-
-	r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
-	pm.proxyRequest(w, r)
-}
-
-func (pm *ProxyManager) proxyRequest(w http.ResponseWriter, r *http.Request) {
-	if pm.currentConfig.Proxy == "" {
-		http.Error(w, "No upstream proxy", http.StatusInternalServerError)
-		return
-	}
-
-	proxyTo := pm.currentConfig.Proxy
-
-	client := &http.Client{}
-	req, err := http.NewRequest(r.Method, proxyTo+r.URL.String(), r.Body)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	req.Header = r.Header
-	resp, err := client.Do(req)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadGateway)
-		return
-	}
-	defer resp.Body.Close()
-	for k, vv := range resp.Header {
-		for _, v := range vv {
-			w.Header().Add(k, v)
-		}
-	}
-	w.WriteHeader(resp.StatusCode)
-
-	// faster than io.Copy when streaming
-	buf := make([]byte, 32*1024)
-	for {
-		n, err := resp.Body.Read(buf)
-		if n > 0 {
-			if _, writeErr := w.Write(buf[:n]); writeErr != nil {
-				http.Error(w, writeErr.Error(), http.StatusInternalServerError)
-				return
-			}
-			if flusher, ok := w.(http.Flusher); ok {
-				flusher.Flush()
-			}
-		}
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadGateway)
-			return
-		}
-	}
-}

proxy/process.go

@@ -0,0 +1,486 @@
+package proxy
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+type ProcessState string
+
+const (
+	StateStopped  ProcessState = ProcessState("stopped")
+	StateStarting ProcessState = ProcessState("starting")
+	StateReady    ProcessState = ProcessState("ready")
+	StateStopping ProcessState = ProcessState("stopping")
+
+	// failed a health check on start and will not be recovered
+	StateFailed ProcessState = ProcessState("failed")
+
+	// process is shutdown and will not be restarted
+	StateShutdown ProcessState = ProcessState("shutdown")
+)
+
+type Process struct {
+	ID     string
+	config ModelConfig
+	cmd    *exec.Cmd
+
+	// for p.cmd.Wait() select { ... }
+	cmdWaitChan chan error
+
+	processLogger *LogMonitor
+	proxyLogger   *LogMonitor
+
+	healthCheckTimeout      int
+	healthCheckLoopInterval time.Duration
+
+	lastRequestHandled time.Time
+
+	stateMutex sync.RWMutex
+	state      ProcessState
+
+	inFlightRequests sync.WaitGroup
+
+	// used to block on multiple start() calls
+	waitStarting sync.WaitGroup
+
+	// for managing shutdown state
+	shutdownCtx    context.Context
+	shutdownCancel context.CancelFunc
+}
+
+func NewProcess(ID string, healthCheckTimeout int, config ModelConfig, processLogger *LogMonitor, proxyLogger *LogMonitor) *Process {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &Process{
+		ID:                      ID,
+		config:                  config,
+		cmd:                     nil,
+		cmdWaitChan:             make(chan error, 1),
+		processLogger:           processLogger,
+		proxyLogger:             proxyLogger,
+		healthCheckTimeout:      healthCheckTimeout,
+		healthCheckLoopInterval: 5 * time.Second, /* default, can not be set by user - used for testing */
+		state:                   StateStopped,
+		shutdownCtx:             ctx,
+		shutdownCancel:          cancel,
+	}
+}
+
+// LogMonitor returns the log monitor associated with the process.
+func (p *Process) LogMonitor() *LogMonitor {
+	return p.processLogger
+}
+
+// custom error types for swapping state
+var (
+	ErrExpectedStateMismatch  = errors.New("expected state mismatch")
+	ErrInvalidStateTransition = errors.New("invalid state transition")
+)
+
+// swapState performs a compare and swap of the state atomically. It returns the current state
+// and an error if the swap failed.
+func (p *Process) swapState(expectedState, newState ProcessState) (ProcessState, error) {
+	p.stateMutex.Lock()
+	defer p.stateMutex.Unlock()
+
+	if p.state != expectedState {
+		p.proxyLogger.Warnf("<%s> swapState() Unexpected current state %s, expected %s", p.ID, p.state, expectedState)
+		return p.state, ErrExpectedStateMismatch
+	}
+
+	if !isValidTransition(p.state, newState) {
+		p.proxyLogger.Warnf("<%s> swapState() Invalid state transition from %s to %s", p.ID, p.state, newState)
+		return p.state, ErrInvalidStateTransition
+	}
+
+	p.state = newState
+	p.proxyLogger.Debugf("<%s> swapState() State transitioned from %s to %s", p.ID, expectedState, newState)
+	return p.state, nil
+}
+
+// Helper function to encapsulate transition rules
+func isValidTransition(from, to ProcessState) bool {
+	switch from {
+	case StateStopped:
+		return to == StateStarting
+	case StateStarting:
+		return to == StateReady || to == StateFailed || to == StateStopping
+	case StateReady:
+		return to == StateStopping
+	case StateStopping:
+		return to == StateStopped || to == StateShutdown
+	case StateFailed, StateShutdown:
+		return false // No transitions allowed from these states
+	}
+	return false
+}
+
+func (p *Process) CurrentState() ProcessState {
+	p.stateMutex.RLock()
+	defer p.stateMutex.RUnlock()
+	return p.state
+}
+
+// start starts the upstream command, checks the health endpoint, and sets the state to Ready
+// it is a private method because starting is automatic but stopping can be called
+// at any time.
+func (p *Process) start() error {
+
+	if p.config.Proxy == "" {
+		return fmt.Errorf("can not start(), upstream proxy missing")
+	}
+
+	args, err := p.config.SanitizedCommand()
+	if err != nil {
+		return fmt.Errorf("unable to get sanitized command: %v", err)
+	}
+
+	if curState, err := p.swapState(StateStopped, StateStarting); err != nil {
+		if err == ErrExpectedStateMismatch {
+			// already starting, just wait for it to complete and expect
+			// it to be be in the Ready start after. If not, return an error
+			if curState == StateStarting {
+				p.waitStarting.Wait()
+				if state := p.CurrentState(); state == StateReady {
+					return nil
+				} else {
+					return fmt.Errorf("process was already starting but wound up in state %v", state)
+				}
+			} else {
+				return fmt.Errorf("processes was in state %v when start() was called", curState)
+			}
+		} else {
+			return fmt.Errorf("failed to set Process state to starting: current state: %v, error: %v", curState, err)
+		}
+	}
+
+	p.waitStarting.Add(1)
+	defer p.waitStarting.Done()
+
+	p.cmd = exec.Command(args[0], args[1:]...)
+	p.cmd.Stdout = p.processLogger
+	p.cmd.Stderr = p.processLogger
+	p.cmd.Env = p.config.Env
+
+	err = p.cmd.Start()
+
+	// Set process state to failed
+	if err != nil {
+		if curState, swapErr := p.swapState(StateStarting, StateFailed); swapErr != nil {
+			return fmt.Errorf(
+				"failed to start command and state swap failed. command error: %v, current state: %v, state swap error: %v",
+				err, curState, swapErr,
+			)
+		}
+		return fmt.Errorf("start() failed: %v", err)
+	}
+
+	// Capture the exit error for later signaling
+	go func() {
+		exitErr := p.cmd.Wait()
+		p.proxyLogger.Debugf("<%s> cmd.Wait() returned error: %v", p.ID, exitErr)
+		p.cmdWaitChan <- exitErr
+	}()
+
+	// One of three things can happen at this stage:
+	// 1. The command exits unexpectedly
+	// 2. The health check fails
+	// 3. The health check passes
+	//
+	// only in the third case will the process be considered Ready to accept
+	<-time.After(250 * time.Millisecond) // give process a bit of time to start
+
+	checkStartTime := time.Now()
+	maxDuration := time.Second * time.Duration(p.healthCheckTimeout)
+	checkEndpoint := strings.TrimSpace(p.config.CheckEndpoint)
+
+	// a "none" means don't check for health ... I could have picked a better word :facepalm:
+	if checkEndpoint != "none" {
+		// keep default behaviour
+		if checkEndpoint == "" {
+			checkEndpoint = "/health"
+		}
+
+		proxyTo := p.config.Proxy
+		healthURL, err := url.JoinPath(proxyTo, checkEndpoint)
+		if err != nil {
+			return fmt.Errorf("failed to create health check URL proxy=%s and checkEndpoint=%s", proxyTo, checkEndpoint)
+		}
+
+		checkDeadline, cancelHealthCheck := context.WithDeadline(
+			context.Background(),
+			checkStartTime.Add(maxDuration),
+		)
+		defer cancelHealthCheck()
+
+	loop:
+		// Ready Check loop
+		for {
+			select {
+			case <-checkDeadline.Done():
+				if curState, err := p.swapState(StateStarting, StateFailed); err != nil {
+					return fmt.Errorf("health check timed out after %vs AND state swap failed: %v, current state: %v", maxDuration.Seconds(), err, curState)
+				} else {
+					return fmt.Errorf("health check timed out after %vs", maxDuration.Seconds())
+				}
+			case <-p.shutdownCtx.Done():
+				return errors.New("health check interrupted due to shutdown")
+			case exitErr := <-p.cmdWaitChan:
+				if exitErr != nil {
+					p.proxyLogger.Warnf("<%s> upstream command exited prematurely with error: %v", p.ID, exitErr)
+					if curState, err := p.swapState(StateStarting, StateFailed); err != nil {
+						return fmt.Errorf("upstream command exited unexpectedly: %s AND state swap failed: %v, current state: %v", exitErr.Error(), err, curState)
+					} else {
+						return fmt.Errorf("upstream command exited unexpectedly: %s", exitErr.Error())
+					}
+				} else {
+					p.proxyLogger.Warnf("<%s> upstream command exited prematurely but successfully", p.ID)
+					if curState, err := p.swapState(StateStarting, StateFailed); err != nil {
+						return fmt.Errorf("upstream command exited prematurely but successfully AND state swap failed: %v, current state: %v", err, curState)
+					} else {
+						return fmt.Errorf("upstream command exited prematurely but successfully")
+					}
+				}
+			default:
+				if err := p.checkHealthEndpoint(healthURL); err == nil {
+					p.proxyLogger.Infof("<%s> Health check passed on %s", p.ID, healthURL)
+					cancelHealthCheck()
+					break loop
+				} else {
+					if strings.Contains(err.Error(), "connection refused") {
+						endTime, _ := checkDeadline.Deadline()
+						ttl := time.Until(endTime)
+						p.proxyLogger.Infof("<%s> Connection refused on %s, giving up in %.0fs", p.ID, healthURL, ttl.Seconds())
+					} else {
+						p.proxyLogger.Infof("<%s> Health check error on %s, %v", p.ID, healthURL, err)
+					}
+				}
+			}
+
+			<-time.After(p.healthCheckLoopInterval)
+		}
+	}
+
+	if p.config.UnloadAfter > 0 {
+		// start a goroutine to check every second if
+		// the process should be stopped
+		go func() {
+			maxDuration := time.Duration(p.config.UnloadAfter) * time.Second
+
+			for range time.Tick(time.Second) {
+				if p.CurrentState() != StateReady {
+					return
+				}
+
+				// wait for all inflight requests to complete and ticker
+				p.inFlightRequests.Wait()
+
+				if time.Since(p.lastRequestHandled) > maxDuration {
+					p.proxyLogger.Infof("<%s> Unloading model, TTL of %ds reached", p.ID, p.config.UnloadAfter)
+					p.Stop()
+					return
+				}
+			}
+		}()
+	}
+
+	if curState, err := p.swapState(StateStarting, StateReady); err != nil {
+		return fmt.Errorf("failed to set Process state to ready: current state: %v, error: %v", curState, err)
+	} else {
+		return nil
+	}
+}
+
+func (p *Process) Stop() {
+	if !isValidTransition(p.CurrentState(), StateStopping) {
+		return
+	}
+
+	// wait for any inflight requests before proceeding
+	p.inFlightRequests.Wait()
+	p.proxyLogger.Debugf("<%s> Stopping process", p.ID)
+
+	// calling Stop() when state is invalid is a no-op
+	if curState, err := p.swapState(StateReady, StateStopping); err != nil {
+		p.proxyLogger.Infof("<%s> Stop() Ready -> StateStopping err: %v, current state: %v", p.ID, err, curState)
+		return
+	}
+
+	// stop the process with a graceful exit timeout
+	p.stopCommand(5 * time.Second)
+
+	if curState, err := p.swapState(StateStopping, StateStopped); err != nil {
+		p.proxyLogger.Infof("<%s> Stop() StateStopping -> StateStopped err: %v, current state: %v", p.ID, err, curState)
+	}
+}
+
+// Shutdown is called when llama-swap is shutting down. It will give a little bit
+// of time for any inflight requests to complete before shutting down. If the Process
+// is in the state of starting, it will cancel it and shut it down
+func (p *Process) Shutdown() {
+	p.shutdownCancel()
+	p.stopCommand(5 * time.Second)
+	p.state = StateShutdown
+}
+
+// stopCommand will send a SIGTERM to the process and wait for it to exit.
+// If it does not exit within 5 seconds, it will send a SIGKILL.
+func (p *Process) stopCommand(sigtermTTL time.Duration) {
+	stopStartTime := time.Now()
+	defer func() {
+		p.proxyLogger.Debugf("<%s> stopCommand took %v", p.ID, time.Since(stopStartTime))
+	}()
+
+	sigtermTimeout, cancelTimeout := context.WithTimeout(context.Background(), sigtermTTL)
+	defer cancelTimeout()
+
+	if p.cmd == nil || p.cmd.Process == nil {
+		p.proxyLogger.Warnf("<%s> cmd or cmd.Process is nil", p.ID)
+		return
+	}
+
+	if err := p.terminateProcess(); err != nil {
+		p.proxyLogger.Infof("<%s> Failed to gracefully terminate process: %v", p.ID, err)
+	}
+
+	select {
+	case <-sigtermTimeout.Done():
+		p.proxyLogger.Infof("<%s> Process timed out waiting to stop, sending KILL signal", p.ID)
+		p.cmd.Process.Kill()
+	case err := <-p.cmdWaitChan:
+		// Note: in start(), p.cmdWaitChan also has a select { ... }. That should be OK
+		// because if we make it here then the cmd has been successfully running and made it
+		// through the health check. There is a possibility that ithe cmd crashed after the health check
+		// succeeded but that's not a case llama-swap is handling for now.
+		if err != nil {
+			if errno, ok := err.(syscall.Errno); ok {
+				p.proxyLogger.Errorf("<%s> errno >> %v", p.ID, errno)
+			} else if exitError, ok := err.(*exec.ExitError); ok {
+				if strings.Contains(exitError.String(), "signal: terminated") {
+					p.proxyLogger.Infof("<%s> Process stopped OK", p.ID)
+				} else if strings.Contains(exitError.String(), "signal: interrupt") {
+					p.proxyLogger.Infof("<%s> Process interrupted OK", p.ID)
+				} else {
+					p.proxyLogger.Warnf("<%s> ExitError >> %v, exit code: %d", p.ID, exitError, exitError.ExitCode())
+				}
+			} else {
+				p.proxyLogger.Errorf("<%s> Process exited >> %v", p.ID, err)
+			}
+		}
+	}
+}
+
+func (p *Process) checkHealthEndpoint(healthURL string) error {
+	client := &http.Client{
+		Timeout: 500 * time.Millisecond,
+	}
+
+	req, err := http.NewRequest("GET", healthURL, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// got a response but it was not an OK
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("status code: %d", resp.StatusCode)
+	}
+
+	return nil
+}
+
+func (p *Process) ProxyRequest(w http.ResponseWriter, r *http.Request) {
+	requestBeginTime := time.Now()
+	var startDuration time.Duration
+
+	// prevent new requests from being made while stopping or irrecoverable
+	currentState := p.CurrentState()
+	if currentState == StateFailed || currentState == StateShutdown || currentState == StateStopping {
+		http.Error(w, fmt.Sprintf("Process can not ProxyRequest, state is %s", currentState), http.StatusServiceUnavailable)
+		return
+	}
+
+	p.inFlightRequests.Add(1)
+	defer func() {
+		p.lastRequestHandled = time.Now()
+		p.inFlightRequests.Done()
+	}()
+
+	// start the process on demand
+	if p.CurrentState() != StateReady {
+		beginStartTime := time.Now()
+		if err := p.start(); err != nil {
+			errstr := fmt.Sprintf("unable to start process: %s", err)
+			http.Error(w, errstr, http.StatusBadGateway)
+			return
+		}
+		startDuration = time.Since(beginStartTime)
+	}
+
+	proxyTo := p.config.Proxy
+	client := &http.Client{}
+	req, err := http.NewRequestWithContext(r.Context(), r.Method, proxyTo+r.URL.String(), r.Body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	req.Header = r.Header.Clone()
+
+	contentLength, err := strconv.ParseInt(req.Header.Get("content-length"), 10, 64)
+	if err == nil {
+		req.ContentLength = contentLength
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadGateway)
+		return
+	}
+	defer resp.Body.Close()
+	for k, vv := range resp.Header {
+		for _, v := range vv {
+			w.Header().Add(k, v)
+		}
+	}
+	w.WriteHeader(resp.StatusCode)
+
+	// faster than io.Copy when streaming
+	buf := make([]byte, 32*1024)
+	for {
+		n, err := resp.Body.Read(buf)
+		if n > 0 {
+			if _, writeErr := w.Write(buf[:n]); writeErr != nil {
+				return
+			}
+			if flusher, ok := w.(http.Flusher); ok {
+				flusher.Flush()
+			}
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+	}
+
+	totalTime := time.Since(requestBeginTime)
+	p.proxyLogger.Debugf("<%s> request %s - start: %v, total: %v",
+		p.ID, r.RequestURI, startDuration, totalTime)
+}

proxy/process_stop.go

@@ -0,0 +1,9 @@
+//go:build !windows
+
+package proxy
+
+import "syscall"
+
+func (p *Process) terminateProcess() error {
+	return p.cmd.Process.Signal(syscall.SIGTERM)
+}

proxy/process_stop_windows.go

@@ -0,0 +1,14 @@
+//go:build windows
+
+package proxy
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+func (p *Process) terminateProcess() error {
+	pid := fmt.Sprintf("%d", p.cmd.Process.Pid)
+	cmd := exec.Command("taskkill", "/f", "/t", "/pid", pid)
+	return cmd.Run()
+}

proxy/process_test.go

@@ -0,0 +1,342 @@
+package proxy
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	debugLogger = NewLogMonitorWriter(os.Stdout)
+)
+
+func init() {
+	// flip to help with debugging tests
+	if false {
+		debugLogger.SetLogLevel(LevelDebug)
+	} else {
+		debugLogger.SetLogLevel(LevelError)
+	}
+}
+
+func TestProcess_AutomaticallyStartsUpstream(t *testing.T) {
+
+	expectedMessage := "testing91931"
+	config := getTestSimpleResponderConfig(expectedMessage)
+
+	// Create a process
+	process := NewProcess("test-process", 5, config, debugLogger, debugLogger)
+	defer process.Stop()
+
+	req := httptest.NewRequest("GET", "/test", nil)
+	w := httptest.NewRecorder()
+
+	// process is automatically started
+	assert.Equal(t, StateStopped, process.CurrentState())
+	process.ProxyRequest(w, req)
+	assert.Equal(t, StateReady, process.CurrentState())
+
+	assert.Equal(t, http.StatusOK, w.Code, "Expected status code %d, got %d", http.StatusOK, w.Code)
+	assert.Contains(t, w.Body.String(), expectedMessage)
+
+	// Stop the process
+	process.Stop()
+
+	req = httptest.NewRequest("GET", "/", nil)
+	w = httptest.NewRecorder()
+
+	// Proxy the request
+	process.ProxyRequest(w, req)
+
+	// should have automatically started the process again
+	if w.Code != http.StatusOK {
+		t.Errorf("Expected status code %d, got %d", http.StatusOK, w.Code)
+	}
+}
+
+// TestProcess_WaitOnMultipleStarts tests that multiple concurrent requests
+// are all handled successfully, even though they all may ask for the process to .start()
+func TestProcess_WaitOnMultipleStarts(t *testing.T) {
+
+	expectedMessage := "testing91931"
+	config := getTestSimpleResponderConfig(expectedMessage)
+
+	process := NewProcess("test-process", 5, config, debugLogger, debugLogger)
+	defer process.Stop()
+
+	var wg sync.WaitGroup
+	for i := 0; i < 5; i++ {
+		wg.Add(1)
+		go func(reqID int) {
+			defer wg.Done()
+			req := httptest.NewRequest("GET", "/test", nil)
+			w := httptest.NewRecorder()
+			process.ProxyRequest(w, req)
+			assert.Equal(t, http.StatusOK, w.Code, "Worker %d got wrong HTTP code", reqID)
+			assert.Contains(t, w.Body.String(), expectedMessage, "Worker %d got wrong message", reqID)
+		}(i)
+	}
+	wg.Wait()
+	assert.Equal(t, StateReady, process.CurrentState())
+}
+
+// test that the automatic start returns the expected error type
+func TestProcess_BrokenModelConfig(t *testing.T) {
+	// Create a process configuration
+	config := ModelConfig{
+		Cmd:           "nonexistent-command",
+		Proxy:         "http://127.0.0.1:9913",
+		CheckEndpoint: "/health",
+	}
+
+	process := NewProcess("broken", 1, config, debugLogger, debugLogger)
+
+	req := httptest.NewRequest("GET", "/", nil)
+	w := httptest.NewRecorder()
+	process.ProxyRequest(w, req)
+	assert.Equal(t, http.StatusBadGateway, w.Code)
+	assert.Contains(t, w.Body.String(), "unable to start process")
+
+	w = httptest.NewRecorder()
+	process.ProxyRequest(w, req)
+	assert.Equal(t, http.StatusServiceUnavailable, w.Code)
+	assert.Contains(t, w.Body.String(), "Process can not ProxyRequest, state is failed")
+}
+
+func TestProcess_UnloadAfterTTL(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping long auto unload TTL test")
+	}
+
+	expectedMessage := "I_sense_imminent_danger"
+	config := getTestSimpleResponderConfig(expectedMessage)
+	assert.Equal(t, 0, config.UnloadAfter)
+	config.UnloadAfter = 3 // seconds
+	assert.Equal(t, 3, config.UnloadAfter)
+
+	process := NewProcess("ttl_test", 2, config, debugLogger, debugLogger)
+	defer process.Stop()
+
+	// this should take 4 seconds
+	req1 := httptest.NewRequest("GET", "/slow-respond?echo=1234&delay=1000ms", nil)
+	req2 := httptest.NewRequest("GET", "/test", nil)
+
+	w := httptest.NewRecorder()
+
+	// Proxy the request (auto start) with a slow response that takes longer than config.UnloadAfter
+	process.ProxyRequest(w, req1)
+
+	t.Log("sending slow first request (4 seconds)")
+	assert.Equal(t, http.StatusOK, w.Code, "Expected status code %d, got %d", http.StatusOK, w.Code)
+	assert.Contains(t, w.Body.String(), "1234")
+	assert.Equal(t, StateReady, process.CurrentState())
+
+	// ensure the TTL timeout does not race slow requests (see issue #25)
+	t.Log("sending second request (1 second)")
+	time.Sleep(time.Second)
+	w = httptest.NewRecorder()
+	process.ProxyRequest(w, req2)
+	assert.Equal(t, http.StatusOK, w.Code, "Expected status code %d, got %d", http.StatusOK, w.Code)
+	assert.Contains(t, w.Body.String(), expectedMessage)
+	assert.Equal(t, StateReady, process.CurrentState())
+
+	// wait 5 seconds
+	t.Log("sleep 5 seconds and check if unloaded")
+	time.Sleep(5 * time.Second)
+	assert.Equal(t, StateStopped, process.CurrentState())
+}
+
+func TestProcess_LowTTLValue(t *testing.T) {
+	if true { // change this code to run this ...
+		t.Skip("skipping test, edit process_test.go to run it ")
+	}
+
+	config := getTestSimpleResponderConfig("fast_ttl")
+	assert.Equal(t, 0, config.UnloadAfter)
+	config.UnloadAfter = 1 // second
+	assert.Equal(t, 1, config.UnloadAfter)
+
+	process := NewProcess("ttl", 2, config, debugLogger, debugLogger)
+	defer process.Stop()
+
+	for i := 0; i < 100; i++ {
+		t.Logf("Waiting before sending request %d", i)
+		time.Sleep(1500 * time.Millisecond)
+
+		expected := fmt.Sprintf("echo=test_%d", i)
+		req := httptest.NewRequest("GET", fmt.Sprintf("/slow-respond?echo=%s&delay=50ms", expected), nil)
+		w := httptest.NewRecorder()
+		process.ProxyRequest(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Contains(t, w.Body.String(), expected)
+	}
+
+}
+
+// issue #19
+// This test makes sure using Process.Stop() does not affect pending HTTP
+// requests. All HTTP requests in this test should complete successfully.
+func TestProcess_HTTPRequestsHaveTimeToFinish(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping slow test")
+	}
+
+	expectedMessage := "12345"
+	config := getTestSimpleResponderConfig(expectedMessage)
+	process := NewProcess("t", 10, config, debugLogger, debugLogger)
+	defer process.Stop()
+
+	results := map[string]string{
+		"12345": "",
+		"abcde": "",
+		"fghij": "",
+	}
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+
+	for key := range results {
+		wg.Add(1)
+		go func(key string) {
+			defer wg.Done()
+			// send a request where simple-responder is will wait 300ms before responding
+			// this will simulate an in-progress request.
+			req := httptest.NewRequest("GET", fmt.Sprintf("/slow-respond?echo=%s&delay=300ms", key), nil)
+			w := httptest.NewRecorder()
+
+			process.ProxyRequest(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Errorf("Expected status OK, got %d for key %s", w.Code, key)
+			}
+
+			mu.Lock()
+			results[key] = w.Body.String()
+			mu.Unlock()
+
+		}(key)
+	}
+
+	// Stop the process while requests are still being processed
+	go func() {
+		<-time.After(150 * time.Millisecond)
+		process.Stop()
+	}()
+
+	wg.Wait()
+
+	for key, result := range results {
+		assert.Equal(t, key, result)
+	}
+}
+
+func TestProcess_SwapState(t *testing.T) {
+	tests := []struct {
+		name           string
+		currentState   ProcessState
+		expectedState  ProcessState
+		newState       ProcessState
+		expectedError  error
+		expectedResult ProcessState
+	}{
+		{"Stopped to Starting", StateStopped, StateStopped, StateStarting, nil, StateStarting},
+		{"Starting to Ready", StateStarting, StateStarting, StateReady, nil, StateReady},
+		{"Starting to Failed", StateStarting, StateStarting, StateFailed, nil, StateFailed},
+		{"Starting to Stopping", StateStarting, StateStarting, StateStopping, nil, StateStopping},
+		{"Ready to Stopping", StateReady, StateReady, StateStopping, nil, StateStopping},
+		{"Stopping to Stopped", StateStopping, StateStopping, StateStopped, nil, StateStopped},
+		{"Stopping to Shutdown", StateStopping, StateStopping, StateShutdown, nil, StateShutdown},
+		{"Stopped to Ready", StateStopped, StateStopped, StateReady, ErrInvalidStateTransition, StateStopped},
+		{"Starting to Stopped", StateStarting, StateStarting, StateStopped, ErrInvalidStateTransition, StateStarting},
+		{"Ready to Starting", StateReady, StateReady, StateStarting, ErrInvalidStateTransition, StateReady},
+		{"Ready to Failed", StateReady, StateReady, StateFailed, ErrInvalidStateTransition, StateReady},
+		{"Stopping to Ready", StateStopping, StateStopping, StateReady, ErrInvalidStateTransition, StateStopping},
+		{"Failed to Stopped", StateFailed, StateFailed, StateStopped, ErrInvalidStateTransition, StateFailed},
+		{"Failed to Starting", StateFailed, StateFailed, StateStarting, ErrInvalidStateTransition, StateFailed},
+		{"Shutdown to Stopped", StateShutdown, StateShutdown, StateStopped, ErrInvalidStateTransition, StateShutdown},
+		{"Shutdown to Starting", StateShutdown, StateShutdown, StateStarting, ErrInvalidStateTransition, StateShutdown},
+		{"Expected state mismatch", StateStopped, StateStarting, StateStarting, ErrExpectedStateMismatch, StateStopped},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			p := NewProcess("test", 10, getTestSimpleResponderConfig("test"), debugLogger, debugLogger)
+			p.state = test.currentState
+
+			resultState, err := p.swapState(test.expectedState, test.newState)
+			if err != nil && test.expectedError == nil {
+				t.Errorf("Unexpected error: %v", err)
+			} else if err == nil && test.expectedError != nil {
+				t.Errorf("Expected error: %v, but got none", test.expectedError)
+			} else if err != nil && test.expectedError != nil {
+				if err.Error() != test.expectedError.Error() {
+					t.Errorf("Expected error: %v, got: %v", test.expectedError, err)
+				}
+			}
+
+			if resultState != test.expectedResult {
+				t.Errorf("Expected state: %v, got: %v", test.expectedResult, resultState)
+			}
+		})
+	}
+}
+
+func TestProcess_ShutdownInterruptsHealthCheck(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping long shutdown test")
+	}
+
+	expectedMessage := "testing91931"
+
+	// make a config where the healthcheck will always fail because port is wrong
+	config := getTestSimpleResponderConfigPort(expectedMessage, 9999)
+	config.Proxy = "http://localhost:9998/test"
+
+	healthCheckTTLSeconds := 30
+	process := NewProcess("test-process", healthCheckTTLSeconds, config, debugLogger, debugLogger)
+
+	// make it a lot faster
+	process.healthCheckLoopInterval = time.Second
+
+	// start a goroutine to simulate a shutdown
+	var wg sync.WaitGroup
+	go func() {
+		defer wg.Done()
+		<-time.After(time.Millisecond * 500)
+		process.Shutdown()
+	}()
+	wg.Add(1)
+
+	// start the process, this is a blocking call
+	err := process.start()
+
+	wg.Wait()
+	assert.ErrorContains(t, err, "health check interrupted due to shutdown")
+	assert.Equal(t, StateShutdown, process.CurrentState())
+}
+
+func TestProcess_ExitInterruptsHealthCheck(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping Exit Interrupts Health Check test")
+	}
+
+	// should run and exit but interrupt the long checkHealthTimeout
+	checkHealthTimeout := 5
+	config := ModelConfig{
+		Cmd:           "sleep 1",
+		Proxy:         "http://127.0.0.1:9913",
+		CheckEndpoint: "/health",
+	}
+
+	process := NewProcess("sleepy", checkHealthTimeout, config, debugLogger, debugLogger)
+	process.healthCheckLoopInterval = time.Second // make it faster
+	err := process.start()
+	assert.Equal(t, "upstream command exited prematurely but successfully", err.Error())
+	assert.Equal(t, process.CurrentState(), StateFailed)
+}

proxy/processgroup.go

@@ -0,0 +1,113 @@
+package proxy
+
+import (
+	"fmt"
+	"net/http"
+	"slices"
+	"sync"
+)
+
+type ProcessGroup struct {
+	sync.Mutex
+
+	config     Config
+	id         string
+	swap       bool
+	exclusive  bool
+	persistent bool
+
+	proxyLogger    *LogMonitor
+	upstreamLogger *LogMonitor
+
+	// map of current processes
+	processes       map[string]*Process
+	lastUsedProcess string
+}
+
+func NewProcessGroup(id string, config Config, proxyLogger *LogMonitor, upstreamLogger *LogMonitor) *ProcessGroup {
+	groupConfig, ok := config.Groups[id]
+	if !ok {
+		panic("Unable to find configuration for group id: " + id)
+	}
+
+	pg := &ProcessGroup{
+		id:             id,
+		config:         config,
+		swap:           groupConfig.Swap,
+		exclusive:      groupConfig.Exclusive,
+		persistent:     groupConfig.Persistent,
+		proxyLogger:    proxyLogger,
+		upstreamLogger: upstreamLogger,
+		processes:      make(map[string]*Process),
+	}
+
+	// Create a Process for each member in the group
+	for _, modelID := range groupConfig.Members {
+		modelConfig, modelID, _ := pg.config.FindConfig(modelID)
+		process := NewProcess(modelID, pg.config.HealthCheckTimeout, modelConfig, pg.upstreamLogger, pg.proxyLogger)
+		pg.processes[modelID] = process
+	}
+
+	return pg
+}
+
+// ProxyRequest proxies a request to the specified model
+func (pg *ProcessGroup) ProxyRequest(modelID string, writer http.ResponseWriter, request *http.Request) error {
+	if !pg.HasMember(modelID) {
+		return fmt.Errorf("model %s not part of group %s", modelID, pg.id)
+	}
+
+	if pg.swap {
+		pg.Lock()
+		if pg.lastUsedProcess != modelID {
+			if pg.lastUsedProcess != "" {
+				pg.processes[pg.lastUsedProcess].Stop()
+			}
+			pg.lastUsedProcess = modelID
+		}
+		pg.Unlock()
+	}
+
+	pg.processes[modelID].ProxyRequest(writer, request)
+	return nil
+}
+
+func (pg *ProcessGroup) HasMember(modelName string) bool {
+	return slices.Contains(pg.config.Groups[pg.id].Members, modelName)
+}
+
+func (pg *ProcessGroup) StopProcesses() {
+	pg.Lock()
+	defer pg.Unlock()
+	pg.stopProcesses()
+}
+
+// stopProcesses stops all processes in the group
+func (pg *ProcessGroup) stopProcesses() {
+	if len(pg.processes) == 0 {
+		return
+	}
+
+	// stop Processes in parallel
+	var wg sync.WaitGroup
+	for _, process := range pg.processes {
+		wg.Add(1)
+		go func(process *Process) {
+			defer wg.Done()
+			process.Stop()
+		}(process)
+	}
+	wg.Wait()
+}
+
+func (pg *ProcessGroup) Shutdown() {
+	var wg sync.WaitGroup
+	for _, process := range pg.processes {
+		wg.Add(1)
+		go func(process *Process) {
+			defer wg.Done()
+			process.Shutdown()
+		}(process)
+	}
+	wg.Wait()
+}

proxy/processgroup_test.go

@@ -0,0 +1,96 @@
+package proxy
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var processGroupTestConfig = AddDefaultGroupToConfig(Config{
+	HealthCheckTimeout: 15,
+	Models: map[string]ModelConfig{
+		"model1": getTestSimpleResponderConfig("model1"),
+		"model2": getTestSimpleResponderConfig("model2"),
+		"model3": getTestSimpleResponderConfig("model3"),
+		"model4": getTestSimpleResponderConfig("model4"),
+		"model5": getTestSimpleResponderConfig("model5"),
+	},
+	Groups: map[string]GroupConfig{
+		"G1": {
+			Swap:      true,
+			Exclusive: true,
+			Members:   []string{"model1", "model2"},
+		},
+		"G2": {
+			Swap:      false,
+			Exclusive: true,
+			Members:   []string{"model3", "model4"},
+		},
+	},
+})
+
+func TestProcessGroup_DefaultHasCorrectModel(t *testing.T) {
+	pg := NewProcessGroup(DEFAULT_GROUP_ID, processGroupTestConfig, testLogger, testLogger)
+	assert.True(t, pg.HasMember("model5"))
+}
+
+func TestProcessGroup_HasMember(t *testing.T) {
+	pg := NewProcessGroup("G1", processGroupTestConfig, testLogger, testLogger)
+	assert.True(t, pg.HasMember("model1"))
+	assert.True(t, pg.HasMember("model2"))
+	assert.False(t, pg.HasMember("model3"))
+}
+
+func TestProcessGroup_ProxyRequestSwapIsTrue(t *testing.T) {
+	pg := NewProcessGroup("G1", processGroupTestConfig, testLogger, testLogger)
+	defer pg.StopProcesses()
+
+	tests := []string{"model1", "model2"}
+
+	for _, modelName := range tests {
+		t.Run(modelName, func(t *testing.T) {
+			reqBody := `{"x", "y"}`
+			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+			w := httptest.NewRecorder()
+
+			assert.NoError(t, pg.ProxyRequest(modelName, w, req))
+			assert.Equal(t, http.StatusOK, w.Code)
+			assert.Contains(t, w.Body.String(), modelName)
+
+			// make sure only one process is in the running state
+			count := 0
+			for _, process := range pg.processes {
+				if process.CurrentState() == StateReady {
+					count++
+				}
+			}
+			assert.Equal(t, 1, count)
+		})
+	}
+}
+
+func TestProcessGroup_ProxyRequestSwapIsFalse(t *testing.T) {
+	pg := NewProcessGroup("G2", processGroupTestConfig, testLogger, testLogger)
+	defer pg.StopProcesses()
+
+	tests := []string{"model3", "model4"}
+
+	for _, modelName := range tests {
+		t.Run(modelName, func(t *testing.T) {
+			reqBody := `{"x", "y"}`
+			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+			w := httptest.NewRecorder()
+			assert.NoError(t, pg.ProxyRequest(modelName, w, req))
+			assert.Equal(t, http.StatusOK, w.Code)
+			assert.Contains(t, w.Body.String(), modelName)
+		})
+	}
+
+	// make sure all the processes are running
+	for _, process := range pg.processes {
+		assert.Equal(t, StateReady, process.CurrentState())
+	}
+}

proxy/proxymanager.go

@@ -0,0 +1,538 @@
+package proxy
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+const (
+	PROFILE_SPLIT_CHAR = ":"
+)
+
+type ProxyManager struct {
+	sync.Mutex
+
+	config    Config
+	ginEngine *gin.Engine
+
+	// logging
+	proxyLogger    *LogMonitor
+	upstreamLogger *LogMonitor
+	muxLogger      *LogMonitor
+
+	processGroups map[string]*ProcessGroup
+}
+
+func New(config Config) *ProxyManager {
+	// set up loggers
+	stdoutLogger := NewLogMonitorWriter(os.Stdout)
+	upstreamLogger := NewLogMonitorWriter(stdoutLogger)
+	proxyLogger := NewLogMonitorWriter(stdoutLogger)
+
+	if config.LogRequests {
+		proxyLogger.Warn("LogRequests configuration is deprecated. Use logLevel instead.")
+	}
+
+	switch strings.ToLower(strings.TrimSpace(config.LogLevel)) {
+	case "debug":
+		proxyLogger.SetLogLevel(LevelDebug)
+		upstreamLogger.SetLogLevel(LevelDebug)
+	case "info":
+		proxyLogger.SetLogLevel(LevelInfo)
+		upstreamLogger.SetLogLevel(LevelInfo)
+	case "warn":
+		proxyLogger.SetLogLevel(LevelWarn)
+		upstreamLogger.SetLogLevel(LevelWarn)
+	case "error":
+		proxyLogger.SetLogLevel(LevelError)
+		upstreamLogger.SetLogLevel(LevelError)
+	default:
+		proxyLogger.SetLogLevel(LevelInfo)
+		upstreamLogger.SetLogLevel(LevelInfo)
+	}
+
+	pm := &ProxyManager{
+		config:    config,
+		ginEngine: gin.New(),
+
+		proxyLogger:    proxyLogger,
+		muxLogger:      stdoutLogger,
+		upstreamLogger: upstreamLogger,
+
+		processGroups: make(map[string]*ProcessGroup),
+	}
+
+	// create the process groups
+	for groupID := range config.Groups {
+		processGroup := NewProcessGroup(groupID, config, proxyLogger, upstreamLogger)
+		pm.processGroups[groupID] = processGroup
+	}
+
+	pm.ginEngine.Use(func(c *gin.Context) {
+		// Start timer
+		start := time.Now()
+
+		// capture these because /upstream/:model rewrites them in c.Next()
+		clientIP := c.ClientIP()
+		method := c.Request.Method
+		path := c.Request.URL.Path
+
+		// Process request
+		c.Next()
+
+		// Stop timer
+		duration := time.Since(start)
+
+		statusCode := c.Writer.Status()
+		bodySize := c.Writer.Size()
+
+		pm.proxyLogger.Infof("Request %s \"%s %s %s\" %d %d \"%s\" %v",
+			clientIP,
+			method,
+			path,
+			c.Request.Proto,
+			statusCode,
+			bodySize,
+			c.Request.UserAgent(),
+			duration,
+		)
+	})
+
+	// see: issue: #81, #77 and #42 for CORS issues
+	// respond with permissive OPTIONS for any endpoint
+	pm.ginEngine.Use(func(c *gin.Context) {
+		if c.Request.Method == "OPTIONS" {
+			c.Header("Access-Control-Allow-Origin", "*")
+			c.Header("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS")
+
+			// allow whatever the client requested by default
+			if headers := c.Request.Header.Get("Access-Control-Request-Headers"); headers != "" {
+				sanitized := SanitizeAccessControlRequestHeaderValues(headers)
+				c.Header("Access-Control-Allow-Headers", sanitized)
+			} else {
+				c.Header(
+					"Access-Control-Allow-Headers",
+					"Content-Type, Authorization, Accept, X-Requested-With",
+				)
+			}
+			c.Header("Access-Control-Max-Age", "86400")
+			c.AbortWithStatus(http.StatusNoContent)
+			return
+		}
+		c.Next()
+	})
+
+	// Set up routes using the Gin engine
+	pm.ginEngine.POST("/v1/chat/completions", pm.proxyOAIHandler)
+	// Support legacy /v1/completions api, see issue #12
+	pm.ginEngine.POST("/v1/completions", pm.proxyOAIHandler)
+
+	// Support embeddings
+	pm.ginEngine.POST("/v1/embeddings", pm.proxyOAIHandler)
+	pm.ginEngine.POST("/v1/rerank", pm.proxyOAIHandler)
+
+	// Support audio/speech endpoint
+	pm.ginEngine.POST("/v1/audio/speech", pm.proxyOAIHandler)
+	pm.ginEngine.POST("/v1/audio/transcriptions", pm.proxyOAIPostFormHandler)
+
+	pm.ginEngine.GET("/v1/models", pm.listModelsHandler)
+
+	// in proxymanager_loghandlers.go
+	pm.ginEngine.GET("/logs", pm.sendLogsHandlers)
+	pm.ginEngine.GET("/logs/stream", pm.streamLogsHandler)
+	pm.ginEngine.GET("/logs/streamSSE", pm.streamLogsHandlerSSE)
+	pm.ginEngine.GET("/logs/stream/:logMonitorID", pm.streamLogsHandler)
+	pm.ginEngine.GET("/logs/streamSSE/:logMonitorID", pm.streamLogsHandlerSSE)
+
+	pm.ginEngine.GET("/upstream", pm.upstreamIndex)
+	pm.ginEngine.Any("/upstream/:model_id/*upstreamPath", pm.proxyToUpstream)
+
+	pm.ginEngine.GET("/unload", pm.unloadAllModelsHandler)
+
+	pm.ginEngine.GET("/running", pm.listRunningProcessesHandler)
+
+	pm.ginEngine.GET("/", func(c *gin.Context) {
+		// Set the Content-Type header to text/html
+		c.Header("Content-Type", "text/html")
+
+		// Write the embedded HTML content to the response
+		htmlData, err := getHTMLFile("index.html")
+		if err != nil {
+			c.String(http.StatusInternalServerError, err.Error())
+			return
+		}
+		_, err = c.Writer.Write(htmlData)
+		if err != nil {
+			c.String(http.StatusInternalServerError, fmt.Sprintf("failed to write response: %v", err))
+			return
+		}
+	})
+
+	pm.ginEngine.GET("/favicon.ico", func(c *gin.Context) {
+		if data, err := getHTMLFile("favicon.ico"); err == nil {
+			c.Data(http.StatusOK, "image/x-icon", data)
+		} else {
+			c.String(http.StatusInternalServerError, err.Error())
+		}
+	})
+
+	// Disable console color for testing
+	gin.DisableConsoleColor()
+
+	return pm
+}
+
+func (pm *ProxyManager) Run(addr ...string) error {
+	return pm.ginEngine.Run(addr...)
+}
+
+func (pm *ProxyManager) HandlerFunc(w http.ResponseWriter, r *http.Request) {
+	pm.ginEngine.ServeHTTP(w, r)
+}
+
+func (pm *ProxyManager) StopProcesses() {
+	pm.Lock()
+	defer pm.Unlock()
+
+	// stop Processes in parallel
+	var wg sync.WaitGroup
+	for _, processGroup := range pm.processGroups {
+		wg.Add(1)
+		go func(processGroup *ProcessGroup) {
+			defer wg.Done()
+			processGroup.stopProcesses()
+		}(processGroup)
+	}
+
+	wg.Wait()
+}
+
+// Shutdown is called to shutdown all upstream processes
+// when llama-swap is shutting down.
+func (pm *ProxyManager) Shutdown() {
+	pm.Lock()
+	defer pm.Unlock()
+
+	pm.proxyLogger.Debug("Shutdown() called in proxy manager")
+
+	var wg sync.WaitGroup
+	// Send shutdown signal to all process in groups
+	for _, processGroup := range pm.processGroups {
+		wg.Add(1)
+		go func(processGroup *ProcessGroup) {
+			defer wg.Done()
+			processGroup.Shutdown()
+		}(processGroup)
+	}
+	wg.Wait()
+}
+
+func (pm *ProxyManager) swapProcessGroup(requestedModel string) (*ProcessGroup, string, error) {
+	// de-alias the real model name and get a real one
+	realModelName, found := pm.config.RealModelName(requestedModel)
+	if !found {
+		return nil, realModelName, fmt.Errorf("could not find real modelID for %s", requestedModel)
+	}
+
+	processGroup := pm.findGroupByModelName(realModelName)
+	if processGroup == nil {
+		return nil, realModelName, fmt.Errorf("could not find process group for model %s", requestedModel)
+	}
+
+	if processGroup.exclusive {
+		pm.proxyLogger.Debugf("Exclusive mode for group %s, stopping other process groups", processGroup.id)
+		for groupId, otherGroup := range pm.processGroups {
+			if groupId != processGroup.id && !otherGroup.persistent {
+				otherGroup.StopProcesses()
+			}
+		}
+	}
+
+	return processGroup, realModelName, nil
+}
+
+func (pm *ProxyManager) listModelsHandler(c *gin.Context) {
+	data := []interface{}{}
+	for id, modelConfig := range pm.config.Models {
+		if modelConfig.Unlisted {
+			continue
+		}
+
+		data = append(data, map[string]interface{}{
+			"id":       id,
+			"object":   "model",
+			"created":  time.Now().Unix(),
+			"owned_by": "llama-swap",
+		})
+	}
+
+	// Set the Content-Type header to application/json
+	c.Header("Content-Type", "application/json")
+
+	if origin := c.Request.Header.Get("Origin"); origin != "" {
+		c.Header("Access-Control-Allow-Origin", origin)
+	}
+
+	// Encode the data as JSON and write it to the response writer
+	if err := json.NewEncoder(c.Writer).Encode(map[string]interface{}{"data": data}); err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error encoding JSON %s", err.Error()))
+		return
+	}
+}
+
+func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
+	requestedModel := c.Param("model_id")
+
+	if requestedModel == "" {
+		pm.sendErrorResponse(c, http.StatusBadRequest, "model id required in path")
+		return
+	}
+
+	processGroup, _, err := pm.swapProcessGroup(requestedModel)
+	if err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+		return
+	}
+
+	// rewrite the path
+	c.Request.URL.Path = c.Param("upstreamPath")
+	processGroup.ProxyRequest(requestedModel, c.Writer, c.Request)
+}
+
+func (pm *ProxyManager) upstreamIndex(c *gin.Context) {
+	var html strings.Builder
+
+	html.WriteString("<!doctype HTML>\n<html><body><h1>Available Models</h1><ul>")
+
+	// Extract keys and sort them
+	var modelIDs []string
+	for modelID, modelConfig := range pm.config.Models {
+		if modelConfig.Unlisted {
+			continue
+		}
+
+		modelIDs = append(modelIDs, modelID)
+	}
+	sort.Strings(modelIDs)
+
+	// Iterate over sorted keys
+	for _, modelID := range modelIDs {
+		html.WriteString(fmt.Sprintf("<li><a href=\"/upstream/%s\">%s</a></li>", modelID, modelID))
+	}
+	html.WriteString("</ul></body></html>")
+	c.Header("Content-Type", "text/html")
+	c.String(http.StatusOK, html.String())
+}
+
+func (pm *ProxyManager) proxyOAIHandler(c *gin.Context) {
+	bodyBytes, err := io.ReadAll(c.Request.Body)
+	if err != nil {
+		pm.sendErrorResponse(c, http.StatusBadRequest, "could not ready request body")
+		return
+	}
+
+	requestedModel := gjson.GetBytes(bodyBytes, "model").String()
+	if requestedModel == "" {
+		pm.sendErrorResponse(c, http.StatusBadRequest, "missing or invalid 'model' key")
+		return
+	}
+
+	processGroup, realModelName, err := pm.swapProcessGroup(requestedModel)
+	if err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+		return
+	}
+
+	// issue #69 allow custom model names to be sent to upstream
+	useModelName := pm.config.Models[realModelName].UseModelName
+	if useModelName != "" {
+		bodyBytes, err = sjson.SetBytes(bodyBytes, "model", useModelName)
+		if err != nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error rewriting model name in JSON: %s", err.Error()))
+			return
+		}
+	}
+
+	c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
+
+	// dechunk it as we already have all the body bytes see issue #11
+	c.Request.Header.Del("transfer-encoding")
+	c.Request.Header.Add("content-length", strconv.Itoa(len(bodyBytes)))
+
+	if err := processGroup.ProxyRequest(realModelName, c.Writer, c.Request); err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error()))
+		pm.proxyLogger.Errorf("Error Proxying Request for processGroup %s and model %s", processGroup.id, realModelName)
+		return
+	}
+}
+
+func (pm *ProxyManager) proxyOAIPostFormHandler(c *gin.Context) {
+	// Parse multipart form
+	if err := c.Request.ParseMultipartForm(32 << 20); err != nil { // 32MB max memory, larger files go to tmp disk
+		pm.sendErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("error parsing multipart form: %s", err.Error()))
+		return
+	}
+
+	// Get model parameter from the form
+	requestedModel := c.Request.FormValue("model")
+	if requestedModel == "" {
+		pm.sendErrorResponse(c, http.StatusBadRequest, "missing or invalid 'model' parameter in form data")
+		return
+	}
+
+	processGroup, realModelName, err := pm.swapProcessGroup(requestedModel)
+	if err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
+		return
+	}
+
+	// We need to reconstruct the multipart form in any case since the body is consumed
+	// Create a new buffer for the reconstructed request
+	var requestBuffer bytes.Buffer
+	multipartWriter := multipart.NewWriter(&requestBuffer)
+
+	// Copy all form values
+	for key, values := range c.Request.MultipartForm.Value {
+		for _, value := range values {
+			fieldValue := value
+			// If this is the model field and we have a profile, use just the model name
+			if key == "model" {
+				// # issue #69 allow custom model names to be sent to upstream
+				useModelName := pm.config.Models[realModelName].UseModelName
+
+				if useModelName != "" {
+					fieldValue = useModelName
+				} else {
+					fieldValue = requestedModel
+				}
+			}
+			field, err := multipartWriter.CreateFormField(key)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error recreating form field")
+				return
+			}
+			if _, err = field.Write([]byte(fieldValue)); err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error writing form field")
+				return
+			}
+		}
+	}
+
+	// Copy all files from the original request
+	for key, fileHeaders := range c.Request.MultipartForm.File {
+		for _, fileHeader := range fileHeaders {
+			formFile, err := multipartWriter.CreateFormFile(key, fileHeader.Filename)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error recreating form file")
+				return
+			}
+
+			file, err := fileHeader.Open()
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error opening uploaded file")
+				return
+			}
+
+			if _, err = io.Copy(formFile, file); err != nil {
+				file.Close()
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error copying file data")
+				return
+			}
+			file.Close()
+		}
+	}
+
+	// Close the multipart writer to finalize the form
+	if err := multipartWriter.Close(); err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, "error finalizing multipart form")
+		return
+	}
+
+	// Create a new request with the reconstructed form data
+	modifiedReq, err := http.NewRequestWithContext(
+		c.Request.Context(),
+		c.Request.Method,
+		c.Request.URL.String(),
+		&requestBuffer,
+	)
+	if err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, "error creating modified request")
+		return
+	}
+
+	// Copy the headers from the original request
+	modifiedReq.Header = c.Request.Header.Clone()
+	modifiedReq.Header.Set("Content-Type", multipartWriter.FormDataContentType())
+
+	// set the content length of the body
+	modifiedReq.Header.Set("Content-Length", strconv.Itoa(requestBuffer.Len()))
+	modifiedReq.ContentLength = int64(requestBuffer.Len())
+
+	// Use the modified request for proxying
+	if err := processGroup.ProxyRequest(realModelName, c.Writer, modifiedReq); err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error()))
+		pm.proxyLogger.Errorf("Error Proxying Request for processGroup %s and model %s", processGroup.id, realModelName)
+		return
+	}
+}
+
+func (pm *ProxyManager) sendErrorResponse(c *gin.Context, statusCode int, message string) {
+	acceptHeader := c.GetHeader("Accept")
+
+	if strings.Contains(acceptHeader, "application/json") {
+		c.JSON(statusCode, gin.H{"error": message})
+	} else {
+		c.String(statusCode, message)
+	}
+}
+
+func (pm *ProxyManager) unloadAllModelsHandler(c *gin.Context) {
+	pm.StopProcesses()
+	c.String(http.StatusOK, "OK")
+}
+
+func (pm *ProxyManager) listRunningProcessesHandler(context *gin.Context) {
+	context.Header("Content-Type", "application/json")
+	runningProcesses := make([]gin.H, 0) // Default to an empty response.
+
+	for _, processGroup := range pm.processGroups {
+		for _, process := range processGroup.processes {
+			if process.CurrentState() == StateReady {
+				runningProcesses = append(runningProcesses, gin.H{
+					"model": process.ID,
+					"state": process.state,
+				})
+			}
+		}
+	}
+
+	// Put the results under the `running` key.
+	response := gin.H{
+		"running": runningProcesses,
+	}
+
+	context.JSON(http.StatusOK, response) // Always return 200 OK
+}
+
+func (pm *ProxyManager) findGroupByModelName(modelName string) *ProcessGroup {
+	for _, group := range pm.processGroups {
+		if group.HasMember(modelName) {
+			return group
+		}
+	}
+	return nil
+}

proxy/proxymanager_loghandlers.go

@@ -0,0 +1,142 @@
+package proxy
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+func (pm *ProxyManager) sendLogsHandlers(c *gin.Context) {
+	accept := c.GetHeader("Accept")
+	if strings.Contains(accept, "text/html") {
+		// Set the Content-Type header to text/html
+		c.Header("Content-Type", "text/html")
+
+		// Write the embedded HTML content to the response
+		logsHTML, err := getHTMLFile("logs.html")
+		if err != nil {
+			c.String(http.StatusInternalServerError, err.Error())
+			return
+		}
+		_, err = c.Writer.Write(logsHTML)
+		if err != nil {
+			c.String(http.StatusInternalServerError, fmt.Sprintf("failed to write response: %v", err))
+			return
+		}
+	} else {
+		c.Header("Content-Type", "text/plain")
+		history := pm.muxLogger.GetHistory()
+		_, err := c.Writer.Write(history)
+		if err != nil {
+			c.AbortWithError(http.StatusInternalServerError, err)
+			return
+		}
+	}
+}
+
+func (pm *ProxyManager) streamLogsHandler(c *gin.Context) {
+	c.Header("Content-Type", "text/plain")
+	c.Header("Transfer-Encoding", "chunked")
+	c.Header("X-Content-Type-Options", "nosniff")
+
+	logMonitorId := c.Param("logMonitorID")
+	logger, err := pm.getLogger(logMonitorId)
+	if err != nil {
+		c.String(http.StatusBadRequest, err.Error())
+		return
+	}
+	ch := logger.Subscribe()
+	defer logger.Unsubscribe(ch)
+
+	notify := c.Request.Context().Done()
+	flusher, ok := c.Writer.(http.Flusher)
+	if !ok {
+		c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("streaming unsupported"))
+		return
+	}
+
+	_, skipHistory := c.GetQuery("no-history")
+	// Send history first if not skipped
+
+	if !skipHistory {
+		history := logger.GetHistory()
+		if len(history) != 0 {
+			c.Writer.Write(history)
+			flusher.Flush()
+		}
+	}
+
+	// Stream new logs
+	for {
+		select {
+		case msg := <-ch:
+			_, err := c.Writer.Write(msg)
+			if err != nil {
+				// just break the loop if we can't write for some reason
+				return
+			}
+			flusher.Flush()
+		case <-notify:
+			return
+		}
+	}
+}
+
+func (pm *ProxyManager) streamLogsHandlerSSE(c *gin.Context) {
+	c.Header("Content-Type", "text/event-stream")
+	c.Header("Cache-Control", "no-cache")
+	c.Header("Connection", "keep-alive")
+	c.Header("X-Content-Type-Options", "nosniff")
+
+	logMonitorId := c.Param("logMonitorID")
+	logger, err := pm.getLogger(logMonitorId)
+	if err != nil {
+		c.String(http.StatusBadRequest, err.Error())
+		return
+	}
+	ch := logger.Subscribe()
+	defer logger.Unsubscribe(ch)
+
+	notify := c.Request.Context().Done()
+
+	// Send history first if not skipped
+	_, skipHistory := c.GetQuery("no-history")
+	if !skipHistory {
+		history := logger.GetHistory()
+		if len(history) != 0 {
+			c.SSEvent("message", string(history))
+			c.Writer.Flush()
+		}
+	}
+
+	// Stream new logs
+	for {
+		select {
+		case msg := <-ch:
+			c.SSEvent("message", string(msg))
+			c.Writer.Flush()
+		case <-notify:
+			return
+		}
+	}
+}
+
+// getLogger searches for the appropriate logger based on the logMonitorId
+func (pm *ProxyManager) getLogger(logMonitorId string) (*LogMonitor, error) {
+	var logger *LogMonitor
+
+	if logMonitorId == "" {
+		// maintain the default
+		logger = pm.muxLogger
+	} else if logMonitorId == "proxy" {
+		logger = pm.proxyLogger
+	} else if logMonitorId == "upstream" {
+		logger = pm.upstreamLogger
+	} else {
+		return nil, fmt.Errorf("invalid logger. Use 'proxy' or 'upstream'")
+	}
+
+	return logger, nil
+}

proxy/proxymanager_test.go

@@ -0,0 +1,622 @@
+package proxy
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProxyManager_SwapProcessCorrectly(t *testing.T) {
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	for _, modelName := range []string{"model1", "model2"} {
+		reqBody := fmt.Sprintf(`{"model":"%s"}`, modelName)
+		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+		w := httptest.NewRecorder()
+
+		proxy.HandlerFunc(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Contains(t, w.Body.String(), modelName)
+	}
+}
+
+func TestProxyManager_SwapMultiProcess(t *testing.T) {
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+		},
+		LogLevel: "error",
+		Groups: map[string]GroupConfig{
+			"G1": {
+				Swap:      true,
+				Exclusive: false,
+				Members:   []string{"model1"},
+			},
+			"G2": {
+				Swap:      true,
+				Exclusive: false,
+				Members:   []string{"model2"},
+			},
+		},
+	})
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	tests := []string{"model1", "model2"}
+	for _, requestedModel := range tests {
+		t.Run(requestedModel, func(t *testing.T) {
+			reqBody := fmt.Sprintf(`{"model":"%s"}`, requestedModel)
+			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+			w := httptest.NewRecorder()
+
+			proxy.HandlerFunc(w, req)
+			assert.Equal(t, http.StatusOK, w.Code)
+			assert.Contains(t, w.Body.String(), requestedModel)
+
+		})
+	}
+
+	// make sure there's two loaded models
+	assert.Equal(t, proxy.findGroupByModelName("model1").processes["model1"].CurrentState(), StateReady)
+	assert.Equal(t, proxy.findGroupByModelName("model2").processes["model2"].CurrentState(), StateReady)
+}
+
+// Test that a persistent group is not affected by the swapping behaviour of
+// other groups.
+func TestProxyManager_PersistentGroupsAreNotSwapped(t *testing.T) {
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"), // goes into the default group
+			"model2": getTestSimpleResponderConfig("model2"),
+		},
+		LogLevel: "error",
+		Groups: map[string]GroupConfig{
+			// the forever group is persistent and should not be affected by model1
+			"forever": {
+				Swap:       true,
+				Exclusive:  false,
+				Persistent: true,
+				Members:    []string{"model2"},
+			},
+		},
+	})
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	// make requests to load all models, loading model1 should not affect model2
+	tests := []string{"model2", "model1"}
+	for _, requestedModel := range tests {
+		reqBody := fmt.Sprintf(`{"model":"%s"}`, requestedModel)
+		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+		w := httptest.NewRecorder()
+
+		proxy.HandlerFunc(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Contains(t, w.Body.String(), requestedModel)
+	}
+
+	assert.Equal(t, proxy.findGroupByModelName("model2").processes["model2"].CurrentState(), StateReady)
+	assert.Equal(t, proxy.findGroupByModelName("model1").processes["model1"].CurrentState(), StateReady)
+}
+
+// When a request for a different model comes in ProxyManager should wait until
+// the first request is complete before swapping. Both requests should complete
+func TestProxyManager_SwapMultiProcessParallelRequests(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping slow test")
+	}
+
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+			"model3": getTestSimpleResponderConfig("model3"),
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	results := map[string]string{}
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+
+	for key := range config.Models {
+		wg.Add(1)
+		go func(key string) {
+			defer wg.Done()
+
+			reqBody := fmt.Sprintf(`{"model":"%s"}`, key)
+			req := httptest.NewRequest("POST", "/v1/chat/completions?wait=1000ms", bytes.NewBufferString(reqBody))
+			w := httptest.NewRecorder()
+
+			proxy.HandlerFunc(w, req)
+
+			if w.Code != http.StatusOK {
+				t.Errorf("Expected status OK, got %d for key %s", w.Code, key)
+			}
+
+			mu.Lock()
+
+			var response map[string]string
+			assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+			results[key] = response["responseMessage"]
+			mu.Unlock()
+		}(key)
+
+		<-time.After(time.Millisecond)
+	}
+
+	wg.Wait()
+	assert.Len(t, results, len(config.Models))
+
+	for key, result := range results {
+		assert.Equal(t, key, result)
+	}
+}
+
+func TestProxyManager_ListModelsHandler(t *testing.T) {
+	config := Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+			"model3": getTestSimpleResponderConfig("model3"),
+		},
+		LogLevel: "error",
+	}
+
+	proxy := New(config)
+
+	// Create a test request
+	req := httptest.NewRequest("GET", "/v1/models", nil)
+	req.Header.Add("Origin", "i-am-the-origin")
+	w := httptest.NewRecorder()
+
+	// Call the listModelsHandler
+	proxy.HandlerFunc(w, req)
+
+	// Check the response status code
+	assert.Equal(t, http.StatusOK, w.Code)
+
+	// Check for Access-Control-Allow-Origin
+	assert.Equal(t, req.Header.Get("Origin"), w.Result().Header.Get("Access-Control-Allow-Origin"))
+
+	// Parse the JSON response
+	var response struct {
+		Data []map[string]interface{} `json:"data"`
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+		t.Fatalf("Failed to parse JSON response: %v", err)
+	}
+
+	// Check the number of models returned
+	assert.Len(t, response.Data, 3)
+
+	// Check the details of each model
+	expectedModels := map[string]struct{}{
+		"model1": {},
+		"model2": {},
+		"model3": {},
+	}
+
+	for _, model := range response.Data {
+		modelID, ok := model["id"].(string)
+		assert.True(t, ok, "model ID should be a string")
+		_, exists := expectedModels[modelID]
+		assert.True(t, exists, "unexpected model ID: %s", modelID)
+		delete(expectedModels, modelID)
+
+		object, ok := model["object"].(string)
+		assert.True(t, ok, "object should be a string")
+		assert.Equal(t, "model", object)
+
+		created, ok := model["created"].(float64)
+		assert.True(t, ok, "created should be a number")
+		assert.Greater(t, created, float64(0)) // Assuming the timestamp is positive
+
+		ownedBy, ok := model["owned_by"].(string)
+		assert.True(t, ok, "owned_by should be a string")
+		assert.Equal(t, "llama-swap", ownedBy)
+	}
+
+	// Ensure all expected models were returned
+	assert.Empty(t, expectedModels, "not all expected models were returned")
+}
+
+func TestProxyManager_Shutdown(t *testing.T) {
+	// make broken model configurations
+	model1Config := getTestSimpleResponderConfigPort("model1", 9991)
+	model1Config.Proxy = "http://localhost:10001/"
+
+	model2Config := getTestSimpleResponderConfigPort("model2", 9992)
+	model2Config.Proxy = "http://localhost:10002/"
+
+	model3Config := getTestSimpleResponderConfigPort("model3", 9993)
+	model3Config.Proxy = "http://localhost:10003/"
+
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": model1Config,
+			"model2": model2Config,
+			"model3": model3Config,
+		},
+		LogLevel: "error",
+		Groups: map[string]GroupConfig{
+			"test": {
+				Swap:    false,
+				Members: []string{"model1", "model2", "model3"},
+			},
+		},
+	})
+
+	proxy := New(config)
+
+	// Start all the processes
+	var wg sync.WaitGroup
+	for _, modelName := range []string{"model1", "model2", "model3"} {
+		wg.Add(1)
+		go func(modelName string) {
+			defer wg.Done()
+			reqBody := fmt.Sprintf(`{"model":"%s"}`, modelName)
+			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+			w := httptest.NewRecorder()
+
+			// send a request to trigger the proxy to load ... this should hang waiting for start up
+			proxy.HandlerFunc(w, req)
+			assert.Equal(t, http.StatusBadGateway, w.Code)
+			assert.Contains(t, w.Body.String(), "health check interrupted due to shutdown")
+		}(modelName)
+	}
+
+	go func() {
+		<-time.After(time.Second)
+		proxy.Shutdown()
+	}()
+	wg.Wait()
+}
+
+func TestProxyManager_Unload(t *testing.T) {
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(config)
+	reqBody := fmt.Sprintf(`{"model":"%s"}`, "model1")
+	req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+	w := httptest.NewRecorder()
+	proxy.HandlerFunc(w, req)
+
+	assert.Equal(t, proxy.processGroups[DEFAULT_GROUP_ID].processes["model1"].CurrentState(), StateReady)
+	req = httptest.NewRequest("GET", "/unload", nil)
+	w = httptest.NewRecorder()
+	proxy.HandlerFunc(w, req)
+	assert.Equal(t, http.StatusOK, w.Code)
+	assert.Equal(t, w.Body.String(), "OK")
+
+	// give it a bit of time to stop
+	<-time.After(time.Millisecond * 250)
+	assert.Equal(t, proxy.processGroups[DEFAULT_GROUP_ID].processes["model1"].CurrentState(), StateStopped)
+}
+
+// Test issue #61 `Listing the current list of models and the loaded model.`
+func TestProxyManager_RunningEndpoint(t *testing.T) {
+
+	// Shared configuration
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+		},
+		LogLevel: "debug",
+	})
+
+	// Define a helper struct to parse the JSON response.
+	type RunningResponse struct {
+		Running []struct {
+			Model string `json:"model"`
+			State string `json:"state"`
+		} `json:"running"`
+	}
+
+	// Create proxy once for all tests
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	t.Run("no models loaded", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/running", nil)
+		w := httptest.NewRecorder()
+		proxy.HandlerFunc(w, req)
+
+		assert.Equal(t, http.StatusOK, w.Code)
+
+		var response RunningResponse
+
+		// Check if this is a valid JSON object.
+		assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+
+		// We should have an empty running array here.
+		assert.Empty(t, response.Running, "expected no running models")
+	})
+
+	t.Run("single model loaded", func(t *testing.T) {
+		// Load just a model.
+		reqBody := `{"model":"model1"}`
+		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+		w := httptest.NewRecorder()
+		proxy.HandlerFunc(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+
+		// Simulate browser call for the `/running` endpoint.
+		req = httptest.NewRequest("GET", "/running", nil)
+		w = httptest.NewRecorder()
+		proxy.HandlerFunc(w, req)
+
+		var response RunningResponse
+		assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+
+		// Check if we have a single array element.
+		assert.Len(t, response.Running, 1)
+
+		// Is this the right model?
+		assert.Equal(t, "model1", response.Running[0].Model)
+
+		// Is the model loaded?
+		assert.Equal(t, "ready", response.Running[0].State)
+	})
+}
+
+func TestProxyManager_AudioTranscriptionHandler(t *testing.T) {
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"TheExpectedModel": getTestSimpleResponderConfig("TheExpectedModel"),
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	// Create a buffer with multipart form data
+	var b bytes.Buffer
+	w := multipart.NewWriter(&b)
+
+	// Add the model field
+	fw, err := w.CreateFormField("model")
+	assert.NoError(t, err)
+	_, err = fw.Write([]byte("TheExpectedModel"))
+	assert.NoError(t, err)
+
+	// Add a file field
+	fw, err = w.CreateFormFile("file", "test.mp3")
+	assert.NoError(t, err)
+	// Generate random content length between 10 and 20
+	contentLength := rand.Intn(11) + 10 // 10 to 20
+	content := make([]byte, contentLength)
+	_, err = fw.Write(content)
+	assert.NoError(t, err)
+	w.Close()
+
+	// Create the request with the multipart form data
+	req := httptest.NewRequest("POST", "/v1/audio/transcriptions", &b)
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	rec := httptest.NewRecorder()
+	proxy.HandlerFunc(rec, req)
+
+	// Verify the response
+	assert.Equal(t, http.StatusOK, rec.Code)
+	var response map[string]string
+	err = json.Unmarshal(rec.Body.Bytes(), &response)
+	assert.NoError(t, err)
+	assert.Equal(t, "TheExpectedModel", response["model"])
+	assert.Equal(t, response["text"], fmt.Sprintf("The length of the file is %d bytes", contentLength)) // matches simple-responder
+	assert.Equal(t, strconv.Itoa(370+contentLength), response["h_content_length"])
+}
+
+// Test useModelName in configuration sends overrides what is sent to upstream
+func TestProxyManager_UseModelName(t *testing.T) {
+	upstreamModelName := "upstreamModel"
+
+	modelConfig := getTestSimpleResponderConfig(upstreamModelName)
+	modelConfig.UseModelName = upstreamModelName
+
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": modelConfig,
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	requestedModel := "model1"
+
+	t.Run("useModelName over rides requested model: /v1/chat/completions", func(t *testing.T) {
+		reqBody := fmt.Sprintf(`{"model":"%s"}`, requestedModel)
+		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+		w := httptest.NewRecorder()
+
+		proxy.HandlerFunc(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+		assert.Contains(t, w.Body.String(), upstreamModelName)
+	})
+
+	t.Run("useModelName over rides requested model: /v1/audio/transcriptions", func(t *testing.T) {
+		// Create a buffer with multipart form data
+		var b bytes.Buffer
+		w := multipart.NewWriter(&b)
+
+		// Add the model field
+		fw, err := w.CreateFormField("model")
+		assert.NoError(t, err)
+		_, err = fw.Write([]byte(requestedModel))
+		assert.NoError(t, err)
+
+		// Add a file field
+		fw, err = w.CreateFormFile("file", "test.mp3")
+		assert.NoError(t, err)
+		_, err = fw.Write([]byte("test"))
+		assert.NoError(t, err)
+		w.Close()
+
+		// Create the request with the multipart form data
+		req := httptest.NewRequest("POST", "/v1/audio/transcriptions", &b)
+		req.Header.Set("Content-Type", w.FormDataContentType())
+		rec := httptest.NewRecorder()
+		proxy.HandlerFunc(rec, req)
+
+		// Verify the response
+		assert.Equal(t, http.StatusOK, rec.Code)
+		var response map[string]string
+		err = json.Unmarshal(rec.Body.Bytes(), &response)
+		assert.NoError(t, err)
+		assert.Equal(t, upstreamModelName, response["model"])
+	})
+}
+
+func TestProxyManager_CORSOptionsHandler(t *testing.T) {
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+		},
+		LogLevel: "error",
+	})
+
+	tests := []struct {
+		name            string
+		method          string
+		requestHeaders  map[string]string
+		expectedStatus  int
+		expectedHeaders map[string]string
+	}{
+		{
+			name:           "OPTIONS with no headers",
+			method:         "OPTIONS",
+			expectedStatus: http.StatusNoContent,
+			expectedHeaders: map[string]string{
+				"Access-Control-Allow-Origin":  "*",
+				"Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
+				"Access-Control-Allow-Headers": "Content-Type, Authorization, Accept, X-Requested-With",
+			},
+		},
+		{
+			name:   "OPTIONS with specific headers",
+			method: "OPTIONS",
+			requestHeaders: map[string]string{
+				"Access-Control-Request-Headers": "X-Custom-Header, Some-Other-Header",
+			},
+			expectedStatus: http.StatusNoContent,
+			expectedHeaders: map[string]string{
+				"Access-Control-Allow-Origin":  "*",
+				"Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
+				"Access-Control-Allow-Headers": "X-Custom-Header, Some-Other-Header",
+			},
+		},
+		{
+			name:           "Non-OPTIONS request",
+			method:         "GET",
+			expectedStatus: http.StatusNotFound, // Since we don't have a GET route defined
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			proxy := New(config)
+			defer proxy.StopProcesses()
+
+			req := httptest.NewRequest(tt.method, "/v1/chat/completions", nil)
+			for k, v := range tt.requestHeaders {
+				req.Header.Set(k, v)
+			}
+
+			w := httptest.NewRecorder()
+			proxy.ginEngine.ServeHTTP(w, req)
+
+			assert.Equal(t, tt.expectedStatus, w.Code)
+
+			for header, expectedValue := range tt.expectedHeaders {
+				assert.Equal(t, expectedValue, w.Header().Get(header))
+			}
+		})
+	}
+}
+
+func TestProxyManager_Upstream(t *testing.T) {
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+	req := httptest.NewRequest("GET", "/upstream/model1/test", nil)
+	rec := httptest.NewRecorder()
+	proxy.HandlerFunc(rec, req)
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, "model1", rec.Body.String())
+}
+
+func TestProxyManager_ChatContentLength(t *testing.T) {
+	config := AddDefaultGroupToConfig(Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+		},
+		LogLevel: "error",
+	})
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	reqBody := fmt.Sprintf(`{"model":"%s", "x": "this is just some content to push the length out a bit"}`, "model1")
+	req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+	w := httptest.NewRecorder()
+
+	proxy.HandlerFunc(w, req)
+	assert.Equal(t, http.StatusOK, w.Code)
+	var response map[string]string
+	assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+	assert.Equal(t, "81", response["h_content_length"])
+	assert.Equal(t, "model1", response["responseMessage"])
+}

proxy/sanitize_cors.go

@@ -0,0 +1,43 @@
+package proxy
+
+import (
+	"strings"
+)
+
+func isTokenChar(r rune) bool {
+	switch {
+	case r >= 'a' && r <= 'z':
+	case r >= 'A' && r <= 'Z':
+	case r >= '0' && r <= '9':
+	case strings.ContainsRune("!#$%&'*+-.^_`|~", r):
+	default:
+		return false
+	}
+	return true
+}
+
+func SanitizeAccessControlRequestHeaderValues(headerValues string) string {
+	parts := strings.Split(headerValues, ",")
+	valid := make([]string, 0, len(parts))
+
+	for _, p := range parts {
+		v := strings.TrimSpace(p)
+		if v == "" {
+			continue
+		}
+
+		validPart := true
+		for _, c := range v {
+			if !isTokenChar(c) {
+				validPart = false
+				break
+			}
+		}
+
+		if validPart {
+			valid = append(valid, v)
+		}
+	}
+
+	return strings.Join(valid, ", ")
+}

proxy/sanitize_cors_test.go

@@ -0,0 +1,77 @@
+package proxy
+
+import "testing"
+
+func TestSanitizeAccessControlRequestHeaderValues(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "empty string",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "whitespace only",
+			input:    "   ",
+			expected: "",
+		},
+		{
+			name:     "single valid value",
+			input:    "content-type",
+			expected: "content-type",
+		},
+		{
+			name:     "multiple valid values",
+			input:    "content-type, authorization, x-requested-with",
+			expected: "content-type, authorization, x-requested-with",
+		},
+		{
+			name:     "values with extra spaces",
+			input:    "  content-type  ,  authorization  ",
+			expected: "content-type, authorization",
+		},
+		{
+			name:     "values with tabs",
+			input:    "content-type,\tauthorization",
+			expected: "content-type, authorization",
+		},
+		{
+			name:     "values with invalid characters",
+			input:    "content-type, auth\n, x-requested-with\r",
+			expected: "content-type, auth, x-requested-with",
+		},
+		{
+			name:     "empty values in list",
+			input:    "content-type,,authorization",
+			expected: "content-type, authorization",
+		},
+		{
+			name:     "leading and trailing commas",
+			input:    ",content-type,authorization,",
+			expected: "content-type, authorization",
+		},
+		{
+			name:     "mixed valid and invalid values",
+			input:    "content-type, \x00invalid, x-requested-with",
+			expected: "content-type, x-requested-with",
+		},
+		{
+			name:     "mixed case values",
+			input:    "Content-Type, my-Valid-Header, Another-hEader",
+			expected: "Content-Type, my-Valid-Header, Another-hEader",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := SanitizeAccessControlRequestHeaderValues(tt.input)
+			if got != tt.expected {
+				t.Errorf("SanitizeAccessControlRequestHeaderValues(%q) = %q, want %q",
+					tt.input, got, tt.expected)
+			}
+		})
+	}
+}